blob: 74d658e854d505c35e4237026fca16907ef660ed [file] [log] [blame]
// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package lib
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"time"
"github.com/maruel/subcommands"
"go.chromium.org/luci/auth"
"go.chromium.org/luci/client/archiver"
"go.chromium.org/luci/client/cas"
"go.chromium.org/luci/client/isolate"
"go.chromium.org/luci/common/data/text/units"
"go.chromium.org/luci/common/errors"
"go.chromium.org/luci/common/isolated"
"go.chromium.org/luci/common/system/signals"
)
// CmdArchive returns an object for the `archive` subcommand.
func CmdArchive(defaultAuthOpts auth.Options) *subcommands.Command {
return &subcommands.Command{
UsageLine: "archive <options>",
ShortDesc: "parses a .isolate file to create a .isolated file, and uploads it and all referenced files to an isolate server",
LongDesc: "All the files listed in the .isolated file are put in the isolate server cache. Small files are combined together in a tar archive before uploading.",
CommandRun: func() subcommands.CommandRun {
c := archiveRun{}
c.commonServerFlags.Init(defaultAuthOpts)
c.isolateFlags.Init(&c.Flags)
c.loggingFlags.Init(&c.Flags)
c.casFlags.Init(&c.Flags)
c.Flags.StringVar(&c.Isolated, "isolated", "", ".isolated file to generate")
c.Flags.StringVar(&c.Isolated, "s", "", "Alias for --isolated")
c.Flags.IntVar(&c.maxConcurrentChecks, "max-concurrent-checks", 1, "The maximum number of in-flight check requests.")
c.Flags.IntVar(&c.maxConcurrentUploads, "max-concurrent-uploads", 8, "The maximum number of in-flight uploads.")
c.Flags.StringVar(&c.dumpJSON, "dump-json", "",
"Write isolated digests of archived trees to this file as JSON")
return &c
},
}
}
type archiveRun struct {
commonServerFlags
isolateFlags
loggingFlags loggingFlags
casFlags cas.Flags
maxConcurrentChecks int
maxConcurrentUploads int
dumpJSON string
}
func (c *archiveRun) Parse(a subcommands.Application, args []string) error {
if err := c.commonServerFlags.Parse(); err != nil {
return err
}
cwd, err := os.Getwd()
if err != nil {
return err
}
if err := c.isolateFlags.Parse(cwd, RequireIsolateFile); err != nil {
return err
}
if err := c.casFlags.Parse(); err != nil {
return err
}
if len(args) != 0 {
return errors.Reason("position arguments not expected").Err()
}
return nil
}
func (c *archiveRun) main(a subcommands.Application, args []string) error {
start := time.Now()
ctx, cancel := context.WithCancel(c.defaultFlags.MakeLoggingContext(os.Stderr))
signals.HandleInterrupt(cancel)
al := &archiveLogger{
start: start,
quiet: c.defaultFlags.Quiet,
}
if c.casFlags.Instance != "" {
roots, err := uploadToCAS(ctx, c.dumpJSON, c.commonServerFlags.parsedAuthOpts, &c.casFlags, al, &c.ArchiveOptions)
if err != nil {
return err
}
al.Printf("uploaded digest: %s\n", roots[0])
return nil
}
return c.archiveToIsolate(ctx, al)
}
// archiveToIsolate performs the archiveToIsolate operation for an isolate specified by opts.
// dumpJSON is the path to write a JSON summary of the uploaded isolate, in the same format as batch_archive.
func (c *archiveRun) archiveToIsolate(ctx context.Context, al *archiveLogger) error {
authCl, err := c.createAuthClient(ctx)
if err != nil {
return err
}
client, err := c.createIsolatedClient(authCl)
if err != nil {
return err
}
opts := &c.ArchiveOptions
// Parse the incoming isolate file.
deps, rootDir, isol, err := isolate.ProcessIsolate(opts)
if err != nil {
return errors.Annotate(err, "isolate %s: failed to process", opts.Isolate).Err()
}
log.Printf("Isolate %s referenced %d deps", opts.Isolate, len(deps))
// Set up a checker and uploader.
checker := archiver.NewChecker(ctx, client, c.maxConcurrentChecks)
uploader := archiver.NewUploader(ctx, client, c.maxConcurrentUploads)
arc := archiver.NewTarringArchiver(checker, uploader)
isolSummary, err := arc.Archive(&archiver.TarringArgs{
Deps: deps,
RootDir: rootDir,
IgnoredPathRe: opts.IgnoredPathFilterRe,
Isolated: opts.Isolated,
Isol: isol})
if err != nil {
return errors.Annotate(err, "isolate %s", opts.Isolate).Err()
}
// Make sure that all pending items have been checked.
if err := checker.Close(); err != nil {
return err
}
// Make sure that all the uploads have completed successfully.
if err := uploader.Close(); err != nil {
return err
}
al.printSummary(isolSummary)
if err := dumpSummaryJSON(c.dumpJSON, isolSummary); err != nil {
return err
}
al.LogSummary(ctx, int64(checker.Hit.Count()), int64(checker.Miss.Count()), units.Size(checker.Hit.Bytes()), units.Size(checker.Miss.Bytes()), []string{string(isolSummary.Digest)})
return nil
}
func (c *archiveRun) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
if err := c.Parse(a, args); err != nil {
fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
return 1
}
if err := c.main(a, args); err != nil {
fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
return 1
}
return 0
}
func dumpSummaryJSON(filename string, summaries ...archiver.IsolatedSummary) error {
if len(filename) == 0 {
return nil
}
f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer f.Close()
m := map[string]isolated.HexDigest{}
for _, summary := range summaries {
m[summary.Name] = summary.Digest
}
return json.NewEncoder(f).Encode(m)
}