isolate: pull exparchive and archive into functions with similar signatures.
BUG=692940
Review-Url: https://codereview.chromium.org/2989173002
diff --git a/client/cmd/isolate/archive.go b/client/cmd/isolate/archive.go
index 77b8cf8..c9d2b5d 100644
--- a/client/cmd/isolate/archive.go
+++ b/client/cmd/isolate/archive.go
@@ -15,6 +15,7 @@
package main
import (
+ "context"
"errors"
"fmt"
"log"
@@ -73,39 +74,57 @@
}
func (c *archiveRun) main(a subcommands.Application, args []string) error {
- out := os.Stdout
- prefix := "\n"
- if c.defaultFlags.Quiet {
- prefix = ""
- }
start := time.Now()
- client, err := c.createAuthClient()
+ authCl, err := c.createAuthClient()
if err != nil {
return err
}
ctx := c.defaultFlags.MakeLoggingContext(os.Stderr)
- arch := archiver.New(ctx, isolatedclient.New(nil, client, c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace, nil, nil), out)
+ client := isolatedclient.New(nil, authCl, c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace, nil, nil)
+
+ eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint)
+
+ archiveDetails, err := doArchive(ctx, client, &c.ArchiveOptions, c.defaultFlags.Quiet, start)
+ if err != nil {
+ return err
+ }
+
+ end := time.Now()
+ op := logpb.IsolateClientEvent_LEGACY_ARCHIVE.Enum()
+ if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err != nil {
+ log.Printf("Failed to log to eventlog: %v", err)
+ }
+ return nil
+}
+
+// doArchive performs the archive operation for an isolate specified by archiveOpts.
+func doArchive(ctx context.Context, client *isolatedclient.Client, archiveOpts *isolate.ArchiveOptions, quiet bool, start time.Time) (*logpb.IsolateClientEvent_ArchiveDetails, error) {
+ prefix := "\n"
+ if quiet {
+ prefix = ""
+ }
+
+ arch := archiver.New(ctx, client, os.Stdout)
CancelOnCtrlC(arch)
- item := isolate.Archive(arch, &c.ArchiveOptions)
+ item := isolate.Archive(arch, archiveOpts)
item.WaitForHashed()
+ var err error
if err = item.Error(); err != nil {
- fmt.Printf("%s%s %s\n", prefix, filepath.Base(c.Isolate), err)
+ fmt.Printf("%s%s %s\n", prefix, filepath.Base(archiveOpts.Isolate), err)
} else {
- fmt.Printf("%s%s %s\n", prefix, item.Digest(), filepath.Base(c.Isolate))
+ fmt.Printf("%s%s %s\n", prefix, item.Digest(), filepath.Base(archiveOpts.Isolate))
}
if err2 := arch.Close(); err == nil {
err = err2
}
stats := arch.Stats()
- if !c.defaultFlags.Quiet {
+ if !quiet {
duration := time.Since(start)
fmt.Fprintf(os.Stderr, "Hits : %5d (%s)\n", stats.TotalHits(), stats.TotalBytesHits())
fmt.Fprintf(os.Stderr, "Misses : %5d (%s)\n", stats.TotalMisses(), stats.TotalBytesPushed())
fmt.Fprintf(os.Stderr, "Duration: %s\n", units.Round(duration, time.Millisecond))
}
- end := time.Now()
-
archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{
HitCount: proto.Int64(int64(stats.TotalHits())),
MissCount: proto.Int64(int64(stats.TotalMisses())),
@@ -115,12 +134,7 @@
if item.Error() != nil {
archiveDetails.IsolateHash = []string{string(item.Digest())}
}
- eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint)
- op := logpb.IsolateClientEvent_LEGACY_ARCHIVE.Enum()
- if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err != nil {
- log.Printf("Failed to log to eventlog: %v", err)
- }
- return err
+ return archiveDetails, nil
}
func (c *archiveRun) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
diff --git a/client/cmd/isolate/exp_archive.go b/client/cmd/isolate/exp_archive.go
index 78cbc2f..7b907f4 100644
--- a/client/cmd/isolate/exp_archive.go
+++ b/client/cmd/isolate/exp_archive.go
@@ -27,6 +27,7 @@
"github.com/maruel/subcommands"
"golang.org/x/net/context"
+ "github.com/luci/luci-go/client/isolate"
"github.com/luci/luci-go/common/auth"
logpb "github.com/luci/luci-go/common/eventlog/proto"
"github.com/luci/luci-go/common/isolated"
@@ -89,6 +90,25 @@
}
client := isolatedclient.New(nil, authCl, c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace, nil, nil)
+ eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint)
+
+ archiveDetails, err := doExpArchive(ctx, client, archiveOpts, c.dumpJSON)
+ if err != nil {
+ return err
+ }
+ end := time.Now()
+
+ op := logpb.IsolateClientEvent_ARCHIVE.Enum()
+ if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err != nil {
+ log.Printf("Failed to log to eventlog: %v", err)
+ }
+
+ return nil
+}
+
+// doExparchive performs the exparchive operation for an isolate specified by archiveOpts.
+// dumpJSON is the path to write a JSON summary of the uploaded isolate, in the same format as batch_archive.
+func doExpArchive(ctx context.Context, client *isolatedclient.Client, archiveOpts *isolate.ArchiveOptions, dumpJSON string) (*logpb.IsolateClientEvent_ArchiveDetails, error) {
// Set up a checker and uploader. We limit the uploader to one concurrent
// upload, since the uploads are all coming from disk (with the exception of
// the isolated JSON itself) and we only want a single goroutine reading from
@@ -98,29 +118,30 @@
archiver := NewTarringArchiver(checker, uploader)
isolSummary, err := archiver.Archive(archiveOpts)
+ if err != nil {
+ return nil, err
+ }
// Make sure that all pending items have been checked.
if err := checker.Close(); err != nil {
- return err
+ return nil, err
}
// Make sure that all the uploads have completed successfully.
if err := uploader.Close(); err != nil {
- return err
+ return nil, err
}
printSummary(isolSummary)
- if c.dumpJSON != "" {
- f, err := os.OpenFile(c.dumpJSON, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+ if dumpJSON != "" {
+ f, err := os.OpenFile(dumpJSON, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
- return err
+ return nil, err
}
writeSummaryJSON(f, isolSummary)
f.Close()
}
- end := time.Now()
-
archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{
HitCount: proto.Int64(int64(checker.Hit.Count)),
MissCount: proto.Int64(int64(checker.Miss.Count)),
@@ -128,13 +149,7 @@
MissBytes: &checker.Miss.Bytes,
IsolateHash: []string{string(isolSummary.Digest)},
}
- eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint)
- op := logpb.IsolateClientEvent_ARCHIVE.Enum()
- if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err != nil {
- log.Printf("Failed to log to eventlog: %v", err)
- }
-
- return nil
+ return archiveDetails, nil
}
func writeSummaryJSON(w io.Writer, summaries ...IsolatedSummary) error {
