client/cmd/isolate/archive.go

// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/maruel/subcommands"

	"github.com/luci/luci-go/client/archiver"
	"github.com/luci/luci-go/client/isolate"
	"github.com/luci/luci-go/common/auth"
	"github.com/luci/luci-go/common/data/text/units"
	logpb "github.com/luci/luci-go/common/eventlog/proto"
	"github.com/luci/luci-go/common/isolatedclient"
)

func cmdArchive(defaultAuthOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "archive <options>",
		ShortDesc: "creates a .isolated file and uploads the tree to an isolate server.",
		LongDesc:  "All the files listed in the .isolated file are put in the isolate server cache",
		CommandRun: func() subcommands.CommandRun {
			c := archiveRun{}
			c.commonServerFlags.Init(defaultAuthOpts)
			c.isolateFlags.Init(&c.Flags)
			c.loggingFlags.Init(&c.Flags)
			return &c
		},
	}
}

type archiveRun struct {
	commonServerFlags
	isolateFlags
	loggingFlags loggingFlags
}

func (c *archiveRun) Parse(a subcommands.Application, args []string) error {
	if err := c.commonServerFlags.Parse(); err != nil {
		return err
	}
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}
	if err := c.isolateFlags.Parse(cwd, RequireIsolatedFile); err != nil {
		return err
	}
	if len(args) != 0 {
		return errors.New("position arguments not expected")
	}
	return nil
}

func (c *archiveRun) main(a subcommands.Application, args []string) error {
	start := time.Now()
	authCl, err := c.createAuthClient()
	if err != nil {
		return err
	}
	ctx := c.defaultFlags.MakeLoggingContext(os.Stderr)
	client := isolatedclient.New(nil, authCl, c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace, nil, nil)

	eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint)

	archiveDetails, err := doArchive(ctx, client, &c.ArchiveOptions, c.defaultFlags.Quiet, start)
	if err != nil {
		return err
	}

	end := time.Now()
	op := logpb.IsolateClientEvent_LEGACY_ARCHIVE.Enum()
	if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err != nil {
		log.Printf("Failed to log to eventlog: %v", err)
	}
	return nil
}

// doArchive performs the archive operation for an isolate specified by archiveOpts.
func doArchive(ctx context.Context, client *isolatedclient.Client, archiveOpts *isolate.ArchiveOptions, quiet bool, start time.Time) (*logpb.IsolateClientEvent_ArchiveDetails, error) {
	prefix := "\n"
	if quiet {
		prefix = ""
	}

	arch := archiver.New(ctx, client, os.Stdout)
	CancelOnCtrlC(arch)
	item := isolate.Archive(arch, archiveOpts)
	item.WaitForHashed()
	var err error
	if err = item.Error(); err != nil {
		fmt.Printf("%s%s  %s\n", prefix, filepath.Base(archiveOpts.Isolate), err)
	} else {
		fmt.Printf("%s%s  %s\n", prefix, item.Digest(), filepath.Base(archiveOpts.Isolate))
	}
	if err2 := arch.Close(); err == nil {
		err = err2
	}
	stats := arch.Stats()
	if !quiet {
		duration := time.Since(start)
		fmt.Fprintf(os.Stderr, "Hits    : %5d (%s)\n", stats.TotalHits(), stats.TotalBytesHits())
		fmt.Fprintf(os.Stderr, "Misses  : %5d (%s)\n", stats.TotalMisses(), stats.TotalBytesPushed())
		fmt.Fprintf(os.Stderr, "Duration: %s\n", units.Round(duration, time.Millisecond))
	}

	archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{
		HitCount:  proto.Int64(int64(stats.TotalHits())),
		MissCount: proto.Int64(int64(stats.TotalMisses())),
		HitBytes:  proto.Int64(int64(stats.TotalBytesHits())),
		MissBytes: proto.Int64(int64(stats.TotalBytesPushed())),
	}
	if item.Error() != nil {
		archiveDetails.IsolateHash = []string{string(item.Digest())}
	}
	return archiveDetails, nil
}

func (c *archiveRun) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.Parse(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	cl, err := c.defaultFlags.StartTracing()
	if err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	defer cl.Close()
	if err := c.main(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	return 0
}

// CancelOnCtrlC is a temporary copy of the CancelOnCtrlC in internal/common/concurrent.go
// This is needed until the old archive and batcharchive code (which uses Cancelers) is removed.
// It operates on a concrete Archiver to avoid the dependency on Canceler.
func CancelOnCtrlC(arch *archiver.Archiver) {
	interrupted := make(chan os.Signal, 1)
	signal.Notify(interrupted, os.Interrupt)
	go func() {
		defer signal.Stop(interrupted)
		select {
		case <-interrupted:
			arch.Cancel(errors.New("Ctrl-C"))
		case <-arch.Channel():
		}
	}()
}