blob: 94ffc63d361fe1ab24e1697565ada3a6dcea98d8 [file] [log] [blame]
// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ledcmd
import (
"context"
"io/ioutil"
"net/http"
"os"
"os/exec"
"sync"
"time"
"github.com/mattn/go-tty"
"go.chromium.org/luci/client/archiver"
"go.chromium.org/luci/client/downloader"
"go.chromium.org/luci/common/errors"
"go.chromium.org/luci/common/isolated"
"go.chromium.org/luci/common/isolatedclient"
"go.chromium.org/luci/common/logging"
"go.chromium.org/luci/common/retry"
"go.chromium.org/luci/led/job"
)
// IsolatedTransformer is a function which receives a directory on the local
// disk with the contents of an isolate and is expected to manipulate the
// contents of that directory however it chooses.
//
// EditIsolated takes these functions as a callback in order to manipulate the
// isolated content of a job.Definition.
type IsolatedTransformer func(ctx context.Context, directory string) error
// ProgramIsolatedTransformer returns an IsolatedTransformer which alters the
// contents of the isolated by running a program specified with `args` in the
// directory where the isolated content has been unpacked.
func ProgramIsolatedTransformer(args ...string) IsolatedTransformer {
return func(ctx context.Context, dir string) error {
logging.Infof(ctx, "Invoking transform_program: %q", args)
tProg := exec.CommandContext(ctx, args[0], args[1:]...)
tProg.Stdout = os.Stderr
tProg.Stderr = os.Stderr
tProg.Dir = dir
return errors.Annotate(tProg.Run(), "running transform_program").Err()
}
}
// PromptIsolatedTransformer returns an IsolatedTransformer which prompts the
// user to navigate to the directory with the isolated content and manipulate
// it manually. When the user is done they should press "enter" to indicate that
// they're finished.
func PromptIsolatedTransformer() IsolatedTransformer {
return func(ctx context.Context, dir string) error {
logging.Infof(ctx, "")
logging.Infof(ctx, "Edit files as you wish in:")
logging.Infof(ctx, "\t%s", dir)
term, err := tty.Open()
if err != nil {
return errors.Annotate(err, "opening terminal").Err()
}
defer term.Close()
logging.Infof(ctx, "When finished, press <enter> here to isolate it.")
_, err = term.ReadString()
return errors.Annotate(err, "reading <enter>").Err()
}
}
// EditIsolated allows you to edit the isolated (input ref) contents of the
// job.Definition.
//
// This implicitly collapses all isolate sources in the job.Definition into
// a single isolate.
func EditIsolated(ctx context.Context, authClient *http.Client, jd *job.Definition, xform IsolatedTransformer) error {
logging.Infof(ctx, "editing isolated")
tdir, err := ioutil.TempDir("", "led-edit-isolated")
if err != nil {
return errors.Annotate(err, "failed to create tempdir").Err()
}
defer func() {
if err = os.RemoveAll(tdir); err != nil {
logging.Errorf(ctx, "failed to cleanup temp dir %q: %s", tdir, err)
}
}()
if err := ConsolidateIsolateSources(ctx, authClient, jd); err != nil {
return err
}
current, err := jd.Info().CurrentIsolated()
if err != nil {
return err
}
err = jd.Edit(func(je job.Editor) {
je.ClearCurrentIsolated()
})
if err != nil {
return err
}
// If we have no current isolate data, default to the default isolate
// server/namespace, which are stored in UserPayload.
isoServerParams := current
if isoServerParams == nil {
isoServerParams = jd.UserPayload
}
rawIsoClient := isolatedclient.NewClient(
isoServerParams.Server,
isolatedclient.WithAuthClient(authClient),
isolatedclient.WithNamespace(isoServerParams.Namespace),
isolatedclient.WithRetryFactory(retry.Default))
if dgst := current.GetDigest(); dgst != "" {
var statMu sync.Mutex
var previousStats *downloader.FileStats
dl := downloader.New(ctx, rawIsoClient, isolated.HexDigest(dgst), tdir, &downloader.Options{
FileStatsCallback: func(s downloader.FileStats, span time.Duration) {
logging.Infof(ctx, "%s", s.StatLine(previousStats, span))
statMu.Lock()
previousStats = &s
statMu.Unlock()
},
})
if err = dl.Wait(); err != nil {
return err
}
}
if err := xform(ctx, tdir); err != nil {
return err
}
logging.Infof(ctx, "uploading new isolated")
hash, err := isolateDirectory(ctx, rawIsoClient, tdir)
if err != nil {
return err
}
logging.Infof(ctx, "isolated upload: done")
jd.UserPayload.Digest = string(hash)
return nil
}
func isolateDirectory(ctx context.Context, isoClient *isolatedclient.Client, dir string) (isolated.HexDigest, error) {
checker := archiver.NewChecker(ctx, isoClient, 32)
uploader := archiver.NewUploader(ctx, isoClient, 8)
arc := archiver.NewTarringArchiver(checker, uploader)
summary, err := arc.Archive(&archiver.TarringArgs{
Deps: []string{dir},
RootDir: dir,
Isol: isolated.New(isoClient.Hash()),
})
if err != nil {
return "", errors.Annotate(err, "isolating directory").Err()
}
if err := checker.Close(); err != nil {
return "", errors.Annotate(err, "closing checker").Err()
}
if err := uploader.Close(); err != nil {
return "", errors.Annotate(err, "closing uploader").Err()
}
return summary.Digest, nil
}