blob: a74be0f5a7a19767e49b5258e783fb5dc242eed4 [file] [log] [blame]
// Copyright 2020 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package cmd
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
uri "path"
"path/filepath"
"sort"
"strings"
"github.com/golang/protobuf/jsonpb"
"github.com/golang/protobuf/proto"
"github.com/maruel/subcommands"
tnProto "go.chromium.org/chromiumos/config/go/api/test/harness/tnull/v1"
rtd "go.chromium.org/chromiumos/config/go/api/test/rtd/v1"
"go.chromium.org/luci/common/cli"
"go.chromium.org/luci/common/errors"
"go.chromium.org/luci/common/logging"
)
type gen struct {
subcommands.CommandRunBase
destBinaryProto string
alsoOutputJson bool
testList string
}
func GenerateFullRequest() *subcommands.Command {
return &subcommands.Command{
UsageLine: "generate-full-request -output_dir /path/to/output-dir/",
ShortDesc: "create Invocations requesting each TNull test, as binaryproto files.",
CommandRun: func() subcommands.CommandRun {
g := &gen{}
g.Flags.StringVar(&g.testList, "input_json", "",
`Optional: path that contains a JSON list of test names to include in invocation.
Each name should look like 'dummy-pass' or look
like "remoteTestDrivers/tnull/tests/dummy-pass"`)
g.Flags.StringVar(&g.destBinaryProto, "output", "", "File to contain binaryproto rtd.Invocation objects")
g.Flags.BoolVar(&g.alsoOutputJson, "also_output_json", true, "Whether to create output JSON rtd.Invocation files in addition to the binaryprotos, in the same directory")
return g
},
}
}
func (g *gen) Run(a subcommands.Application, args []string, env subcommands.Env) int {
if err := g.innerRun(cli.GetContext(a, g, env)); err != nil {
fmt.Fprintf(a.GetErr(), "%s\n", err)
return 1
}
return 0
}
func (g *gen) innerRun(ctx context.Context) error {
lookup, err := extractTestsFromSpecification()
if err != nil {
return err
}
names, err := getNames(g.testList)
if err != nil {
return err
}
nameStructs, err := parseNames(names)
if err != nil {
return err
}
validMap, err := filterTestMap(lookup.Lookup, nameStructs)
if err != nil {
return err
}
err = WriteInvocation(ctx, validMap, g.destBinaryProto, g.alsoOutputJson)
if err != nil {
return err
}
return nil
}
func getNames(path string) ([]string, error) {
var names []string
if path == "" {
return names, nil
}
bs, err := ioutil.ReadFile(path)
if err != nil {
return nil, errors.Annotate(err, "read in JSON list").Err()
}
err = json.Unmarshal(bs, &names)
if err != nil {
return nil, errors.Annotate(err, "read in JSON list").Err()
}
return names, nil
}
// WriteInvocation writes a binary-encoded Invocation proto to destFile.
func WriteInvocation(ctx context.Context, tests map[string]*tnProto.Steps, destFile string, alsoOutputJson bool) error {
var reqs []*rtd.Request
for _, name := range mapKeysToSortedList(tests) {
req := rtd.Request{
Name: "request_" + uri.Base(name),
Test: name,
}
reqs = append(reqs, &req)
}
Inv := rtd.Invocation{
Requests: reqs,
}
dir := filepath.Dir(destFile)
// Create the directory if it doesn't exist.
if err := os.MkdirAll(dir, 0755); err != nil {
return errors.Annotate(err, "write invocation pb").Err()
}
{
protoBytes, err := proto.Marshal(&Inv)
if err != nil {
return errors.Annotate(err, "write Invocation binaryproto").Err()
}
if err = ioutil.WriteFile(destFile, protoBytes, 0644); err != nil {
return errors.Annotate(err, "write Invocation binaryproto").Err()
}
logging.Infof(ctx, "Wrote binaryproto Invocation to %v", destFile)
}
if alsoOutputJson {
jsonFile := strings.TrimSuffix(destFile, ".binaryproto") + ".json"
w, err := os.Create(jsonFile)
if err != nil {
return errors.Annotate(err, "write Invocation pb").Err()
}
defer w.Close()
marshaler := jsonpb.Marshaler{Indent: " "}
if err := marshaler.Marshal(w, &Inv); err != nil {
return errors.Annotate(err, "write JSON pb").Err()
}
logging.Infof(ctx, "Wrote JSON Invocation to %v", jsonFile)
}
return nil
}
func mapKeysToSortedList(m map[string]*tnProto.Steps) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// We have two canonical name formats; short, e.g. "dummy-pass", and full, e.g.
// "remoteTestDrivers/tnull/tests/dummy-pass". Requiring either is a serious hassle
// for users, so convert to a single format. This accepts either and translates them
// to simple structs which keep the original value for the purposes of debugging
func parseNames(names []string) ([]NameStruct, error) {
snames := make([]NameStruct, 0, len(names))
errs := errors.MultiError{}
for _, name := range names {
split := strings.SplitN(name, "/", 4)
if len(split) != 4 {
snames = append(snames, NameStruct{givenName: name, shortName: name})
continue
}
if uri.Join(split[0:2]...) != "remoteTestDrivers/tnull/tests" {
errs = append(errs, errors.Reason("test name %s is malformed", name).Err())
continue
}
snames = append(snames, NameStruct{givenName: name, shortName: split[3]})
}
if len(errs) > 0 {
return nil, errs
}
return snames, nil
}
func filterTestMap(spec map[string]*tnProto.Steps, names []NameStruct) (map[string]*tnProto.Steps, error) {
if len(names) == 0 {
return spec, nil
}
errs := errors.MultiError{}
newLookup := make(map[string]*tnProto.Steps)
for _, ns := range names {
name := ns.inferredName()
if t, present := spec[name]; present {
newLookup[name] = t
} else {
e := errors.Reason("Could not find test %s specified by name %s", name, ns.givenName).Err()
errs = append(errs, e)
}
}
if len(errs) > 0 {
return nil, errs
}
return newLookup, nil
}
type NameStruct struct {
givenName string
shortName string
}
func (n *NameStruct) inferredName() string {
return uri.Join(TNullName, "tests", n.shortName)
}