Add automatic metadata generation

BUG=chromium:1051691
TEST=emerge and run_tests.sh

Change-Id: I482e2cb4763dd7ac253b537578dad0e0c393f994
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/infra/tnull/+/2298662
Tested-by: Jacob Kopczynski <jkop@chromium.org>
Reviewed-by: Prathmesh Prabhu <pprabhu@chromium.org>
Commit-Queue: Jacob Kopczynski <jkop@chromium.org>
Auto-Submit: Jacob Kopczynski <jkop@chromium.org>
diff --git a/metadata/all-tests.json b/metadata/all-tests.json
new file mode 100644
index 0000000..2838d98
--- /dev/null
+++ b/metadata/all-tests.json
@@ -0,0 +1,76 @@
+{
+  "requests": [
+    {
+      "name": "request_parallel-log",
+      "test": "remoteTestDrivers/tnull/tests/parallel-log"
+    },
+    {
+      "name": "request_mixed-logs",
+      "test": "remoteTestDrivers/tnull/tests/mixed-logs"
+    },
+    {
+      "name": "request_multiline-artifact",
+      "test": "remoteTestDrivers/tnull/tests/multiline-artifact"
+    },
+    {
+      "name": "request_fail-on-second-err",
+      "test": "remoteTestDrivers/tnull/tests/fail-on-second-err"
+    },
+    {
+      "name": "request_report-failure",
+      "test": "remoteTestDrivers/tnull/tests/report-failure"
+    },
+    {
+      "name": "request_report-skip",
+      "test": "remoteTestDrivers/tnull/tests/report-skip"
+    },
+    {
+      "name": "request_simple-log",
+      "test": "remoteTestDrivers/tnull/tests/simple-log"
+    },
+    {
+      "name": "request_no-artifact",
+      "test": "remoteTestDrivers/tnull/tests/no-artifact"
+    },
+    {
+      "name": "request_simple-artifact",
+      "test": "remoteTestDrivers/tnull/tests/simple-artifact"
+    },
+    {
+      "name": "request_two-artifacts",
+      "test": "remoteTestDrivers/tnull/tests/two-artifacts"
+    },
+    {
+      "name": "request_report-success",
+      "test": "remoteTestDrivers/tnull/tests/report-success"
+    },
+    {
+      "name": "request_success-with-warnings",
+      "test": "remoteTestDrivers/tnull/tests/success-with-warnings"
+    },
+    {
+      "name": "request_no-log",
+      "test": "remoteTestDrivers/tnull/tests/no-log"
+    },
+    {
+      "name": "request_dummy-pass",
+      "test": "remoteTestDrivers/tnull/tests/dummy-pass"
+    },
+    {
+      "name": "request_two-logs",
+      "test": "remoteTestDrivers/tnull/tests/two-logs"
+    },
+    {
+      "name": "request_two-requests",
+      "test": "remoteTestDrivers/tnull/tests/two-requests"
+    },
+    {
+      "name": "request_duped-artifacts",
+      "test": "remoteTestDrivers/tnull/tests/duped-artifacts"
+    },
+    {
+      "name": "request_multiline-logs",
+      "test": "remoteTestDrivers/tnull/tests/multiline-logs"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/run_tests.sh b/run_tests.sh
index 731ce0a..5b0fafd 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -3,4 +3,13 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-tnull run-steps -input_json metadata/dummy-pass.json
+set -x
+MY_PATH=$(readlink -f ~/trunk/infra/tnull/metadata/all-tests.json)
+tnull generate-full-request -output_json "${MY_PATH}"
+tnull run-steps -input_json "${MY_PATH}"
+if $?
+then
+  echo "pass"
+else
+  echo "fail"
+fi
diff --git a/src/tnull/cmd/execute.go b/src/tnull/cmd/execute.go
new file mode 100644
index 0000000..29210c8
--- /dev/null
+++ b/src/tnull/cmd/execute.go
@@ -0,0 +1,188 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"tnull/driver"
+
+	"github.com/golang/protobuf/jsonpb"
+	"github.com/golang/protobuf/proto"
+	"github.com/maruel/subcommands"
+	tnProto "go.chromium.org/chromiumos/config/go/api/test/harness/tnull/v1"
+	metadata "go.chromium.org/chromiumos/config/go/api/test/metadata/v1"
+	rtd "go.chromium.org/chromiumos/config/go/api/test/rtd/v1"
+	"go.chromium.org/luci/common/errors"
+)
+
+const specRelPath = "trunk/infra/tnull/metadata/generated/config.cfg"
+const tnullName = "remoteTestDrivers/tnull"
+
+type run struct {
+	subcommands.CommandRunBase
+	invocationPath string
+}
+
+func RunSteps() *subcommands.Command {
+	return &subcommands.Command{
+		UsageLine: "run-steps -input_json /path/to/input.json",
+		ShortDesc: "run an Invocation object.",
+		CommandRun: func() subcommands.CommandRun {
+			r := &run{}
+			r.Flags.StringVar(&r.invocationPath, "input_json", "", "Path that contains a JSON-encoded rtd.Invocation object")
+			return r
+		},
+	}
+}
+
+func (r *run) Run(a subcommands.Application, args []string, env subcommands.Env) int {
+	if err := r.innerRun(a, args, env); err != nil {
+		fmt.Fprintf(a.GetErr(), "%s\n", err)
+		return 1
+	}
+	return 0
+}
+
+func (r *run) innerRun(a subcommands.Application, args []string, env subcommands.Env) error {
+	if err := r.validateArgs(); err != nil {
+		return errors.Annotate(err, "validation").Err()
+	}
+
+	var inv rtd.Invocation
+	if err := readJSONPb(r.invocationPath, &inv); err != nil {
+		return errors.Annotate(err, "reading in the Invocation").Err()
+	}
+	if inv.GetRequests() == nil || len(inv.GetRequests()) == 0 {
+		return errors.Reason("no requests in invocation").Err()
+	}
+
+	lookup, err := extractTestsFromSpecification()
+	if err != nil {
+		return err
+	}
+
+	conf := inv.ProgressSinkClientConfig
+	reqs := inv.Requests
+	errs := errors.MultiError{}
+
+	for _, req := range reqs {
+		fmt.Fprintf(a.GetOut(), "req %s: 'executing' test %s\n", req.Name, req.Test)
+		if steps, present := lookup.Lookup[req.Test]; !present {
+			errs = append(errs, errors.Reason("no test with the name %s", req.Test).Err())
+		} else {
+			steps.Setup.Config = conf
+			r.Execute(req.Test, steps)
+		}
+	}
+	if len(errs) == 0 {
+		return nil
+	}
+	return errs
+}
+
+func extractTestsFromSpecification() (*tnProto.TestMap, error) {
+	p, err := specPath()
+	if err != nil {
+		return nil, errors.Annotate(err, "extracting tests from spec").Err()
+	}
+	var s metadata.Specification
+	if err := readJSONPb(p, &s); err != nil {
+		return nil, errors.Annotate(err, "extracting tests from spec").Err()
+	}
+	testMap := &tnProto.TestMap{Lookup: map[string]*tnProto.Steps{}}
+	for _, rtd := range s.RemoteTestDrivers {
+		if rtd.Name != tnullName {
+			return nil, errors.Reason(
+				"bad specification, RTD was %s, should be %s",
+				rtd.Name, tnullName).Err()
+		}
+		marshaler := jsonpb.Marshaler{}
+		for _, test := range rtd.Tests {
+			steps, err := detailsToSteps(marshaler, test.Informational)
+			if err != nil {
+				return nil, errors.Annotate(
+					err, "extracting tests from spec").Err()
+			}
+			testMap.Lookup[test.Name] = steps
+		}
+	}
+	return testMap, nil
+}
+
+func detailsToSteps(m jsonpb.Marshaler, info *metadata.Informational) (*tnProto.Steps, error) {
+	jsonSteps, err := m.MarshalToString(info.Details.GetFields()["steps"])
+	if err != nil {
+		return nil, errors.Annotate(err, "Details>json").Err()
+	}
+	var steps tnProto.Steps
+	if err := jsonpb.UnmarshalString(jsonSteps, &steps); err != nil {
+		return nil, errors.Annotate(err, "json>Steps").Err()
+	}
+	return &steps, nil
+}
+
+func (r *run) Execute(name string, s *tnProto.Steps) error {
+	var d driver.Driver
+	if err := d.Setup(name, *s.Setup); err != nil {
+		return err
+	}
+	for _, st := range s.Steps {
+		ste := st.GetStep()
+		switch t := ste.(type) {
+		case *tnProto.Step_Archive:
+			req := st.GetArchive().GetCommonArgs().GetRequestName()
+			return d.Archive(req)
+		case *tnProto.Step_Log:
+			req := st.GetLog().GetCommonArgs().GetRequestName()
+			return d.Log(req)
+		case *tnProto.Step_Result:
+			req := st.GetResult().GetCommonArgs().GetRequestName()
+			return d.Result(req)
+		case *tnProto.Step_Other:
+			step := st.GetOther()
+			return errors.Reason(
+				"driver has no method %s or support for args %v",
+				step.GetMethodName(), step.GetArgs(),
+			).Err()
+		default:
+			_ = t
+			return errors.Reason("no step specified").Err()
+		}
+	}
+	return nil
+}
+
+func (r *run) validateArgs() error {
+	if r.invocationPath == "" {
+		return fmt.Errorf("input_json must specify an rtd.Invocation")
+	}
+	return nil
+}
+
+// readJSONPb reads a JSON string from inFile and unpacks it as a proto.
+// Unexpected fields are ignored.
+func readJSONPb(inFile string, payload proto.Message) error {
+	r, err := os.Open(inFile)
+	if err != nil {
+		return errors.Annotate(err, "read JSON pb").Err()
+	}
+	defer r.Close()
+
+	unmarshaler := jsonpb.Unmarshaler{AllowUnknownFields: true}
+	if err := unmarshaler.Unmarshal(r, payload); err != nil {
+		return errors.Annotate(err, "read JSON pb").Err()
+	}
+	return nil
+}
+
+func specPath() (string, error) {
+	if home, err := os.UserHomeDir(); err != nil {
+		return "", err
+	} else {
+		return filepath.Join(home, specRelPath), nil
+	}
+}
diff --git a/src/tnull/cmd/generate.go b/src/tnull/cmd/generate.go
new file mode 100644
index 0000000..c66a788
--- /dev/null
+++ b/src/tnull/cmd/generate.go
@@ -0,0 +1,80 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/golang/protobuf/jsonpb"
+	"github.com/maruel/subcommands"
+	tnProto "go.chromium.org/chromiumos/config/go/api/test/harness/tnull/v1"
+	rtd "go.chromium.org/chromiumos/config/go/api/test/rtd/v1"
+	"go.chromium.org/luci/common/errors"
+)
+
+type gen struct {
+	subcommands.CommandRunBase
+	dest string
+}
+
+func GenerateFullRequest() *subcommands.Command {
+	return &subcommands.Command{
+		UsageLine: "generate-full-request -output_json /path/to/output/json",
+		ShortDesc: "create an Invocation requesting every TNull test, as a JSON file.",
+		CommandRun: func() subcommands.CommandRun {
+			g := &gen{}
+			g.Flags.StringVar(&g.dest, "output_json", "", "Path to contain a JSON-encoded rtd.Invocation object")
+			return g
+		},
+	}
+}
+
+func (g *gen) Run(a subcommands.Application, args []string, env subcommands.Env) int {
+	lookup, err := extractTestsFromSpecification()
+	if err != nil {
+		fmt.Fprintf(a.GetErr(), "%s\n", err)
+		return 1
+	}
+	err = WriteJSONInvocation(lookup.Lookup, g.dest)
+	if err != nil {
+		fmt.Fprintf(a.GetErr(), "%s\n", err)
+		return 1
+	}
+	return 0
+}
+
+// WriteJSONInvocation writes a JSON encoded Invocation proto to destFile.
+func WriteJSONInvocation(tests map[string]*tnProto.Steps, destFile string) error {
+	reqs := []*rtd.Request{}
+	for name := range tests {
+		req := rtd.Request{
+			Name: "request_" + filepath.Base(name),
+			Test: name,
+		}
+		reqs = append(reqs, &req)
+	}
+	Inv := rtd.Invocation{
+		Requests: reqs,
+	}
+	dir := filepath.Dir(destFile)
+	// Create the directory if it doesn't exist.
+	if err := os.MkdirAll(dir, 0777); err != nil {
+		return errors.Annotate(err, "write JSON pb").Err()
+	}
+
+	w, err := os.Create(destFile)
+	if err != nil {
+		return errors.Annotate(err, "write JSON pb").Err()
+	}
+	defer w.Close()
+
+	marshaler := jsonpb.Marshaler{Indent: "  "}
+	if err := marshaler.Marshal(w, &Inv); err != nil {
+		return errors.Annotate(err, "write JSON pb").Err()
+	}
+	return nil
+}
diff --git a/src/tnull/main.go b/src/tnull/main.go
index b9bd44a..b3edb0e 100644
--- a/src/tnull/main.go
+++ b/src/tnull/main.go
@@ -6,187 +6,16 @@
 
 import (
 	"context"
-	"fmt"
 	"os"
-	"path/filepath"
-	"tnull/driver"
+	"tnull/cmd"
 
-	"github.com/golang/protobuf/jsonpb"
-	"github.com/golang/protobuf/proto"
 	"github.com/maruel/subcommands"
 	"go.chromium.org/luci/auth/client/authcli"
 	"go.chromium.org/luci/common/cli"
-	"go.chromium.org/luci/common/errors"
 	"go.chromium.org/luci/common/logging/gologger"
 	"go.chromium.org/luci/hardcoded/chromeinfra"
-
-	tnProto "go.chromium.org/chromiumos/config/go/api/test/harness/tnull/v1"
-	metadata "go.chromium.org/chromiumos/config/go/api/test/metadata/v1"
-	rtd "go.chromium.org/chromiumos/config/go/api/test/rtd/v1"
 )
 
-const tnullName = "remoteTestDrivers/tnull"
-const specRelPath = "trunk/infra/tnull/metadata/generated/config.cfg"
-
-type run struct {
-	subcommands.CommandRunBase
-
-	invocationPath string
-}
-
-func runSteps() *subcommands.Command {
-	return &subcommands.Command{
-		UsageLine: "run-steps -input_json /path/to/input.json",
-		ShortDesc: "run a Steps object.",
-		CommandRun: func() subcommands.CommandRun {
-			r := &run{}
-			r.Flags.StringVar(&r.invocationPath, "input_json", "", "Path that contains-JSON encoded rtd.Invocation object")
-			return r
-		},
-	}
-}
-
-func (r *run) Run(a subcommands.Application, args []string, env subcommands.Env) int {
-	if err := r.innerRun(a, args, env); err != nil {
-		fmt.Fprintf(a.GetErr(), "%s\n", err)
-		return 1
-	}
-	return 0
-}
-
-func (r *run) innerRun(a subcommands.Application, args []string, env subcommands.Env) error {
-	if err := r.validateArgs(); err != nil {
-		return errors.Annotate(err, "validation").Err()
-	}
-
-	var inv rtd.Invocation
-	if err := readJSONPb(r.invocationPath, &inv); err != nil {
-		return errors.Annotate(err, "reading in the Invocation").Err()
-	}
-
-	lookup, err := extractTestsFromSpecification()
-	if err != nil {
-		return err
-	}
-
-	conf := inv.ProgressSinkClientConfig
-	reqs := inv.Requests
-
-	for _, req := range reqs {
-		fmt.Fprintf(a.GetOut(), "req %s: 'executing' test %s\n", req.Name, req.Test)
-		if steps, present := lookup.Lookup[req.Test]; !present {
-			return errors.Reason("no test with the name %s", req.Test).Err()
-		} else {
-			steps.Setup.Config = conf
-			r.Execute(req.Test, steps)
-		}
-	}
-	return nil
-}
-
-func extractTestsFromSpecification() (*tnProto.TestMap, error) {
-	p, err := specPath()
-	if err != nil {
-		return nil, errors.Annotate(err, "extracting tests from spec").Err()
-	}
-	var s metadata.Specification
-	if err := readJSONPb(p, &s); err != nil {
-		return nil, errors.Annotate(err, "extracting tests from spec").Err()
-	}
-	testMap := &tnProto.TestMap{Lookup: map[string]*tnProto.Steps{}}
-	for _, rtd := range s.RemoteTestDrivers {
-		if rtd.Name != tnullName {
-			return nil, errors.Reason(
-				"bad specification, RTD was %s, should be %s",
-				rtd.Name, tnullName).Err()
-		}
-		marshaler := jsonpb.Marshaler{}
-		for _, test := range rtd.Tests {
-			steps, err := detailsToSteps(marshaler, test.Informational)
-			if err != nil {
-				return nil, errors.Annotate(
-					err, "extracting tests from spec").Err()
-			}
-			testMap.Lookup[test.Name] = steps
-		}
-	}
-	return testMap, nil
-}
-
-func detailsToSteps(m jsonpb.Marshaler, info *metadata.Informational) (*tnProto.Steps, error) {
-	jsonSteps, err := m.MarshalToString(info.Details.GetFields()["steps"])
-	if err != nil {
-		return nil, errors.Annotate(err, "Details>json").Err()
-	}
-	var steps tnProto.Steps
-	if err := jsonpb.UnmarshalString(jsonSteps, &steps); err != nil {
-		return nil, errors.Annotate(err, "json>Steps").Err()
-	}
-	return &steps, nil
-}
-
-func (r *run) Execute(name string, s *tnProto.Steps) error {
-	var d driver.Driver
-	if err := d.Setup(name, *s.Setup); err != nil {
-		return err
-	}
-	for _, st := range s.Steps {
-		ste := st.GetStep()
-		switch t := ste.(type) {
-		case *tnProto.Step_Archive:
-			req := st.GetArchive().GetCommonArgs().GetRequestName()
-			return d.Archive(req)
-		case *tnProto.Step_Log:
-			req := st.GetLog().GetCommonArgs().GetRequestName()
-			return d.Log(req)
-		case *tnProto.Step_Result:
-			req := st.GetResult().GetCommonArgs().GetRequestName()
-			return d.Result(req)
-		case *tnProto.Step_Other:
-			step := st.GetOther()
-			return errors.Reason(
-				"driver has no method %s or support for args %v",
-				step.GetMethodName(), step.GetArgs(),
-			).Err()
-		default:
-			_ = t
-			return errors.Reason("no step specified").Err()
-		}
-	}
-	return nil
-}
-
-func (r *run) validateArgs() error {
-	if r.invocationPath == "" {
-		return fmt.Errorf("input_json must specify an rtd.Invocation")
-	}
-	return nil
-}
-
-// readJSONPb reads a JSON string from inFile and unpacks it as a proto.
-// Unexpected fields are ignored.
-func readJSONPb(inFile string, payload proto.Message) error {
-	r, err := os.Open(inFile)
-	if err != nil {
-		return errors.Annotate(err, "read JSON pb").Err()
-	}
-	defer r.Close()
-
-	unmarshaler := jsonpb.Unmarshaler{AllowUnknownFields: true}
-	if err := unmarshaler.Unmarshal(r, payload); err != nil {
-		return errors.Annotate(err, "read JSON pb").Err()
-	}
-	return nil
-}
-
-func specPath() (string, error) {
-	if home, err := os.UserHomeDir(); err != nil {
-		return "", err
-	} else {
-		return filepath.Join(home, specRelPath), nil
-	}
-}
-
 func main() {
 	application := &cli.Application{
 		Name:  "TNull",
@@ -199,7 +28,8 @@
 			authcli.SubcommandInfo(chromeinfra.DefaultAuthOptions(), "whoami", false),
 			authcli.SubcommandLogin(chromeinfra.DefaultAuthOptions(), "login", false),
 			authcli.SubcommandLogout(chromeinfra.DefaultAuthOptions(), "logout", false),
-			runSteps(),
+			cmd.RunSteps(),
+			cmd.GenerateFullRequest(),
 		},
 	}
 	os.Exit(subcommands.Run(application, nil))
diff --git a/src/tnull/test/dummy-pass.go b/src/tnull/test/dummy-pass.go
deleted file mode 100644
index 6853624..0000000
--- a/src/tnull/test/dummy-pass.go
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2020 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package test