tast: Check dependencies before building tests.
When run with -build (currently the default), "tast run ..."
rebuilds the test executable. To do this, it relies on the
executable's dependencies' source code having already been
checked out as a side effect of the dependencies having
previously been built by Portage. (/usr/lib/gopath is used
by default, although this change adds a -sysgopath flag that
can be used to specify a different workspace.)
This change makes tast use Portage's equery command to check
that the test package's dependencies are installed before
performing the build. If they aren't, a command that can be
used to install them is printed. Checking dependencies can
take close to a second; -checkdeps=false can be passed to
skip the check.
BUG=chromium:737628
TEST=unit tests pass; also manually verified that for both
local and remote tests, an error is printed if
dependencies are missing and that the check is skipped
when -checkdeps=false is passed
Change-Id: I532821307e638f27b7dfe013e840b371c63b7547
Reviewed-on: https://chromium-review.googlesource.com/664379
Commit-Ready: Dan Erat <derat@chromium.org>
Tested-by: Dan Erat <derat@chromium.org>
Reviewed-by: Jason Clinton <jclinton@chromium.org>
diff --git a/src/chromiumos/tast/tast/build/build.go b/src/chromiumos/tast/tast/build/build.go
index 1d524c4..2b71a26 100644
--- a/src/chromiumos/tast/tast/build/build.go
+++ b/src/chromiumos/tast/tast/build/build.go
@@ -6,6 +6,7 @@
package build
import (
+ "bytes"
"context"
"fmt"
"os/exec"
@@ -15,10 +16,6 @@
"chromiumos/tast/tast/timing"
)
-const (
- sysGopath = "/usr/lib/gopath" // readonly Go workspace where source for system packages are stored
-)
-
// GetLocalArch returns the local system's architecture as described by "uname -m".
func GetLocalArch() (string, error) {
b, err := exec.Command("uname", "-m").Output()
@@ -51,11 +48,27 @@
return out, fmt.Errorf("unknown arch %q", cfg.Arch)
}
+ if cfg.PortagePkg != "" {
+ if missing, err := checkDeps(ctx, cfg.PortagePkg); err != nil {
+ return out, fmt.Errorf("failed checking deps for %s: %v", cfg.PortagePkg, err)
+ } else if len(missing) > 0 {
+ b := bytes.NewBufferString("To install missing dependencies, run:\n\n sudo emerge -j 16 \\\n")
+ for i, dep := range missing {
+ suffix := ""
+ if i < len(missing)-1 {
+ suffix = " \\"
+ }
+ fmt.Fprintf(b, " =%s%s\n", dep, suffix)
+ }
+ return b.Bytes(), fmt.Errorf("%s has missing dependencies", cfg.PortagePkg)
+ }
+ }
+
pkgDir := filepath.Join(cfg.OutDir, cfg.Arch)
cmd := exec.Command(comp, "build", "-i", "-ldflags=-s -w", "-pkgdir", pkgDir, "-o", path, pkg)
cmd.Env = []string{
"PATH=/usr/bin",
- "GOPATH=" + strings.Join([]string{cfg.TestWorkspace, sysGopath}, ":"),
+ "GOPATH=" + strings.Join([]string{cfg.TestWorkspace, cfg.SysGopath}, ":"),
}
if out, err = cmd.CombinedOutput(); err != nil {
return out, err
diff --git a/src/chromiumos/tast/tast/build/build_test.go b/src/chromiumos/tast/tast/build/build_test.go
index bbf3d37..8c8842f 100644
--- a/src/chromiumos/tast/tast/build/build_test.go
+++ b/src/chromiumos/tast/tast/build/build_test.go
@@ -12,6 +12,8 @@
"os/exec"
"path/filepath"
"testing"
+
+ "chromiumos/tast/common/testutil"
)
func TestBuildTests(t *testing.T) {
@@ -21,21 +23,31 @@
}
defer os.RemoveAll(tempDir)
+ sysGopath := filepath.Join(tempDir, "gopath")
cfg := &Config{
TestWorkspace: tempDir,
+ SysGopath: sysGopath,
OutDir: filepath.Join(tempDir, "out"),
}
if cfg.Arch, err = GetLocalArch(); err != nil {
t.Fatal("Failed to get local arch: ", err)
}
- srcDir := filepath.Join(tempDir, "src", "foo", "cmd")
- if err = os.MkdirAll(srcDir, 0755); err != nil {
- t.Fatal(err)
- }
- const exp = "success!"
- code := fmt.Sprintf("package main\nfunc main() { print(\"%s\") }", exp)
- if err := ioutil.WriteFile(filepath.Join(srcDir, "main.go"), []byte(code), 0644); err != nil {
+ // In order to test that the supplied system GOPATH is used, build a main
+ // package that prints a constant exported by a system package.
+ const (
+ pkgName = "testpkg" // system package's name (without chromiumos/tast prefix)
+ constName = "Msg" // name of const exported by system package
+ constVal = "success!" // value of const exported by system package
+ )
+ pkgCode := fmt.Sprintf("package %s\nconst %s = %q", pkgName, constName, constVal)
+ mainCode := fmt.Sprintf("package main\nimport %q\nfunc main() { print(%s.%s) }",
+ pkgName, pkgName, constName)
+
+ if err := testutil.WriteFiles(tempDir, map[string]string{
+ filepath.Join("src", pkgName, "lib.go"): pkgCode,
+ "src/foo/cmd/main.go": mainCode,
+ }); err != nil {
t.Fatal(err)
}
@@ -46,7 +58,7 @@
if out, err := exec.Command(bin).CombinedOutput(); err != nil {
t.Errorf("Failed to run %s: %v", bin, err)
- } else if string(out) != exp {
- t.Errorf("%s printed %q; want %q", bin, string(out), exp)
+ } else if string(out) != constVal {
+ t.Errorf("%s printed %q; want %q", bin, string(out), constVal)
}
}
diff --git a/src/chromiumos/tast/tast/build/config.go b/src/chromiumos/tast/tast/build/config.go
index fe8cbba..a282cd7 100644
--- a/src/chromiumos/tast/tast/build/config.go
+++ b/src/chromiumos/tast/tast/build/config.go
@@ -19,10 +19,18 @@
// TestWorkspace is the path to the Go workspace where test source code is stored (i.e.
// containing a top-level directory named "src").
TestWorkspace string
+ // SysGopath is the path to the Go workspace containing source for test executables'
+ // emerged dependencies. This is typically /usr/lib/gopath.
+ SysGopath string
// Arch is the architecture to build for (as a machine name or processor given by "uname -m").
Arch string
// OutDir is the path to a directory where compiled code is stored (after appending arch).
OutDir string
+ // PortagePkg is the Portage package that contains the test executable (when tests are
+ // included in a system image rather than being compiled by the tast command).
+ // If non-empty, BuildTests checks that the package's direct dependencies are installed
+ // in the host sysroot before building tests.
+ PortagePkg string
}
// OutPath returns the path to a file named fn within cfg's architecture-specific output dir.
@@ -35,6 +43,8 @@
func (c *Config) SetFlags(f *flag.FlagSet, trunkDir string) {
f.StringVar(&c.Arch, "arch", "", "target architecture (per \"uname -m\")")
f.StringVar(&c.OutDir, "outdir", defaultBuildOutDir, "directory storing build artifacts")
+ f.StringVar(&c.SysGopath, "sysgopath", "/usr/lib/gopath",
+ "Go workspace containing system package source code (if empty, chosen automatically)")
f.StringVar(&c.TestWorkspace, "testdir", filepath.Join(trunkDir, defaultTestDir),
"Go workspace containing test source code")
}
diff --git a/src/chromiumos/tast/tast/build/portage.go b/src/chromiumos/tast/tast/build/portage.go
new file mode 100644
index 0000000..1aa7edb
--- /dev/null
+++ b/src/chromiumos/tast/tast/build/portage.go
@@ -0,0 +1,119 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package build
+
+import (
+ "context"
+ "fmt"
+ "os/exec"
+ "regexp"
+ "strings"
+ "syscall"
+
+ "chromiumos/tast/tast/timing"
+)
+
+var equeryDepsRegexp *regexp.Regexp
+
+func init() {
+ // Matches lines containing first-level dependencies printed by an
+ // "equery -q -C g --depth=1 <pkg>" command, which produces output
+ // similar to the following (preceded by a blank line):
+ //
+ // chromeos-base/tast-local-tests-9999:
+ // [ 0] chromeos-base/tast-local-tests-9999
+ // [ 1] chromeos-base/tast-common-9999
+ // [ 1] dev-go/cdp-0.9.1
+ // [ 1] dev-go/dbus-0.0.2-r5
+ // [ 1] dev-lang/go-1.8.3-r1
+ // [ 1] dev-vcs/git-2.12.2
+ equeryDepsRegexp = regexp.MustCompile("^\\s*\\[\\s*1\\]\\s+([\\S]+)")
+}
+
+// depInfo contains information about one of a package's dependencies.
+type depInfo struct {
+ pkg string // dependency's package name
+ installed bool // true if dependency is installed
+ err error // non-nil if error encountered while getting status
+}
+
+// checkDeps checks if all of portagePkg's direct dependencies are installed.
+// Missing dependencies are returned in badDeps, with package names in
+// the format "<category>/<package>-<version>" as keys and descriptive error
+// messages as values. err is set if a more-serious error is encountered while
+// trying to check dependencies.
+func checkDeps(ctx context.Context, portagePkg string) (missing []string, err error) {
+ if tl, ok := timing.FromContext(ctx); ok {
+ st := tl.Start("check_deps")
+ defer st.End()
+ }
+
+ cmd := exec.Command("equery", "-q", "-C", "g", "--depth=1", portagePkg)
+ out, err := cmd.Output()
+ if err != nil {
+ return nil, fmt.Errorf("%q failed: %v", strings.Join(cmd.Args, " "), err)
+ }
+
+ deps := parseEqueryDeps(out)
+ if len(deps) == 0 {
+ return nil, fmt.Errorf("no deps found in output from %q", strings.Join(cmd.Args, " "))
+ }
+
+ // "equery l" doesn't appear to accept multiple package names, so run queries in parallel.
+ ch := make(chan *depInfo, len(deps))
+ for _, dep := range deps {
+ go func(pkg string) {
+ info := &depInfo{pkg: pkg}
+ info.installed, info.err = portagePkgInstalled(pkg)
+ ch <- info
+ }(dep)
+ }
+
+ missing = make([]string, 0)
+ for range deps {
+ info := <-ch
+ if info.err != nil {
+ return missing, fmt.Errorf("failed getting status of %s: %v", info.pkg, info.err)
+ } else if !info.installed {
+ missing = append(missing, info.pkg)
+ }
+ }
+ return missing, nil
+}
+
+// parseEqueryDeps parses the output of checkDeps's "equery g" command and returns
+// the names (as "<category>/<package>-<version>") of first-level dependencies.
+func parseEqueryDeps(out []byte) []string {
+ deps := make([]string, 0)
+ for _, ln := range strings.Split(string(out), "\n") {
+ if matches := equeryDepsRegexp.FindStringSubmatch(ln); matches != nil {
+ deps = append(deps, matches[1])
+ }
+ }
+ return deps
+}
+
+// portagePkgInstalled runs "equery l" to check if pkg is installed.
+func portagePkgInstalled(pkg string) (bool, error) {
+ cmd := exec.Command("equery", "-q", "-C", "l", pkg)
+ out, err := cmd.Output()
+ if err != nil {
+ // equery (in "quiet mode") exits with 3 if the package isn't installed.
+ if exitErr, ok := err.(*exec.ExitError); ok {
+ if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+ if status.ExitStatus() == 3 {
+ return false, nil
+ }
+ }
+ }
+ return false, fmt.Errorf("%q failed: %v", strings.Join(cmd.Args, " "), err)
+ }
+
+ // equery should print the package name.
+ if str := strings.TrimSpace(string(out)); str != pkg {
+ return false, fmt.Errorf("%q returned %q", strings.Join(cmd.Args, " "), str)
+ }
+ return true, nil
+}
diff --git a/src/chromiumos/tast/tast/build/portage_test.go b/src/chromiumos/tast/tast/build/portage_test.go
new file mode 100644
index 0000000..d6b5498
--- /dev/null
+++ b/src/chromiumos/tast/tast/build/portage_test.go
@@ -0,0 +1,35 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package build
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestParseEqueryDeps(t *testing.T) {
+ // Copy-and-pasted output (including trailing whitespace) from
+ // "equery -q -C g --depth=1 chromeos-base/tast-local-tests-9999".
+ out := `
+chromeos-base/tast-local-tests-9999:
+ [ 0] chromeos-base/tast-local-tests-9999
+ [ 1] chromeos-base/tast-common-9999
+ [ 1] dev-go/cdp-0.9.1
+ [ 1] dev-go/dbus-0.0.2-r5
+ [ 1] dev-lang/go-1.8.3-r1
+ [ 1] dev-vcs/git-2.12.2
+`
+
+ exp := []string{
+ "chromeos-base/tast-common-9999",
+ "dev-go/cdp-0.9.1",
+ "dev-go/dbus-0.0.2-r5",
+ "dev-lang/go-1.8.3-r1",
+ "dev-vcs/git-2.12.2",
+ }
+ if act := parseEqueryDeps([]byte(out)); !reflect.DeepEqual(act, exp) {
+ t.Errorf("parseEqueryDeps(%q) = %v; want %v", out, act, exp)
+ }
+}
diff --git a/src/chromiumos/tast/tast/build_cmd.go b/src/chromiumos/tast/tast/build_cmd.go
index 1deb863..55e2965 100644
--- a/src/chromiumos/tast/tast/build_cmd.go
+++ b/src/chromiumos/tast/tast/build_cmd.go
@@ -38,13 +38,15 @@
fmt.Fprintf(os.Stderr, b.Usage())
return subcommands.ExitUsageError
}
+
if b.cfg.Arch == "" {
var err error
if b.cfg.Arch, err = build.GetLocalArch(); err != nil {
- lg.Logf("Failed to get local arch: %v\n", err)
+ lg.Log("Failed to get local arch: ", err)
return subcommands.ExitFailure
}
}
+
if out, err := build.BuildTests(ctx, &b.cfg, f.Args()[0], f.Args()[1]); err != nil {
lg.Logf("Failed building tests: %v\n%s", err, string(out))
return subcommands.ExitFailure
diff --git a/src/chromiumos/tast/tast/run_cmd.go b/src/chromiumos/tast/tast/run_cmd.go
index 577ea0b..7a74a97 100644
--- a/src/chromiumos/tast/tast/run_cmd.go
+++ b/src/chromiumos/tast/tast/run_cmd.go
@@ -33,8 +33,9 @@
// runCmd implements subcommands.Command to support running tests.
type runCmd struct {
- testType string // type of tests to run (either "local" or "remote")
- cfg run.Config // shared config for running tests
+ testType string // type of tests to run (either "local" or "remote")
+ checkDeps bool // true if test package's dependencies should be checked before building
+ cfg run.Config // shared config for running tests
}
func (*runCmd) Name() string { return "run" }
@@ -47,6 +48,7 @@
func (r *runCmd) SetFlags(f *flag.FlagSet) {
f.StringVar(&r.testType, "testtype", "local", "type of tests to run (either \"local\" or \"remote\")")
+ f.BoolVar(&r.checkDeps, "checkdeps", true, "checks test package's dependencies before building")
r.cfg.SetFlags(f)
r.cfg.BuildCfg.SetFlags(f, getTrunkDir())
}
@@ -111,8 +113,14 @@
lg.Log("Writing results to ", r.cfg.ResDir)
switch r.testType {
case localType:
+ if r.cfg.Build && r.checkDeps {
+ r.cfg.BuildCfg.PortagePkg = "chromeos-base/tast-local-tests-9999"
+ }
return run.Local(ctx, &r.cfg)
case remoteType:
+ if r.cfg.Build && r.checkDeps {
+ r.cfg.BuildCfg.PortagePkg = "chromeos-base/tast-remote-tests-9999"
+ }
return run.Remote(ctx, &r.cfg)
}
lg.Logf(fmt.Sprintf("Invalid test type %q\n\n%s", r.testType, r.Usage()))
