| // Copyright 2020 The ChromiumOS Authors |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| package planner |
| |
| import ( |
| "bufio" |
| "bytes" |
| "context" |
| "fmt" |
| "path/filepath" |
| "runtime" |
| "runtime/pprof" |
| "sort" |
| "sync" |
| "time" |
| |
| "go.chromium.org/tast/core/errors" |
| "go.chromium.org/tast/core/internal/logging" |
| "go.chromium.org/tast/core/internal/planner/internal/entity" |
| "go.chromium.org/tast/core/internal/planner/internal/fixture" |
| "go.chromium.org/tast/core/internal/planner/internal/output" |
| "go.chromium.org/tast/core/internal/protocol" |
| "go.chromium.org/tast/core/internal/testcontext" |
| "go.chromium.org/tast/core/internal/testing" |
| "go.chromium.org/tast/core/internal/timing" |
| "go.chromium.org/tast/core/internal/usercode" |
| |
| frameworkprotocol "go.chromium.org/tast/core/framework/protocol" |
| ) |
| |
| // OutputStream is an interface to report streamed outputs of multiple entity runs. |
| type OutputStream = output.Stream |
| |
| // FixtureStack maintains a stack of fixtures and their states. |
| type FixtureStack = fixture.InternalStack |
| |
| // NewFixtureStack creates a new empty fixture stack. |
| func NewFixtureStack(cfg *Config, out OutputStream) *FixtureStack { |
| return fixture.NewInternalStack(cfg.FixtureConfig(), out) |
| } |
| |
| const ( |
| preTestTimeout = 3 * time.Minute // timeout for RuntimeConfig.TestHook |
| postTestTimeout = 3 * time.Minute // timeout for a closure returned by RuntimeConfig.TestHook |
| |
| // DefaultGracePeriod is default recommended grace period for SafeCall. |
| DefaultGracePeriod = 30 * time.Second |
| ) |
| |
| // Config contains details about how the planner should run tests. |
| type Config struct { |
| // Dirs holds several directory paths important for running tests. |
| Dirs *protocol.RunDirectories |
| |
| // Features contains software/hardware features the DUT has, and runtime variables. |
| Features *protocol.Features |
| |
| // ServiceConfig contains configurations of external services available to |
| // Tast framework and Tast tests. |
| Service *protocol.ServiceConfig |
| |
| // DataFileConfig contains configurations about data files. |
| DataFile *protocol.DataFileConfig |
| |
| // RemoteData contains information relevant to remote tests. |
| // It is nil for local tests. |
| RemoteData *testing.RemoteData |
| // TestHook is run before TestInstance.Func (and TestInstance.Pre.Prepare, when applicable) if non-nil. |
| // The returned closure is executed after a test if not nil. |
| TestHook func(context.Context, *testing.TestHookState) func(context.Context, *testing.TestHookState) |
| |
| // BeforeDownload specifies a function called before downloading external data files. |
| // It is ignored if it is nil. |
| BeforeDownload func(context.Context) |
| // Tests is a map from a test name to its metadata. |
| Tests map[string]*testing.TestInstance |
| // Fixtures is a map from a fixture name to its metadata. |
| Fixtures map[string]*testing.FixtureInstance |
| // StartFixtureName is a name of a fixture to start test execution. |
| // Tests requested to run should depend on the start fixture directly or |
| // indirectly. |
| // Since a start fixture is treated specially (e.g. no output directory is |
| // created), metadata of a start fixture must not be contained in |
| // Config.Fixtures. Instead, StartFixtureImpl gives an implementation of |
| // a start fixture. |
| StartFixtureName string |
| // StartFixtureImpl gives an implementation of a start fixture. |
| // If it is nil, a default stub implementation is used. |
| StartFixtureImpl testing.FixtureImpl |
| // CustomGracePeriod specifies custom grace period after entity timeout. |
| // If nil reasonable default will be used. Config.GracePeriod() returns |
| // the grace period to use. This field exists for unit testing. |
| CustomGracePeriod *time.Duration |
| |
| // ExternalTarget represents configs for running an external bundle from |
| // current bundle. (i.e. local bundle from remote bundle). |
| ExternalTarget *ExternalTarget |
| |
| //MaxSysMsgLogSize is a size of flag for truncate log file. |
| MaxSysMsgLogSize int64 |
| } |
| |
| // GracePeriod returns grace period after entity timeout. |
| func (c *Config) GracePeriod() time.Duration { |
| if c.CustomGracePeriod != nil { |
| return *c.CustomGracePeriod |
| } |
| return DefaultGracePeriod |
| } |
| |
| // FixtureConfig returns a fixture config derived from c. |
| func (c *Config) FixtureConfig() *fixture.Config { |
| // Features contains software/hardware features each DUT has, and runtime variables. |
| // Its key for the map is the role of the DUT such as "cd1". |
| // The role for primary DUT should be "". |
| features := make(map[string]*frameworkprotocol.DUTFeatures) |
| features[""] = c.Features.GetDut() |
| for role, dutFeatures := range c.Features.GetCompanionFeatures() { |
| features[role] = dutFeatures |
| } |
| return &fixture.Config{ |
| DataDir: c.Dirs.GetDataDir(), |
| OutDir: c.Dirs.GetOutDir(), |
| Vars: c.Features.GetInfra().GetVars(), |
| Service: c.Service, |
| BuildArtifactsURL: c.DataFile.GetBuildArtifactsUrl(), |
| RemoteData: c.RemoteData, |
| StartFixtureName: c.StartFixtureName, |
| Features: features, |
| GracePeriod: c.GracePeriod(), |
| DUTLabConfig: c.Features.GetInfra().GetDUTLabConfig(), |
| } |
| } |
| |
| // RunTestsLegacy runs a set of tests, writing outputs to out. |
| // |
| // RunTestsLegacy is responsible for building an efficient plan to run the given tests. |
| // Therefore the order of tests in the argument is ignored; it just specifies |
| // a set of tests to run. |
| // |
| // RunTestsLegacy runs tests on goroutines. If a test does not finish after reaching |
| // its timeout, this function returns with an error without waiting for its finish. |
| func RunTestsLegacy(ctx context.Context, tests []*testing.TestInstance, out OutputStream, pcfg *Config) error { |
| if pcfg.Tests != nil { |
| return fmt.Errorf("BUG: RunTestsLegacy pcfg.Tests = %v, want nil", pcfg.Tests) |
| } |
| // HACK: modify Tests field. This code should soon go away along with the |
| // removal of this function. |
| defer func() { |
| pcfg.Tests = nil |
| }() |
| pcfg.Tests = make(map[string]*testing.TestInstance) |
| for _, t := range tests { |
| pcfg.Tests[t.Name] = t |
| } |
| ts := make([]*protocol.ResolvedEntity, len(tests)) |
| for i, t := range tests { |
| ts[i] = &protocol.ResolvedEntity{ |
| Hops: 0, |
| Entity: t.EntityProto(), |
| } |
| } |
| return RunTests(ctx, ts, out, pcfg) |
| } |
| |
| // RunTests runs a set of tests, writing outputs to out. |
| // |
| // RunTests is responsible for building an efficient plan to run the given tests. |
| // Therefore the order of tests in the argument is ignored; it just specifies |
| // a set of tests to run. |
| // |
| // RunTests runs tests on goroutines. If a test does not finish after reaching |
| // its timeout, this function returns with an error without waiting for its finish. |
| func RunTests(ctx context.Context, tests []*protocol.ResolvedEntity, out OutputStream, pcfg *Config) error { |
| plan, err := buildPlan(tests, pcfg) |
| if err != nil { |
| return err |
| } |
| return plan.run(ctx, out) |
| } |
| |
| // plan holds a top-level plan of test execution. |
| type plan struct { |
| skips []*skippedTest |
| fixtPlan *fixtPlan |
| prePlans []*prePlan |
| pcfg *Config |
| } |
| |
| type skippedTest struct { |
| test *testing.TestInstance |
| reasons []string |
| err error |
| } |
| |
| func buildPlan(tests []*protocol.ResolvedEntity, pcfg *Config) (*plan, error) { |
| var testsWithFixture []*protocol.ResolvedEntity |
| preMap := make(map[string][]*testing.TestInstance) |
| var skips []*skippedTest |
| for _, t := range tests { |
| if t.GetHops() > 0 { |
| testsWithFixture = append(testsWithFixture, t) |
| continue |
| } |
| ti, ok := pcfg.Tests[t.GetEntity().GetName()] |
| if !ok { |
| return nil, fmt.Errorf("BUG: test %v does not exist", t.GetEntity().GetName()) |
| } |
| reasons, err := ti.Deps().Check(pcfg.Features) |
| if err != nil || len(reasons) > 0 { |
| skips = append(skips, &skippedTest{test: ti, reasons: reasons, err: err}) |
| continue |
| } |
| if ti.Pre != nil { |
| preName := ti.Pre.String() |
| preMap[preName] = append(preMap[preName], ti) |
| continue |
| } |
| // A test which is not skipped nor depending on a precondition is |
| // fixture-ready, possibly depending on an empty fixture. |
| testsWithFixture = append(testsWithFixture, t) |
| } |
| sort.Slice(skips, func(i, j int) bool { |
| return skips[i].test.Name < skips[j].test.Name |
| }) |
| |
| preNames := make([]string, 0, len(preMap)) |
| for preName := range preMap { |
| preNames = append(preNames, preName) |
| } |
| sort.Strings(preNames) |
| |
| prePlans := make([]*prePlan, len(preNames)) |
| for i, preName := range preNames { |
| prePlans[i] = buildPrePlan(preMap[preName], pcfg) |
| } |
| |
| fixtPlan, err := buildFixtPlan(testsWithFixture, pcfg) |
| if err != nil { |
| return nil, err |
| } |
| return &plan{skips, fixtPlan, prePlans, pcfg}, nil |
| } |
| |
| func (p *plan) run(ctx context.Context, out output.Stream) error { |
| dl, err := newDownloader(ctx, p.pcfg) |
| if err != nil { |
| return errors.Wrap(err, "failed to create new downloader") |
| } |
| defer dl.TearDown() |
| dl.BeforeRun(ctx, p.entitiesToRun()) |
| |
| for _, s := range p.skips { |
| tout := output.NewEntityStream(out, s.test.EntityProto()) |
| reportSkippedTest(tout, s.reasons, s.err) |
| } |
| |
| if err := p.fixtPlan.run(ctx, out, dl); err != nil { |
| return err |
| } |
| |
| for _, pp := range p.prePlans { |
| if err := pp.run(ctx, out, dl); err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| func (p *plan) entitiesToRun() []*protocol.Entity { |
| var res = p.fixtPlan.entitiesToRun() |
| for _, pp := range p.prePlans { |
| for _, t := range pp.testsToRun() { |
| res = append(res, t.EntityProto()) |
| } |
| } |
| return res |
| } |
| |
| // fixtTree represents a fixture tree. |
| // At the beginning of fixture plan execution, a clone of a fixture tree is |
| // created, and finished tests are removed from the tree as we execute the plan |
| // to remember which tests are still to be run. |
| // A fixture tree is considered empty if it contains no test. An empty fixture |
| // tree must not appear as a subtree of another fixture tree so that we can |
| // check if a fixture tree is empty in O(1). |
| type fixtTree struct { |
| fixt *testing.FixtureInstance |
| |
| // Following fields are updated as we execute a plan. |
| tests []*testing.TestInstance |
| externalTests []string |
| children []*fixtTree |
| } |
| |
| // Empty returns if a tree has no test. |
| // An empty fixture tree must not appear as a subtree of another fixture tree |
| // so that we can check if a fixture tree is empty in O(1). |
| func (t *fixtTree) Empty() bool { |
| return len(t.tests) == 0 && len(t.externalTests) == 0 && len(t.children) == 0 |
| } |
| |
| // Clone returns a deep copy of fixtTree. |
| func (t *fixtTree) Clone() *fixtTree { |
| children := make([]*fixtTree, len(t.children)) |
| for i, child := range t.children { |
| children[i] = child.Clone() |
| } |
| return &fixtTree{ |
| fixt: t.fixt, |
| tests: append([]*testing.TestInstance(nil), t.tests...), |
| externalTests: append([]string(nil), t.externalTests...), |
| children: children, |
| } |
| } |
| |
| // orphanTest represents a test that depends on a missing fixture directly or |
| // indirectly. |
| type orphanTest struct { |
| test *protocol.ResolvedEntity |
| missingFixtName string |
| } |
| |
| // fixtPlan holds an execution plan of fixture-ready tests. |
| type fixtPlan struct { |
| pcfg *Config |
| tree *fixtTree // original fixture tree; must not be modified |
| orphans []*orphanTest |
| } |
| |
| // buildFixtPlan builds an execution plan of fixture-ready tests. Tests passed |
| // to this function must not depend on preconditions. |
| func buildFixtPlan(tests []*protocol.ResolvedEntity, pcfg *Config) (*fixtPlan, error) { |
| var orphans []*orphanTest |
| testsToRun := make(map[string][]*testing.TestInstance) // keyed by fixture |
| externalTestsToRun := make(map[string][]string) |
| |
| // Build a graph of fixtures relevant to the given tests. |
| graph := make(map[string][]string) // fixture name to its children names |
| added := make(map[string]struct{}) // set of fixtures added to graph as children |
| var traverse func(string) (bool, string) |
| traverse = func(fixture string) (rooted bool, missingFixtureName string) { |
| if fixture == pcfg.StartFixtureName { |
| return true, "" |
| } |
| if _, ok := added[fixture]; ok { |
| return true, "" |
| } |
| f, ok := pcfg.Fixtures[fixture] |
| if !ok { |
| return false, fixture |
| } |
| rooted, missing := traverse(f.Parent) |
| if rooted { |
| added[fixture] = struct{}{} |
| graph[f.Parent] = append(graph[f.Parent], fixture) |
| } |
| return rooted, missing |
| } |
| for _, t := range tests { |
| f := fixtTreeParent(t) |
| rooted, missing := traverse(f) |
| if !rooted { |
| orphans = append(orphans, &orphanTest{ |
| test: t, |
| missingFixtName: missing, |
| }) |
| } else if t.Hops == 0 { |
| // Existence of the test instance is already checked in buildPlan. |
| testsToRun[f] = append(testsToRun[f], pcfg.Tests[t.GetEntity().GetName()]) |
| } else if len(t.GetSkip().GetReasons()) > 0 { |
| // If a test is going to be skipped, don't run fixture setup. |
| externalTestsToRun[""] = append(externalTestsToRun[""], t.GetEntity().GetName()) |
| } else { |
| externalTestsToRun[f] = append(externalTestsToRun[f], t.GetEntity().GetName()) |
| } |
| } |
| // Normalize |
| sort.Slice(orphans, func(i, j int) bool { |
| ei := orphans[i].test |
| ej := orphans[j].test |
| if ei.Hops != ej.Hops { |
| return ei.Hops < ej.Hops |
| } |
| return ei.Entity.Name < ej.Entity.Name |
| }) |
| for _, ts := range testsToRun { |
| sort.Slice(ts, func(i, j int) bool { |
| return ts[i].Name < ts[j].Name |
| }) |
| } |
| for _, ts := range externalTestsToRun { |
| sort.Strings(ts) |
| } |
| for _, children := range graph { |
| sort.Strings(children) |
| } |
| |
| impl := pcfg.StartFixtureImpl |
| if impl == nil { |
| impl = &stubFixture{} |
| } |
| const infinite = 24 * time.Hour // a day is considered infinite |
| rootFixture := &testing.FixtureInstance{ |
| // Do not set Name of a start fixture. output.EntityOutputStream do not |
| // emit EntityStart/EntityEnd for unnamed entities. |
| Impl: impl, |
| // Set infinite timeouts to all lifecycle methods. In production, the |
| // start fixture may communicate with the host machine to trigger remote |
| // fixtures, which would take quite some time but timeouts are responsibly |
| // handled by the host binary. In unit tests, we may set the custom grace |
| // period to very small values (like a millisecond) to test the behavior |
| // when user code ignore timeouts, where we need long timeouts to avoid |
| // hitting timeouts in the start fixture. |
| SetUpTimeout: infinite, |
| TearDownTimeout: infinite, |
| ResetTimeout: infinite, |
| PreTestTimeout: infinite, |
| PostTestTimeout: infinite, |
| } |
| |
| // Traverse the graph to build a fixture tree. |
| var newTree func(name string) *fixtTree |
| newTree = func(name string) *fixtTree { |
| var f *testing.FixtureInstance |
| if name == pcfg.StartFixtureName { |
| f = rootFixture |
| } else { |
| f = pcfg.Fixtures[name] |
| } |
| var children []*fixtTree |
| for _, c := range graph[name] { |
| children = append(children, newTree(c)) |
| } |
| return &fixtTree{ |
| fixt: f, |
| tests: testsToRun[name], |
| externalTests: externalTestsToRun[name], |
| children: children, |
| } |
| } |
| tree := newTree(pcfg.StartFixtureName) |
| return &fixtPlan{pcfg: pcfg, tree: tree, orphans: orphans}, nil |
| } |
| |
| func fixtTreeParent(test *protocol.ResolvedEntity) string { |
| if test.Hops > 0 { |
| return test.StartFixtureName |
| } |
| return test.Entity.Fixture |
| } |
| |
| func (p *fixtPlan) run(ctx context.Context, out output.Stream, dl *downloader) error { |
| for _, o := range p.orphans { |
| tout := output.NewEntityStream(out, o.test.GetEntity()) |
| reportOrphanTest(tout, o.missingFixtName) |
| } |
| |
| tree := p.tree.Clone() |
| internalStack := fixture.NewInternalStack(p.pcfg.FixtureConfig(), out) |
| |
| var stack *internalOrCombinedStack |
| if p.pcfg.ExternalTarget != nil { |
| externalStack, err := fixture.NewExternalStack(ctx, out) |
| if err != nil { |
| return err |
| } |
| combinedStack := fixture.NewCombinedStack(externalStack, internalStack) |
| stack = &internalOrCombinedStack{combined: combinedStack} |
| internalStack.ParentFixtSerializedValue = func() ([]byte, error) { |
| return externalStack.SerializedVal(ctx) |
| } |
| } else { |
| // TODO(oka): Remove this code after migration for full remote fixtures. |
| // After migration is fully finished, only CombinedStack should be used. |
| stack = &internalOrCombinedStack{internal: internalStack} |
| } |
| |
| return runFixtTree(ctx, tree, stack, p.pcfg, out, dl) |
| } |
| |
| func (p *fixtPlan) entitiesToRun() []*protocol.Entity { |
| var entities []*protocol.Entity |
| |
| for _, o := range p.orphans { |
| entities = append(entities, o.test.GetEntity()) |
| } |
| |
| var traverse func(tree *fixtTree) |
| traverse = func(tree *fixtTree) { |
| if tree.fixt.Name != "" { |
| entities = append(entities, tree.fixt.EntityProto()) |
| } |
| for _, t := range tree.tests { |
| entities = append(entities, t.EntityProto()) |
| } |
| for _, subtree := range tree.children { |
| traverse(subtree) |
| } |
| } |
| traverse(p.tree) |
| |
| return entities |
| } |
| |
| // runFixtTree runs tests in a fixture tree. |
| // tree is modified as tests are run. |
| func runFixtTree(ctx context.Context, tree *fixtTree, stack *internalOrCombinedStack, pcfg *Config, out output.Stream, dl *downloader) error { |
| // Note about invariants: |
| // On entering this function, if the fixture stack is green, it is clean. |
| // Thus we don't need to reset fixtures before running a next test. |
| // On returning from this function, if the fixture stack was green and the |
| // fixture tree was non-empty on entering this function, the stack is dirty. |
| for !tree.Empty() && stack.Status() != fixture.StatusYellow { |
| if err := func() error { |
| // Create a fixture-scoped context. |
| ctx, cancel := context.WithCancel(ctx) |
| defer cancel() |
| |
| release := dl.BeforeEntity(ctx, tree.fixt.EntityProto()) |
| defer release() |
| |
| // Push a fixture to the stack. This will call SetUp if the fixture stack is green. |
| if err := stack.Push(ctx, tree.fixt); err != nil { |
| return err |
| } |
| // Do not defer stack.Pop call here. It is correct to not call TearDown when |
| // returning an error because it happens only when the timeout is ignored. |
| |
| // Run direct child tests first. |
| for stack.Status() != fixture.StatusYellow && len(tree.tests) > 0 { |
| t := tree.tests[0] |
| tree.tests = tree.tests[1:] |
| tout := output.NewEntityStream(out, t.EntityProto()) |
| if err := runTest(ctx, t, tout, pcfg, &preConfig{}, stack, dl); err != nil { |
| return err |
| } |
| if !tree.Empty() { |
| if err := stack.Reset(ctx); err != nil { |
| return err |
| } |
| } |
| } |
| // Run external tests then. |
| for stack.Status() != fixture.StatusYellow && len(tree.externalTests) > 0 { |
| unstarted, err := runExternalTests(ctx, tree.externalTests, stack.combined, pcfg, out) |
| if err != nil { |
| return err |
| } |
| if len(unstarted) == len(tree.externalTests) { |
| return fmt.Errorf("BUG: runExternalTests succeeds but no external test has run") |
| } |
| tree.externalTests = unstarted |
| } |
| |
| // Run child fixtures recursively. |
| for stack.Status() != fixture.StatusYellow && len(tree.children) > 0 { |
| subtree := tree.children[0] |
| if err := runFixtTree(ctx, subtree, stack, pcfg, out, dl); err != nil { |
| return err |
| } |
| // It is possible that a recursive call of runFixtTree aborted in middle of |
| // execution due to reset failures. Remove the subtree only when it is empty. |
| if subtree.Empty() { |
| tree.children = tree.children[1:] |
| } |
| if stack.Status() != fixture.StatusYellow && !tree.Empty() { |
| if err := stack.Reset(ctx); err != nil { |
| return err |
| } |
| } |
| } |
| |
| // Pop the fixture from the stack. This will call TearDown if it is not red. |
| if err := stack.Pop(ctx); err != nil { |
| return err |
| } |
| return nil |
| }(); err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| // prePlan holds execution plan of tests using the same precondition. |
| type prePlan struct { |
| pre testing.Precondition |
| tests []*testing.TestInstance |
| pcfg *Config |
| } |
| |
| func buildPrePlan(tests []*testing.TestInstance, pcfg *Config) *prePlan { |
| sort.Slice(tests, func(i, j int) bool { |
| return tests[i].Name < tests[j].Name |
| }) |
| return &prePlan{tests[0].Pre, tests, pcfg} |
| } |
| |
| func (p *prePlan) run(ctx context.Context, out output.Stream, dl *downloader) error { |
| // Create a precondition-scoped context. |
| ec := &testcontext.CurrentEntity{ |
| // OutDir is not available for a precondition-scoped context. |
| HasSoftwareDeps: true, |
| SoftwareDeps: append([]string(nil), p.tests[0].SoftwareDeps[""]...), |
| ServiceDeps: append([]string(nil), p.tests[0].ServiceDeps...), |
| PrivateAttr: append([]string(nil), p.tests[0].PrivateAttr...), |
| } |
| plog := newPreLogger(out) |
| pctx, cancel := context.WithCancel(testing.NewContext(ctx, ec, logging.NewFuncLogger(plog.Log))) |
| defer cancel() |
| |
| // Create an empty fixture stack. Tests using preconditions can't depend on |
| // fixture. |
| internalStack := fixture.NewInternalStack(p.pcfg.FixtureConfig(), out) |
| stack := &internalOrCombinedStack{internal: internalStack} |
| |
| for i, t := range p.tests { |
| ti := t.EntityProto() |
| plog.SetCurrentTest(ti) |
| tout := output.NewEntityStream(out, ti) |
| precfg := &preConfig{ |
| ctx: pctx, |
| close: p.pre != nil && i == len(p.tests)-1, |
| } |
| if err := runTest(ctx, t, tout, p.pcfg, precfg, stack, dl); err != nil { |
| return err |
| } |
| if i < len(p.tests)-1 { |
| if err := stack.Reset(ctx); err != nil { |
| return err |
| } |
| } |
| } |
| return nil |
| } |
| |
| func (p *prePlan) testsToRun() []*testing.TestInstance { |
| return append([]*testing.TestInstance(nil), p.tests...) |
| } |
| |
| // preLogger is a logger behind precondition-scoped contexts. It emits |
| // precondition logs to output.OutputStream just as if they are emitted by a currently |
| // running test. Call SetCurrentTest to set a current test. |
| type preLogger struct { |
| out output.Stream |
| |
| mu sync.Mutex |
| ti *protocol.Entity |
| } |
| |
| func newPreLogger(out output.Stream) *preLogger { |
| return &preLogger{out: out} |
| } |
| |
| // Log emits a log message to output.OutputStream just as if it is emitted by the |
| // current test. SetCurrentTest must be called before calling this method. |
| func (l *preLogger) Log(level logging.Level, ts time.Time, msg string) { |
| l.mu.Lock() |
| defer l.mu.Unlock() |
| l.out.EntityLog(l.ti, level, ts, msg) |
| } |
| |
| // SetCurrentTest sets the current test. |
| func (l *preLogger) SetCurrentTest(ti *protocol.Entity) { |
| l.mu.Lock() |
| defer l.mu.Unlock() |
| l.ti = ti |
| } |
| |
| // preConfig contains information needed to interact with a precondition for |
| // a single test. |
| type preConfig struct { |
| // ctx is a context that lives as long as the precondition. It is available |
| // to preconditions as testing.PreState.PreCtx. |
| ctx context.Context |
| // close is true if the test is the last test using the precondition and thus |
| // it should be closed. |
| close bool |
| } |
| |
| // runTest runs a single test, writing outputs messages to tout. |
| // |
| // runTest runs a test on a goroutine. If a test does not finish after reaching |
| // its timeout, this function returns with an error without waiting for its finish. |
| func runTest(ctx context.Context, t *testing.TestInstance, tout *output.EntityStream, pcfg *Config, precfg *preConfig, stack *internalOrCombinedStack, dl *downloader) error { |
| fixtCtx := ctx |
| |
| // Attach a log that the test can use to report timing events. |
| timingLog := timing.NewLog() |
| ctx = timing.NewContext(ctx, timingLog) |
| |
| outDir, err := entity.CreateOutDir(pcfg.Dirs.GetOutDir(), t.Name) |
| if err != nil { |
| return err |
| } |
| |
| tout.Start(outDir) |
| defer tout.End(nil, timingLog) |
| |
| afterTest := dl.BeforeEntity(ctx, t.EntityProto()) |
| defer afterTest() |
| |
| if err := stack.MarkDirty(ctx); err != nil { |
| return err |
| } |
| |
| switch stack.Status() { |
| case fixture.StatusGreen: |
| case fixture.StatusRed: |
| for _, e := range stack.Errors() { |
| tout.Error(e) |
| } |
| return nil |
| case fixture.StatusYellow: |
| return errors.New("BUG: Cannot run a test on a yellow fixture stack") |
| } |
| |
| tcfg := &testConfig{ |
| test: t, |
| outDir: outDir, |
| fixtCtx: fixtCtx, |
| // TODO(crbug.com/1106218): Make sure this approach is scalable. |
| // Recomputing purgeable on each test costs O(|purgeable| * |tests|) overall. |
| purgeable: dl.m.Purgeable(), |
| } |
| if err := runTestWithConfig(ctx, tcfg, pcfg, stack, precfg, tout); err != nil { |
| // If runTestWithRoot reported that the test didn't finish, print diagnostic messages. |
| msg := fmt.Sprintf("%v (see log for goroutine dump)", err) |
| tout.Error(testing.NewError(nil, msg, msg, 0)) |
| dumpGoroutines(tout) |
| return err |
| } |
| |
| return nil |
| } |
| |
| type testConfig struct { |
| test *testing.TestInstance |
| outDir string |
| fixtCtx context.Context |
| purgeable []string |
| } |
| |
| // runTestWithConfig runs a test on the given configs. |
| // |
| // The time allotted to the test is generally the sum of t.Timeout and t.ExitTimeout, but |
| // additional time may be allotted for preconditions and pre/post-test hooks. |
| func runTestWithConfig(ctx context.Context, tcfg *testConfig, pcfg *Config, stack *internalOrCombinedStack, precfg *preConfig, out testing.OutputStream) error { |
| // codeName is included in error messages if the user code ignores the timeout. |
| // For compatibility, the same fixed name is used for tests, preconditions and test hooks. |
| const codeName = "Test" |
| |
| var postTestFunc func(ctx context.Context, s *testing.TestHookState) |
| |
| condition := testing.NewEntityCondition() |
| |
| features := make(map[string]*frameworkprotocol.DUTFeatures) |
| features[""] = pcfg.Features.GetDut() |
| for role, dutFeatures := range pcfg.Features.GetCompanionFeatures() { |
| features[role] = dutFeatures |
| } |
| rcfg := &testing.RuntimeConfig{ |
| DataDir: filepath.Join(pcfg.Dirs.GetDataDir(), testing.RelativeDataDir(tcfg.test.Pkg)), |
| OutDir: tcfg.outDir, |
| Vars: pcfg.Features.GetInfra().GetVars(), |
| Features: features, |
| DUTLabConfig: pcfg.Features.GetInfra().GetDUTLabConfig(), |
| CloudStorage: testing.NewCloudStorage( |
| pcfg.Service.GetDevservers(), |
| pcfg.Service.GetTlwServer(), |
| pcfg.Service.GetTlwSelfName(), |
| pcfg.Service.GetDutServer(), |
| pcfg.DataFile.GetBuildArtifactsUrl(), |
| pcfg.Service.GetSwarmingTaskID(), |
| pcfg.Service.GetBuildBucketID(), |
| ), |
| RemoteData: pcfg.RemoteData, |
| FixtCtx: tcfg.fixtCtx, |
| FixtValue: stack.Val(), |
| FixtSerializedValue: func() ([]byte, error) { |
| return stack.SerializedVal(ctx) |
| }, |
| PreCtx: precfg.ctx, |
| Purgeable: tcfg.purgeable, |
| MaxSysMsgLogSize: pcfg.MaxSysMsgLogSize, |
| } |
| troot := testing.NewTestEntityRoot(tcfg.test, rcfg, out, condition) |
| ctx = troot.NewContext(ctx) |
| testState := troot.NewTestState() |
| |
| // First, perform setup and run the pre-test function. |
| if err := usercode.SafeCall(ctx, codeName, preTestTimeout, pcfg.GracePeriod(), usercode.ErrorOnPanic(testState), func(ctx context.Context) { |
| // The test bundle is responsible for ensuring t.Timeout is nonzero before calling Run, |
| // but we call s.Fatal instead of panicking since it's arguably nicer to report individual |
| // test failures instead of aborting the entire run. |
| if tcfg.test.Timeout <= 0 { |
| testState.Fatal("Invalid timeout ", tcfg.test.Timeout) |
| } |
| |
| entity.PreCheck(tcfg.test.Data, testState) |
| if testState.HasError() { |
| return |
| } |
| |
| // In remote tests, reconnect to the DUT if needed. |
| if pcfg.RemoteData != nil && testState.DUT() != nil { |
| dt := testState.DUT() |
| if !dt.Connected(ctx) { |
| testState.Log("Reconnecting to DUT") |
| if err := dt.Connect(ctx); err != nil { |
| testState.Error(testing.TestDidNotRunMsg) |
| testState.Fatal("Failed to reconnect to DUT: ", err) |
| } |
| } |
| } |
| |
| if pcfg.TestHook != nil { |
| postTestFunc = pcfg.TestHook(ctx, troot.NewTestHookState()) |
| } |
| }); err != nil { |
| return err |
| } |
| |
| // Prepare the test's precondition (if any) if setup was successful. |
| if !condition.HasError() && tcfg.test.Pre != nil { |
| preState := troot.NewPreState() |
| if err := usercode.SafeCall(ctx, codeName, tcfg.test.Pre.Timeout(), pcfg.GracePeriod(), usercode.ErrorOnPanic(preState), func(ctx context.Context) { |
| preState.Logf("Preparing precondition %q", tcfg.test.Pre) |
| troot.SetPreValue(tcfg.test.Pre.Prepare(ctx, preState)) |
| }); err != nil { |
| return err |
| } |
| } |
| |
| if err := func() error { |
| ctx, cancel := context.WithCancel(ctx) |
| defer cancel() |
| |
| // Run fixture pre-test hooks. |
| postTest, err := stack.PreTest(ctx, tcfg.test.EntityProto(), tcfg.outDir, out, condition) |
| if err != nil { |
| return err |
| } |
| |
| if !condition.HasError() { |
| // Run the test function itself. |
| if err := usercode.SafeCall(ctx, codeName, tcfg.test.Timeout, timeoutOrDefault(tcfg.test.ExitTimeout, pcfg.GracePeriod()), usercode.ErrorOnPanic(testState), func(ctx context.Context) { |
| tcfg.test.Func(ctx, testState) |
| }); err != nil { |
| return err |
| } |
| } |
| |
| // Run fixture post-test hooks. |
| if err := postTest(ctx); err != nil { |
| return err |
| } |
| return nil |
| }(); err != nil { |
| return err |
| } |
| |
| // If this is the final test using this precondition, close it |
| // (even if setup, tcfg.test.Pre.Prepare, or tcfg.test.Func failed). |
| if precfg.close { |
| preState := troot.NewPreState() |
| if err := usercode.SafeCall(ctx, codeName, tcfg.test.Pre.Timeout(), pcfg.GracePeriod(), usercode.ErrorOnPanic(preState), func(ctx context.Context) { |
| preState.Logf("Closing precondition %q", tcfg.test.Pre.String()) |
| tcfg.test.Pre.Close(ctx, preState) |
| }); err != nil { |
| return err |
| } |
| } |
| |
| // Finally, run the post-test functions unconditionally. |
| if postTestFunc != nil { |
| if err := usercode.SafeCall(ctx, codeName, postTestTimeout, pcfg.GracePeriod(), usercode.ErrorOnPanic(testState), func(ctx context.Context) { |
| postTestFunc(ctx, troot.NewTestHookState()) |
| }); err != nil { |
| return err |
| } |
| } |
| |
| return nil |
| } |
| |
| // timeoutOrDefault returns timeout if positive or def otherwise. |
| func timeoutOrDefault(timeout, def time.Duration) time.Duration { |
| if timeout > 0 { |
| return timeout |
| } |
| return def |
| } |
| |
| // reportOrphanTest is called instead of runTest for a test that depends on |
| // a missing fixture directly or indirectly. |
| func reportOrphanTest(tout *output.EntityStream, missingFixtName string) { |
| tout.Start("") |
| _, fn, ln, _ := runtime.Caller(0) |
| tout.Error(&protocol.Error{ |
| Reason: fmt.Sprintf("Fixture %q not found", missingFixtName), |
| Location: &protocol.ErrorLocation{ |
| File: fn, |
| Line: int64(ln), |
| }, |
| }) |
| tout.End(nil, timing.NewLog()) |
| } |
| |
| // reportSkippedTest is called instead of runTest for a test that is skipped due to |
| // having unsatisfied dependencies. |
| func reportSkippedTest(tout *output.EntityStream, reasons []string, err error) { |
| tout.Start("") |
| if err == nil { |
| tout.End(reasons, timing.NewLog()) |
| return |
| } |
| |
| _, fn, ln, _ := runtime.Caller(0) |
| tout.Error(&protocol.Error{ |
| Reason: err.Error(), |
| Location: &protocol.ErrorLocation{ |
| File: fn, |
| Line: int64(ln), |
| }, |
| }) |
| // Do not report a test as skipped if we encounter errors while checking |
| // dependencies. There is ambiguity if a test is skipped while reporting |
| // errors, and in the worst case, dependency check errors can be |
| // silently discarded. |
| tout.End(nil, timing.NewLog()) |
| } |
| |
| // dumpGoroutines dumps all goroutines to tout. |
| func dumpGoroutines(tout *output.EntityStream) { |
| tout.Log(logging.LevelInfo, time.Now(), "Dumping all goroutines") |
| if err := func() error { |
| p := pprof.Lookup("goroutine") |
| if p == nil { |
| return errors.New("goroutine pprof not found") |
| } |
| var buf bytes.Buffer |
| if err := p.WriteTo(&buf, 2); err != nil { |
| return err |
| } |
| sc := bufio.NewScanner(&buf) |
| for sc.Scan() { |
| tout.Log(logging.LevelInfo, time.Now(), sc.Text()) |
| } |
| return sc.Err() |
| }(); err != nil { |
| tout.Error(&protocol.Error{ |
| Reason: fmt.Sprintf("Failed to dump goroutines: %v", err), |
| }) |
| } |
| } |
| |
| // stubFixture is a stub implementation of testing.FixtureImpl. |
| type stubFixture struct{} |
| |
| func (f *stubFixture) SetUp(ctx context.Context, s *testing.FixtState) interface{} { return nil } |
| func (f *stubFixture) Reset(ctx context.Context) error { return nil } |
| func (f *stubFixture) PreTest(ctx context.Context, s *testing.FixtTestState) {} |
| func (f *stubFixture) PostTest(ctx context.Context, s *testing.FixtTestState) {} |
| func (f *stubFixture) TearDown(ctx context.Context, s *testing.FixtState) {} |