| // Copyright 2020 The ChromiumOS Authors |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| package testing |
| |
| import ( |
| "context" |
| "regexp" |
| "time" |
| |
| "go.chromium.org/tast/core/errors" |
| "go.chromium.org/tast/core/internal/protocol" |
| ) |
| |
| // Fixture represents fixtures to register to the framework. |
| type Fixture struct { |
| // Name is the name of the fixture. |
| // Tests and fixtures use the name to specify the fixture. |
| // The name must be camelCase starting with a lowercase letter and containing only digits and letters. |
| Name string |
| |
| // Desc is the description of the fixture. |
| Desc string |
| |
| // Contacts is a list of email addresses of persons and groups who are familiar with the |
| // fixture. At least one personal email address of an active committer should be specified so |
| // that we can file bugs or ask for code review. |
| Contacts []string |
| |
| // Impl is the implementation of the fixture. |
| Impl FixtureImpl |
| |
| // Parent specifies the parent fixture name, or empty if it has no parent. |
| Parent string |
| |
| // SetUpTimeout is the timeout applied to SetUp. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // This timeout is by default 0. |
| SetUpTimeout time.Duration |
| |
| // ResetTimeout is the timeout applied to Reset. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // This timeout is by default 0. |
| ResetTimeout time.Duration |
| |
| // PreTestTimeout is the timeout applied to PreTest. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // This timeout is by default 0. |
| PreTestTimeout time.Duration |
| |
| // PostTestTimeout is the timeout applied to PostTest. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // This timeout is by default 0. |
| PostTestTimeout time.Duration |
| |
| // TearDownTimeout is the timeout applied to TearDown. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // This timeout is by default 0. |
| TearDownTimeout time.Duration |
| |
| // Params lists the Param structs for parameterized fixtures. |
| Params []FixtureParam |
| |
| // ServiceDeps contains a list of RPC service names in local test bundles that this remote fixture |
| // will access. This field is valid only for remote fixtures. |
| ServiceDeps []string |
| |
| // Vars contains the names of runtime variables used to pass out-of-band data to tests. |
| // Values are supplied using "tast run -var=name=value", and tests can access values via State.Var. |
| Vars []string |
| |
| // Data contains paths of data files needed by the fixture, relative to a |
| // "data" subdirectory within the directory in which the fixture is registered. |
| Data []string |
| |
| // PrivateAttr contains freeform text private attributres describing the fixture. |
| PrivateAttr []string |
| } |
| |
| // FixtureParam defines parameters for a parameterized fixture. |
| type FixtureParam struct { |
| // Name is the name of this parameterized fixture. |
| // Full name of the fixture will be category.FixtureName.param_name, |
| // or category.FixtureName if Name is empty. |
| // Name should match with [a-z0-9_]*. |
| Name string |
| |
| // ExtraContacts is a list of extra email addresses of persons and groups who are familiar |
| // with the parameter. At least one personal email address of an active committer |
| // should be specified so that we can file bugs or ask for code review. |
| ExtraContacts []string |
| |
| // Parent specifies the parent fixture name, or empty if it has no parent. |
| // Can only be set if the enclosing fixture doesn't have one already set. |
| Parent string |
| |
| // SetUpTimeout is the timeout applied to SetUp. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // Can only be set if the enclosing fixture doesn't have one already set. |
| SetUpTimeout time.Duration |
| |
| // ResetTimeout is the timeout applied to Reset. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // Can only be set if the enclosing fixture doesn't have one already set. |
| ResetTimeout time.Duration |
| |
| // PreTestTimeout is the timeout applied to PreTest. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // Can only be set if the enclosing fixture doesn't have one already set. |
| PreTestTimeout time.Duration |
| |
| // PostTestTimeout is the timeout applied to PostTest. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // Can only be set if the enclosing fixture doesn't have one already set. |
| PostTestTimeout time.Duration |
| |
| // TearDownTimeout is the timeout applied to TearDown. |
| // Even if fixtures are nested, the timeout is applied only to this stage. |
| // Can only be set if the enclosing fixture doesn't have one already set. |
| TearDownTimeout time.Duration |
| |
| // Val is the value which can be retrieved from testing.FixtState.Param() method. |
| Val interface{} |
| |
| // ExtraServiceDeps contains a list of extra RPC service names in local test bundles |
| // that this remote fixture parameter will access. |
| // This field is valid only for remote fixtures. |
| ExtraServiceDeps []string |
| |
| // ExtraData contains paths of extra data files needed by the fixture parameter, |
| // relative to a "data" subdirectory within the directory in which the fixture is registered. |
| ExtraData []string |
| |
| // ExtraPrivateAttr contains extra freeform text private attributes describing the |
| // fixture parameter. |
| ExtraPrivateAttr []string |
| } |
| |
| func (f *Fixture) instantiate(pkg string) ([]*FixtureInstance, error) { |
| if err := validateFixture(f); err != nil { |
| return nil, err |
| } |
| // Empty Params is equivalent to one Param with all default values. |
| ps := f.Params |
| if len(ps) == 0 { |
| ps = []FixtureParam{{}} |
| } |
| |
| fis := make([]*FixtureInstance, 0, len(ps)) |
| for _, p := range ps { |
| fi, err := newFixtureInstance(f, pkg, &p) |
| if err != nil { |
| return nil, err |
| } |
| fis = append(fis, fi) |
| } |
| return fis, nil |
| } |
| |
| func newFixtureInstance(f *Fixture, pkg string, p *FixtureParam) (*FixtureInstance, error) { |
| name := f.Name |
| if p.Name != "" { |
| name += "." + p.Name |
| } |
| |
| contacts := append(f.Contacts, p.ExtraContacts...) |
| |
| parent := f.Parent |
| if p.Parent != "" { |
| parent = p.Parent |
| } |
| |
| setUpTimeout, err := timeout(p.SetUpTimeout, f.SetUpTimeout, "SetUpTimeout") |
| if err != nil { |
| return nil, err |
| } |
| resetTimeout, err := timeout(p.ResetTimeout, f.ResetTimeout, "ResetTimeout") |
| if err != nil { |
| return nil, err |
| } |
| preTestTimeout, err := timeout(p.PreTestTimeout, f.PreTestTimeout, "PreTestTimeout") |
| if err != nil { |
| return nil, err |
| } |
| postTestTimeout, err := timeout(p.PostTestTimeout, f.PostTestTimeout, "PostTestTimeout") |
| if err != nil { |
| return nil, err |
| } |
| tearDownTimeout, err := timeout(p.TearDownTimeout, f.TearDownTimeout, "TearDownTimeout") |
| if err != nil { |
| return nil, err |
| } |
| |
| serviceDeps := append(f.ServiceDeps, p.ExtraServiceDeps...) |
| |
| data := append(f.Data, p.ExtraData...) |
| |
| privateAttr := append(f.PrivateAttr, p.ExtraPrivateAttr...) |
| |
| return &FixtureInstance{ |
| Pkg: pkg, |
| Name: name, |
| Desc: f.Desc, |
| Contacts: contacts, |
| Impl: f.Impl, |
| Parent: parent, |
| SetUpTimeout: setUpTimeout, |
| ResetTimeout: resetTimeout, |
| PreTestTimeout: preTestTimeout, |
| PostTestTimeout: postTestTimeout, |
| TearDownTimeout: tearDownTimeout, |
| Val: p.Val, |
| ServiceDeps: serviceDeps, |
| Data: data, |
| Vars: f.Vars, |
| PrivateAttr: privateAttr, |
| }, nil |
| } |
| |
| func timeout(paramTimeout, fixtureTimeout time.Duration, timeoutType string) (time.Duration, error) { |
| if paramTimeout != 0 { |
| if fixtureTimeout != 0 { |
| return 0, |
| errors.Errorf("Param has %s specified and its enclosing fixture also has %s specified, but only one can be specified", |
| timeoutType, timeoutType) |
| } |
| return paramTimeout, nil |
| } |
| return fixtureTimeout, nil |
| } |
| |
| // FixtureInstance represents a fixture instance registered to the framework. |
| // |
| // FixtureInstance is to Fixture what TestInstance is to Test. |
| type FixtureInstance struct { |
| // Pkg is the package from which the fixture is registered. |
| Pkg string |
| |
| // Following fields are copied from Fixture. |
| Name string |
| Desc string |
| Contacts []string |
| Impl FixtureImpl |
| Parent string |
| SetUpTimeout time.Duration |
| ResetTimeout time.Duration |
| PreTestTimeout time.Duration |
| PostTestTimeout time.Duration |
| TearDownTimeout time.Duration |
| Val interface{} |
| Data []string |
| ServiceDeps []string |
| Vars []string |
| PrivateAttr []string |
| |
| // Bundle is the name of the test bundle this test belongs to. |
| // This field is empty initially, and later set when the test is added |
| // to testing.Registry. |
| Bundle string |
| } |
| |
| // Constraints returns EntityConstraints for this fixture. |
| func (f *FixtureInstance) Constraints() *EntityConstraints { |
| return &EntityConstraints{ |
| allVars: append([]string(nil), f.Vars...), |
| allData: append([]string(nil), f.Data...), |
| } |
| } |
| |
| // EntityProto returns a protocol buffer message representation of f. |
| func (f *FixtureInstance) EntityProto() *protocol.Entity { |
| return &protocol.Entity{ |
| Type: protocol.EntityType_FIXTURE, |
| Package: f.Pkg, |
| Name: f.Name, |
| Description: f.Desc, |
| Fixture: f.Parent, |
| Dependencies: &protocol.EntityDependencies{ |
| DataFiles: append([]string(nil), f.Data...), |
| Services: append([]string(nil), f.ServiceDeps...), |
| }, |
| Contacts: &protocol.EntityContacts{ |
| Emails: append([]string(nil), f.Contacts...), |
| }, |
| LegacyData: &protocol.EntityLegacyData{ |
| Variables: append([]string(nil), f.Vars...), |
| Bundle: f.Bundle, |
| }, |
| } |
| } |
| |
| // fixtureNameRegexp defines the valid fixture name pattern. |
| var fixtureNameRegexp = regexp.MustCompile(`^[a-z][A-Za-z0-9]*$`) |
| |
| // validateFixture validates a user-supplied Fixture metadata. |
| func validateFixture(f *Fixture) error { |
| if !fixtureNameRegexp.MatchString(f.Name) { |
| return errors.Errorf("invalid fixture name: %q", f.Name) |
| } |
| return nil |
| } |
| |
| // FixtureImpl provides implementation of the fixture registered to the framework. |
| type FixtureImpl interface { |
| // SetUp is called by the framework to set up the environment with possibly heavy-weight |
| // operations. |
| // |
| // The context and state passed to SetUp are associated with the fixture metadata. For example, |
| // testing.ContextOutDir(ctx) and s.OutDir() return the output directory allocated for the |
| // fixture itself. testing.ContextSoftwareDeps(ctx) fails since fixtures can't declare software |
| // dependencies. |
| // |
| // TODO(oka): Consider updating ContextSoftwareDeps API so that it's meaningful for fixture |
| // scoped contexts. For tests, ContextSoftwareDeps returns the dependencies the test declares, |
| // and utility methods (e.g. arc.New) use it to check that tests declare proper software |
| // dependencies. For fixtures, it is uncertain what ContextSoftwareDeps should return. One |
| // might think it could return the intersection of the software deps of the tests depending on |
| // the fixture, but it doesn't work considering arc.New checks OR condition of the software |
| // deps. Still, fixtures can call functions like arc.New that calls ContextSoftwareDeps. It |
| // indicates that we need to reconsider ContextSoftwareDeps API. |
| // |
| // The return value is made available to the direct children of this fixture in the entity |
| // graph, as long as this fixture and a child live in the same process. |
| // If any resource is associated with the value (e.g. Chrome browser connection), it must not |
| // be released in Reset, PreTest and PostTest because descendant fixtures may cache it or |
| // construct subresources derived from it (e.g. Chrome tab connection). |
| // |
| // SetUp is called in descending order (parents to children) when fixtures are nested. |
| // |
| // SetUp can be called multiple times. This happens when this fixture's TearDown is called |
| // before completing all tests depending on it in the following cases: |
| // - This fixture's Reset requested it by returning an error. |
| // - Ascendant fixtures' Reset requested it by returning an error. |
| // In any case, TearDown is called in a pair with a successful SetUp call. |
| // |
| // Errors in this method are reported as errors of the fixture itself, rather than tests |
| // depending on it. |
| // |
| // If one or more errors are reported in SetUp by s.Error or s.Fatal, all remaining tests |
| // depending on this fixture are marked failed without actually running. TearDown is not called |
| // in this case. If s.Fatal is called, SetUp immediately aborts. |
| // |
| // This method is the best place to do a heavy-weight setup of the system environment, e.g. |
| // restarting a Chrome session. |
| // |
| // Note that SetUpTimeout is by default 0. Change it to have a valid context. |
| SetUp(ctx context.Context, s *FixtState) interface{} |
| |
| // Reset is called by the framework after each test (except for the last one) to do a |
| // light-weight reset of the environment to the original state. |
| // |
| // The context passed to Reset is associated with the fixture metadata. See SetUp for details. |
| // |
| // If Reset returns a non-nil error, the framework tears down and re-sets up the fixture to |
| // recover. To be accurate, the following methods are called in order: |
| // - Descendant fixtures' TearDown in ascending order |
| // - This fixture's TearDown |
| // - This fixture's SetUp |
| // - Descendant fixtures' SetUp in descending order |
| // Consequently, errors Reset returns don't affect ascendant fixtures. |
| // |
| // Returning an error from Reset is valid when light-weight reset doesn't restore the |
| // condition the fixture declares. |
| // |
| // Reset is called in descending order (parents to children) when fixtures are nested. |
| // |
| // This method is the best place to do a light-weight cleanup of the system environment to the |
| // original one when the fixture was set up, e.g. closing open Chrome tabs. |
| // |
| // Note that ResetTimeout is by default 0. Change it to have a valid context. |
| Reset(ctx context.Context) error |
| |
| // PreTest is called by the framework before each test to do a light-weight set up for the test. |
| // |
| // The context and state passed to PreTest are associated with the test metadata. For example, |
| // testing.ContextOutDir(ctx) and s.OutDir() return the output directory allocated to the test. |
| // |
| // PreTest is called in descending order (parents to children) when fixtures are nested. |
| // |
| // s.Error or s.Fatal can be used to report errors in PreTest. When s.Fatal is called PreTest |
| // immediately aborts. The error is marked as the test failure. |
| // |
| // This method is always called if fixture is successfully reset by SetUp or Reset. |
| // |
| // In any case, PostTest is called in a pair with a successful PreTest call. |
| // |
| // If errors are reported in PreTest, it is reported as the test failure. |
| // |
| // If errors are reported in PreTest, the test and PostTest are not run. |
| // |
| // This method is the best place to do a setup for the test runs next. e.g. redirect logs to a |
| // file in the test's output directory. |
| // |
| // Note that PreTestTimeout is by default 0. Change it to have a valid context. |
| PreTest(ctx context.Context, s *FixtTestState) |
| |
| // PostTest is called by the framework after each test to tear down changes PreTest made. |
| // |
| // The context and state passed to PostTest are associated with the test metadata. For example, |
| // testing.ContextOutDir(ctx) and s.OutDir() return the output directory allocated to the test. |
| // |
| // PostTest is called in ascending order (children to parent) when fixtures are nested. |
| // |
| // s.Error or s.Fatal can be used to report errors in PostTest. When s.Fatal is called PostTest |
| // immediately aborts. The error is marked as the test failure. |
| // |
| // This method is always called if PreTest succeeds. |
| // |
| // PostTest is always called in a pair with a successful PreTest call. |
| // |
| // If errors are reported in PostTest, it is reported as the test failure. |
| // |
| // The errors PostTest reports don't affect the order of the fixture methods the framework |
| // calls. |
| // |
| // This method is the best place to tear down changes PreTest made. e.g. close log files in the |
| // test output directory. |
| // |
| // Note that PostTestTimeout is by default 0. Change it to have a valid context. |
| PostTest(ctx context.Context, s *FixtTestState) |
| |
| // TearDown is called by the framework to tear down the environment SetUp set up. |
| // |
| // The context and state passed to TearDown are associated with the fixture metadata. See SetUp |
| // for details. |
| // |
| // TearDown is called in an ascending order (children to parents) when fixtures are nested. |
| // |
| // TearDown is always called in a pair with a successful SetUp call. |
| // |
| // Errors in this method are reported as errors of the fixture itself, rather than tests |
| // depending on it. |
| // |
| // Errors in TearDown doesn't affect the order of the fixture methods the framework calls. |
| // That is, even if this fixture's TearDown reports errors, its ascendants' TearDown are still |
| // called. |
| // |
| // This method is the best place to tear down changes SetUp made. e.g. Unenroll enterprise |
| // enrollment. |
| // Changes that shouldn't hinder healthy execution of succeeding tests are not necessarily to |
| // be teared down. e.g. Chrome session can be left open. |
| // |
| // Note that TearDownTimeout is by default 0. Change it to have a valid context. |
| TearDown(ctx context.Context, s *FixtState) |
| } |