src/chromiumos/tast/local/crostini/params.go - chromiumos/platform/tast-tests - Git at Google

 // Copyright 2020 The Chromium OS Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 package crostini

 // This file contains the generators for crostini test
 // parameters. Crostini tests generally require multiple sub-tests
 // parameterized over how crostini is set up. As these parameters are
 // complex and *change over time* you should strongly prefer to use
 // these generators rather then copying and pasting from other tests
 // or writing your own. This allows for changes to be made in one
 // central location (this file) and automatically propagated to all
 // tests that rely on them.
 //
 // Usage:
 //
 // In general, to generate test parameters for a test you add a unit
 // test file to the package containing the test (the name must end in
 // "_test.go"). This file should contain a go unit test which creates
 // a string containing the test parameters and passes it to
 // genparams.Ensure, along with the name of the test file. For
 // crostini tests, generate this string by calling one of the
 // functions in this file.
 //
 // By default, this unit test will ensure that the parameters in the
 // test match up with the string generated by the unit tests. These
 // tests are run on the CQ, so unexpected changes will block CLs from
 // landing. When run with TAST_GENERATE_UPDATE=1 set, it will instead
 // update the parameters in the test file.
 //
 // go unit tests can be run using the command
 // ~/trunk/src/platform/tast/tools/go.sh test -count=1 chromiumos/tast/local/bundles/...
 // parameter regeneration can be done with
 // TAST_GENERATE_UPDATE=1 ~/trunk/src/platform/tast/tools/go.sh test -count=1 chromiumos/tast/local/bundles/...
 //
 // The crostini package has a mega-test in params_test.go which
 // generates parameters for most of its tests. If you are adding a
 // test to that package, consider if it can be added to that
 // file. This should be the case for most tests, which only need the
 // "normal" set of parameters. Otherwise, or if you are writing a test
 // outside of the crostini package, you will likely need to create a
 // new test file. See any of the file in the crostini package ending
 // in "_test.go" for examples.
 //
 // Tests using these generators should not specify timeouts or
 // preconditions, as we expect to be able to specify those freely in
 // the parameters.
 //
 // Any test which is controlled by a generator will have a comment
 // above its test parameters indicating which file contains the
 // relevant unit test. To modify the parameters, update the test and
 // run the above command to regenerate the results.

 import (
 	"fmt"
 	"strings"
 	"time"

 	"chromiumos/tast/common/genparams"
 	"chromiumos/tast/local/vm"
 )

 // Param specifies how each set of crostini tests should be generated.
 type Param struct {
 	// Name of the test case. Generated tests will look like
 	// "name_amd64_buster_stable", "name_arm_stretch_unstable"
 	// etc.
 	Name string

 	// ExtraAttr contains additional attributes to add to the
 	// generated test's ExtraAttr field beyond what the generator
 	// function adds. For example, if you want this test case to
 	// be in the "graphics_daily" group without putting the whole
 	// test in, you can add that label here.
 	ExtraAttr []string

 	// ExtraData contains paths of additional data files needed by
 	// the test case. Note that data files required for specific
 	// crostini preconditions are added automatically to the
 	// generated tests and should not be added here.
 	ExtraData []string

 	// ExtraSoftwareDeps lists software features that are required
 	// to run this test case.
 	ExtraSoftwareDeps []string

 	// Timeout indicates the timeout for this test case. If
 	// unspecified, defaults to 7 * time.Minute.
 	Timeout time.Duration

 	// Val is a freeform value that can be retrieved from
 	// testing.State.Param() method. This string is inserted
 	// unmodified and unquoted into the generated test case code
 	// as the Val for each test case generated for this object.
 	Val string

 	// SelfManagedInstall indicates that this test will be
 	// installing crostini itself, and therefore there should be
 	// no crostini install precondition set.
 	SelfManagedInstall bool

 	// StableHardwareDep contains a go expression that evaluates
 	// to a hardware dependency which controls the collection of
 	// boards considered stable.
 	StableHardwareDep string

 	// UnstableHardwareDep contains a go expression that evaluates
 	// to a hardware dependency which controls the collection of
 	// boards considered unstable. It should be the inverse of
 	// StableHardwareDep.
 	UnstableHardwareDep string

 	// MinimalSet - if true, generate only a minimal set of test
 	// parameters such that each device will have at most one test
 	// case it can run. This is useful for things like performance
 	// tests, which are too expensive to be run in every possible
 	// configuration.
 	MinimalSet bool

 	// IsNotMainline indicates whether the test case is in
 	// group:mainline or not. This is important to get right
 	// because we can't add the "informational" attribute to
 	// non-mainline tests, but leaving it off of a mainline test
 	// will cause the test case to become CQ critical. If in
 	// doubt, set to false, and if you're wrong you will get an
 	// error message when you try to run your tests.
 	//
 	// This also controls whether separate stable/unstable tests
 	// are generated, since this distinction is only relevant to
 	// the CQ.
 	IsNotMainline bool

 	// UseLargeContainer controls whether to use the normal test
 	// container, or a larger container with more applications
 	// pre-installed.
 	UseLargeContainer bool

 	// OnlyStableBoards controls whether to only use the stable
 	// board variants and exclude all the unstable variants.
 	OnlyStableBoards bool

 	// UseGaiaLogin controls whether using gaia user to login
 	// to the DUT.
 	UseGaiaLogin bool
 }

 type generatedParam struct {
 	Name              string
 	ExtraAttr         []string
 	ExtraData         []string
 	ExtraSoftwareDeps []string
 	ExtraHardwareDeps string
 	Pre               string
 	Timeout           time.Duration
 	Val               string
 }

 const template = `{{range .}} {
 	{{if .Name}}              Name:              {{fmt .Name}},                                             {{end}}
 	{{if .ExtraAttr}}         ExtraAttr:         []string{ {{range .ExtraAttr}}{{fmt .}},{{end}} },         {{end}}
 	{{if .ExtraData}}         ExtraData:         []string{ {{range .ExtraData}}  {{.}},  {{end}} },         {{end}}
 	{{if .ExtraSoftwareDeps}} ExtraSoftwareDeps: []string{ {{range .ExtraSoftwareDeps}}{{fmt .}},{{end}} }, {{end}}
 	{{if .ExtraHardwareDeps}} ExtraHardwareDeps: {{.ExtraHardwareDeps}},                                    {{end}}
 	{{if .Pre}}               Pre:               {{.Pre}},                                                  {{end}}
 	{{if .Timeout}}           Timeout:           {{fmt .Timeout}},                                          {{end}}
 	{{if .Val}}               Val:               {{.Val}},                                                  {{end}}
 }, {{end}}`

 func combineName(first, second string) string {
 	if first == "" {
 		return second
 	}
 	if second == "" {
 		return first
 	}
 	return first + "_" + second
 }

 // MakeTestParamsFromList takes a list of test cases (in the form of
 // crostini.Param objects) and generates a set of crostini test
 // parameters for each. See the documentation for crostini.Param for
 // how these values effect the results. Each crostini.Param object is
 // treated independently, producing its own set of sub-tests.
 //
 // Normally you should use MakeTestParams instead, but if your test is
 // parameterized beyond which crostini preconditions it uses, you will
 // need this.
 func MakeTestParamsFromList(t genparams.TestingT, baseCases []Param) string {
 	var result []generatedParam

 	type iterator struct {
 		debianVersion vm.ContainerDebianVersion
 		stable        bool
 	}
 	var it = []iterator{}

 	for _, debianVersion := range []vm.ContainerDebianVersion{vm.DebianStretch, vm.DebianBuster, vm.DebianBullseye} {
 		for _, stable := range []bool{true, false} {
 			it = append(it, iterator{
 				debianVersion: debianVersion,
 				stable:        stable,
 			})
 		}
 	}

 	for _, testCase := range baseCases {

 		if testCase.UseLargeContainer && !testCase.MinimalSet {
 			t.Fatalf("Test %q: Testing apps on stretch is not supported", testCase.Name)
 		}

 		// Check here if it's possible for any iteration of
 		// this test to be critical, i.e. if it doesn't
 		// already have the "informational" attribute, and is
 		// a mainline test.
 		canBeCritical := true
 		for _, attr := range testCase.ExtraAttr {
 			if attr == "informational" {
 				canBeCritical = false
 			}
 		}

 		for _, i := range it {

 			if (testCase.IsNotMainline || testCase.OnlyStableBoards) && !i.stable {
 				// The stable/unstable distinction is only important for mainline tests
 				continue
 			}

 			if testCase.MinimalSet && i.debianVersion != vm.DebianBuster {
 				// The minimal set is currently buster
 				continue
 			}

 			name := testCase.Name
 			if !testCase.MinimalSet {
 				// If we're generating a minimal set
 				// then the debian version is always
 				// the same and we don't need to
 				// include it in the test name.
 				name = combineName(name, string(i.debianVersion))
 			}
 			if !testCase.IsNotMainline && !testCase.OnlyStableBoards {
 				if i.stable {
 					name = combineName(name, "stable")
 				} else {
 					name = combineName(name, "unstable")
 				}
 			}
 			if testCase.UseGaiaLogin {
 				name = combineName(name, "gaia")
 			}

 			// _unstable tests can never be CQ critical.
 			// stretch is informational while we phase it out.
 			var extraAttr []string
 			if (!i.stable || i.debianVersion == vm.DebianStretch) && canBeCritical {
 				extraAttr = append(extraAttr, "informational")
 			}

 			var extraData []string
 			extraData = append(extraData,
 				fmt.Sprintf("crostini.GetContainerMetadataArtifact(%q, %t)", i.debianVersion, testCase.UseLargeContainer),
 				fmt.Sprintf("crostini.GetContainerRootfsArtifact(%q, %t)", i.debianVersion, testCase.UseLargeContainer),
 			)

 			// Quote the extra data strings we got passed,
 			// so we can define the container and VM
 			// artifacts with runtime functions while
 			// still taking string literals from the
 			// outside world.
 			for _, data := range testCase.ExtraData {
 				extraData = append(extraData,
 					fmt.Sprintf("%q", data))
 			}

 			var extraSoftwareDeps []string
 			extraSoftwareDeps = append(extraSoftwareDeps, "dlc")

 			var hardwareDeps string
 			if !testCase.IsNotMainline {
 				if i.stable {
 					if testCase.StableHardwareDep != "" {
 						hardwareDeps = testCase.StableHardwareDep
 					} else if testCase.UseLargeContainer {
 						hardwareDeps = "crostini.CrostiniAppTest"
 					} else {
 						hardwareDeps = "crostini.CrostiniStable"
 					}
 				} else {
 					if testCase.UnstableHardwareDep != "" {
 						hardwareDeps = testCase.UnstableHardwareDep
 					} else {
 						hardwareDeps = "crostini.CrostiniUnstable"
 					}
 				}
 			}

 			var precondition string
 			if testCase.SelfManagedInstall {
 				precondition = ""
 			} else if testCase.UseLargeContainer {
 				precondition = fmt.Sprintf("crostini.StartedBy%s%sLargeContainer()", "Dlc", strings.Title(string(i.debianVersion)))
 			} else if testCase.UseGaiaLogin {
 				precondition = fmt.Sprintf("crostini.StartedBy%s%sGaia()", "Dlc", strings.Title(string(i.debianVersion)))
 			} else {
 				precondition = fmt.Sprintf("crostini.StartedBy%s%s()", "Dlc", strings.Title(string(i.debianVersion)))
 			}

 			var timeout time.Duration
 			if testCase.Timeout != time.Duration(0) {
 				timeout = testCase.Timeout
 			} else {
 				timeout = 7 * time.Minute
 			}

 			testParam := generatedParam{
 				Name:              name,
 				ExtraAttr:         append(testCase.ExtraAttr, extraAttr...),
 				ExtraData:         extraData,
 				ExtraSoftwareDeps: append(testCase.ExtraSoftwareDeps, extraSoftwareDeps...),
 				ExtraHardwareDeps: hardwareDeps,
 				Pre:               precondition,
 				Timeout:           timeout,
 				Val:               testCase.Val,
 			}
 			result = append(result, testParam)
 		}
 	}
 	return genparams.Template(t, template, result)
 }

 // MakeTestParams generates the default set of crostini test
 // parameters using MakeTestParamsFromList. If your test only needs to
 // be parameterized over how crostini is acquired and which version is
 // installed, use this. Otherwise, you may need to use
 // MakeTestParamsFromList.
 //
 // Sub-tests which are not eligible for being on the CQ (unstable or
 // download tests) will be tagged informational. Whether the test as a
 // whole is CQ-critical should be controlled by a test-level
 // informational attribute.
 func MakeTestParams(t genparams.TestingT) string {
 	defaultTest := Param{}
 	return MakeTestParamsFromList(t, []Param{defaultTest})
 }
	// Copyright 2020 The Chromium OS Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	package crostini

	// This file contains the generators for crostini test
	// parameters. Crostini tests generally require multiple sub-tests
	// parameterized over how crostini is set up. As these parameters are
	// complex and change over time you should strongly prefer to use
	// these generators rather then copying and pasting from other tests
	// or writing your own. This allows for changes to be made in one
	// central location (this file) and automatically propagated to all
	// tests that rely on them.
	//
	// Usage:
	//
	// In general, to generate test parameters for a test you add a unit
	// test file to the package containing the test (the name must end in
	// "_test.go"). This file should contain a go unit test which creates
	// a string containing the test parameters and passes it to
	// genparams.Ensure, along with the name of the test file. For
	// crostini tests, generate this string by calling one of the
	// functions in this file.
	//
	// By default, this unit test will ensure that the parameters in the
	// test match up with the string generated by the unit tests. These
	// tests are run on the CQ, so unexpected changes will block CLs from
	// landing. When run with TAST_GENERATE_UPDATE=1 set, it will instead
	// update the parameters in the test file.
	//
	// go unit tests can be run using the command
	// ~/trunk/src/platform/tast/tools/go.sh test -count=1 chromiumos/tast/local/bundles/...
	// parameter regeneration can be done with
	// TAST_GENERATE_UPDATE=1 ~/trunk/src/platform/tast/tools/go.sh test -count=1 chromiumos/tast/local/bundles/...
	//
	// The crostini package has a mega-test in params_test.go which
	// generates parameters for most of its tests. If you are adding a
	// test to that package, consider if it can be added to that
	// file. This should be the case for most tests, which only need the
	// "normal" set of parameters. Otherwise, or if you are writing a test
	// outside of the crostini package, you will likely need to create a
	// new test file. See any of the file in the crostini package ending
	// in "_test.go" for examples.
	//
	// Tests using these generators should not specify timeouts or
	// preconditions, as we expect to be able to specify those freely in
	// the parameters.
	//
	// Any test which is controlled by a generator will have a comment
	// above its test parameters indicating which file contains the
	// relevant unit test. To modify the parameters, update the test and
	// run the above command to regenerate the results.

	import (
	"fmt"
	"strings"
	"time"

	"chromiumos/tast/common/genparams"
	"chromiumos/tast/local/vm"
	)

	// Param specifies how each set of crostini tests should be generated.
	type Param struct {
	// Name of the test case. Generated tests will look like
	// "name_amd64_buster_stable", "name_arm_stretch_unstable"
	// etc.
	Name string

	// ExtraAttr contains additional attributes to add to the
	// generated test's ExtraAttr field beyond what the generator
	// function adds. For example, if you want this test case to
	// be in the "graphics_daily" group without putting the whole
	// test in, you can add that label here.
	ExtraAttr []string

	// ExtraData contains paths of additional data files needed by
	// the test case. Note that data files required for specific
	// crostini preconditions are added automatically to the
	// generated tests and should not be added here.
	ExtraData []string

	// ExtraSoftwareDeps lists software features that are required
	// to run this test case.
	ExtraSoftwareDeps []string

	// Timeout indicates the timeout for this test case. If
	// unspecified, defaults to 7 * time.Minute.
	Timeout time.Duration

	// Val is a freeform value that can be retrieved from
	// testing.State.Param() method. This string is inserted
	// unmodified and unquoted into the generated test case code
	// as the Val for each test case generated for this object.
	Val string

	// SelfManagedInstall indicates that this test will be
	// installing crostini itself, and therefore there should be
	// no crostini install precondition set.
	SelfManagedInstall bool

	// StableHardwareDep contains a go expression that evaluates
	// to a hardware dependency which controls the collection of
	// boards considered stable.
	StableHardwareDep string

	// UnstableHardwareDep contains a go expression that evaluates
	// to a hardware dependency which controls the collection of
	// boards considered unstable. It should be the inverse of
	// StableHardwareDep.
	UnstableHardwareDep string

	// MinimalSet - if true, generate only a minimal set of test
	// parameters such that each device will have at most one test
	// case it can run. This is useful for things like performance
	// tests, which are too expensive to be run in every possible
	// configuration.
	MinimalSet bool

	// IsNotMainline indicates whether the test case is in
	// group:mainline or not. This is important to get right
	// because we can't add the "informational" attribute to
	// non-mainline tests, but leaving it off of a mainline test
	// will cause the test case to become CQ critical. If in
	// doubt, set to false, and if you're wrong you will get an
	// error message when you try to run your tests.
	//
	// This also controls whether separate stable/unstable tests
	// are generated, since this distinction is only relevant to
	// the CQ.
	IsNotMainline bool

	// UseLargeContainer controls whether to use the normal test
	// container, or a larger container with more applications
	// pre-installed.
	UseLargeContainer bool

	// OnlyStableBoards controls whether to only use the stable
	// board variants and exclude all the unstable variants.
	OnlyStableBoards bool

	// UseGaiaLogin controls whether using gaia user to login
	// to the DUT.
	UseGaiaLogin bool
	}

	type generatedParam struct {
	Name string
	ExtraAttr []string
	ExtraData []string
	ExtraSoftwareDeps []string
	ExtraHardwareDeps string
	Pre string
	Timeout time.Duration
	Val string
	}

	const template = `{{range .}} {
	{{if .Name}} Name: {{fmt .Name}}, {{end}}
	{{if .ExtraAttr}} ExtraAttr: []string{ {{range .ExtraAttr}}{{fmt .}},{{end}} }, {{end}}
	{{if .ExtraData}} ExtraData: []string{ {{range .ExtraData}} {{.}}, {{end}} }, {{end}}
	{{if .ExtraSoftwareDeps}} ExtraSoftwareDeps: []string{ {{range .ExtraSoftwareDeps}}{{fmt .}},{{end}} }, {{end}}
	{{if .ExtraHardwareDeps}} ExtraHardwareDeps: {{.ExtraHardwareDeps}}, {{end}}
	{{if .Pre}} Pre: {{.Pre}}, {{end}}
	{{if .Timeout}} Timeout: {{fmt .Timeout}}, {{end}}
	{{if .Val}} Val: {{.Val}}, {{end}}
	}, {{end}}`

	func combineName(first, second string) string {
	if first == "" {
	return second
	}
	if second == "" {
	return first
	}
	return first + "_" + second
	}

	// MakeTestParamsFromList takes a list of test cases (in the form of
	// crostini.Param objects) and generates a set of crostini test
	// parameters for each. See the documentation for crostini.Param for
	// how these values effect the results. Each crostini.Param object is
	// treated independently, producing its own set of sub-tests.
	//
	// Normally you should use MakeTestParams instead, but if your test is
	// parameterized beyond which crostini preconditions it uses, you will
	// need this.
	func MakeTestParamsFromList(t genparams.TestingT, baseCases []Param) string {
	var result []generatedParam

	type iterator struct {
	debianVersion vm.ContainerDebianVersion
	stable bool
	}
	var it = []iterator{}

	for _, debianVersion := range []vm.ContainerDebianVersion{vm.DebianStretch, vm.DebianBuster, vm.DebianBullseye} {
	for _, stable := range []bool{true, false} {
	it = append(it, iterator{
	debianVersion: debianVersion,
	stable: stable,
	})
	}
	}

	for _, testCase := range baseCases {

	if testCase.UseLargeContainer && !testCase.MinimalSet {
	t.Fatalf("Test %q: Testing apps on stretch is not supported", testCase.Name)
	}

	// Check here if it's possible for any iteration of
	// this test to be critical, i.e. if it doesn't
	// already have the "informational" attribute, and is
	// a mainline test.
	canBeCritical := true
	for _, attr := range testCase.ExtraAttr {
	if attr == "informational" {
	canBeCritical = false
	}
	}

	for _, i := range it {

	if (testCase.IsNotMainline \|\| testCase.OnlyStableBoards) && !i.stable {
	// The stable/unstable distinction is only important for mainline tests
	continue
	}

	if testCase.MinimalSet && i.debianVersion != vm.DebianBuster {
	// The minimal set is currently buster
	continue
	}

	name := testCase.Name
	if !testCase.MinimalSet {
	// If we're generating a minimal set
	// then the debian version is always
	// the same and we don't need to
	// include it in the test name.
	name = combineName(name, string(i.debianVersion))
	}
	if !testCase.IsNotMainline && !testCase.OnlyStableBoards {
	if i.stable {
	name = combineName(name, "stable")
	} else {
	name = combineName(name, "unstable")
	}
	}
	if testCase.UseGaiaLogin {
	name = combineName(name, "gaia")
	}

	// _unstable tests can never be CQ critical.
	// stretch is informational while we phase it out.
	var extraAttr []string
	if (!i.stable \|\| i.debianVersion == vm.DebianStretch) && canBeCritical {
	extraAttr = append(extraAttr, "informational")
	}

	var extraData []string
	extraData = append(extraData,
	fmt.Sprintf("crostini.GetContainerMetadataArtifact(%q, %t)", i.debianVersion, testCase.UseLargeContainer),
	fmt.Sprintf("crostini.GetContainerRootfsArtifact(%q, %t)", i.debianVersion, testCase.UseLargeContainer),
	)

	// Quote the extra data strings we got passed,
	// so we can define the container and VM
	// artifacts with runtime functions while
	// still taking string literals from the
	// outside world.
	for _, data := range testCase.ExtraData {
	extraData = append(extraData,
	fmt.Sprintf("%q", data))
	}

	var extraSoftwareDeps []string
	extraSoftwareDeps = append(extraSoftwareDeps, "dlc")

	var hardwareDeps string
	if !testCase.IsNotMainline {
	if i.stable {
	if testCase.StableHardwareDep != "" {
	hardwareDeps = testCase.StableHardwareDep
	} else if testCase.UseLargeContainer {
	hardwareDeps = "crostini.CrostiniAppTest"
	} else {
	hardwareDeps = "crostini.CrostiniStable"
	}
	} else {
	if testCase.UnstableHardwareDep != "" {
	hardwareDeps = testCase.UnstableHardwareDep
	} else {
	hardwareDeps = "crostini.CrostiniUnstable"
	}
	}
	}

	var precondition string
	if testCase.SelfManagedInstall {
	precondition = ""
	} else if testCase.UseLargeContainer {
	precondition = fmt.Sprintf("crostini.StartedBy%s%sLargeContainer()", "Dlc", strings.Title(string(i.debianVersion)))
	} else if testCase.UseGaiaLogin {
	precondition = fmt.Sprintf("crostini.StartedBy%s%sGaia()", "Dlc", strings.Title(string(i.debianVersion)))
	} else {
	precondition = fmt.Sprintf("crostini.StartedBy%s%s()", "Dlc", strings.Title(string(i.debianVersion)))
	}

	var timeout time.Duration
	if testCase.Timeout != time.Duration(0) {
	timeout = testCase.Timeout
	} else {
	timeout = 7 * time.Minute
	}

	testParam := generatedParam{
	Name: name,
	ExtraAttr: append(testCase.ExtraAttr, extraAttr...),
	ExtraData: extraData,
	ExtraSoftwareDeps: append(testCase.ExtraSoftwareDeps, extraSoftwareDeps...),
	ExtraHardwareDeps: hardwareDeps,
	Pre: precondition,
	Timeout: timeout,
	Val: testCase.Val,
	}
	result = append(result, testParam)
	}
	}
	return genparams.Template(t, template, result)
	}

	// MakeTestParams generates the default set of crostini test
	// parameters using MakeTestParamsFromList. If your test only needs to
	// be parameterized over how crostini is acquired and which version is
	// installed, use this. Otherwise, you may need to use
	// MakeTestParamsFromList.
	//
	// Sub-tests which are not eligible for being on the CQ (unstable or
	// download tests) will be tagged informational. Whether the test as a
	// whole is CQ-critical should be controlled by a test-level
	// informational attribute.
	func MakeTestParams(t genparams.TestingT) string {
	defaultTest := Param{}
	return MakeTestParamsFromList(t, []Param{defaultTest})
	}