blob: 781328ebfba7975d5cf1b4ff72565c910e1a3bcd [file] [log] [blame]
// Copyright 2018 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package platform
import (
func init() {
Func: CrosConfig,
Desc: "Check cros_config commands match the golden file built during image creation",
Contacts: []string{"", ""},
Attr: []string{"group:mainline", "group:labqual", "informational"},
// crosConfigDir defines where all of the input files and golden files are located
// on the device under test.
const crosConfigDir = "/usr/local/tmp/chromeos-config/tast/"
// deviceRecords is a JSON struct for reading and writing the golden file. A new
// golden output file will be created and added to the output dir making it
// easier to update golden file in build configuration.
type deviceRecords struct {
DeviceName string `json:"device_name"`
Records []cmdAndOutput `json:"records"`
// cmdAndOutput is a JSON struct for each command and its output.
type cmdAndOutput struct {
CommandKey string `json:"command_key"`
Value string `json:"value"`
func CrosConfig(ctx context.Context, s *testing.State) {
cmdFile := filepath.Join(crosConfigDir, "cros_config_test_commands.json")
// Check to see if the commands file exists.
_, err := os.Stat(cmdFile)
if os.IsNotExist(err) {
s.Log("No commands file, ignoring")
deviceName, err := getDeviceIdentity(ctx)
if err != nil {
s.Fatal("Failed to get device identity: ", err)
s.Logf("Checking configs for device %q", deviceName)
// Get all of the commands from the cros_config_test_commands.json for the DUT. Commands
// are generated during the build step for the dev and test images.
commandsToRun, err := buildCommands(ctx, cmdFile, deviceName)
if err != nil {
s.Fatal("Failed to build commands: ", err)
// Run all of the commands on the DUT, capturing the output and building
// a JSON file with all of the data.
var recs []cmdAndOutput
for _, c := range commandsToRun {
s.Logf("Running command %q", c)
out, err := testexec.CommandContext(ctx, "sh", "-c", c).Output()
if err != nil {
s.Errorf("Failed to run %q: %v", c, err)
// Use ':' as key delim between commands for readability.
recs = append(recs, cmdAndOutput{strings.Replace(c, " ", ":", -1), strings.TrimSpace(string(out))})
sort.Slice(recs, func(i, j int) bool { return recs[i].CommandKey < recs[j].CommandKey })
output := deviceRecords{deviceName, recs}
jsonOutput, err := json.MarshalIndent(output, "", " ")
if err != nil {
s.Fatal("Failed to generate output JSON: ", err)
fn := fmt.Sprintf("%s_output.json", deviceName)
err = ioutil.WriteFile(filepath.Join(s.OutDir(), fn), jsonOutput, 0644)
if err != nil {
s.Error("Failed to write output JSON: ", err)
// Compare the new output with the existing golden file.
goldenFn := fmt.Sprintf("%s_golden_file.json", deviceName)
compareGoldenOutput(s, output, filepath.Join(crosConfigDir, goldenFn))
// getDeviceIdentity determines the DUT's identity by getting /:name from cros_config.
// This identity will be expanded in the future to include SKU and whitelabels.
// See go/cros-domain-model for details and naming.
// Returns output such as "eve", "scarlet", or "nautilus".
func getDeviceIdentity(ctx context.Context) (string, error) {
// NOTE: we are using some of the config programs to determine
// device identity that we are trying to test.
c := testexec.CommandContext(ctx, "cros_config", "/", "name")
out, err := c.Output()
if err != nil {
return "", err
return string(out), nil
// buildCommands returns the shell quoted command lines rom the JSON input file for the
// device to be tested.
func buildCommands(ctx context.Context, cmdPath, deviceFilter string) ([]string, error) {
// JSON struct for each command to run.
commandRecs := struct {
ChromeOS struct {
Devices []struct {
DeviceName string `json:"device-name"`
CommandGroups []struct {
Args []string `json:"args"`
Name string `json:"name"`
} `json:"command-groups"`
} `json:"devices"`
} `json:"chromeos"`
f, err := os.Open(cmdPath)
if err != nil {
return nil, err
defer f.Close()
if err := json.NewDecoder(f).Decode(&commandRecs); err != nil {
return nil, err
var commands []string
for _, device := range commandRecs.ChromeOS.Devices {
if device.DeviceName != deviceFilter {
for _, cg := range device.CommandGroups {
for _, a := range cg.Args {
commands = append(commands, shutil.Escape(cg.Name)+" "+a)
return commands, nil
// buildRecordsMap builds a map of the records.
func buildRecordsMap(recs []cmdAndOutput) map[string]string {
m := make(map[string]string)
for _, rec := range recs {
m[rec.CommandKey] = rec.Value
return m
// compareGoldenOutput compares the existing golden file to the newly built output structure.
func compareGoldenOutput(s *testing.State, output deviceRecords, goldenFilename string) {
b, err := ioutil.ReadFile(goldenFilename)
if os.IsNotExist(err) {
// By design (go/cros-config-test-coverage), no golden file will only warn. This
// is most likely a device in development and should not fail CQ runs.
s.Logf("WARNING: No existing golden file %v, ignoring", goldenFilename)
if err != nil {
s.Fatalf("Failed to read golden file %v: %v", goldenFilename, err)
var golden deviceRecords
if err = json.Unmarshal(b, &golden); err != nil {
s.Fatalf("Failed to unmarshal golden file %v: %v", goldenFilename, err)
// Do a deep compare of the two JSON files, if they match all is good.
if reflect.DeepEqual(output, golden) {
s.Log("All keys and values matched")
// Verify the device names match.
if output.DeviceName != golden.DeviceName {
s.Fatalf("Tested device %q; want %q", output.DeviceName, golden.DeviceName)
outputMap := buildRecordsMap(output.Records)
goldenMap := buildRecordsMap(golden.Records)
// Look for keys only in the new output map and missing from the golden.
var missingGoldenKeys []string
for k := range outputMap {
if _, ok := goldenMap[k]; !ok {
missingGoldenKeys = append(missingGoldenKeys, k)
// Iterate through the golden map checking all of the keys and values. Value differences
// will generate an error returned in the list of errors.
var missingOutputKeys []string
var matchingComparisons int
for k, v := range goldenMap {
if ov, ok := outputMap[k]; !ok {
missingOutputKeys = append(missingOutputKeys, k)
} else if v != ov {
s.Errorf("Key %q is %q; want %q", k, ov, v)
} else {
// By design (go/cros-config-test-coverage), missing keys will only warn. This
// is most likely a device in development and should not fail CQ runs.
if len(missingOutputKeys) > 0 {
s.Logf("WARNING: keys %v only exist in golden file, ignoring", missingOutputKeys)
if len(missingGoldenKeys) > 0 {
s.Logf("WARNING: keys %v only exist in new output file, ignoring", missingGoldenKeys)
s.Logf("Matched %d keys and values", matchingComparisons)