tools/generate_cuj_metadata.py

#!/usr/bin/env python3
# Copyright 2023 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Tool for generating autogenerated-metadata.json for performance CUJs.

This tool can scrape information from performance CUJ
tests, combine them with information defined in tests-info.json, and
output a autogenerated-metadata.json file.

Input JSON file:
src/go.chromium.org/tast-tests/cros/local/bundles/cros/ui/metadata/tests-info.json

Output JSON file:
src/go.chromium.org/tast-tests/cros/local/bundles/cros/ui/metadata/autogenerated-metadata.json

Performance CUJ File Directory:
src/go.chromium.org/tast-tests/cros/local/bundles/cros/ui/

Usage Example (for adding DesksCUJ metadata):
1. Add the desired CUJ path to TESTS_PATHS in generate_cuj_metadata.py.
2. Run `python3 generate_cuj_metadata.py`
3. The script will autodetect all of the variants in desks_cuj.go, and
   will add the defaults to tests-info.json.
4. Update tests-info.json to fit your needs. For example, add variants
   to individual groups to show Ash and Lacros test pairs.
5. Rerun `python3 generate_cuj_metadata.py` with your newly updated
   tests-info.json, and the output autogenerated-metadata.json should
   now contain all desired information about DesksCUJ.

tests-info.json JSON is an object with the outermost key being the CUJ test
name. Each test is made up of "aliases", "groups", "majorTestVersion", and
"variantsInfo".

1. "aliases" is a dictionary, where for each key value pair (k, v), k is a name
   for the currently active variant, and v is a list of former names for the
   test.

   For example, if we renamed VideoCUJ.clamshell to VideoCUJ, then aliases
   would be:
   {
      "": ["clamshell"]
   }.

   If we also updated VideoCUJ.tablet_mode to be VideoCUJ.tablet, aliases
   would be:
   {
      "":       ["clamshell"],
      "tablet": ["tablet_mode"],
    }
2. "groups" is a list of test groups, that help give context to which variants
   are related to each other. Each group consists of a "groupName", which is
   usually just a common feature between all of the tests, and a list of
   "variantNames", which is a list of strings of all variants within the group.
   For example, if MeetCUJ has a 2p test and a lacros_2p test, tests-info.json
   might have a group:
   {
      "groupName": "2p",
      "variantNames": [
         "2p",
         "lacros_2p"
      ]
   }
   Each variant in "variantNames" must have a variant in the corresponding go
   file for that specific test.
3. "majorTestVersion" is a number that indicates the current version of the
   test. This number should be increased if there is a test change that might
   make previous metrics uncomparable.
4. "variantsInfo" contains a list of information for each variant in the test.
   This information includes a string "id", which should contain a unique ID
   that can withstand name changes for this variant. It also contains a string
   "variantName", which is the current name of the variant.

Complete example for DesksCUJ:
{
  "DesksCUJ": {
    "aliases": [],
    "groups": [
      {
        "groupName": "ungrouped",
        "variantNames": [
          "",
          "lacros"
        ]
      }
    ],
    "majorTestVersion": 1,
    "variantsInfo": [
      {
        "id": "deskscuj.",
        "variantName": ""
      },
      {
        "id": "deskscuj.lacros",
        "variantName": "lacros"
      }
    ]
  }
}

autogenerated-metadata.json JSON is an object similar to tests-info.json,
but includes a few additional fields. These added fields are:
1. "description", which is the test description found in the test file.
2. "variantsInfo.fixture", which is the test fixture in the test file.
3. "browserType", which is the browser used in the test. This defaults to Ash
   if no browser type is specified.

Complete example for DesksCUJ
{
  "DesksCUJ": {
    "aliases": {},
    "description": "Measures the performance of critical user journey for
    virtual desks",
    "groups": [
      {
        "groupName": "default",
        "variantNames": [
          "",
          "lacros"
        ]
      }
    ],
    "majorTestVersion": 1,
    "variantsInfo": [
      {
        "id": "deskscuj.",
        "attributes": [
          "group:cuj"
        ],
        "fixture": "loggedInToCUJUser",
        "browserType": "ash",
        "variantName": ""
      },
      {
        "id": "deskscuj.lacros",
        "attributes": [
          "group:cuj"
        ],
        "fixture": "loggedInToCUJUserLacros",
        "browserType": "lacros",
        "variantName": "lacros"
      }
    ]
  }
}
"""

import argparse
import json
import logging
import os
import re


# Default location for the tast-tests repo. This can be overridden with the
# --tast-tests-path arg.
DEFAULT_TAST_TEST_REPO = os.path.expanduser(
    "~/chromiumos/src/platform/tast-tests/"
)

# Location for all of the CUJ tests.
UI_PATH = "src/go.chromium.org/tast-tests/cros/local/bundles/cros/ui/"

# Add new CUJ tests to this list. Each test must be found in the UI_PATH folder.
TEST_PATHS = [
    "arc_youtube_cuj.go",
    "benchmark_cuj.go",
    "desks_cuj.go",
    "debug_lacros_perf.go",
    "docs_cuj.go",
    "example_cuj.go",
    "google_sheets_cuj.go",
    "google_slides_cuj.go",
    "login_perf.go",
    "meet_multi_tasking_cuj.go",
    "meet_cuj.go",
    "tab_switch_cuj.go",
    "task_switch_cuj.go",
    "video_cuj.go",
    "window_arrangement_cuj.go",
]

# Location of where to store autogenerated-metadata.json, and where to find
# tests-info.json.
CUJ_PATH = "src/go.chromium.org/tast-tests/cros/local/chrome/cuj/metadata/"

# Location where the generated autogenerated-metadata.json will be saved.
OUTPUT_PATH = os.path.join(CUJ_PATH, "autogenerated-metadata.json")

# Location where to read tests-info.json.
TESTS_INFO_PATH = os.path.join(CUJ_PATH, "tests-info.json")

# Names of keys within autogenerated-metadata.json and tests-info.json.
GROUPS = "groups"
GROUP_NAME = "groupName"
VARIANT_NAMES = "variantNames"
MAJOR_TEST_VERSION = "majorTestVersion"
ALIASES = "aliases"
VARIANTS_INFO = "variantsInfo"
VARIANTS_INFO_ID = "id"
VARIANTS_INFO_NAME = "variantName"
DESCRIPTION = "description"
UNGROUPED_GROUP_NAME = "ungrouped"


class BrowserType:
    """Browser type used within the test.

    Attributes:
      LACROS: A string signifying the Lacros browser.
      ASH: A string signifying the Ash browser.
    """

    LACROS = "lacros"
    ASH = "ash"


def get_variant_full_name(test_name: str, variant_name: str) -> str:
    """Get the full name of the test variant.

    Returns:
      A string in the format <test name>.<variant name>
    """
    return f"{test_name}.{variant_name}"


class Variant:
    """Variant contains all metadata for a single performance CUJ variant.

    Attributes:
      name: A string of the name of this variant.
      fixture: A string of the fixture for this variant found in the test file.
      browser_type: A BrowserType indicating which browser is used.
      attributes: A string list of test attributes taken from the test file.
      ID: A unique string for this specific variant.
    """

    def __init__(self, raw: str, attributes: list, test_name: str) -> str:
        """Initializes this instance based on the go test file.

        Args:
          raw: The raw text to parse the metadata from. If "Name: " is not
            found, the variant name is assumed to be an empty string. If no
            fixture is found, we default to an empty string. If the word
            "lacros" is not found in the variant name or fixture, the test is
            assumed to be an Ash.
          attributes: A list of global attributes that should be added to the
            variant. These attributes are usually part of "Attr: " outside of
            "testing.Param", and the individual attributes are part of
            "ExtraAttr: ", which is inside "testing.Param".
          test_name: A string of the test name for this variant, like
            "VideoCUJ".
        """
        self.name = self._parse_variant_name(raw)
        self.fixture = self._parse_fixture(raw)
        self.browser_type = self._parse_browser_type(self.name, self.fixture)
        self.attributes = self._parse_attributes(raw) + attributes

        # Set a default ID. This will be overwritten if tests-info.json
        # has a different ID for this variant.
        self.ID = get_variant_full_name(test_name, self.name).lower()

    def _parse_variant_name(self, raw: str) -> str:
        """Parse the variant name from the raw string of the variant.

        Check the raw string for a section like `Name: "2p"`, and parse the
        name portion of it ("2p").

        Args:
          raw: Raw sting used the parse the variant name.

        Returns:
          A string name of the variant, or an empty string if no name is found.
        """
        name_match = re.search('Name:.*"(.*)"', raw)
        if name_match is not None:
            return name_match.group(1)
        return ""

    def _parse_fixture(self, raw: str) -> str:
        """Parse the fixture from the raw string of the variant.

        Check for a section of the string in the format
        `Fixture: "loggedInToCUJUser"`, and parse the name portion of it
        ("loggedInToCUJUser").

        Args:
          raw: Raw sting used the parse the fixture name.

        Returns:
          A string of the fixture, or an empty string if no fixture is found.
        """
        fixture_match = re.search('Fixture:.*"(.*)"', raw)
        if fixture_match is not None:
            return fixture_match.group(1)
        return ""

    def _parse_browser_type(self, name: str, fixture: str) -> BrowserType:
        """Determine the BrowserType of the test variant.

        Args:
          name: A string name of the test variant.
          fixture: A string name of the test fixture.

        Returns:
          Either BrowserType.ASH, or BrowserType.LACROS, depending on if
          "lacros" is found in the test name or fixture name
          (case insensitive).
        """
        type_match = re.search("(?i)lacros", f"{name} {fixture}")
        if type_match is not None:
            return BrowserType.LACROS
        return BrowserType.ASH

    def _parse_attributes(self, raw: str) -> list:
        """Parse the attributes from the raw string of the variant.

        Check for a section of the string in the format
        `Attr: []string{"group:cuj"}`, and parse the name portion of it
        (i.e. ["group:cuj"]).

        Args:
          raw: Raw sting used the parse the fixture name.

        Returns:
          A list of strings, where each string is an attribute for this variant.
        """
        attr_match = re.search('Attr:.*\[\]string{(".*")*}', raw)

        # Some tests don't have any attributes defined for each variant.
        if attr_match is None:
            return []

        # Some tests have attributes defined, but the attributes list is empty.
        if attr_match[1] is None:
            logging.warning(
                '[%s] has an unexpected number of attributes: "%s"',
                self.name,
                attr_match[0],
            )
            return []

        attributes = attr_match.group(1).replace('"', "").split(",")
        return [attr.strip() for attr in attributes]

    def to_dict(self) -> dict:
        """Create a dictionary formatted for autogenerated-metadata.json.

        Returns:
          A dictionary including the variantName, fixture, browserType,
          attributes, and variant id.
        """
        return {
            "variantName": self.name,
            "fixture": self.fixture,
            "browserType": self.browser_type,
            "attributes": self.attributes,
            "id": self.ID,
        }

    def __str__(self):
        return (
            f"Name: {self.name}\n"
            f"Fixture: {self.fixture}\n"
            f"Type: {self.browser_type}\n"
            f"Attributes: {self.attributes}"
        )


class Test:
    """All information about a tast-test and its corresponding variants.

    Attributes:
      name: A string indicating the test name.
      description: A string description of the test.
      variants: A dictionary mapping a string variant name to a Variant.
      groups: A list of groups, where each group is a dictionary that includes
        a string groupName, and a list of variantNames.
      major_test_verstion: A float representing the major test version.
      aliases: A dictionary where each keys are variant names and values are
        lists of variant names that the variant were formerly known by.
    """

    def __init__(self, full_test_path: str):
        """Parses test and variant information from a tast-test file.

        Args:
          full_test_path:
            The complete path to the test go file.
        """

        # Set defaults.
        self.name = ""
        self.description = ""
        self.variants = {}
        self.groups = [get_default_group()]
        self.major_test_version = 1
        self.aliases = {}

        # Parse the go file.
        self._parse_test(full_test_path)

    def _parse_test(self, full_test_path: str):
        """Parse the test found at the given test path.

        Parse the name, description, variants, and attributes from the test
        file. Use regex to find certain unique sections of the code, such as
        "Func: " for the name, or "Desc: " for the description. Parse through
        testing.Param (if it exists) to get information about each variant.
        """
        with open(full_test_path) as file:
            test_txt = file.read()

            # Extract overall test related information, such as test name
            # and description.
            test_name_match = re.search("Func:(.*),", test_txt)
            if test_name_match is None:
                raise Exception(
                    "Invalid file, cannot find the test function name"
                )
            self.name = test_name_match.group(1).strip()

            desc_match = re.search('Desc:.*"(.*)"', test_txt)
            if desc_match is not None:
                self.description = desc_match.group(1).strip()

            # Extract the test variants found within the {} of
            # Params: []testing.Param.
            start_match = re.search("Params:.*\[\]testing\.Param", test_txt)
            if start_match is None:
                # We found a test that doesn't define any variants, so use the
                # whole file to extract the variant information, as opposed to
                # just the `Params` field. Leave the overall attributes as
                # empty, because they will be parsed in the Variant constructor.
                variant = Variant(test_txt, [], self.name)
                self.add_variant(variant)
                return

            # Retrieve the overall test attributes to determine which suite the
            # variants are running in. Check for `Attr: ` in the file -- if
            # this doesn't exist, then no test is running in the lab.
            # If the match is found within `Params: testing.Param`, then the
            # group is a variant group that will be parsed later. Otherwise,
            # parse the group now.
            attributes = []
            start_group_match = re.search(
                'Attr:.*\[\]string{(".*")*}', test_txt
            )
            if start_group_match is None:
                logging.warning(
                    "[%s] None of the tests in this file are "
                    "running in the lab. Is this correct?",
                    self.name,
                )
            elif start_group_match.start() < start_match.start():
                attributes = (
                    start_group_match.group(1).replace('"', "").split(",")
                )
            for i in range(len(attributes)):
                attributes[i] = attributes[i].strip()

            # Parse testing.Params using { and }. Each matching pair at the
            # first level of testing.Param{} will be a variant.
            curly_braces_re = re.compile("[{]|[}]")
            open_braces = []

            # The first brace to look at is one after the opening brace
            # in `testing.Param{ ... }`.
            start_brace = start_match.end()

            start_variant = -1
            for brace in re.finditer(curly_braces_re, test_txt[start_brace:]):
                # If we found an open brace, then we either have a new variant,
                # or some unimportant struct that we wish to ignore. If our
                # open_braces stack has a length of 1, then we know that this
                # new open brace is a variant, since it is directly inside
                # testing.Param { ... }.
                if brace.group(0) == "{":
                    if len(open_braces) == 1:
                        start_variant = brace.start()
                    open_braces.append(brace)
                else:
                    # If you reach a closing brace, and after popping the top
                    # element of the stack open_brace has a length of 1, then
                    # we finished finding a full variant.
                    open_braces.pop()
                    if len(open_braces) == 1:
                        variant = Variant(
                            test_txt[
                                (start_variant + start_brace) : (
                                    brace.end() + start_brace
                                )
                            ],
                            attributes,
                            self.name,
                        )
                        self.add_variant(variant)
                # All variants have been found once the outermost { is popped.
                if len(open_braces) == 0:
                    break

    def add_variant(self, variant: Variant):
        """Add the variant to this test instance.

        Add the variant to self.variants, and update the default group to
        include this new variant.

        Args:
          variant: A Variant instance.
        """
        self.variants[variant.name] = variant

        # Add the variant to the "ungrouped" group. This will be overwritten if
        # tests-info.json already defines its own groups.
        self.groups[0][VARIANT_NAMES].append(variant.name)

    def add_tests_info(self, test_info: dict):
        """Add info from tests-info.json to this test instance.

        Update this instance to include group, test versioning, aliases, and
        variant information found in tests-info.json.
        """
        self.test_info = test_info
        self.groups = test_info[GROUPS]
        self.major_test_version = test_info[MAJOR_TEST_VERSION]
        self.aliases = test_info[ALIASES]

        # Update our Variant objects to have the ID defined in
        # tests-info.json.
        for variant in test_info[VARIANTS_INFO]:
            self.variants[variant[VARIANTS_INFO_NAME]].ID = variant[
                VARIANTS_INFO_ID
            ]

    def get_user_defined_test_info(self) -> dict:
        """Get a dict of user defined test attributes.

        Returns:
          A dict mapping user-defined fields to their repective values. For
          example:
          {
            "groups": [
              {
                "groupName": "ungrouped",
                "variantNames": [
                  "",
                  "lacros"
                ]
              }
            ]
            "major_test_version": 1,
            "aliases": {},
            "variantsInfo": [
              {
                "id": "deskscuj.",
                "variantName": ""
              },
              {
                "id": "deskscuj.lacros",
                "variantName": "lacros"
              },
            ]
          }
        """
        return {
            GROUPS: self.groups,
            MAJOR_TEST_VERSION: self.major_test_version,
            ALIASES: self.aliases,
            VARIANTS_INFO: [
                self.get_variant_info(v_name) for v_name in self.variants
            ],
        }

    def get_variant_info(self, variant_name: str) -> dict:
        """Get a dict for all info needed for variantsInfo.

        Returns:
        A dict containing the variant ID and the variant name.
        """
        return {
            VARIANTS_INFO_ID: self.variants[variant_name].ID,
            VARIANTS_INFO_NAME: variant_name,
        }

    def to_dict(self):
        """Get a dict fully representing this test instance.

        Returns:
          A dict containing the exact format that should be used in
          autogenerated-metadata.json to represent this test.
        """
        res = self.get_user_defined_test_info()
        res[DESCRIPTION] = self.description
        res[VARIANTS_INFO] = [
            self.variants[v_name].to_dict() for v_name in self.variants
        ]
        return res


def get_default_group():
    """Get a dict test group for the "ungrouped" tests.

    Returns:
      The following dictionary:
      {
        "groupName": "ungrouped",
        "variantNames": []
      }
    """
    return {GROUP_NAME: UNGROUPED_GROUP_NAME, VARIANT_NAMES: []}


def ensure_tests_info_format(tests_info: dict, all_variants: set):
    """Verify that all information in tests-info.json is properly formatted.

    Verify that all required fields are present, as well as that all referenced
    variant names are properly scraped from the test go files.

    Raises:
      Exception: If a malformed portion of the tests-info.json is found.
    """
    variants_in_tests_info = set()
    for test_name in tests_info:
        # Each test in test_info must have fields for groups, major test
        # version, aliases, and variants info.
        for expected_field in [
            GROUPS,
            MAJOR_TEST_VERSION,
            ALIASES,
            VARIANTS_INFO,
        ]:
            if expected_field not in tests_info[test_name]:
                raise Exception(
                    f"Test {test_name} does not have the required "
                    f'attributes "{expected_field}"'
                )

        group_names = set()
        for test_group in tests_info[test_name][GROUPS]:
            # Each group must have a group name field.
            if GROUP_NAME not in test_group:
                raise Exception(
                    f"Test {test_name} does not have the required "
                    f"attribute {GROUP_NAME}"
                )

            # Each group name must be unique,
            group_name = test_group[GROUP_NAME]
            if group_name in group_names:
                raise Exception(
                    f"Test {test_name} has multiple groups with the name"
                    f' "{group_name}"'
                )
            group_names.add(group_name)

            # Each group must have a field with a list of variant names.
            if VARIANT_NAMES not in test_group:
                raise Exception(
                    f"Test {test_name} group does not have the required "
                    f'attribute "{VARIANT_NAMES}"'
                )

            # Every variant in a group must have been parsed from a go file.
            # Keep track of all variant names we've seen in a single test, so
            # we can compare them against all the variant names parsed from
            # the go files later.
            for variant_name in test_group[VARIANT_NAMES]:
                variants_in_tests_info.add(
                    get_variant_full_name(test_name, variant_name)
                )

        # Every variant must have the ID field and variant name field.
        for variant_info in tests_info[test_name][VARIANTS_INFO]:
            for expected_field in [
                VARIANTS_INFO_ID,
                VARIANTS_INFO_NAME,
            ]:
                if expected_field not in variant_info:
                    raise Exception(
                        f"Test {test_name} has variant {variant_info} missing "
                        f'the required field "{expected_field}"'
                    )
            variants_in_tests_info.add(
                get_variant_full_name(
                    test_name, variant_info[VARIANTS_INFO_NAME]
                )
            )

        for alias in tests_info[test_name][ALIASES]:
            if (
                get_variant_full_name(test_name, alias)
                not in variants_in_tests_info
            ):
                raise Exception(
                    f"Test {test_name} has alias {alias}"
                    " that is not in variantsInfo"
                )

    # Every test found in tests-info.json must be found in a corresponding file
    # that we parsed in parsed_tests.
    tests_info_diff = variants_in_tests_info.difference(all_variants)
    if len(tests_info_diff) > 0:
        raise Exception(
            f"Found tests {tests_info_diff} that are found in "
            "tests-info.json, but not in the test files"
        )


def cleanup_tests_info(
    tests_info: dict, parsed_tests: dict, should_add_defaults: bool
):
    """Clean up tests-info.json file.

    Depending on should_add_defaults, either directly add default tests and
    test variants to tests_info, or log which tests need to be updated. Cleanup
    steps include:
    1. Adding default missing tests / test variants.
    2. Removing any tests in "ungrouped" that appears in another group.
    3. Removing any variant names that appear multiple times in the same group.
    4. Adding variants to the "ungrouped" group, if they aren't in another
       group.

    Raises:
      Exception: If there is a variant that doesn't have a corresponding value
      in variantsInfo.
    """
    logging.info("Beginning tests-info.json cleanup")

    # All tests must have a value in tests-info.json. If it is not present,
    # give an option to add in basic default values for the
    missing_tests_names = [
        test_name for test_name in parsed_tests if test_name not in tests_info
    ]

    issues = []

    if len(missing_tests_names) > 0:
        issue = (
            f"{missing_tests_names} do(es) not have any info in tests-info.json"
        )
        if should_add_defaults:
            logging.info("Adding in defaults")
            # For each missing test, get the default test information and add
            # it to the original tests-info.json file.
            for missing_test_name in missing_tests_names:
                tests_info[missing_test_name] = parsed_tests[
                    missing_test_name
                ].get_user_defined_test_info()
        else:
            # If they choose not to add the test into tests-info.json, add this
            # issue to our list of overall issues with the file.
            issues.append(issue)

    for test_name in tests_info:
        # Check if there is a group called "ungrouped". These tests should not
        # appear in any other group. For example, if "ungrouped" contains the
        # test "lacros", and another group also contains the test "lacros",
        # offer to remove "lacros" from "ungrouped".
        ungrouped_variants_idx = -1
        ungrouped_names = set()
        grouped_names = set()
        for i, test_group in enumerate(tests_info[test_name][GROUPS]):
            # Each variant name should only appear once in each group.
            variant_names = set(test_group[VARIANT_NAMES])
            test_group[VARIANT_NAMES] = sorted(variant_names)

            # Find which group (if it exists) has the name "ungrouped".
            if test_group[GROUP_NAME] == UNGROUPED_GROUP_NAME:
                ungrouped_names = set(variant_names)
                ungrouped_variants_idx = i
            else:
                grouped_names.update(variant_names)

        # Any tests that are in "ungrouped", but appear in another group,
        # will be removed from "ungrouped".
        extra_ungrouped_tests = ungrouped_names.intersection(grouped_names)
        if len(extra_ungrouped_tests) != 0:
            logging.info(
                'Removing tests %s.%s from "ungrouped", '
                " since they appeared elsewhere",
                test_name,
                ", ".join(extra_ungrouped_tests),
            )

            new_ungrouped = [
                v_name
                for v_name in tests_info[test_name][GROUPS][
                    ungrouped_variants_idx
                ][VARIANT_NAMES]
                if v_name not in grouped_names
            ]
            if len(new_ungrouped) > 0:
                tests_info[test_name][GROUPS][ungrouped_variants_idx][
                    VARIANT_NAMES
                ] = new_ungrouped
            else:
                tests_info[test_name][GROUPS].pop(ungrouped_variants_idx)

        # Get a set of the variant names defined in tests-info.json under
        # "variantsInfo".
        variant_names_tests_info = set(
            [
                v[VARIANTS_INFO_NAME]
                for v in tests_info[test_name][VARIANTS_INFO]
            ]
        )

        # If a variant is parsed from a go file and not found in
        # "variantsInfo", offer to add it in.
        for parsed_variant_name in parsed_tests[test_name].variants:
            if parsed_variant_name not in variant_names_tests_info:
                issue = (
                    f"[{get_variant_full_name(test_name, parsed_variant_name)}"
                    "] found variant that is not in tests-info.json"
                )
                if should_add_defaults:
                    tests_info[test_name][VARIANTS_INFO].append(
                        parsed_tests[test_name].get_variant_info(
                            parsed_variant_name
                        )
                    )
                    variant_names_tests_info.add(parsed_variant_name)
                else:
                    issues.append(issue)

        # All variants must be in at least 1 group, either the "ungrouped"
        # group or another user defined group. If a variant exists that's not
        # in either, put it in the ungrouped category.
        variants_in_a_group = ungrouped_names.union(grouped_names)
        missing_variants = variant_names_tests_info.difference(
            variants_in_a_group
        )
        if len(missing_variants) != 0:
            logging.warning(
                "[%s.%s] found variant that is not "
                "in any group -- adding them to the 'ungrouped' group",
                test_name,
                missing_variants,
            )

            # If no "ungrouped" group exists, create one at the start of the
            # group list.
            if ungrouped_variants_idx < 0:
                tests_info[test_name][GROUPS].insert(0, get_default_group())
                ungrouped_variants_idx = 0

            for missing_variant_name in missing_variants:
                tests_info[test_name][GROUPS][ungrouped_variants_idx][
                    VARIANT_NAMES
                ].append(missing_variant_name)

        # Every variant in a group should also have a corresponding field in
        # "variantsInfo". This error is thrown here and not in
        # ensure_tests_info_format, to give users a chance to automatically add
        # in a variant using this script.
        extra_variants = variants_in_a_group.difference(
            variant_names_tests_info
        )
        if len(extra_variants) > 0:
            raise Exception(
                f"{extra_variants}] Found test(s) in a group that do not have "
                f"a value in {VARIANTS_INFO}"
            )

    logging.info("Completed tests-info.json cleanup")
    return issues


def main():
    logging.basicConfig(level=logging.DEBUG)

    parser = argparse.ArgumentParser(
        description="Generate autogenerated-metadata.json"
    )
    parser.add_argument(
        "--tast-tests-path",
        help="Location of the tast-tests directory",
        default=DEFAULT_TAST_TEST_REPO,
    )
    parser.add_argument(
        "--dry-run",
        help=(
            "Do not edit tests-info.json or"
            " generate autogenerated-metadata.json"
        ),
        dest="update",
        action="store_false",
    )

    args = parser.parse_args()

    # Extract data from all the CUJs, and store them in `parsed_tests`.
    parsed_tests = {}

    # Keep a set of all variants - this will be useful when verifying the
    # correctness of tests-info.json.
    all_variants = set()
    for test_path in TEST_PATHS:
        test = Test(os.path.join(args.tast_tests_path, UI_PATH, test_path))
        for variant_name in test.variants:
            all_variants.add(get_variant_full_name(test.name, variant_name))
        parsed_tests[test.name] = test

    # Read the user generated test-info.json.
    tests_info_issues = []
    with open(
        os.path.join(args.tast_tests_path, TESTS_INFO_PATH), "r"
    ) as tests_info_file:
        tests_info = json.load(tests_info_file)

        # Ensure that tests-info.json has the correct fields, and
        # only references variants that exist in our go files.
        ensure_tests_info_format(tests_info, all_variants)

        # Provide cleanup options to tests-info.json, such as adding defaults
        # for tests, and removing variant names from the "ungrouped" group.
        tests_info_issues = cleanup_tests_info(
            tests_info, parsed_tests, args.update
        )

    if len(tests_info_issues) != 0:
        issues_list = "\n".join(tests_info_issues)
        logging.warning(
            "Not generating autogenerated-metadata.json until the following "
            "issues are resolved: \n%s",
            issues_list,
        )
        return

    if not args.update:
        logging.warning("Not updating any files because --dry-run was given.")
        return

    # Update tests-info.json with any updates we performed.
    with open(
        os.path.join(args.tast_tests_path, TESTS_INFO_PATH), "w"
    ) as tests_info_file:
        json.dump(tests_info, tests_info_file, indent=2, sort_keys=True)
        logging.info("Updated %s", TESTS_INFO_PATH)

    # If there were no issues with tests-info.json, combine info from that file
    # to the info scraped from our CUJ tests.
    for test_name in parsed_tests:
        parsed_tests[test_name].add_tests_info(tests_info[test_name])

    # Create autogenerated-metadata.json by converting all Test objects to
    # dictionaries and dumping a json file.
    with open(os.path.join(args.tast_tests_path, OUTPUT_PATH), "w") as metadata:
        for test_name in parsed_tests:
            parsed_tests[test_name] = parsed_tests[test_name].to_dict()

        json.dump(parsed_tests, metadata, indent=2, sort_keys=True)
        logging.info("Generated metadata file can be found at: %s", OUTPUT_PATH)


if __name__ == "__main__":
    main()