script/stable_delegate_accel_test_conf.py

#!/usr/bin/env python3
# Copyright 2024 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A script to generate accel_test.conf for stable_delegate_test_suite."""

import argparse
import logging
from pathlib import Path
import subprocess
import sys
from typing import List, Optional, TextIO


def list_test(
    stable_delegate_test_suite: Path,
    gtest_filter: str,
) -> List[str]:
    stable_delegate_test_suite_cmd = [
        str(stable_delegate_test_suite.absolute()),
        "--gtest_list_tests",
        "--gtest_filter=%s" % gtest_filter,
    ]
    logging.debug("Running: %s", " ".join(stable_delegate_test_suite_cmd))
    result = subprocess.check_output(stable_delegate_test_suite_cmd, text=True)
    prefix = ""
    test_list: List[str] = []
    for line in result.splitlines():
        # ignoring comments for test parameters
        item = line.split("#", 1)[0].strip()
        if item.endswith("."):
            prefix = item
        else:
            test_list.append(prefix + item)
    return test_list


def generate_accel_test_conf(
    stable_delegate_test_suite: Path,
    stable_delegate_settings_file: Path,
    output_file: TextIO,
    crash_output_file: TextIO,
    gtest_filter: str,
    list_passed_tests: bool,
):
    assert stable_delegate_test_suite.is_file()
    assert stable_delegate_settings_file.is_file()

    passed_tests: List[str] = []
    failed_tests: List[str] = []
    crashed_tests: List[str] = []
    for test in list_test(stable_delegate_test_suite, gtest_filter):
        stable_delegate_test_suite_cmd = [
            str(stable_delegate_test_suite.absolute()),
            "--stable_delegate_settings_file=%s"
            % (stable_delegate_settings_file.absolute(),),
            "--gtest_filter=%s" % test,
        ]
        result = subprocess.run(
            stable_delegate_test_suite_cmd,
            stdout=subprocess.DEVNULL,
            stderr=subprocess.PIPE,
            check=False,
        )
        if result.returncode == 0:
            logging.debug("%s: PASS", test)
            passed_tests.append(test)
        elif result.returncode == 1:
            logging.debug("%s: FAIL", test)
            failed_tests.append(test)
        else:
            logging.debug("%s: CRASH", test)
            crashed_tests.append(test)
    logging.debug("Finished running stable_delegate_test_suite")
    logging.debug(
        "pass: %d, fail %d, crash, %d",
        len(passed_tests),
        len(failed_tests),
        len(crashed_tests),
    )

    output_file.write(
        "# Auto-generated by stable_delegate_accel_test_conf.py\n\n"
    )
    output_file.write("# Failed tests\n")
    for test in failed_tests:
        output_file.write("-%s\n" % test.replace(".", "/"))

    output_file.write("# Passed tests\n")
    if list_passed_tests:
        for test in passed_tests:
            output_file.write("%s\n" % test.replace(".", "/"))
    else:
        output_file.write(".*\n")

    crash_output_file.write("-%s" % ":".join(crashed_tests))

    logging.debug("Finished generating accel_test_conf")


def setup_argument_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    parser.add_argument(
        "--stable_delegate_settings_file",
        required=True,
        type=Path,
        help="The path to the delegate settings JSON file.",
    )
    parser.add_argument(
        "--output",
        default="accel_test.conf",
        type=argparse.FileType("w"),
        help="output path for generated acceleration test config",
    )
    parser.add_argument(
        "--crash_output",
        type=argparse.FileType("w"),
        default="crash_filter.txt",
        help="output path for generated gtest filter for crashed tests",
    )
    parser.add_argument(
        "--gtest_filter",
        type=str,
        default="*",
        help="gtest filter for quick verifying subset of tests",
    )
    parser.add_argument(
        "--stable_delegate_test_suite",
        default="/usr/local/bin/stable_delegate_test_suite",
        type=Path,
        help="path to the stable_delegate_test_suite binary",
    )
    parser.add_argument(
        "--list_passed_tests",
        action="store_true",
        help="list passed tests in the output config file",
    )
    parser.add_argument(
        "--debug",
        action="store_true",
        help="enable debug logging",
    )
    return parser


def main(argv: Optional[List[str]] = None) -> Optional[int]:
    parser = setup_argument_parser()
    args = parser.parse_args(argv)

    log_level = logging.DEBUG if args.debug else logging.INFO
    log_format = "%(asctime)s - %(levelname)s - %(funcName)s: %(message)s"
    logging.basicConfig(level=log_level, format=log_format)

    generate_accel_test_conf(
        args.stable_delegate_test_suite,
        args.stable_delegate_settings_file,
        args.output,
        args.crash_output,
        args.gtest_filter,
        args.list_passed_tests,
    )


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))