tools/code_coverage/ci_only_validator_test.py

#!/usr/bin/env vpython3
# Copyright 2026 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import io
import json
import os
import pathlib
import subprocess
import sys
import textwrap
import unittest
from unittest import mock
from pyfakefs import fake_filesystem_unittest

import ci_only_validator


class TestCIOnlyValidator(fake_filesystem_unittest.TestCase):
  def setUp(self):
    # Resolve the path on the real filesystem first before pyfakefs is active.
    self.real_src_root = (
      pathlib.Path(ci_only_validator.__file__).resolve().parents[2]
    )
    self.setUpPyfakefs()
    self.fs.create_dir(self.real_src_root)

    # Set up global mocking for subprocess.run.
    self.subprocess_run_patcher = mock.patch('subprocess.run')
    self.mock_run = self.subprocess_run_patcher.start()
    self.addCleanup(self.subprocess_run_patcher.stop)

  def test_discover_trybots_for_test_suite(self):
    self.mock_run.return_value = mock.MagicMock(
      returncode=0,
      stdout=textwrap.dedent("""\

            infra/config/generated/builders/try/android-code-coverage/targets/chromium.coverage.json
            infra/config/generated/builders/try/android-x86-code-coverage/targets/chromium.coverage.json
            """),
    )

    src_root = pathlib.Path('/mock/src')
    entries = ci_only_validator.discover_trybots_for_test_suite(
      src_root, 'chrome_junit_tests'
    )
    self.assertEqual(
      entries,
      [
        ci_only_validator.TrybotEntry(
          'android-code-coverage', 'chromium.coverage'
        ),
        ci_only_validator.TrybotEntry(
          'android-x86-code-coverage', 'chromium.coverage'
        ),
      ],
    )

  def test_discover_trybots_for_test_suite_error(self):
    self.mock_run.return_value = mock.MagicMock(returncode=1)
    src_root = pathlib.Path('/mock/src')
    entries = ci_only_validator.discover_trybots_for_test_suite(
      src_root, 'chrome_junit_tests'
    )
    self.assertEqual(entries, [])

  def test_discover_trybots_for_test_suite_with_revision(self):
    self.mock_run.return_value = mock.MagicMock(
      returncode=0,
      stdout=textwrap.dedent("""\
            myrev:infra/config/generated/builders/try/android-code-coverage/targets/chromium.coverage.json
            """),
    )

    src_root = pathlib.Path('/mock/src')
    entries = ci_only_validator.discover_trybots_for_test_suite(
      src_root, 'chrome_junit_tests', 'myrev'
    )
    self.assertEqual(
      entries,
      [
        ci_only_validator.TrybotEntry(
          'android-code-coverage', 'chromium.coverage'
        )
      ],
    )

  def test_read_local_success(self):
    src_root = pathlib.Path('/mock/src')
    dummy_file = src_root / 'dummy_path'
    self.fs.create_file(dummy_file, contents='file_content')

    content = ci_only_validator.read_local(src_root, 'dummy_path')
    self.assertEqual(content, 'file_content')

  def test_read_local_not_found(self):
    src_root = pathlib.Path('/mock/src')
    content = ci_only_validator.read_local(src_root, 'non_existent_file')
    self.assertIsNone(content)

  def test_read_local_os_error(self):
    src_root = pathlib.Path('/mock/src')
    dummy_file = src_root / 'dummy_path'
    # Create an unreadable file to raise OSError/PermissionError natively.
    self.fs.create_file(dummy_file, contents='')
    os.chmod(dummy_file, 0o000)

    # Redirect stderr to suppress prints in test output
    with mock.patch('sys.stderr', new=io.StringIO()):
      content = ci_only_validator.read_local(src_root, 'dummy_path')
      self.assertIsNone(content)

  def test_read_git_success(self):
    self.mock_run.return_value = mock.MagicMock(
      returncode=0, stdout='git_file_content'
    )
    src_root = pathlib.Path('/mock/src')

    content = ci_only_validator.read_git(src_root, 'myrev', 'dummy_path')
    self.assertEqual(content, 'git_file_content')

  def test_read_git_error(self):
    self.mock_run.return_value = mock.MagicMock(returncode=1)
    src_root = pathlib.Path('/mock/src')

    content = ci_only_validator.read_git(src_root, 'myrev', 'dummy_path')
    self.assertIsNone(content)

  def test_check_coverage_enabled_true(self):
    mock_read = mock.MagicMock(
      return_value=json.dumps({'gn_args': {'use_clang_coverage': True}})
    )
    enabled = ci_only_validator.check_coverage_enabled(
      mock_read, 'android-code-coverage'
    )
    self.assertTrue(enabled)

  def test_check_coverage_enabled_false(self):
    mock_read = mock.MagicMock(
      return_value=json.dumps({'gn_args': {'use_clang_coverage': False}})
    )
    enabled = ci_only_validator.check_coverage_enabled(
      mock_read, 'android-code-coverage'
    )
    self.assertFalse(enabled)

  def test_check_coverage_enabled_none_content(self):
    mock_read = mock.MagicMock(return_value=None)
    enabled = ci_only_validator.check_coverage_enabled(
      mock_read, 'android-code-coverage'
    )
    self.assertFalse(enabled)

  def test_check_coverage_enabled_invalid_json(self):
    mock_read = mock.MagicMock(return_value='{invalid_json')
    enabled = ci_only_validator.check_coverage_enabled(
      mock_read, 'android-code-coverage'
    )
    self.assertFalse(enabled)

  def test_check_trybot_coverage_config_ci_only_true(self):
    targets_json = {
      'android-code-coverage': {
        'isolated_scripts': [{'name': 'chrome_junit_tests', 'ci_only': True}]
      }
    }
    mock_read = mock.MagicMock(return_value=json.dumps(targets_json))

    results = ci_only_validator.check_trybot_coverage_config(
      mock_read,
      'android-code-coverage',
      'chromium.coverage',
      'chrome_junit_tests',
    )
    self.assertEqual(
      results,
      [ci_only_validator.BuilderConfig('android-code-coverage', True)],
    )

  def test_check_trybot_coverage_config_ci_only_false(self):
    targets_json = {
      'android-code-coverage': {
        'isolated_scripts': [{'name': 'chrome_junit_tests', 'ci_only': False}]
      }
    }
    mock_read = mock.MagicMock(return_value=json.dumps(targets_json))

    results = ci_only_validator.check_trybot_coverage_config(
      mock_read,
      'android-code-coverage',
      'chromium.coverage',
      'chrome_junit_tests',
    )
    self.assertEqual(
      results,
      [ci_only_validator.BuilderConfig('android-code-coverage', False)],
    )

  def test_check_trybot_coverage_config_ci_only_omitted(self):
    # Tests that when 'ci_only' is omitted entirely, it defaults to False.
    targets_json = {
      'android-code-coverage': {
        'isolated_scripts': [{'name': 'chrome_junit_tests'}]
      }
    }
    mock_read = mock.MagicMock(return_value=json.dumps(targets_json))

    results = ci_only_validator.check_trybot_coverage_config(
      mock_read,
      'android-code-coverage',
      'chromium.coverage',
      'chrome_junit_tests',
    )
    self.assertEqual(
      results,
      [ci_only_validator.BuilderConfig('android-code-coverage', False)],
    )

  def test_check_trybot_coverage_config_none_content(self):
    mock_read = mock.MagicMock(return_value=None)
    results = ci_only_validator.check_trybot_coverage_config(
      mock_read,
      'android-code-coverage',
      'chromium.coverage',
      'chrome_junit_tests',
    )
    self.assertEqual(results, [])

  def test_check_trybot_coverage_config_invalid_json(self):
    mock_read = mock.MagicMock(return_value='{invalid_json')
    # Redirect stderr to suppress prints in test output
    with mock.patch('sys.stderr', new=io.StringIO()):
      results = ci_only_validator.check_trybot_coverage_config(
        mock_read,
        'android-code-coverage',
        'chromium.coverage',
        'chrome_junit_tests',
      )
      self.assertEqual(results, [])

  @mock.patch('ci_only_validator.discover_trybots_for_test_suite')
  def test_main_success_disk(self, mock_discover):
    mock_discover.return_value = [
      ci_only_validator.TrybotEntry(
        'android-code-coverage', 'chromium.coverage'
      )
    ]

    targets_json = {
      'android-code-coverage': {
        'isolated_scripts': [{'name': 'chrome_junit_tests', 'ci_only': True}]
      }
    }
    # Create the files on the fake filesystem relative to real_src_root
    targets_file = (
      self.real_src_root
      / 'infra/config/generated/builders/try/android-code-coverage/targets/'
      'chromium.coverage.json'
    )
    self.fs.create_file(targets_file, contents=json.dumps(targets_json))

    gn_args_file = (
      self.real_src_root
      / 'infra/config/generated/builders/try/android-code-coverage/'
      'gn-args.json'
    )
    self.fs.create_file(
      gn_args_file,
      contents=json.dumps({'gn_args': {'use_clang_coverage': True}}),
    )

    test_args = ['ci_only_validator.py', 'chrome_junit_tests']
    with (
      mock.patch('sys.argv', test_args),
      mock.patch('sys.stdout', new=io.StringIO()) as mock_stdout,
    ):
      ci_only_validator.main()
      output = mock_stdout.getvalue()
      self.assertIn("Checking test suite 'chrome_junit_tests'", output)
      self.assertIn('android-code-coverage', output)
      self.assertIn('ci_only = True', output)
      self.assertIn('Coverage: Enabled', output)

  @mock.patch('ci_only_validator.discover_trybots_for_test_suite')
  def test_main_success_git(self, mock_discover):
    mock_discover.return_value = [
      ci_only_validator.TrybotEntry(
        'android-code-coverage', 'chromium.coverage'
      )
    ]

    targets_json = {
      'android-code-coverage': {
        'isolated_scripts': [{'name': 'chrome_junit_tests', 'ci_only': False}]
      }
    }

    # Setup mock subprocess runs for git show
    def mock_run_side_effect(args, **kwargs):
      # git rev-parse --verify myrev
      if 'rev-parse' in args:
        return mock.MagicMock(returncode=0)

      # git show myrev:path/to/targets/chromium.coverage.json
      if 'show' in args:
        rel_path = args[2].split(':', 1)[1]
        if 'targets' in rel_path:
          return mock.MagicMock(returncode=0, stdout=json.dumps(targets_json))
        if 'gn-args.json' in rel_path:
          return mock.MagicMock(
            returncode=0,
            stdout=json.dumps({'gn_args': {'use_jacoco_coverage': True}}),
          )
      return mock.MagicMock(returncode=1)

    self.mock_run.side_effect = mock_run_side_effect

    test_args = [
      'ci_only_validator.py',
      'chrome_junit_tests',
      '--revision',
      'myrev',
    ]
    with (
      mock.patch('sys.argv', test_args),
      mock.patch('sys.stdout', new=io.StringIO()) as mock_stdout,
    ):
      ci_only_validator.main()
      output = mock_stdout.getvalue()
      self.assertIn(
        "Checking test suite 'chrome_junit_tests' across 1 configured"
        ' trybots at revision myrev...',
        output,
      )
      self.assertIn('android-code-coverage', output)
      self.assertIn('ci_only = False', output)
      self.assertIn('Coverage: Enabled', output)

  def test_main_src_root_not_exists(self):
    test_args = ['ci_only_validator.py', 'chrome_junit_tests']
    with (
      mock.patch('pathlib.Path.exists', return_value=False),
      mock.patch('sys.argv', test_args),
      mock.patch('sys.stderr', new=io.StringIO()) as mock_stderr,
    ):
      with self.assertRaises(SystemExit) as cm:
        ci_only_validator.main()
      self.assertEqual(cm.exception.code, 1)
      self.assertIn('Error: Src path', mock_stderr.getvalue())

  def test_main_invalid_revision(self):
    self.mock_run.return_value = mock.MagicMock(returncode=1)
    test_args = [
      'ci_only_validator.py',
      'chrome_junit_tests',
      '--revision',
      'invalidrev',
    ]
    with (
      mock.patch('sys.argv', test_args),
      mock.patch('sys.stderr', new=io.StringIO()) as mock_stderr,
    ):
      with self.assertRaises(SystemExit) as cm:
        ci_only_validator.main()
      self.assertEqual(cm.exception.code, 1)
      self.assertIn('Error: Invalid revision', mock_stderr.getvalue())

  @mock.patch('ci_only_validator.discover_trybots_for_test_suite')
  def test_main_no_trybots(self, mock_discover):
    mock_discover.return_value = []
    test_args = ['ci_only_validator.py', 'chrome_junit_tests']
    with (
      mock.patch('sys.argv', test_args),
      mock.patch('sys.stderr', new=io.StringIO()) as mock_stderr,
    ):
      with self.assertRaises(SystemExit) as cm:
        ci_only_validator.main()
      self.assertEqual(cm.exception.code, 0)
      self.assertIn("Warning: Test suite", mock_stderr.getvalue())


class TestCIOnlyValidatorExecutable(unittest.TestCase):
  def test_main_executable_run(self):
    import runpy

    script_path = pathlib.Path(ci_only_validator.__file__).resolve()
    test_args = ['ci_only_validator.py', '--help']
    with (
      mock.patch('sys.argv', test_args),
      mock.patch('sys.stdout', new=io.StringIO()) as mock_stdout,
    ):
      try:
        runpy.run_path(str(script_path), run_name='__main__')
      except SystemExit as e:
        self.assertEqual(e.code, 0)
      self.assertIn(
        'Check if a test suite is barred by ci_only', mock_stdout.getvalue()
      )


if __name__ == '__main__':
  unittest.main()