blob: 14d4d0c2478f95412c193408a585d8c90830b448 [file] [log] [blame]
# Copyright 2012 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import functools
import io
import json
import os
import unittest
import subprocess
import sys
import optparse
from unittest.mock import Mock, MagicMock, mock_open, patch
bisect_builds = __import__('bisect-builds')
if 'NO_MOCK_SERVER' not in os.environ:
maybe_patch = patch
else:
# SetupEnvironment for gsutil to connect to real server.
options, _ = bisect_builds.ParseCommandLine(['-a', 'linux64'])
bisect_builds.SetupEnvironment(options)
bisect_builds.SetupAndroidEnvironment()
# Mock object that always wraps for the spec.
# This will pass the call through and ignore the return_value and side_effect.
class WrappedMock(MagicMock):
def __init__(self,
spec=None,
return_value=None,
side_effect=None,
*args,
**kwargs):
super().__init__(spec, *args, **kwargs, wraps=spec)
maybe_patch = functools.partial(patch, spec=True, new_callable=WrappedMock)
maybe_patch.object = functools.partial(patch.object,
spec=True,
new_callable=WrappedMock)
class FakeProcess:
called_num_times = 0
def __init__(self, returncode):
self.returncode = returncode
FakeProcess.called_num_times += 1
def communicate(self):
return ('', '')
class BisectTest(unittest.TestCase):
patched = []
max_rev = 10000
fake_process_return_code = 0
def monkey_patch(self, obj, name, new):
patcher = patch.object(obj, name, new)
patcher.start()
def clear_patching(self):
patch.stopall()
def setUp(self):
FakeProcess.called_num_times = 0
self.fake_process_return_code = 0
self.monkey_patch(bisect_builds.DownloadJob, 'Start', lambda *args: None)
self.monkey_patch(bisect_builds.DownloadJob, 'Stop', lambda *args: None)
self.monkey_patch(bisect_builds.DownloadJob, 'WaitFor', lambda *args: None)
self.monkey_patch(bisect_builds, 'UnzipFilenameToDir', lambda *args: None)
self.monkey_patch(
subprocess, 'Popen',
lambda *args, **kwargs: FakeProcess(self.fake_process_return_code))
self.monkey_patch(bisect_builds.SnapshotBuild, '_get_rev_list',
lambda *args: range(self.max_rev))
def tearDown(self):
self.clear_patching()
def bisect(self, good_rev, bad_rev, evaluate, num_runs=1):
base_url = bisect_builds.CHROMIUM_BASE_URL
archive = 'linux'
asan = False
use_local_cache = False
options = optparse.Values()
options.good = good_rev
options.bad = bad_rev
options.archive = 'linux64'
options.release_builds = False
options.official_builds = False
options.asan = False
options.use_local_cache = False
options.blink = False
options.apk = None
options.signed = False
options.times = num_runs
context = bisect_builds.PathContext(options)
archive_build = bisect_builds.create_archive_build(options)
(minrev, maxrev, _) = bisect_builds.Bisect(context=context,
archive_build=archive_build,
evaluate=evaluate,
num_runs=num_runs,
profile=None,
try_args=[])
return (minrev, maxrev)
def testBisectConsistentAnswer(self):
self.assertEqual(self.bisect(1000, 100, lambda *args: 'g'), (100, 101))
self.assertEqual(self.bisect(100, 1000, lambda *args: 'b'), (100, 101))
self.assertEqual(self.bisect(2000, 200, lambda *args: 'b'), (1999, 2000))
self.assertEqual(self.bisect(200, 2000, lambda *args: 'g'), (1999, 2000))
def testBisectMultipleRunsEarlyReturn(self):
self.fake_process_return_code = 1
self.assertEqual(self.bisect(1, 3, lambda *args: 'b', num_runs=10), (1, 2))
self.assertEqual(FakeProcess.called_num_times, 1)
@unittest.skipIf(sys.platform == 'win32', 'Test fails on Windows due to '
'https://crbug.com/1393138')
def testBisectAllRunsWhenAllSucceed(self):
self.assertEqual(self.bisect(1, 3, lambda *args: 'b', num_runs=10), (1, 2))
self.assertEqual(FakeProcess.called_num_times, 10)
class ArchiveBuildTest(unittest.TestCase):
def setUp(self):
patch.multiple(bisect_builds.ArchiveBuild,
__abstractmethods__=set(),
build_type='release',
_get_rev_list=Mock(return_value=list(map(str, range(10)))),
_rev_list_cache_key='abc').start()
def tearDown(self):
patch.stopall()
def create_build(self, args=None):
if args is None:
args = ['-a', 'linux64', '-g', '0', '-b', '9']
options, args = bisect_builds.ParseCommandLine(args)
return bisect_builds.ArchiveBuild(options)
def test_cache_should_not_work_if_not_enabled(self):
build = self.create_build()
self.assertFalse(build.use_local_cache)
with patch('builtins.open') as m:
self.assertEqual(build.get_rev_list(), [str(x) for x in range(10)])
bisect_builds.ArchiveBuild._get_rev_list.assert_called_once()
m.assert_not_called()
def test_cache_should_save_and_load(self):
build = self.create_build(
['-a', 'linux64', '-g', '0', '-b', '9', '--use-local-cache'])
self.assertTrue(build.use_local_cache)
# Load the non-existent cache and write to it.
cached_data = []
# The cache file would be opened 3 times:
# 1. read by _load_rev_list_cache
# 2. read by _save_rev_list_cache for existing cache
# 3. write by _save_rev_list_cache
write_mock = MagicMock()
write_mock.__enter__().write.side_effect = lambda d: cached_data.append(d)
with patch('builtins.open',
side_effect=[FileNotFoundError, FileNotFoundError, write_mock]):
self.assertEqual(build.get_rev_list(), [str(x) for x in range(10)])
bisect_builds.ArchiveBuild._get_rev_list.assert_called_once()
cached_json = json.loads(''.join(cached_data))
self.assertDictEqual(cached_json, {'abc': [str(x) for x in range(10)]})
# Load cache with cached data.
build = self.create_build(
['-a', 'linux64', '-g', '0', '-b', '9', '--use-local-cache'])
bisect_builds.ArchiveBuild._get_rev_list.reset_mock()
with patch('builtins.open', mock_open(read_data=''.join(cached_data))):
self.assertEqual(build.get_rev_list(), [str(x) for x in range(10)])
bisect_builds.ArchiveBuild._get_rev_list.assert_not_called()
@patch.object(bisect_builds.ArchiveBuild, '_load_rev_list_cache')
@patch.object(bisect_builds.ArchiveBuild, '_save_rev_list_cache')
@patch.object(bisect_builds.ArchiveBuild,
'_get_rev_list',
return_value=[str(x) for x in range(10)])
def test_should_request_partial_rev_list(self, mock_get_rev_list,
mock_save_rev_list_cache,
mock_load_rev_list_cache):
build = self.create_build()
# missing latest
mock_load_rev_list_cache.return_value = [str(x) for x in range(5)]
self.assertEqual(build.get_rev_list(), [str(x) for x in range(10)])
mock_get_rev_list.assert_called_with('4', '9')
# missing old and latest
mock_load_rev_list_cache.return_value = [str(x) for x in range(1, 5)]
self.assertEqual(build.get_rev_list(), [str(x) for x in range(10)])
mock_get_rev_list.assert_called_with('0', '9')
# missing old
mock_load_rev_list_cache.return_value = [str(x) for x in range(3, 10)]
self.assertEqual(build.get_rev_list(), [str(x) for x in range(10)])
mock_get_rev_list.assert_called_with('0', '3')
# no intersect
mock_load_rev_list_cache.return_value = ['c', 'd', 'e']
self.assertEqual(build.get_rev_list(), [str(x) for x in range(10)])
mock_save_rev_list_cache.assert_called_with([str(x) for x in range(10)] +
['c', 'd', 'e'])
mock_get_rev_list.assert_called_with('0', 'c')
@patch.object(bisect_builds.ArchiveBuild, '_get_rev_list', return_value=[])
def test_should_raise_error_when_no_rev_list(self, mock_get_rev_list):
build = self.create_build()
with self.assertRaises(bisect_builds.BisectException):
build.get_rev_list()
mock_get_rev_list.assert_any_call('0', '9')
mock_get_rev_list.assert_any_call()
@unittest.skipIf('NO_MOCK_SERVER' not in os.environ,
'The test is to ensure NO_MOCK_SERVER working correctly')
@maybe_patch('bisect-builds.GetRevisionFromVersion', return_value=123)
def test_no_mock(self, mock_GetRevisionFromVersion):
self.assertEqual(bisect_builds.GetRevisionFromVersion('127.0.6533.74'),
1313161)
mock_GetRevisionFromVersion.assert_called()
class ReleaseBuildTest(unittest.TestCase):
def test_should_look_up_path_context(self):
options, args = bisect_builds.ParseCommandLine(
['-r', '-a', 'linux64', '-g', '127.0.6533.74', '-b', '127.0.6533.88'])
self.assertEqual(options.archive, 'linux64')
build = bisect_builds.create_archive_build(options)
self.assertIsInstance(build, bisect_builds.ReleaseBuild)
self.assertEqual(build.binary_name, 'chrome')
self.assertEqual(build.listing_platform_dir, 'linux64/')
self.assertEqual(build.archive_name, 'chrome-linux64.zip')
self.assertEqual(build.archive_extract_dir, 'chrome-linux64')
@maybe_patch(
'bisect-builds.GsutilList',
return_value=[
'gs://chrome-unsigned/desktop-5c0tCh/%s/linux64/chrome-linux64.zip' %
x for x in ['127.0.6533.74', '127.0.6533.75', '127.0.6533.76']
])
def test_get_rev_list(self, mock_GsutilList):
options, args = bisect_builds.ParseCommandLine(
['-r', '-a', 'linux64', '-g', '127.0.6533.74', '-b', '127.0.6533.76'])
build = bisect_builds.create_archive_build(options)
self.assertIsInstance(build, bisect_builds.ReleaseBuild)
self.assertEqual(build.get_rev_list(),
['127.0.6533.74', '127.0.6533.75', '127.0.6533.76'])
mock_GsutilList.assert_any_call('gs://chrome-unsigned/desktop-5c0tCh')
mock_GsutilList.assert_any_call(*[
'gs://chrome-unsigned/desktop-5c0tCh/%s/linux64/chrome-linux64.zip' % x
for x in ['127.0.6533.74', '127.0.6533.75', '127.0.6533.76']
],
ignore_fail=True)
self.assertEqual(mock_GsutilList.call_count, 2)
@patch('bisect-builds.GsutilList',
return_value=['127.0.6533.74', '127.0.6533.75', '127.0.6533.76'])
def test_should_save_and_load_cache(self, mock_GsutilList):
options, args = bisect_builds.ParseCommandLine([
'-r', '-a', 'linux64', '-g', '127.0.6533.74', '-b', '127.0.6533.76',
'--use-local-cache'
])
build = bisect_builds.create_archive_build(options)
self.assertIsInstance(build, bisect_builds.ReleaseBuild)
# Load the non-existent cache and write to it.
cached_data = []
write_mock = MagicMock()
write_mock.__enter__().write.side_effect = lambda d: cached_data.append(d)
with patch('builtins.open',
side_effect=[FileNotFoundError, FileNotFoundError, write_mock]):
self.assertEqual(build.get_rev_list(),
['127.0.6533.74', '127.0.6533.75', '127.0.6533.76'])
mock_GsutilList.assert_called()
cached_json = json.loads(''.join(cached_data))
self.assertDictEqual(
cached_json, {
build._rev_list_cache_key:
['127.0.6533.74', '127.0.6533.75', '127.0.6533.76']
})
# Load cache with cached data.
mock_GsutilList.reset_mock()
with patch('builtins.open', mock_open(read_data=''.join(cached_data))):
self.assertEqual(build.get_rev_list(),
['127.0.6533.74', '127.0.6533.75', '127.0.6533.76'])
mock_GsutilList.assert_not_called()
class AndroidReleaseBuildTest(unittest.TestCase):
@maybe_patch(
'bisect-builds.GsutilList',
return_value=[
'gs://chrome-signed/android-B0urB0N/%s/arm_64/MonochromeStable.apk' %
x for x in ['127.0.6533.76', '127.0.6533.78', '127.0.6533.79']
])
@maybe_patch('bisect-builds._GetMappingFromAndroidApk',
return_value=bisect_builds.MONOCHROME_APK_FILENAMES)
def test_get_android_rev_list(self, mock_GetMapping, mock_GsutilList):
options, args = bisect_builds.ParseCommandLine([
'-r', '-a', 'android-arm64', '--apk', 'chrome_stable', '-g',
'127.0.6533.76', '-b', '127.0.6533.79', '--signed'
])
device = Mock()
device.build_version_sdk = 31 # version_codes.S
build = bisect_builds.create_archive_build(options, device)
self.assertIsInstance(build, bisect_builds.AndroidReleaseBuild)
self.assertEqual(build.get_rev_list(),
['127.0.6533.76', '127.0.6533.78', '127.0.6533.79'])
mock_GsutilList.assert_any_call('gs://chrome-signed/android-B0urB0N')
mock_GsutilList.assert_any_call(*[
'gs://chrome-signed/android-B0urB0N/%s/arm_64/MonochromeStable.apk' % x
for x in ['127.0.6533.76', '127.0.6533.78', '127.0.6533.79']
],
ignore_fail=True)
self.assertEqual(mock_GsutilList.call_count, 2)
class OfficialBuildTest(unittest.TestCase):
@maybe_patch.object(bisect_builds,
'GetRevisionFromVersion',
return_value=1313161)
@maybe_patch.object(bisect_builds,
'GetChromiumRevision',
return_value=999999999)
def test_should_convert_revision_as_commit_position(
self, mock_GetChromiumRevision, mock_GetRevisionFromVersion):
options, args = bisect_builds.ParseCommandLine(
['-a', 'linux64', '-g', '127.0.6533.74'])
build = bisect_builds.OfficialBuild(options)
self.assertEqual(build.good_revision, 1313161)
self.assertEqual(build.bad_revision, 999999999)
mock_GetRevisionFromVersion.assert_called_once_with('127.0.6533.74')
mock_GetChromiumRevision.assert_called()
def test_should_lookup_path_context(self):
options, args = bisect_builds.ParseCommandLine(
['-a', 'linux64', '-g', '0', '-b', '10'])
self.assertEqual(options.archive, 'linux64')
build = bisect_builds.OfficialBuild(options)
self.assertEqual(build.binary_name, 'chrome')
self.assertEqual(build.listing_platform_dir, 'linux-builder-perf/')
self.assertEqual(build.archive_name, 'chrome-perf-linux.zip')
self.assertEqual(build.archive_extract_dir, 'full-build-linux')
@maybe_patch('bisect-builds.GsutilList',
return_value=[
'full-build-linux_%d.zip' % x
for x in range(1313161, 1313164)
])
def test_get_rev_list(self, mock_GsutilList):
options, args = bisect_builds.ParseCommandLine(
['-a', 'linux64', '-g', '1313161', '-b', '1313163'])
build = bisect_builds.OfficialBuild(options)
self.assertEqual(build._get_rev_list(None), list(range(1313161, 1313164)))
mock_GsutilList.assert_called_once_with(
'gs://chrome-test-builds/official-by-commit/linux-builder-perf/')
class SnapshotBuildTest(unittest.TestCase):
def test_should_lookup_path_context(self):
options, args = bisect_builds.ParseCommandLine(
['-a', 'linux64', '-g', '0', '-b', '10'])
self.assertEqual(options.archive, 'linux64')
build = bisect_builds.SnapshotBuild(options)
self.assertEqual(build.binary_name, 'chrome')
self.assertEqual(build.listing_platform_dir, 'Linux_x64/')
self.assertEqual(build.archive_name, 'chrome-linux.zip')
self.assertEqual(build.archive_extract_dir, 'chrome-linux')
CommonDataXMLContent = '''<?xml version='1.0' encoding='UTF-8'?>
<ListBucketResult xmlns='http://doc.s3.amazonaws.com/2006-03-01'>
<Name>chromium-browser-snapshots</Name>
<Prefix>Linux_x64/</Prefix>
<Marker></Marker>
<NextMarker></NextMarker>
<Delimiter>/</Delimiter>
<IsTruncated>true</IsTruncated>
<CommonPrefixes>
<Prefix>Linux_x64/1313161/</Prefix>
</CommonPrefixes>
<CommonPrefixes>
<Prefix>Linux_x64/1313163/</Prefix>
</CommonPrefixes>
<CommonPrefixes>
<Prefix>Linux_x64/1313185/</Prefix>
</CommonPrefixes>
</ListBucketResult>
'''
@maybe_patch('urllib.request.urlopen',
return_value=io.StringIO(CommonDataXMLContent))
@patch.object(bisect_builds, 'GetChromiumRevision', return_value=1313185)
def test_get_rev_list(self, mock_GetChromiumRevision, mock_urlopen):
options, args = bisect_builds.ParseCommandLine(
['-a', 'linux64', '-g', '1313161', '-b', '1313185'])
build = bisect_builds.SnapshotBuild(options)
rev_list = build.get_rev_list()
mock_urlopen.assert_any_call(
'http://commondatastorage.googleapis.com/chromium-browser-snapshots/'
'?delimiter=/&prefix=Linux_x64/&marker=Linux_x64/1313161')
self.assertEqual(mock_urlopen.call_count, 1)
self.assertEqual(rev_list, [1313161, 1313163, 1313185])
class ASANBuildTest(unittest.TestCase):
CommonDataXMLContent = '''<?xml version='1.0' encoding='UTF-8'?>
<ListBucketResult xmlns='http://doc.s3.amazonaws.com/2006-03-01'>
<Name>chromium-browser-asan</Name>
<Prefix>linux-release</Prefix>
<Marker/>
<NextMarker></NextMarker>
<IsTruncated>true</IsTruncated>
<Contents>
<Key>linux-release/asan-symbolized-linux-release-131722.zip</Key>
<Generation>1394037704890000</Generation>
<MetaGeneration>3</MetaGeneration>
<LastModified>2014-03-05T16:41:44.883Z</LastModified>
<ETag>"0a9571ae451f4b510aebd38b1810ffce"</ETag>
<Size>1020894259</Size>
</Contents>
<Contents>
<Key>linux-release/asan-symbolized-linux-release-131727.zip</Key>
<Generation>1394037705035000</Generation>
<MetaGeneration>3</MetaGeneration>
<LastModified>2014-03-05T16:41:44.923Z</LastModified>
<ETag>"ee91342f9745640479146b5bb32fb1d4"</ETag>
<Size>1020872693</Size>
</Contents>
<Contents>
<Key>linux-release/asan-symbolized-linux-release-131728.zip</Key>
<Generation>1394037705141000</Generation>
<MetaGeneration>3</MetaGeneration>
<LastModified>2014-03-05T16:41:45.047Z</LastModified>
<ETag>"cfea42b6f3d5ca7f75d8a2fdb94a0df6"</ETag>
<Size>1020871851</Size>
</Contents>
</ListBucketResult>
'''
@maybe_patch('urllib.request.urlopen',
return_value=io.StringIO(CommonDataXMLContent))
def test_get_rev_list(self, mock_urlopen):
# TODO: Last available bisect revision for linux is 398598.
options, args = bisect_builds.ParseCommandLine(
['-a', 'linux64', '-g', '131722', '-b', '131730'])
# TODO: The archive name of linux platform for ASAN is actually linux,
# however it is not listed in option supported list. Will fix in following
# CLs.
options.archive = 'linux'
build = bisect_builds.ASANBuild(options)
rev_list = build.get_rev_list()
print(mock_urlopen.call_args_list)
mock_urlopen.assert_any_call(
'http://commondatastorage.googleapis.com/chromium-browser-asan/'
'?delimiter=&prefix=linux-release'
'&marker=linux-release/asan-symbolized-linux-release-131722')
self.assertEqual(rev_list, [131722, 131727, 131728])
if __name__ == '__main__':
unittest.main()