| #!/usr/bin/env vpython3 |
| # Copyright 2013 The LUCI Authors. All rights reserved. |
| # Use of this source code is governed under the Apache License, Version 2.0 |
| # that can be found in the LICENSE file. |
| |
| from __future__ import print_function |
| from __future__ import unicode_literals |
| |
| import base64 |
| import contextlib |
| import functools |
| import json |
| import logging |
| import os |
| import sys |
| import tempfile |
| import unittest |
| |
| import mock |
| |
| # Mutates sys.path. |
| import test_env |
| |
| # third_party/ |
| from depot_tools import auto_stub |
| |
| import cipdserver_fake |
| |
| import cas_util |
| import cipd |
| import errors |
| import local_caching |
| import run_isolated |
| from libs import luci_context |
| from utils import file_path |
| from utils import fs |
| from utils import large |
| from utils import logging_utils |
| from utils import subprocess42 |
| from utils import tools |
| |
| |
| ROOT_DIR = os.path.dirname( |
| os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| LUCI_GO_CLIENT_DIR = os.path.join(ROOT_DIR, 'luci-go') |
| |
| |
| def json_dumps(data): |
| return json.dumps(data, sort_keys=True, separators=(',', ':')) |
| |
| |
| @contextlib.contextmanager |
| def init_named_caches_stub(_run_dir, _stats): |
| yield |
| |
| |
| def trim_caches_stub(_stats): |
| pass |
| |
| |
| class StorageFake: |
| def __init__(self, files, server_ref): |
| self._files = files.copy() |
| self._server_ref = server_ref |
| |
| def __enter__(self, *_): |
| return self |
| |
| def __exit__(self, *_): |
| pass |
| |
| @property |
| def server_ref(self): |
| return self._server_ref |
| |
| def async_fetch(self, channel, _priority, digest, _size, sink): |
| sink([self._files[digest]]) |
| channel.send_result(digest) |
| |
| def upload_items(self, items_to_upload, _verify_push): |
| # Return all except the first one. |
| return list(items_to_upload)[1:] |
| |
| |
| class RunIsolatedTestBase(auto_stub.TestCase): |
| # These tests fail with the following error |
| # 'AssertionError: Items in the first set but not the second' |
| # Need to run in sequential_test_runner.py as an executable |
| no_run = 1 |
| DISABLE_CIPD_FOR_TESTS = ['--cipd-enabled', False] |
| |
| @classmethod |
| def setUpClass(cls): |
| if not file_path.enable_symlink(): |
| raise Exception( |
| 'Failed to enable symlink; this test requires it. On Windows, maybe ' |
| 'try running as Administrator') |
| |
| def setUp(self): |
| super(RunIsolatedTestBase, self).setUp() |
| os.environ.pop('LUCI_CONTEXT', None) |
| os.environ['LUCI_GO_CLIENT_DIR'] = LUCI_GO_CLIENT_DIR |
| self._previous_dir = os.getcwd() |
| self.tempdir = tempfile.mkdtemp(prefix='run_isolated_test') |
| logging.debug('Temp dir: %s', self.tempdir) |
| cwd = os.path.join(self.tempdir, 'cwd') |
| fs.mkdir(cwd) |
| os.chdir(cwd) |
| self.mock(run_isolated, 'make_temp_dir', self.fake_make_temp_dir) |
| self.mock(run_isolated.auth, 'ensure_logged_in', lambda _: None) |
| self.mock( |
| logging_utils.OptionParserWithLogging, 'logger_root', |
| logging.Logger('unittest')) |
| |
| self._cipd_server = None # initialized lazily |
| |
| def tearDown(self): |
| # Remove mocks. |
| super(RunIsolatedTestBase, self).tearDown() |
| fs.chdir(self._previous_dir) |
| file_path.rmtree(self.tempdir) |
| if self._cipd_server: |
| self._cipd_server.close() |
| |
| @property |
| def cipd_server(self): |
| if not self._cipd_server: |
| self._cipd_server = cipdserver_fake.FakeCipdServer() |
| return self._cipd_server |
| |
| def fake_make_temp_dir(self, prefix, _root_dir): |
| """Predictably returns directory for run_tha_test (one per test case).""" |
| self.assertIn(prefix, |
| (run_isolated.ISOLATED_OUT_DIR, run_isolated.ISOLATED_RUN_DIR, |
| run_isolated.ISOLATED_TMP_DIR, run_isolated._CAS_CLIENT_DIR, |
| 'cipd_site_root')) |
| temp_dir = os.path.join(self.tempdir, prefix) |
| self.assertFalse(fs.isdir(temp_dir)) |
| fs.makedirs(temp_dir) |
| return temp_dir |
| |
| def ir_dir(self, *args): |
| """Shortcut for joining path with ISOLATED_RUN_DIR. |
| |
| Where to map all files in run_isolated.run_tha_test(). |
| """ |
| return os.path.join(self.tempdir, run_isolated.ISOLATED_RUN_DIR, *args) |
| |
| def assertExpectedTree(self, expected, root_dir=None): |
| # Assume expected path are relative to root if not specified. |
| root_dir = root_dir or os.path.join(self.tempdir, 'io') |
| |
| # Return True is the entries in out_dir are exactly the same as entries in |
| # expected. Return False otherwise. |
| count = 0 |
| for path in expected: |
| content = expected[path] |
| full_path = os.path.join(root_dir, path) |
| self.assertTrue(fs.exists(full_path), "%s doesn't exist" % full_path) |
| while fs.islink(full_path): |
| full_path = fs.readlink(full_path) |
| # If we expect a non-empty directory, check the entries in dir. |
| # If we expect an empty dir, its existence (checked above) is sufficient. |
| if not fs.isdir(full_path): |
| with open(full_path, 'r') as f: |
| self.assertEqual(f.read(), content) |
| count += 1 |
| self.assertEqual(count, len(expected)) |
| |
| |
| class RunIsolatedTest(RunIsolatedTestBase): |
| # Mocked Popen so no subprocess is started. |
| def setUp(self): |
| super(RunIsolatedTest, self).setUp() |
| # list of func(args, **kwargs) -> retcode |
| # if the func returns None, then it's skipped. The first function to return |
| # non-None is taken as the retcode for the mocked Popen call. |
| self.popen_fakes = [] |
| self.popen_calls = [] |
| |
| self.capture_popen_env = False |
| self.capture_luci_ctx = False |
| |
| # pylint: disable=no-self-argument |
| class Popen: |
| def __init__(self2, args, **kwargs): |
| if not self.capture_popen_env: |
| kwargs.pop('env', None) |
| if self.capture_luci_ctx: |
| with open(os.environ['LUCI_CONTEXT']) as f: |
| kwargs['luci_ctx'] = json.load(f) |
| self2.returncode = None |
| self2.args = args |
| self2.kwargs = kwargs |
| self.popen_calls.append((args, kwargs)) |
| |
| def yield_any_line(self2, timeout=None): |
| self.assertEqual(0.1, timeout) |
| return () |
| |
| def wait(self2, timeout=None): |
| if timeout is not None: |
| self.assertTrue(timeout > 0) |
| self2.returncode = 0 |
| for mock_fn in self.popen_fakes: |
| ret = mock_fn(self2.args, **self2.kwargs) |
| if ret is not None: |
| self2.returncode = ret |
| break |
| return self2.returncode |
| |
| def kill(self): |
| pass |
| |
| self.mock(subprocess42, 'Popen', Popen) |
| |
| def test_copy_recusrsively(self): |
| src = os.path.join(self.tempdir, 'src') |
| dst = os.path.join(self.tempdir, 'dst') |
| with open(src, 'w'): |
| pass |
| |
| run_isolated.copy_recursively(src, dst) |
| self.assertTrue(os.path.isfile(dst)) |
| |
| def test_copy_recusrsively_not_exist(self): |
| src = os.path.join(self.tempdir, 'src') |
| dst = os.path.join(self.tempdir, 'dst') |
| run_isolated.copy_recursively(src, dst) |
| self.assertFalse(os.path.exists(dst)) |
| |
| def test_get_command_env(self): |
| old_env = os.environ.copy() |
| try: |
| os.environ.pop('B', None) |
| self.assertNotIn('B', os.environ) |
| os.environ['C'] = 'foo' |
| os.environ['D'] = 'bar' |
| os.environ['E'] = 'baz' |
| env = run_isolated.get_command_env( |
| '/a', |
| None, |
| '/b', |
| { |
| 'A': 'a', |
| 'B': None, |
| 'C': None, |
| 'E': '${ISOLATED_OUTDIR}/eggs' |
| }, |
| {'D': ['foo']}, |
| '/spam', |
| None) |
| self.assertNotIn('B', env) |
| self.assertNotIn('C', env) |
| if sys.platform == 'win32': |
| self.assertEqual('\\b\\foo;bar', env['D']) |
| else: |
| self.assertEqual('/b/foo:bar', env['D']) |
| self.assertEqual(os.sep + os.path.join('spam', 'eggs'), env['E']) |
| finally: |
| os.environ.clear() |
| os.environ.update(old_env) |
| |
| @mock.patch.dict(os.environ, {'SWARMING_TASK_ID': '4242'}) |
| def test_main(self): |
| self.mock(tools, 'disable_buffering', lambda: None) |
| |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), '--root-dir', self.tempdir, |
| '--', 'foo.exe', 'cmd with space', '-task-id', '${SWARMING_TASK_ID}' |
| ] |
| ret = run_isolated.main(cmd) |
| self.assertEqual(0, ret) |
| self.assertEqual([ |
| ( |
| [self.ir_dir('foo.exe'), 'cmd with space', '-task-id', '4242'], |
| { |
| 'cwd': self.ir_dir(), |
| 'detached': True, |
| 'close_fds': True, |
| 'lower_priority': False, |
| 'containment': subprocess42.Containment(), |
| }, |
| ), |
| ], self.popen_calls) |
| |
| def test_main_args(self): |
| self.mock(tools, 'disable_buffering', lambda: None) |
| |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--root-dir', |
| self.tempdir, |
| '--', |
| 'foo.exe', |
| 'cmd w/ space', |
| ] |
| ret = run_isolated.main(cmd) |
| self.assertEqual(0, ret) |
| self.assertEqual([ |
| ( |
| [self.ir_dir('foo.exe'), 'cmd w/ space'], |
| { |
| 'cwd': self.ir_dir(), |
| 'detached': True, |
| 'close_fds': True, |
| 'lower_priority': False, |
| 'containment': subprocess42.Containment(), |
| }, |
| ), |
| ], self.popen_calls) |
| |
| def _run_tha_test(self, |
| command=None, |
| lower_priority=False, |
| relative_cwd=None): |
| make_tree_call = [] |
| def add(i, _): |
| make_tree_call.append(i) |
| |
| for i in ('make_tree_files_read_only', 'make_tree_deleteable'): |
| self.mock(file_path, i, functools.partial(add, i)) |
| |
| data = run_isolated.TaskData( |
| command=command or [], |
| relative_cwd=relative_cwd, |
| cas_instance=None, |
| cas_digest=None, |
| outputs=None, |
| install_named_caches=init_named_caches_stub, |
| leak_temp_dir=False, |
| root_dir=self.tempdir, |
| hard_timeout=60, |
| grace_period=30, |
| bot_file=None, |
| switch_to_account=False, |
| install_packages_fn=run_isolated.copy_local_packages, |
| cas_cache_dir=None, |
| cas_cache_policies=None, |
| cas_kvs='', |
| env={}, |
| env_prefix={}, |
| lower_priority=lower_priority, |
| containment=None, |
| trim_caches_fn=trim_caches_stub) |
| ret = run_isolated.run_tha_test(data, None) |
| self.assertEqual(0, ret) |
| return make_tree_call |
| |
| def test_run_tha_test_naked(self): |
| self._run_tha_test(command=['invalid', 'command']) |
| self.assertEqual([ |
| ( |
| [self.ir_dir('invalid'), 'command'], |
| { |
| 'cwd': self.ir_dir(), |
| 'detached': True, |
| 'close_fds': True, |
| 'lower_priority': False, |
| 'containment': None, |
| }, |
| ), |
| ], self.popen_calls) |
| |
| def mock_popen_with_oserr(self): |
| def r(self, args, **kwargs): |
| old_init(self, args, **kwargs) |
| raise OSError('Unknown') |
| old_init = self.mock(subprocess42.Popen, '__init__', r) |
| |
| def test_main_naked(self): |
| self.mock_popen_with_oserr() |
| # The most naked .isolated file that can exist. |
| self.mock(tools, 'disable_buffering', lambda: None) |
| |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), '--root-dir', self.tempdir, |
| '--', 'invalid', 'command' |
| ] |
| ret = run_isolated.main(cmd) |
| self.assertEqual(1, ret) |
| self.assertEqual(1, len(self.popen_calls)) |
| self.assertEqual([ |
| ( |
| [self.ir_dir('invalid'), 'command'], |
| { |
| 'cwd': self.ir_dir(), |
| 'detached': True, |
| 'close_fds': True, |
| 'lower_priority': False, |
| 'containment': subprocess42.Containment(), |
| }, |
| ), |
| ], self.popen_calls) |
| |
| @unittest.skipIf(sys.platform == 'win32', 'crbug.com/1148174') |
| def test_main_naked_without_isolated(self): |
| self.mock_popen_with_oserr() |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--root-dir', |
| self.tempdir, |
| '--', |
| '/bin/echo', |
| 'hello', |
| 'world', |
| ] |
| ret = run_isolated.main(cmd) |
| self.assertEqual(1, ret) |
| self.assertEqual( |
| [ |
| ( |
| ['/bin/echo', 'hello', 'world'], |
| { |
| 'cwd': self.ir_dir(), |
| 'detached': True, |
| 'close_fds': True, |
| 'lower_priority': False, |
| 'containment': subprocess42.Containment(), |
| }, |
| ), |
| ], |
| self.popen_calls) |
| |
| @unittest.skipIf(sys.platform == 'win32', 'crbug.com/1148174') |
| def test_main_naked_with_account_switch(self): |
| self.capture_luci_ctx = True |
| self.mock_popen_with_oserr() |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--switch-to-account', |
| 'task', |
| '--', |
| '/bin/echo', |
| 'hello', |
| 'world', |
| ] |
| root_ctx = { |
| 'accounts': [{'id': 'bot'}, {'id': 'task'}], |
| 'default_account_id' : 'bot', |
| 'secret': 'sekret', |
| 'rpc_port': 12345, |
| } |
| with luci_context.write(local_auth=root_ctx): |
| run_isolated.main(cmd) |
| # Switched default account to task. |
| task_ctx = root_ctx.copy() |
| task_ctx['default_account_id'] = 'task' |
| self.assertEqual(task_ctx, self.popen_calls[0][1]['luci_ctx']['local_auth']) |
| |
| @unittest.skipIf(sys.platform == 'win32', 'crbug.com/1148174') |
| def test_main_naked_with_account_pop(self): |
| self.capture_luci_ctx = True |
| self.mock_popen_with_oserr() |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--switch-to-account', |
| 'task', |
| '--', |
| '/bin/echo', |
| 'hello', |
| 'world', |
| ] |
| root_ctx = { |
| 'accounts': [{'id': 'bot'}], # only 'bot', there's no 'task' |
| 'default_account_id' : 'bot', |
| 'secret': 'sekret', |
| 'rpc_port': 12345, |
| } |
| with luci_context.write(local_auth=root_ctx): |
| run_isolated.main(cmd) |
| # Unset default account, since 'task' account is not defined. |
| task_ctx = root_ctx.copy() |
| task_ctx.pop('default_account_id') |
| self.assertEqual(task_ctx, self.popen_calls[0][1]['luci_ctx']['local_auth']) |
| |
| @unittest.skipIf(sys.platform == 'win32', 'crbug.com/1148174') |
| def test_main_naked_leaking(self): |
| workdir = tempfile.mkdtemp() |
| try: |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', |
| '--root-dir', |
| workdir, |
| '--leak-temp-dir', |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--', |
| '/bin/echo', |
| 'hello', |
| 'world', |
| ] |
| ret = run_isolated.main(cmd) |
| self.assertEqual(0, ret) |
| finally: |
| fs.rmtree(workdir) |
| |
| def test_main_naked_with_packages(self): |
| self.mock(cipd, 'get_platform', lambda: 'linux-amd64') |
| |
| def pins_generator(): |
| yield { |
| '': [ |
| ('infra/data/x', 'badc0fee' * 5), |
| ('infra/data/y', 'cafebabe' * 5), |
| ], |
| 'bin': [('infra/tools/echo/linux-amd64', 'deadbeef' * 5),], |
| } |
| yield { |
| '': [('infra/tools/luci/cas/linux-amd64', |
| run_isolated._LUCI_GO_REVISION)], |
| } |
| |
| pins_gen = pins_generator() |
| |
| suffix = '.exe' if sys.platform == 'win32' else '' |
| def fake_ensure(args, **kwargs): |
| if (args[0].endswith(os.path.join('bin', 'cipd' + suffix)) and |
| args[1] == 'ensure' |
| and '-json-output' in args): |
| idx = args.index('-json-output') |
| with open(args[idx+1], 'w') as json_out: |
| json.dump( |
| { |
| 'result': { |
| subdir: [{ |
| 'package': pkg, |
| 'instance_id': ver |
| } for pkg, ver in packages] |
| for subdir, packages in next(pins_gen).items() |
| } |
| }, json_out) |
| return 0 |
| if args[0].endswith(os.sep + 'echo' + suffix): |
| return 0 |
| self.fail('unexpected: %s, %s' % (args, kwargs)) |
| return 1 |
| |
| self.popen_fakes.append(fake_ensure) |
| cipd_cache = os.path.join(self.tempdir, 'cipd_cache') |
| cmd = [ |
| '--no-log', |
| '--cipd-client-version', |
| 'git:wowza', |
| '--cipd-package', |
| 'bin:infra/tools/echo/${platform}:latest', |
| '--cipd-package', |
| '.:infra/data/x:latest', |
| '--cipd-package', |
| '.:infra/data/y:canary', |
| '--cipd-server', |
| self.cipd_server.url, |
| '--cipd-cache', |
| cipd_cache, |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--', |
| 'bin/echo${EXECUTABLE_SUFFIX}', |
| 'hello', |
| 'world', |
| ] |
| ret = run_isolated.main(cmd) |
| self.assertEqual(0, ret) |
| |
| self.assertEqual(3, len(self.popen_calls)) |
| |
| # Test cipd-ensure command for installing packages. |
| cipd_ensure_cmd, _ = self.popen_calls[0] |
| self.assertEqual(cipd_ensure_cmd[:2], [ |
| os.path.join(cipd_cache, 'bin', 'cipd' + cipd.EXECUTABLE_SUFFIX), |
| 'ensure', |
| ]) |
| cache_dir_index = cipd_ensure_cmd.index('-cache-dir') |
| self.assertEqual( |
| cipd_ensure_cmd[cache_dir_index+1], |
| os.path.join(cipd_cache, 'cache')) |
| |
| # Test cipd client cache. `git:wowza` was a tag and so is cacheable. |
| self.assertEqual(len(fs.listdir(os.path.join(cipd_cache, 'versions'))), 2) |
| version_file = os.path.join( |
| cipd_cache, 'versions', |
| '5c2ee864d65c435dfa1eb06bf4c52f58854db39392340a91fc91e88d6000d737') |
| self.assertTrue(fs.isfile(version_file)) |
| with open(version_file) as f: |
| self.assertEqual(f.read(), 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') |
| |
| client_binary_file = os.path.join( |
| cipd_cache, 'clients', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') |
| self.assertTrue(fs.isfile(client_binary_file)) |
| |
| # Test echo call. |
| echo_cmd, _ = self.popen_calls[2] |
| self.assertTrue(echo_cmd[0].endswith( |
| os.path.sep + 'bin' + os.path.sep + 'echo' + cipd.EXECUTABLE_SUFFIX), |
| echo_cmd[0]) |
| self.assertEqual(echo_cmd[1:], ['hello', 'world']) |
| |
| def test_main_naked_with_invalid_cas_input(self): |
| def dump_bad_digest_json(cmd, _): |
| json_path = cmd[cmd.index('-dump-json') + 1] |
| with open(json_path, 'w') as fp: |
| json.dump( |
| { |
| 'result': 'digest_invalid', |
| 'error_details': { |
| 'digest': 'abcd1234/21' |
| } |
| }, fp) |
| |
| self.mock(run_isolated, "_run_go_cmd_and_wait", dump_bad_digest_json) |
| result_json_path = os.path.join(self.tempdir, 'result.json') |
| |
| cmd = [ |
| '--json', |
| result_json_path, |
| '--no-log', |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--cipd-enabled', |
| 'False', |
| '--cas-digest', |
| 'not_valid', |
| '--cas-instance', |
| 'some_cas_instance', |
| '--', |
| 'bin/echo${EXECUTABLE_SUFFIX}', |
| 'hello', |
| 'world', |
| ] |
| |
| ret = run_isolated.main(cmd) |
| self.assertEqual(1, ret) |
| with open(result_json_path, 'r') as fp: |
| result_json = json.load(fp) |
| |
| missing_cas = result_json['missing_cas'][0] |
| self.assertEqual('digest_invalid', missing_cas['status']) |
| self.assertEqual('abcd1234/21', missing_cas['digest']) |
| self.assertEqual('some_cas_instance', missing_cas['instance']) |
| |
| def test_main_naked_with_invalid_cipd_package(self): |
| self.mock(cipd, 'get_platform', lambda: 'linux-amd64') |
| suffix = '.exe' if sys.platform == 'win32' else '' |
| |
| def fake_ensure(args, **kwargs): |
| if (args[0].endswith(os.path.join('bin', 'cipd' + suffix)) |
| and args[1] == 'ensure' and '-json-output' in args): |
| idx = args.index('-json-output') |
| with open(args[idx + 1], 'w') as json_out: |
| json.dump( |
| { |
| "error": |
| "failed to resolve does/not/exists/linux-amd64@latest", |
| "error_code": "invalid_version_error", |
| "error_details": { |
| "package": "does/not/exists/linux-amd64", |
| "version": "latest" |
| }, |
| "result": None |
| }, json_out) |
| return 0 |
| if args[0].endswith(os.sep + 'echo' + suffix): |
| return 0 |
| self.fail('unexpected: %s, %s' % (args, kwargs)) |
| return 1 |
| |
| self.popen_fakes.append(fake_ensure) |
| result_json_path = os.path.join(self.tempdir, 'result.json') |
| cipd_cache = os.path.join(self.tempdir, 'cipd_cache') |
| cmd = [ |
| '--json', |
| result_json_path, |
| '--no-log', |
| '--cipd-package', |
| 'bin:does/not/exists/${platform}:latest', |
| '--cipd-server', |
| self.cipd_server.url, |
| '--cipd-cache', |
| cipd_cache, |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--', |
| 'bin/echo${EXECUTABLE_SUFFIX}', |
| 'hello', |
| 'world', |
| ] |
| ret = run_isolated.main(cmd) |
| self.assertEqual(1, ret) |
| with open(result_json_path, 'r') as fp: |
| result_json = json.load(fp) |
| self.assertEqual(1, len(result_json['missing_cipd'])) |
| missing_cipd = result_json['missing_cipd'][0] |
| self.assertEqual('does/not/exists/linux-amd64', |
| missing_cipd['package_name']) |
| self.assertEqual('invalid_version_error', missing_cipd['status']) |
| self.assertEqual('latest', missing_cipd['version']) |
| self.assertIsNone(missing_cipd['path']) |
| |
| def test_main_naked_with_cipd_client_no_packages(self): |
| self.mock(cipd, 'get_platform', lambda: 'linux-amd64') |
| |
| cipd_cache = os.path.join(self.tempdir, 'cipd_cache') |
| cmd = [ |
| '--no-log', |
| '--cipd-client-version', |
| 'git:wowza', |
| '--cipd-server', |
| self.cipd_server.url, |
| '--cipd-cache', |
| cipd_cache, |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--relative-cwd', |
| 'a', |
| '--root-dir', |
| self.tempdir, |
| '--', |
| 'bin/echo${EXECUTABLE_SUFFIX}', |
| 'hello', |
| 'world', |
| ] |
| |
| pins = { |
| '': [('infra/tools/luci/isolated/linux-amd64', |
| run_isolated._LUCI_GO_REVISION)], |
| } |
| |
| suffix = '.exe' if sys.platform == 'win32' else '' |
| |
| def fake_ensure(args, **kwargs): |
| if (args[0].endswith(os.path.join('bin', 'cipd' + suffix)) and |
| args[1] == 'ensure' and '-json-output' in args): |
| idx = args.index('-json-output') |
| with open(args[idx + 1], 'w') as json_out: |
| json.dump({ |
| 'result': { |
| subdir: [{ |
| 'package': pkg, |
| 'instance_id': ver |
| } for pkg, ver in packages |
| ] for subdir, packages in pins.items() |
| } |
| }, json_out) |
| return 0 |
| if args[0].endswith(os.sep + 'echo' + suffix): |
| return 0 |
| self.fail('unexpected: %s, %s' % (args, kwargs)) |
| return 1 |
| |
| self.popen_fakes.append(fake_ensure) |
| |
| self.capture_popen_env = True |
| ret = run_isolated.main(cmd) |
| self.assertEqual(0, ret) |
| |
| # The CIPD client was bootstrapped and hardlinked (or copied on Win). |
| client_binary_file = os.path.join( |
| cipd_cache, 'clients', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') |
| self.assertTrue(fs.isfile(client_binary_file)) |
| client_binary_link = os.path.join(cipd_cache, 'bin', |
| 'cipd' + cipd.EXECUTABLE_SUFFIX) |
| self.assertTrue(fs.isfile(client_binary_link)) |
| |
| env = self.popen_calls[1][1].pop('env') |
| exec_path = self.ir_dir('a', 'bin', 'echo') |
| if sys.platform == 'win32': |
| exec_path += '.exe' |
| self.assertEqual( |
| [ |
| ( |
| [exec_path, 'hello', 'world'], |
| { |
| 'cwd': self.ir_dir('a'), |
| 'detached': True, |
| 'close_fds': True, |
| 'lower_priority': False, |
| 'containment': subprocess42.Containment(), |
| }, |
| ), |
| ], |
| # Ignore `cipd ensure` for isolated client here. |
| self.popen_calls[1:]) |
| |
| # Directory with cipd client is in front of PATH. |
| path = env['PATH'].split(os.pathsep) |
| self.assertEqual(os.path.join(cipd_cache, 'bin'), path[0]) |
| |
| # CIPD_CACHE_DIR is set. |
| self.assertEqual(os.path.join(cipd_cache, 'cache'), env['CIPD_CACHE_DIR']) |
| |
| def test_main_relative_cwd_no_cmd(self): |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--relative-cwd', |
| 'a', |
| ] |
| with self.assertRaises(SystemExit): |
| run_isolated.main(cmd) |
| |
| def test_main_bad_relative_cwd(self): |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--relative-cwd', |
| 'a/../../b', |
| '--', |
| 'bin/echo${EXECUTABLE_SUFFIX}', |
| 'hello', |
| 'world', |
| ] |
| with self.assertRaises(SystemExit): |
| run_isolated.main(cmd) |
| |
| def test_main_naked_with_caches(self): |
| # An empty named cache is not kept! |
| # Interestingly, because we would need to put something in the named cache |
| # for it to be kept, we need the tool to write to it. This is tested in the |
| # smoke test. |
| trimmed = [] |
| def trim_caches(caches, root, min_free_space, max_age_secs): |
| trimmed.append(True) |
| self.assertEqual(2, len(caches)) |
| self.assertTrue(root) |
| # The name cache root is increased by the sum of the two hints and buffer. |
| self.assertEqual( |
| run_isolated._FREE_SPACE_BUFFER_FOR_CIPD_PACKAGES + |
| 2 * 1024 * 1024 * 1024 + 1100, min_free_space) |
| self.assertEqual(1814400, max_age_secs) |
| self.mock(local_caching, 'trim_caches', trim_caches) |
| nc = os.path.join(self.tempdir, 'named_cache') |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', |
| '--leak-temp-dir', |
| '100', |
| '--named-cache-root', |
| nc, |
| '--named-cache', |
| 'cache_foo', |
| 'foo', |
| '100', |
| '--named-cache', |
| 'cache_bar', |
| 'bar', |
| '1000', |
| '--', |
| 'bin/echo${EXECUTABLE_SUFFIX}', |
| 'hello', |
| 'world', |
| ] |
| ret = run_isolated.main(cmd) |
| self.assertEqual(0, ret) |
| |
| for cache_name in ('cache_foo', 'cache_bar'): |
| named_path = os.path.join(nc, 'named', cache_name) |
| self.assertFalse(fs.exists(named_path)) |
| self.assertTrue(trimmed) |
| |
| def test_main_clean(self): |
| cas_cache_dir = os.path.join(self.tempdir, 'cas_cache') |
| named_cache_dir = os.path.join(self.tempdir, 'named_cache') |
| kvs_dir = os.path.join(self.tempdir, 'kvs_dir') |
| os.mkdir(kvs_dir) |
| with open(os.path.join(kvs_dir, 'dummy'), 'w') as f: |
| f.write('0' * 100) |
| |
| # override size threshold. |
| self.mock(run_isolated, '_CAS_KVS_CACHE_THRESHOLD', 99) |
| |
| min_free_space = 1 |
| max_cache_size = 2 |
| |
| cmd = [ |
| '--no-log', |
| '--clean', |
| # Shared options. |
| '--min-free-space', |
| str(min_free_space), |
| '--max-cache-size', |
| str(max_cache_size), |
| # CAS cache option. |
| '--cas-cache', |
| cas_cache_dir, |
| # Named cache option. |
| '--named-cache-root', |
| named_cache_dir, |
| '--kvs-dir', |
| kvs_dir, |
| ] |
| |
| def trim_caches_mock(caches, _root_dir, min_free_space, max_age_secs): |
| self.assertEqual(min_free_space, min_free_space) |
| self.assertEqual(max_age_secs, run_isolated.MAX_AGE_SECS) |
| |
| # CAS cache. |
| cas_cache = caches[0] |
| self.assertEqual(cas_cache.state_file, |
| os.path.join(cas_cache_dir, cas_cache.STATE_FILE)) |
| self.assertEqual(cas_cache.policies.max_cache_size, max_cache_size) |
| self.assertEqual(cas_cache.policies.min_free_space, min_free_space) |
| self.assertIsNone(cas_cache.policies.max_items) |
| |
| # Named cache. |
| named_cache = caches[1] |
| self.assertEqual(named_cache.state_file, |
| os.path.join(named_cache_dir, named_cache.STATE_FILE)) |
| # max_cache_size, max_items, max_age_secs are hardcoded in |
| # run_isolated.process_named_cache_options(). |
| self.assertEqual(named_cache._policies.max_cache_size, 1024**4) |
| self.assertEqual(named_cache._policies.min_free_space, min_free_space) |
| self.assertEqual(named_cache._policies.max_items, 50) |
| self.assertEqual(named_cache._policies.max_age_secs, |
| run_isolated.MAX_AGE_SECS) |
| |
| self.mock(local_caching, 'trim_caches', trim_caches_mock) |
| |
| ret = run_isolated.main(cmd) |
| self.assertEqual(0, ret) |
| |
| with self.assertRaises(OSError): |
| # kvs dir should be removed. |
| fs.stat(kvs_dir) |
| |
| def test_modified_cwd(self): |
| self._run_tha_test(command=['../out/some.exe', 'arg'], relative_cwd='some') |
| self.assertEqual([ |
| ( |
| [self.ir_dir('out', 'some.exe'), 'arg'], |
| { |
| 'cwd': self.ir_dir('some'), |
| 'detached': True, |
| 'close_fds': True, |
| 'lower_priority': False, |
| 'containment': None, |
| }, |
| ), |
| ], self.popen_calls) |
| |
| @unittest.skipIf(sys.platform == 'win32', 'crbug.com/1148174') |
| def test_run_tha_test_non_isolated(self): |
| _ = self._run_tha_test(command=['/bin/echo', 'hello', 'world']) |
| self.assertEqual([ |
| ( |
| ['/bin/echo', 'hello', 'world'], |
| { |
| 'cwd': self.ir_dir(), |
| 'detached': True, |
| 'close_fds': True, |
| 'lower_priority': False, |
| 'containment': None, |
| }, |
| ), |
| ], self.popen_calls) |
| |
| @unittest.skipIf(sys.platform == 'win32', 'crbug.com/1148174') |
| def test_main_containment(self): |
| def fake_wait(args, **kwargs): # pylint: disable=unused-argument |
| # Success. |
| return 0 |
| self.popen_fakes.append(fake_wait) |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', |
| '--lower-priority', |
| '--containment-type', |
| 'JOB_OBJECT', |
| '--limit-processes', |
| '42', |
| '--limit-total-committed-memory', |
| '1024', |
| '--root-dir', |
| self.tempdir, |
| '--', |
| '/bin/echo', |
| 'hello', |
| 'world', |
| ] |
| ret = run_isolated.main(cmd) |
| self.assertEqual(0, ret) |
| self.assertEqual([ |
| ( |
| ['/bin/echo', 'hello', 'world'], |
| { |
| 'cwd': |
| self.ir_dir(), |
| 'detached': |
| True, |
| 'close_fds': |
| True, |
| 'lower_priority': |
| True, |
| 'containment': |
| subprocess42.Containment( |
| containment_type=subprocess42.Containment.JOB_OBJECT, |
| limit_processes=42, |
| limit_total_committed_memory=1024, |
| ), |
| }, |
| ), |
| ], self.popen_calls) |
| |
| |
| class RunIsolatedTestRun(RunIsolatedTestBase): |
| |
| def setUp(self): |
| super(RunIsolatedTestRun, self).setUp() |
| # Starts a full CAS server mock and have run_tha_test() uploads results |
| # back after the task completed. |
| self._server = cas_util.LocalCAS(self.tempdir) |
| self._server.start() |
| |
| def tearDown(self): |
| self._server.stop() |
| super(RunIsolatedTestRun, self).tearDown() |
| |
| # Runs the actual command requested. |
| def test_output(self): |
| digest = self._server.archive_files( |
| {'cmd.py': b'import sys\n' |
| b'open(sys.argv[1], "w").write("bar")\n'}) |
| |
| os.environ['RUN_ISOLATED_CAS_ADDRESS'] = self._server.address |
| data = run_isolated.TaskData( |
| command=['python3', 'cmd.py', '${ISOLATED_OUTDIR}/foo'], |
| relative_cwd=None, |
| cas_instance=None, |
| cas_digest=digest, |
| outputs=None, |
| install_named_caches=init_named_caches_stub, |
| leak_temp_dir=False, |
| root_dir=self.tempdir, |
| hard_timeout=60, |
| grace_period=30, |
| bot_file=None, |
| switch_to_account=False, |
| install_packages_fn=run_isolated.copy_local_packages, |
| cas_cache_dir='', |
| cas_cache_policies=local_caching.CachePolicies(0, 0, 0, 0), |
| cas_kvs='', |
| env={}, |
| env_prefix={}, |
| lower_priority=False, |
| containment=None, |
| trim_caches_fn=trim_caches_stub) |
| |
| result_json = os.path.join(self.tempdir, 'result.json') |
| ret = run_isolated.run_tha_test(data, result_json) |
| self.assertEqual(0, ret) |
| with open(result_json) as f: |
| result = json.load(f) |
| |
| output_digest = result['cas_output_root']['digest'] |
| dest = os.path.join(self.tempdir, 'output') |
| self._server.download( |
| output_digest['hash'] + '/' + str(output_digest['size_bytes']), dest) |
| |
| self.assertEqual(os.listdir(dest), ['foo']) |
| with open(os.path.join(dest, 'foo'), 'rb') as f: |
| self.assertEqual(f.read(), b'bar') |
| |
| def test_write_without_empty_cas_does_not_crash(self): |
| """Test is here to prevent regression of: https://crbug.com/1504567 |
| """ |
| cmd = """ |
| import os |
| outpath = os.path.join("${ISOLATED_OUTDIR}", "foo") |
| with open(outpath, "w") as f: |
| f.write("dont crash") |
| """ |
| os.environ.pop('RUN_ISOLATED_CAS_ADDRESS', None) |
| data = run_isolated.TaskData( |
| command=['python3', '-c', cmd], |
| relative_cwd=None, |
| cas_instance=None, |
| cas_digest=None, |
| outputs=None, |
| install_named_caches=init_named_caches_stub, |
| leak_temp_dir=False, |
| root_dir=self.tempdir, |
| hard_timeout=60, |
| grace_period=30, |
| bot_file=None, |
| switch_to_account=False, |
| install_packages_fn=run_isolated.copy_local_packages, |
| cas_cache_dir='', |
| cas_cache_policies=local_caching.CachePolicies(0, 0, 0, 0), |
| cas_kvs='', |
| env={}, |
| env_prefix={}, |
| lower_priority=False, |
| containment=None, |
| trim_caches_fn=trim_caches_stub) |
| |
| result_json = os.path.join(self.tempdir, 'result.json') |
| ret = run_isolated.run_tha_test(data, result_json) |
| self.assertEqual(0, ret) |
| |
| |
| class RunIsolatedTestCase(RunIsolatedTestRun): |
| def test_bad_cas_json_output(self): |
| def dump_bad_json(cmd, _): |
| json_path = cmd[cmd.index('-dump-json') + 1] |
| with open(json_path, 'w') as fp: |
| fp.write("{i[]nv[[]alid js{}on") |
| |
| digest = self._server.archive_files( |
| {'cmd.py': b'import sys\n' |
| b'open(sys.argv[1], "w").write("bar")\n'}) |
| |
| self.mock(run_isolated, "_run_go_cmd_and_wait", dump_bad_json) |
| |
| data = run_isolated.TaskData( |
| command=['python3', 'cmd.py', '${ISOLATED_OUTDIR}/foo'], |
| relative_cwd=None, |
| cas_instance="some_instance", |
| cas_digest=digest, |
| outputs=None, |
| install_named_caches=init_named_caches_stub, |
| leak_temp_dir=False, |
| root_dir=self.tempdir, |
| hard_timeout=60, |
| grace_period=30, |
| bot_file=None, |
| switch_to_account=False, |
| install_packages_fn=run_isolated.copy_local_packages, |
| cas_cache_dir='', |
| cas_cache_policies=local_caching.CachePolicies(0, 0, 0, 0), |
| cas_kvs=None, |
| env={}, |
| env_prefix={}, |
| lower_priority=False, |
| containment=None, |
| trim_caches_fn=trim_caches_stub) |
| |
| result_json = os.path.join(self.tempdir, 'result.json') |
| run_isolated.run_tha_test(data, result_json) |
| with open(result_json) as f: |
| result = json.load(f) |
| # Don't care the exact error, so long as its not reported as a missing_cas |
| self.assertTrue(result.get('internal_failure')) |
| self.assertIsNone(result.get('missing_cas')) |
| |
| def test_bad_cas_digest(self): |
| digest = "notvaliddigest" |
| os.environ['RUN_ISOLATED_CAS_ADDRESS'] = self._server.address |
| data = run_isolated.TaskData( |
| command=['python3', 'cmd.py', '${ISOLATED_OUTDIR}/foo'], |
| relative_cwd=None, |
| cas_instance="some_instance", |
| cas_digest=digest, |
| outputs=None, |
| install_named_caches=init_named_caches_stub, |
| leak_temp_dir=False, |
| root_dir=self.tempdir, |
| hard_timeout=60, |
| grace_period=30, |
| bot_file=None, |
| switch_to_account=False, |
| install_packages_fn=run_isolated.copy_local_packages, |
| cas_cache_dir='', |
| cas_cache_policies=local_caching.CachePolicies(0, 0, 0, 0), |
| cas_kvs='', |
| env={}, |
| env_prefix={}, |
| lower_priority=False, |
| containment=None, |
| trim_caches_fn=trim_caches_stub) |
| |
| result_json = os.path.join(self.tempdir, 'result.json') |
| ret = run_isolated.run_tha_test(data, result_json) |
| with open(result_json) as f: |
| result = json.load(f) |
| self.assertEqual(1, ret) |
| self.assertEqual(digest, result['missing_cas'][0]['digest']) |
| self.assertEqual('some_instance', result['missing_cas'][0]['instance']) |
| |
| def test_bad_cipd_package(self): |
| def emulate_bad_cipd(_run_dir, cas_dir): |
| raise errors.NonRecoverableCipdException("missing_cipd", "foo_package", |
| "not/found/foo", "deadbeef") |
| |
| data = run_isolated.TaskData( |
| command=['python3', 'cmd.py', '${ISOLATED_OUTDIR}/foo'], |
| relative_cwd=None, |
| cas_instance=None, |
| cas_digest=None, |
| outputs=None, |
| install_named_caches=init_named_caches_stub, |
| leak_temp_dir=False, |
| root_dir=self.tempdir, |
| hard_timeout=60, |
| grace_period=30, |
| bot_file=None, |
| switch_to_account=False, |
| install_packages_fn=emulate_bad_cipd, |
| cas_cache_dir='', |
| cas_cache_policies=local_caching.CachePolicies(0, 0, 0, 0), |
| cas_kvs='', |
| env={}, |
| env_prefix={}, |
| lower_priority=False, |
| containment=None, |
| trim_caches_fn=trim_caches_stub) |
| |
| result_json = os.path.join(self.tempdir, 'result.json') |
| ret = run_isolated.run_tha_test(data, result_json) |
| with open(result_json) as f: |
| result = json.load(f) |
| |
| self.assertEqual(1, ret) |
| self.assertEqual('foo_package', result['missing_cipd'][0]['package_name']) |
| self.assertEqual('not/found/foo', result['missing_cipd'][0]['path']) |
| self.assertEqual('deadbeef', result['missing_cipd'][0]['version']) |
| |
| |
| FILE, LINK, RELATIVE_LINK, DIR = range(4) |
| |
| |
| class RunIsolatedTestOutputs(RunIsolatedTestBase): |
| # Unit test for link_outputs_to_outdir function. |
| |
| def create_src_tree(self, run_dir, src_dir): |
| # Create files and directories specified by src_dir in run_dir. |
| for path in src_dir: |
| full_path = os.path.join(run_dir, path) |
| (t, content) = src_dir[path] |
| if t == FILE: |
| with open(full_path, 'w') as f: |
| f.write(content) |
| elif t == RELATIVE_LINK: |
| fs.symlink(content, full_path) |
| elif t == LINK: |
| root_dir = os.path.join(self.tempdir, 'ir') |
| real_path = os.path.join(root_dir, content) |
| fs.symlink(real_path, full_path) |
| else: |
| fs.mkdir(full_path) |
| self.create_src_tree(os.path.join(run_dir, path), content) |
| |
| def link_outputs_test(self, src_dir, outputs): |
| run_dir = os.path.join(self.tempdir, 'ir') |
| out_dir = os.path.join(self.tempdir, 'io') |
| fs.mkdir(run_dir) |
| fs.mkdir(out_dir) |
| self.create_src_tree(run_dir, src_dir) |
| run_isolated.link_outputs_to_outdir(run_dir, out_dir, outputs) |
| |
| def test_file(self): |
| src_dir = { |
| 'foo_file': (FILE, 'contents of foo'), |
| } |
| outputs = ['foo_file'] |
| expected = { |
| 'foo_file': 'contents of foo', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_file(self): |
| src_dir = { |
| 'foo_file': (FILE, 'contents of foo'), |
| 'foo_link': (LINK, 'foo_file'), |
| } |
| outputs = ['foo_link'] |
| expected = { |
| 'foo_link': 'contents of foo', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_dir_containing_files(self): |
| src_dir = { |
| 'subdir': (DIR, { |
| 'child_a': (FILE, 'contents of a'), |
| 'child_b': (FILE, 'contents of b'), |
| }) |
| } |
| outputs = [os.path.join('subdir', '')] |
| expected = { |
| os.path.join('subdir', 'child_a'): 'contents of a', |
| os.path.join('subdir', 'child_b'): 'contents of b', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_relative_symlink(self): |
| src_dir = { |
| 'foo_file': (FILE, 'contents of foo'), |
| 'subdir': (DIR, { |
| 'foo_link': (RELATIVE_LINK, '../foo_file'), |
| 'subsubdir': (DIR, { |
| 'bar_link': (RELATIVE_LINK, '../foo_link'), |
| }), |
| }), |
| } |
| outputs = [os.path.join('subdir', 'subsubdir', 'bar_link')] |
| expected = { |
| os.path.join('subdir', 'subsubdir', 'bar_link'): 'contents of foo', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_dir_containing_files(self): |
| src_dir = { |
| 'subdir_link': (LINK, 'subdir'), |
| 'subdir': (DIR, { |
| 'child_a': (FILE, 'contents of a'), |
| }), |
| } |
| outputs = ['subdir_link'] |
| expected = { |
| os.path.join('subdir_link', 'child_a'): 'contents of a', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_symlink_to_dir_containing_files(self): |
| src_dir = { |
| 'subdir_link': (LINK, 'subdir_link2'), |
| 'subdir_link2': (LINK, 'subdir'), |
| 'subdir': (DIR, { |
| 'child_a': (FILE, 'contents of a'), |
| 'child_b': (FILE, 'contents of b'), |
| }), |
| } |
| outputs = ['subdir_link'] |
| expected = { |
| os.path.join('subdir_link', 'child_a'): 'contents of a', |
| os.path.join('subdir_link', 'child_b'): 'contents of b', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_empty_dir(self): |
| src_dir = { |
| 'subdir': (DIR, {}), |
| } |
| outputs = [os.path.join('subdir', '')] |
| expected = { |
| os.path.join('subdir', ''): '', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_dir_ignore_trailing_slash(self): |
| src_dir = { |
| 'subdir': (DIR, {}), |
| } |
| outputs = [os.path.join('subdir', '')] |
| expected = { |
| 'subdir': '', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_dir_containing_empty_dir(self): |
| src_dir = { |
| 'subdir': (DIR, { |
| 'subsubdir': (DIR, ''), |
| }), |
| } |
| outputs = [os.path.join('subdir', '')] |
| expected = { |
| os.path.join('subdir', 'subsubdir', ''): '', |
| os.path.join('subdir', 'subsubdir', ''): '', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_empty_dir(self): |
| src_dir = { |
| 'subdir': (DIR, {}), |
| 'subdir_link': (LINK, 'subdir'), |
| } |
| outputs = ['subdir_link'] |
| expected = { |
| os.path.join('subdir_link', ''): '', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_nonexistent_file(self): |
| src_dir = { |
| 'bad_link': (LINK, 'nonexistent_file'), |
| } |
| outputs = ['bad_link'] |
| expected = {} |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_symlink_to_file(self): |
| src_dir = { |
| 'first_link': (LINK, 'second_link'), |
| 'second_link': (LINK, 'foo_file'), |
| 'foo_file': (FILE, 'contents of foo'), |
| } |
| outputs = ['first_link'] |
| expected = { |
| 'first_link': 'contents of foo', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_symlink_to_nonexistent_file(self): |
| src_dir = { |
| 'first_link': (LINK, 'second_link'), |
| 'second_link': (LINK, 'nonexistent_file'), |
| } |
| outputs = ['first_link'] |
| expected = {} |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_file_in_parent(self): |
| src_dir = { |
| 'subdir': (DIR, { |
| 'subsubdir': (DIR, { |
| 'foo_link': (LINK, 'subdir/foo_file'), |
| }), |
| 'foo_file': (FILE, 'contents of foo'), |
| }), |
| } |
| outputs = [ |
| os.path.join('subdir', 'subsubdir', 'foo_link'), |
| os.path.join('subdir', 'foo_file'), |
| ] |
| expected = { |
| os.path.join('subdir', 'subsubdir', 'foo_link'): 'contents of foo', |
| os.path.join('subdir', 'foo_file'): 'contents of foo', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_file_in_dir(self): |
| src_dir = { |
| 'subdir_link': (LINK, 'subdir/child_a'), |
| 'subdir': (DIR, { |
| 'child_a': (FILE, 'contents of a'), |
| }), |
| } |
| outputs = ['subdir_link'] |
| expected = { |
| 'subdir_link': 'contents of a', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| def test_symlink_to_symlink_to_file_in_dir(self): |
| src_dir = { |
| 'first_link': (LINK, 'subdir_link'), |
| 'subdir_link': (LINK, 'subdir/child_a'), |
| 'subdir': (DIR, { |
| 'child_a': (FILE, 'contents of a'), |
| }), |
| } |
| outputs = ['first_link'] |
| expected = { |
| 'first_link': 'contents of a', |
| } |
| self.link_outputs_test(src_dir, outputs) |
| self.assertExpectedTree(expected) |
| |
| |
| class RunIsolatedTestOutputFiles(RunIsolatedTestBase): |
| |
| def setUp(self): |
| super(RunIsolatedTestOutputFiles, self).setUp() |
| |
| # Starts a full CAS server mock and have run_tha_test() uploads results |
| # back after the task completed. |
| self._server = cas_util.LocalCAS(self.tempdir) |
| self._server.start() |
| os.environ['RUN_ISOLATED_CAS_ADDRESS'] = self._server.address |
| |
| def tearDown(self): |
| self._server.stop() |
| super(RunIsolatedTestOutputFiles, self).tearDown() |
| |
| # Like RunIsolatedTestRun, but ensures that specific output files |
| # (as opposed to anything in $(ISOLATED_OUTDIR)) are returned. |
| def test_output_cmd(self): |
| # Output the following structure: |
| # |
| # foo1 |
| # foodir --> foo2_sl (symlink to "foo2_content" file) |
| # bardir --> bar1 |
| # |
| # Create the symlinks only on Linux. |
| cas_digest = self._server.archive_files({ |
| 'cmd.py': |
| b'import os\n' |
| b'import sys\n' |
| b'open(sys.argv[1], "w").write("foo1")\n' |
| b'bar1_path = os.path.join(sys.argv[3], "bar1")\n' |
| b'open(bar1_path, "w").write("bar1")\n' |
| b'if sys.platform == "linux":\n' |
| b' foo_realpath = os.path.abspath("foo2_content")\n' |
| b' open(foo_realpath, "w").write("foo2")\n' |
| b' os.symlink(foo_realpath, sys.argv[2])\n' |
| b'else:\n' |
| b' open(sys.argv[2], "w").write("foo2")\n' |
| }) |
| |
| data = run_isolated.TaskData( |
| command=['python3', 'cmd.py', 'foo1', 'foodir/foo2_sl', 'bardir/'], |
| relative_cwd=None, |
| cas_instance=None, |
| cas_digest=cas_digest, |
| outputs=[ |
| 'foo1', |
| # They must be in OS native path. |
| os.path.join('foodir', 'foo2_sl'), |
| os.path.join('bardir', ''), |
| ], |
| install_named_caches=init_named_caches_stub, |
| leak_temp_dir=False, |
| root_dir=self.tempdir, |
| hard_timeout=60, |
| grace_period=30, |
| bot_file=None, |
| switch_to_account=False, |
| install_packages_fn=run_isolated.copy_local_packages, |
| cas_cache_dir='', |
| cas_cache_policies=local_caching.CachePolicies(0, 0, 0, 0), |
| cas_kvs='', |
| env={}, |
| env_prefix={}, |
| lower_priority=False, |
| containment=None, |
| trim_caches_fn=trim_caches_stub) |
| result_json = os.path.join(self.tempdir, 'result.json') |
| ret = run_isolated.run_tha_test(data, result_json) |
| self.assertEqual(0, ret) |
| |
| # It uploaded back. Assert the store has a new item containing foo. |
| |
| with open(result_json) as f: |
| result = json.load(f) |
| |
| output_digest = result['cas_output_root']['digest'] |
| dest = os.path.join(self.tempdir, 'outputs') |
| self._server.download( |
| output_digest['hash'] + '/' + str(output_digest['size_bytes']), dest) |
| |
| self.assertExpectedTree( |
| { |
| os.path.join('foodir', 'foo2_sl'): 'foo2', |
| 'foo1': 'foo1', |
| os.path.join('bardir', 'bar1'): 'bar1', |
| }, dest) |
| |
| self.assertCountEqual(os.listdir(dest), ['foodir', 'foo1', 'bardir']) |
| self.assertEqual(os.listdir(os.path.join(dest, 'foodir')), ['foo2_sl']) |
| self.assertEqual(os.listdir(os.path.join(dest, 'bardir')), ['bar1']) |
| |
| |
| class RunIsolatedJsonTest(RunIsolatedTestBase): |
| # Similar to RunIsolatedTest but adds the hacks to process ISOLATED_OUTDIR to |
| # generate a json result file. |
| |
| def setUp(self): |
| super(RunIsolatedJsonTest, self).setUp() |
| |
| # Starts a full CAS server mock and have run_tha_test() uploads results |
| # back after the task completed. |
| self._server = cas_util.LocalCAS(self.tempdir) |
| self._server.start() |
| os.environ['RUN_ISOLATED_CAS_ADDRESS'] = self._server.address |
| |
| def tearDown(self): |
| self._server.stop() |
| super(RunIsolatedJsonTest, self).tearDown() |
| |
| def test_main_json(self): |
| sub_cmd = [ |
| 'python3', |
| '-c', |
| 'import sys; open(sys.argv[1], "wb").write(b"generated data\\n")', |
| '${ISOLATED_OUTDIR}/out.txt', |
| ] |
| |
| out = os.path.join(self.tempdir, 'res.json') |
| cmd = self.DISABLE_CIPD_FOR_TESTS + [ |
| '--no-log', |
| '--named-cache-root', |
| os.path.join(self.tempdir, 'named_cache'), |
| '--json', |
| out, |
| '--root-dir', |
| self.tempdir, |
| '--', |
| ] + sub_cmd |
| ret = run_isolated.main(cmd) |
| self.assertEqual(0, ret) |
| |
| cas_hash = ( |
| '5868195adddf130eeb09a939bc40a17033fb058288b7b0bec0144dcb07e9cf78') |
| |
| expected = { |
| 'exit_code': 0, |
| 'had_hard_timeout': False, |
| 'internal_failure': None, |
| 'cas_output_root': { |
| # Local cas server uses its own default instance. |
| 'cas_instance': None, |
| 'digest': { |
| 'hash': cas_hash, |
| 'size_bytes': 81 |
| }, |
| }, |
| 'outputs_ref': None, |
| 'stats': { |
| 'trim_caches': {}, |
| 'isolated': { |
| 'download': {}, |
| 'upload': { |
| 'items_cold': [15, 81], |
| 'items_hot': None, |
| 'result': 'success', |
| 'size_cold': 96, |
| 'size_hot': 0, |
| }, |
| }, |
| 'named_caches': { |
| 'install': {}, |
| 'uninstall': {}, |
| }, |
| 'cleanup': {}, |
| }, |
| 'version': 5, |
| } |
| actual = tools.read_json(out) |
| # duration can be exactly 0 due to low timer resolution, especially but not |
| # exclusively on Windows. |
| self.assertLessEqual(0, actual.pop('duration')) |
| self.assertLessEqual(0, actual['stats']['trim_caches'].pop('duration')) |
| actual_upload_stats = actual['stats']['isolated']['upload'] |
| self.assertLessEqual(0, actual_upload_stats.pop('duration')) |
| named_caches_stats = actual['stats']['named_caches'] |
| self.assertLessEqual(0, named_caches_stats['install'].pop('duration')) |
| self.assertLessEqual(0, named_caches_stats['uninstall'].pop('duration')) |
| self.assertLessEqual(0, actual['stats']['cleanup'].pop('duration')) |
| for i in ('items_cold', 'items_hot'): |
| if actual_upload_stats[i]: |
| actual_upload_stats[i] = large.unpack( |
| base64.b64decode(actual_upload_stats[i])) |
| self.assertEqual(expected, actual) |
| |
| |
| if __name__ == '__main__': |
| test_env.main() |
