mac signing: Parallelize notarization submissions.
Notarization submissions take a fair amount of time. By parallelizing
them I hope to decrease the latency of producing a signed Chrome
build.
Because we can now take advantage of more sophisticated python
parallelism, I combined the submission and the wait steps, which led
to less code overall.
Bug: 411130935
Change-Id: I92901e3ed2da6e0ef9f982dd664f3b3904ca6fbf
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/6470354
Reviewed-by: Keybo Qian <keybo@google.com>
Auto-Submit: Joshua Pawlicki <waffles@chromium.org>
Commit-Queue: Keybo Qian <keybo@google.com>
Cr-Commit-Position: refs/branch-heads/7118@{#6}
Cr-Branched-From: d143a2e5953b2083b230257a6ae5302f7e678283-refs/heads/main@{#1444677}
diff --git a/chrome/installer/mac/signing/commands.py b/chrome/installer/mac/signing/commands.py
index 398a4ca..c1f1212 100644
--- a/chrome/installer/mac/signing/commands.py
+++ b/chrome/installer/mac/signing/commands.py
@@ -5,6 +5,7 @@
The commands module wraps operations that have side-effects.
"""
+import asyncio
import os
import platform
import plistlib
@@ -102,6 +103,21 @@
return subprocess.check_output(args, **kwargs)
+async def run_command_output_async(args, **kwargs):
+ logger.info('Running command: %s', args)
+ process = await asyncio.create_subprocess_exec(
+ *args,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ **kwargs)
+ stdout, stderr = await process.communicate()
+ if process.returncode:
+ logger.error('%s failed. stdout: %s stderr: %s', args, stdout, stderr)
+ raise subprocess.CalledProcessError(
+ process.returncode, args, output=stdout, stderr=stderr)
+ return stdout
+
+
def lenient_run_command_output(args, **kwargs):
"""Runs a command, being fairly tolerant of errors.
diff --git a/chrome/installer/mac/signing/driver.py b/chrome/installer/mac/signing/driver.py
index 2fd03817..a4addbb4 100644
--- a/chrome/installer/mac/signing/driver.py
+++ b/chrome/installer/mac/signing/driver.py
@@ -6,7 +6,9 @@
"""
import argparse
+import asyncio
import os
+import subprocess
from signing import config_factory, commands, invoker, logger, model, pipeline
@@ -70,7 +72,12 @@
def _show_tool_versions():
logger.info('Showing macOS and tool versions.')
commands.run_command(['sw_vers'])
- commands.run_command(['xcodebuild', '-version'])
+ try:
+ # xcodebuild -version fails with command-line xcode, but the rest of the
+ # signing script works.
+ commands.run_command(['xcodebuild', '-version'])
+ except subprocess.CalledProcessError as e:
+ print("xcodebuild failed: %s" % e.stderr)
commands.run_command(['xcrun', '-show-sdk-path'])
@@ -171,9 +178,10 @@
_show_tool_versions()
- pipeline.sign_all(
- paths,
- config,
- disable_packaging=args.disable_packaging,
- skip_brands=args.skip_brands,
- channels=args.channels)
+ asyncio.run(
+ pipeline.sign_all(
+ paths,
+ config,
+ disable_packaging=args.disable_packaging,
+ skip_brands=args.skip_brands,
+ channels=args.channels))
diff --git a/chrome/installer/mac/signing/driver_test.py b/chrome/installer/mac/signing/driver_test.py
index 8ee9534..6ec48b26 100644
--- a/chrome/installer/mac/signing/driver_test.py
+++ b/chrome/installer/mac/signing/driver_test.py
@@ -181,7 +181,7 @@
'--identity',
'G',
'--notarize',
- 'nowait',
+ 'wait-nostaple',
'--notary-arg=--key',
'--notary-arg',
'/path/to/key',
@@ -192,7 +192,8 @@
])
self.assertEquals(1, sign_all.call_count)
config = sign_all.call_args.args[1]
- self.assertEquals(model.NotarizeAndStapleLevel.NOWAIT, config.notarize)
+ self.assertEquals(model.NotarizeAndStapleLevel.WAIT_NOSTAPLE,
+ config.notarize)
self.assertEquals(
['--key', '/path/to/key', '--key-id=KeyId', '--issuer', 'Issuer'],
config.invoker.notarizer.notary_args)
diff --git a/chrome/installer/mac/signing/invoker.py b/chrome/installer/mac/signing/invoker.py
index e38dee9c..2c2df88 100644
--- a/chrome/installer/mac/signing/invoker.py
+++ b/chrome/installer/mac/signing/invoker.py
@@ -88,7 +88,7 @@
notarization-related operations.
"""
- def submit(self, path, config):
+ async def submit(self, path, config):
"""Submits an artifact to Apple for notarization.
Args:
diff --git a/chrome/installer/mac/signing/model.py b/chrome/installer/mac/signing/model.py
index b65876c..461bfcb 100644
--- a/chrome/installer/mac/signing/model.py
+++ b/chrome/installer/mac/signing/model.py
@@ -191,9 +191,6 @@
`NONE` means no notarization tasks should be performed.
- `NOWAIT` means to submit the signed application and packaging to Apple for
- notarization, but not to wait for a reply.
-
`WAIT_NOSTAPLE` means to submit the signed application and packaging to
Apple for notarization, and wait for a reply, but not to staple the
resulting notarization ticket.
@@ -203,16 +200,12 @@
ticket.
"""
NONE = 0
- NOWAIT = 1
- WAIT_NOSTAPLE = 2
- STAPLE = 3
+ WAIT_NOSTAPLE = 1
+ STAPLE = 2
def should_notarize(self):
return self.value > self.NONE.value
- def should_wait(self):
- return self.value > self.NOWAIT.value
-
def should_staple(self):
return self.value > self.WAIT_NOSTAPLE.value
diff --git a/chrome/installer/mac/signing/notarize.py b/chrome/installer/mac/signing/notarize.py
index 802da61a..9d02d5cb 100644
--- a/chrome/installer/mac/signing/notarize.py
+++ b/chrome/installer/mac/signing/notarize.py
@@ -6,6 +6,7 @@
for results, and stapling Apple Notary notarization tickets.
"""
+import asyncio
import collections
import enum
import os
@@ -47,7 +48,8 @@
def notary_args(self):
return self._notary_args
- def submit(self, path, config):
+ async def submit(self, path, config):
+ # Submit the notarization.
command = [
'xcrun',
'notarytool',
@@ -57,15 +59,36 @@
'--output-format',
'plist',
] + self.notary_args
-
- output = commands.run_command_output(command)
+ output = await commands.run_command_output_async(command)
try:
- plist = plistlib.loads(output)
- return plist['id']
+ uuid = plistlib.loads(output)['id']
except:
raise NotarizationError(
'xcrun notarytool returned output that could not be parsed: {}'
.format(output))
+ logger.info('Submitted %s for notarization, request UUID: %s.', path,
+ uuid)
+
+ # Wait for notarization to complete.
+ while True:
+ result = config.invoker.notarizer.get_result(uuid, config)
+ if result.status == Status.IN_PROGRESS:
+ await asyncio.sleep(5)
+ continue
+ elif result.status == Status.SUCCESS:
+ logger.info('Successfully notarized request %s. Log file: %s',
+ uuid, result.log_file)
+ return
+ else:
+ logger.error(
+ 'Failed to notarize request %s.\n'
+ 'Output:\n%s\n'
+ 'Log file:\n%s', uuid, result.output, result.log_file)
+ raise NotarizationError(
+ 'Notarization request {} failed with status: "{}".'.format(
+ uuid,
+ result.status_string,
+ ))
def get_result(self, uuid, config):
command = [
@@ -98,19 +121,14 @@
return commands.run_command_output(command)
-def submit(path, config):
- """Submits an artifact to Apple for notarization.
+async def submit(path, config):
+ """Submits an artifact to Apple for notarization and awaits its success.
Args:
path: The path to the artifact that will be uploaded for notarization.
config: The |config.CodeSignConfig| for the artifact.
-
- Returns:
- A UUID from the notary service that represents the request.
"""
- uuid = config.invoker.notarizer.submit(path, config)
- logger.info('Submitted %s for notarization, request UUID: %s.', path, uuid)
- return uuid
+ await config.invoker.notarizer.submit(path, config)
class Status(enum.Enum):
@@ -127,66 +145,6 @@
'NotarizationResult', ['status', 'status_string', 'output', 'log_file'])
-def wait_for_results(uuids, config):
- """Waits for results from the notarization service. This iterates the list
- of UUIDs and checks the status of each one. For each successful result, the
- function yields to the caller. If a request failed, this raises a
- NotarizationError. If no requests are ready, this operation blocks and
- retries until a result is ready. After a certain amount of time, the
- operation will time out with a NotarizationError if no results are
- produced.
-
- Args:
- uuids: List of UUIDs to check for results. The list must not be empty.
- config: The |config.CodeSignConfig| object.
-
- Yields:
- The UUID of a successful notarization request.
- """
- assert len(uuids)
-
- wait_set = set(uuids)
-
- sleep_time_seconds = 5
- total_sleep_time_seconds = 0
-
- while len(wait_set) > 0:
- for uuid in list(wait_set):
- result = config.invoker.notarizer.get_result(uuid, config)
- if result.status == Status.IN_PROGRESS:
- continue
- elif result.status == Status.SUCCESS:
- logger.info('Successfully notarized request %s. Log file: %s',
- uuid, result.log_file)
- wait_set.remove(uuid)
- yield uuid
- else:
- logger.error(
- 'Failed to notarize request %s.\n'
- 'Output:\n%s\n'
- 'Log file:\n%s', uuid, result.output, result.log_file)
- raise NotarizationError(
- 'Notarization request {} failed with status: "{}".'.format(
- uuid,
- result.status_string,
- ))
-
- if len(wait_set) > 0:
- # Do not wait more than 60 minutes for all the operations to
- # complete.
- if total_sleep_time_seconds < 60 * 60:
- # No results were available, so wait and try again in some
- # number of seconds. Do not wait more than 1 minute for any
- # iteration.
- time.sleep(sleep_time_seconds)
- total_sleep_time_seconds += sleep_time_seconds
- sleep_time_seconds = min(sleep_time_seconds * 2, 60)
- else:
- raise NotarizationError(
- 'Timed out waiting for notarization requests: {}'.format(
- wait_set))
-
-
def staple_bundled_parts(parts, paths):
"""Staples all the bundled executable components of the app bundle.
diff --git a/chrome/installer/mac/signing/notarize_test.py b/chrome/installer/mac/signing/notarize_test.py
index 8a2abb2..9d4909a 100644
--- a/chrome/installer/mac/signing/notarize_test.py
+++ b/chrome/installer/mac/signing/notarize_test.py
@@ -2,6 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+import asyncio
import os
import plistlib
import subprocess
@@ -14,39 +15,53 @@
class TestSubmitNotarytool(unittest.TestCase):
+ @mock.patch('signing.commands.run_command_output_async')
@mock.patch('signing.commands.run_command_output')
- def test_valid_upload(self, run_command_output):
- run_command_output.return_value = plistlib.dumps({
+ def test_valid_upload(self, run_command_output, run_command_output_async):
+ run_command_output_async.return_value = plistlib.dumps({
'id': '13d6aa9b-d204-4f0d-9164-4bda5e730258',
'message': 'Successfully uploaded file',
'path': '/tmp/file.dmg'
})
+ run_command_output.return_value = plistlib.dumps({'status': 'Accepted'})
config = test_config.TestConfig()
- uuid = notarize.submit('/tmp/file.dmg', config)
+ asyncio.run(notarize.submit('/tmp/file.dmg', config))
- self.assertEqual('13d6aa9b-d204-4f0d-9164-4bda5e730258', uuid)
- run_command_output.assert_called_once_with([
+ run_command_output_async.assert_awaited_once_with([
'xcrun', 'notarytool', 'submit', '/tmp/file.dmg', '--no-wait',
'--output-format', 'plist'
])
+ run_command_output.assert_called_once_with([
+ 'xcrun', 'notarytool', 'info',
+ '13d6aa9b-d204-4f0d-9164-4bda5e730258', '--output-format', 'plist'
+ ])
+
+ @mock.patch('signing.commands.run_command_output_async')
@mock.patch('signing.commands.run_command_output')
- def test_valid_upload_with_args(self, run_command_output):
- run_command_output.return_value = plistlib.dumps(
+ def test_valid_upload_with_args(self, run_command_output,
+ run_command_output_async):
+ run_command_output_async.return_value = plistlib.dumps(
{'id': 'b53b3ed1-82cb-41b4-9e12-b097b2c05f64'})
+ run_command_output.return_value = plistlib.dumps({'status': 'Accepted'})
config = test_config.TestConfig(
invoker=test_config.TestInvoker.factory_with_args(notary_arg=[
'--apple-id', '[NOTARY-USER]', '--team-id', '[NOTARY-TEAM]',
'--password', 'hunter2'
]))
- uuid = notarize.submit('/tmp/file.dmg', config)
+ asyncio.run(notarize.submit('/tmp/file.dmg', config))
- self.assertEqual('b53b3ed1-82cb-41b4-9e12-b097b2c05f64', uuid)
- run_command_output.assert_called_once_with([
+ run_command_output_async.assert_awaited_once_with([
'xcrun', 'notarytool', 'submit', '/tmp/file.dmg', '--no-wait',
'--output-format', 'plist', '--apple-id', '[NOTARY-USER]',
'--team-id', '[NOTARY-TEAM]', '--password', 'hunter2'
])
+ run_command_output.assert_called_once_with([
+ 'xcrun', 'notarytool', 'info',
+ 'b53b3ed1-82cb-41b4-9e12-b097b2c05f64', '--output-format', 'plist',
+ '--apple-id', '[NOTARY-USER]', '--team-id', '[NOTARY-TEAM]',
+ '--password', 'hunter2'
+ ])
class TestGetResultNotarytool(unittest.TestCase):
@@ -183,69 +198,6 @@
])
-class TestWaitForResults(unittest.TestCase):
-
- @mock.patch('signing.notarize.Invoker.get_result')
- def test_success(self, get_result):
- get_result.return_value = notarize.NotarizationResult(
- notarize.Status.SUCCESS, 'success', None,
- 'https://example.com/log.json')
- config = test_config.TestConfig()
- uuid = 'cca0aec2-7c64-4ea4-b895-051ea3a17311'
- uuids = [uuid]
- self.assertEqual(uuids, list(notarize.wait_for_results(uuids, config)))
- get_result.assert_called_once_with(uuid, config)
-
- @mock.patch('signing.notarize.Invoker.get_result')
- def test_failure(self, get_result):
- get_result.return_value = notarize.NotarizationResult(
- notarize.Status.ERROR, 'invalid', 'Package Invalid',
- 'https://example.com/log.json')
- config = test_config.TestConfig()
- uuid = 'cca0aec2-7c64-4ea4-b895-051ea3a17311'
- uuids = [uuid]
- with self.assertRaises(notarize.NotarizationError) as cm:
- list(notarize.wait_for_results(uuids, config))
-
- self.assertEqual(
- 'Notarization request cca0aec2-7c64-4ea4-b895-051ea3a17311 failed '
- 'with status: "invalid".', str(cm.exception))
-
- @mock.patch('signing.notarize.Invoker.get_result')
- def test_subprocess_errors(self, get_result):
- get_result.side_effect = subprocess.CalledProcessError(
- 1, 'notarytool', 'A mysterious error occurred.')
-
- with self.assertRaises(subprocess.CalledProcessError):
- uuids = ['77c0ad17-479e-4b82-946a-73739cf6ca16']
- list(notarize.wait_for_results(uuids, test_config.TestConfig()))
-
- @mock.patch.multiple('time', **{'sleep': mock.DEFAULT})
- @mock.patch.multiple('signing.notarize.Invoker',
- **{'get_result': mock.DEFAULT})
- def test_timeout(self, **kwargs):
- kwargs['get_result'].return_value = notarize.NotarizationResult(
- notarize.Status.IN_PROGRESS, None, None, None)
- config = test_config.TestConfig()
- uuid = '0c652bb4-7d44-4904-8c59-1ee86a376ece'
- uuids = [uuid]
- with self.assertRaises(notarize.NotarizationError) as cm:
- list(notarize.wait_for_results(uuids, config))
-
- # Python 2 and 3 stringify set() differently.
- self.assertIn(
- str(cm.exception), [
- "Timed out waiting for notarization requests: set(['0c652bb4-7d44-4904-8c59-1ee86a376ece'])",
- "Timed out waiting for notarization requests: {'0c652bb4-7d44-4904-8c59-1ee86a376ece'}"
- ])
-
- for call in kwargs['get_result'].mock_calls:
- self.assertEqual(call, mock.call(uuid, config))
-
- total_time = sum([call[1][0] for call in kwargs['sleep'].mock_calls])
- self.assertLess(total_time, 61 * 60)
-
-
class TestStaple(unittest.TestCase):
@mock.patch('signing.commands.run_command')
diff --git a/chrome/installer/mac/signing/pipeline.py b/chrome/installer/mac/signing/pipeline.py
index fbebaf4d..fea4466 100644
--- a/chrome/installer/mac/signing/pipeline.py
+++ b/chrome/installer/mac/signing/pipeline.py
@@ -9,6 +9,7 @@
4. Signing and packaging the installer tools.
"""
+import asyncio
import os.path
from signing import commands, model, modification, notarize, parts, signing
@@ -696,11 +697,11 @@
return filtered_distributions
-def sign_all(orig_paths,
- config,
- disable_packaging=False,
- skip_brands=[],
- channels=[]):
+async def sign_all(orig_paths,
+ config,
+ disable_packaging=False,
+ skip_brands=[],
+ channels=[]):
"""For each distribution in |config|, performs customization, signing, and
DMG packaging and places the resulting signed DMG in |orig_paths.output|.
The |paths.input| must contain the products to customize and sign.
@@ -725,32 +726,33 @@
# First, sign all the distributions and optionally submit the
# notarization requests.
- uuids_to_config = _sign_and_maybe_notarize_distributions(
- config, distributions, notary_paths, disable_packaging)
+ dist_configs = await asyncio.wait_for(
+ _sign_and_maybe_notarize_distributions(config, distributions,
+ notary_paths,
+ disable_packaging),
+ timeout=60 * 60 * 2)
- # If needed, wait for app notarization results to come back, and staple
- # if required.
- if config.notarize.should_wait():
- for result in notarize.wait_for_results(uuids_to_config.keys(),
- config):
- if config.notarize.should_staple():
- dist_config = uuids_to_config[result]
- dest_dir = os.path.join(
- notary_paths.work,
- _intermediate_work_dir_name(dist_config.distribution))
- _staple_chrome(
- notary_paths.replace_work(dest_dir), dist_config)
+ # Staple if required.
+ if config.notarize.should_staple():
+ for dist_config in dist_configs:
+ dest_dir = os.path.join(
+ notary_paths.work,
+ _intermediate_work_dir_name(dist_config.distribution))
+ _staple_chrome(notary_paths.replace_work(dest_dir), dist_config)
# After all apps are optionally notarized, package as required.
if not disable_packaging:
- _package_and_maybe_notarize_distributions(config, distributions,
- notary_paths)
+ await asyncio.wait_for(
+ _package_and_maybe_notarize_distributions(
+ config, distributions, notary_paths),
+ timeout=60 * 60 * 2)
_package_installer_tools(orig_paths, config)
-def _sign_and_maybe_notarize_distributions(config, distributions, notary_paths,
- disable_packaging):
+async def _sign_and_maybe_notarize_distributions(config, distributions,
+ notary_paths,
+ disable_packaging):
"""Iterates each distribution in |distributions|, codesigns it according to
the |config|, and potentially uploads it for notarization.
@@ -766,53 +768,60 @@
|config.CodeSignConfig.dist_config| for the |model.Distribution|. If
notarization is not performed, returns an empty dict.
"""
- uuids_to_config = {}
+ dist_configs = []
signed_frameworks = {}
created_app_bundles = set()
- for dist in distributions:
- with commands.WorkDirectory(notary_paths) as paths:
- dist_config = dist.to_config(config)
- do_packaging = (dist.package_as_dmg or dist.package_as_pkg or
- dist.package_as_zip) and not disable_packaging
+ async with asyncio.TaskGroup() as tasks:
+ for dist in distributions:
+ with commands.WorkDirectory(notary_paths) as paths:
+ dist_config = dist.to_config(config)
+ dist_configs.append(dist_config)
+ do_packaging = (dist.package_as_dmg or dist.package_as_pkg or
+ dist.package_as_zip) and not disable_packaging
- # If not packaging and not notarizing, then simply drop the
- # signed bundle in the output directory when done signing.
- if not do_packaging and not config.notarize.should_notarize():
- dest_dir = paths.output
- else:
- dest_dir = notary_paths.work
+ # If not packaging and not notarizing, then simply drop the
+ # signed bundle in the output directory when done signing.
+ if not do_packaging and not config.notarize.should_notarize():
+ dest_dir = paths.output
+ else:
+ dest_dir = notary_paths.work
- dest_dir = os.path.join(dest_dir, _intermediate_work_dir_name(dist))
+ dest_dir = os.path.join(dest_dir,
+ _intermediate_work_dir_name(dist))
- # Different distributions might share the same underlying app
- # bundle, and if they do, then the _intermediate_work_dir_name
- # function will return the same value. Skip creating another app
- # bundle if that is the case.
- if dest_dir in created_app_bundles:
- continue
- created_app_bundles.add(dest_dir)
+ # Different distributions might share the same underlying app
+ # bundle, and if they do, then the _intermediate_work_dir_name
+ # function will return the same value. Skip creating another app
+ # bundle if that is the case.
+ if dest_dir in created_app_bundles:
+ continue
+ created_app_bundles.add(dest_dir)
- _customize_and_sign_chrome(paths, dist_config, dest_dir,
- signed_frameworks)
+ _customize_and_sign_chrome(paths, dist_config, dest_dir,
+ signed_frameworks)
- # If the build products are to be notarized, ZIP the app bundle
- # and submit it for notarization.
- if config.notarize.should_notarize():
- zip_file = os.path.join(notary_paths.work,
- dist_config.packaging_basename + '.zip')
- commands.run_command([
- 'zip', '--recurse-paths', '--symlinks', '--quiet', zip_file,
- dist_config.app_dir
- ],
- cwd=dest_dir)
- uuid = notarize.submit(zip_file, dist_config)
- uuids_to_config[uuid] = dist_config
- return uuids_to_config
+ # If the build products are to be notarized, ZIP the app bundle
+ # and submit it for notarization.
+ if config.notarize.should_notarize():
+ zip_file = os.path.join(
+ notary_paths.work,
+ dist_config.packaging_basename + '.zip')
+ commands.run_command([
+ 'zip', '--recurse-paths', '--symlinks', '--quiet',
+ zip_file, dist_config.app_dir
+ ],
+ cwd=dest_dir)
+ tasks.create_task(notarize.submit(zip_file, dist_config))
+
+ # Yield the event loop to let the notarization subprocesses start
+ # before continuing to the next distribution.
+ asyncio.sleep(0)
+ return dist_configs
-def _package_and_maybe_notarize_distributions(config, distributions,
- notary_paths):
+async def _package_and_maybe_notarize_distributions(config, distributions,
+ notary_paths):
"""Iterates each |model.Distribution| in |distributions| and packages it
according to its specification. If notarization is requested, that is
performed on the assembled package.
@@ -823,43 +832,44 @@
notary_paths: A |model.Paths| object where artifacts will be placed when
notarizing.
"""
- uuids_to_package_path = {}
- for dist in distributions:
- dist_config = dist.to_config(config)
- paths = notary_paths.replace_work(
- os.path.join(notary_paths.work,
- _intermediate_work_dir_name(dist_config.distribution)))
+ staple_paths = []
+ async with asyncio.TaskGroup() as tasks:
+ for dist in distributions:
+ dist_config = dist.to_config(config)
+ paths = notary_paths.replace_work(
+ os.path.join(
+ notary_paths.work,
+ _intermediate_work_dir_name(dist_config.distribution)))
- if dist.inflation_kilobytes:
- inflation_path = os.path.join(
- paths.packaging_dir(config), 'inflation.bin')
- commands.run_command([
- 'dd', 'if=/dev/urandom', 'of=' + inflation_path, 'bs=1000',
- 'count={}'.format(dist.inflation_kilobytes)
- ])
+ if dist.inflation_kilobytes:
+ inflation_path = os.path.join(
+ paths.packaging_dir(config), 'inflation.bin')
+ commands.run_command([
+ 'dd', 'if=/dev/urandom', 'of=' + inflation_path, 'bs=1000',
+ 'count={}'.format(dist.inflation_kilobytes)
+ ])
- if dist.package_as_dmg:
- dmg_path = _package_and_sign_dmg(paths, dist_config)
+ if dist.package_as_dmg:
+ dmg_path = _package_and_sign_dmg(paths, dist_config)
- if config.notarize.should_notarize():
- uuid = notarize.submit(dmg_path, dist_config)
- uuids_to_package_path[uuid] = dmg_path
+ if config.notarize.should_notarize():
+ tasks.create_task(notarize.submit(dmg_path, dist_config))
+ staple_paths.append(dmg_path)
- if dist.package_as_pkg:
- pkg_path = _package_and_sign_pkg(paths, dist_config)
+ if dist.package_as_pkg:
+ pkg_path = _package_and_sign_pkg(paths, dist_config)
- if config.notarize.should_notarize():
- uuid = notarize.submit(pkg_path, dist_config)
- uuids_to_package_path[uuid] = pkg_path
+ if config.notarize.should_notarize():
+ tasks.create_task(notarize.submit(pkg_path, dist_config))
+ staple_paths.append(pkg_path)
- if dist.package_as_zip:
- _package_zip(paths, dist_config)
+ if dist.package_as_zip:
+ _package_zip(paths, dist_config)
- # If needed, wait for package notarization results to come back, and
- # staple if required.
- if config.notarize.should_wait():
- for result in notarize.wait_for_results(uuids_to_package_path.keys(),
- config):
- if config.notarize.should_staple():
- package_path = uuids_to_package_path[result]
- notarize.staple(package_path)
+ # Yield the event loop to let the notarization subprocesses start
+ # before continuing to the next distribution.
+ asyncio.sleep(0)
+
+ if config.notarize.should_staple():
+ for path in staple_paths:
+ notarize.staple(path)
diff --git a/chrome/installer/mac/signing/pipeline_test.py b/chrome/installer/mac/signing/pipeline_test.py
index d422527..7f4c988 100644
--- a/chrome/installer/mac/signing/pipeline_test.py
+++ b/chrome/installer/mac/signing/pipeline_test.py
@@ -1107,9 +1107,8 @@
m: mock.DEFAULT for m in ('move_file', 'copy_files', 'run_command',
'make_dir', 'shutil', 'os')
})
-@mock.patch.multiple('signing.notarize', **{
- m: mock.DEFAULT for m in ('submit', 'wait_for_results', 'staple')
-})
+@mock.patch.multiple('signing.notarize',
+ **{m: mock.DEFAULT for m in ('submit', 'staple')})
@mock.patch.multiple(
'signing.pipeline', **{
m: mock.DEFAULT
@@ -1129,12 +1128,6 @@
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = 'f38ee49c-c55b-4a10-a4f5-aaaa17636b76'
- dmg_uuid = '9f49067e-a13d-436a-8016-3a22a4f6ef92'
- kwargs['submit'].side_effect = [app_uuid, dmg_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]), iter([dmg_uuid])
- ]
kwargs[
'_package_and_sign_dmg'].return_value = '/$O/AppProduct-99.0.9999.99.dmg'
@@ -1167,7 +1160,6 @@
cwd='/$W_1/stable'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
mock.call._staple_chrome(
self.paths.replace_work('/$W_1/stable'), mock.ANY),
@@ -1176,7 +1168,6 @@
# Notarize the DMG.
mock.call.submit('/$O/AppProduct-99.0.9999.99.dmg', mock.ANY),
- mock.call.wait_for_results({dmg_uuid: None}.keys(), mock.ANY),
mock.call.staple('/$O/AppProduct-99.0.9999.99.dmg'),
mock.call.shutil.rmtree('/$W_1'),
@@ -1189,12 +1180,6 @@
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = 'f38ee49c-c55b-4a10-a4f5-aaaa17636b76'
- kwargs['submit'].side_effect = [app_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]),
- iter([]),
- ]
kwargs['_package_zip'].return_value = '/$O/AppProduct-99.0.9999.99.zip'
class Config(test_config.TestConfig):
@@ -1226,7 +1211,6 @@
cwd='/$W_1/stable'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
mock.call._staple_chrome(
self.paths.replace_work('/$W_1/stable'), mock.ANY),
@@ -1234,7 +1218,6 @@
mock.call._package_zip(mock.ANY, mock.ANY),
# Notarize the DMG.
- mock.call.wait_for_results({}.keys(), mock.ANY),
mock.call.shutil.rmtree('/$W_1'),
# Package the installer tools.
@@ -1246,12 +1229,6 @@
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = 'f38ee49c-c55b-4a10-a4f5-aaaa17636b76'
- dmg_uuid = '9f49067e-a13d-436a-8016-3a22a4f6ef92'
- kwargs['submit'].side_effect = [app_uuid, dmg_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]), iter([dmg_uuid])
- ]
kwargs[
'_package_and_sign_dmg'].return_value = '/$O/AppProduct-99.0.9999.99.dmg'
@@ -1286,7 +1263,6 @@
cwd='/$W_1/stable-5000'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
mock.call._staple_chrome(
self.paths.replace_work('/$W_1/stable-5000'), mock.ANY),
mock.call.run_command([
@@ -1300,7 +1276,6 @@
# Notarize the DMG.
mock.call.submit('/$O/AppProduct-99.0.9999.99.dmg', mock.ANY),
- mock.call.wait_for_results({dmg_uuid: None}.keys(), mock.ANY),
mock.call.staple('/$O/AppProduct-99.0.9999.99.dmg'),
mock.call.shutil.rmtree('/$W_1'),
@@ -1313,12 +1288,6 @@
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = 'b2ce64e5-4fae-4043-9c20-d9ff53065b2a'
- pkg_uuid = 'cb811baf-5d35-4caa-adf4-1f61b4991eed'
- kwargs['submit'].side_effect = [app_uuid, pkg_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]), iter([pkg_uuid])
- ]
kwargs[
'_package_and_sign_pkg'].return_value = '/$O/AppProduct-99.0.9999.99.pkg'
@@ -1351,7 +1320,6 @@
cwd='/$W_1/stable'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
mock.call._staple_chrome(
self.paths.replace_work('/$W_1/stable'), mock.ANY),
@@ -1360,7 +1328,6 @@
# Notarize the DMG.
mock.call.submit('/$O/AppProduct-99.0.9999.99.pkg', mock.ANY),
- mock.call.wait_for_results({pkg_uuid: None}.keys(), mock.ANY),
mock.call.staple('/$O/AppProduct-99.0.9999.99.pkg'),
mock.call.shutil.rmtree('/$W_1'),
@@ -1373,12 +1340,6 @@
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = '6de7df90-cf07-4213-9ce6-45f83588a386'
- dmg_uuid = '7f77eefd-6c9d-4271-9367-760dc78a49dd'
- kwargs['submit'].side_effect = [app_uuid, dmg_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]), iter([dmg_uuid])
- ]
kwargs[
'_package_and_sign_dmg'].return_value = '/$O/AppProduct-99.0.9999.99.dmg'
kwargs['_package_zip'].return_value = '/$O/AppProduct-99.0.9999.99.zip'
@@ -1412,7 +1373,6 @@
cwd='/$W_1/stable'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
mock.call._staple_chrome(
self.paths.replace_work('/$W_1/stable'), mock.ANY),
@@ -1424,7 +1384,6 @@
mock.call._package_zip(mock.ANY, mock.ANY),
# Notarize the DMG.
- mock.call.wait_for_results({dmg_uuid: None}.keys(), mock.ANY),
mock.call.staple('/$O/AppProduct-99.0.9999.99.dmg'),
mock.call.shutil.rmtree('/$W_1'),
@@ -1437,13 +1396,6 @@
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = '6de7df90-cf07-4213-9ce6-45f83588a386'
- dmg_uuid = '7f77eefd-6c9d-4271-9367-760dc78a49dd'
- pkg_uuid = '364d9b29-a0a0-4661-b366-e35449197671'
- kwargs['submit'].side_effect = [app_uuid, dmg_uuid, pkg_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]), iter([dmg_uuid, pkg_uuid])
- ]
kwargs[
'_package_and_sign_dmg'].return_value = '/$O/AppProduct-99.0.9999.99.dmg'
kwargs[
@@ -1478,7 +1430,6 @@
cwd='/$W_1/stable'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
mock.call._staple_chrome(
self.paths.replace_work('/$W_1/stable'), mock.ANY),
@@ -1491,10 +1442,6 @@
mock.call.submit('/$O/AppProduct-99.0.9999.99.pkg', mock.ANY),
# Wait for notarization results.
- mock.call.wait_for_results({
- dmg_uuid: None,
- pkg_uuid: None
- }.keys(), mock.ANY),
mock.call.staple('/$O/AppProduct-99.0.9999.99.dmg'),
mock.call.staple('/$O/AppProduct-99.0.9999.99.pkg'),
mock.call.shutil.rmtree('/$W_1'),
@@ -1508,12 +1455,6 @@
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = '6de7df90-cf07-4213-9ce6-45f83588a386'
- pkg_uuid = '364d9b29-a0a0-4661-b366-e35449197671'
- kwargs['submit'].side_effect = [app_uuid, pkg_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]), iter([pkg_uuid])
- ]
kwargs[
'_package_and_sign_pkg'].return_value = '/$O/AppProduct-99.0.9999.99.pkg'
kwargs['_package_zip'].return_value = '/$O/AppProduct-99.0.9999.99.zip'
@@ -1547,7 +1488,6 @@
cwd='/$W_1/stable'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
mock.call._staple_chrome(
self.paths.replace_work('/$W_1/stable'), mock.ANY),
@@ -1559,7 +1499,6 @@
mock.call._package_zip(mock.ANY, mock.ANY),
# Notarize the PKG.
- mock.call.wait_for_results({pkg_uuid: None}.keys(), mock.ANY),
mock.call.staple('/$O/AppProduct-99.0.9999.99.pkg'),
mock.call.shutil.rmtree('/$W_1'),
@@ -1572,13 +1511,6 @@
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = '6de7df90-cf07-4213-9ce6-45f83588a386'
- dmg_uuid = '7f77eefd-6c9d-4271-9367-760dc78a49dd'
- pkg_uuid = '364d9b29-a0a0-4661-b366-e35449197671'
- kwargs['submit'].side_effect = [app_uuid, dmg_uuid, pkg_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]), iter([dmg_uuid, pkg_uuid])
- ]
kwargs[
'_package_and_sign_dmg'].return_value = '/$O/AppProduct-99.0.9999.99.dmg'
kwargs[
@@ -1614,7 +1546,6 @@
cwd='/$W_1/stable'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
mock.call._staple_chrome(
self.paths.replace_work('/$W_1/stable'), mock.ANY),
@@ -1630,10 +1561,6 @@
mock.call._package_zip(mock.ANY, mock.ANY),
# Notarize the DMG.
- mock.call.wait_for_results({
- dmg_uuid: None,
- pkg_uuid: None
- }.keys(), mock.ANY),
mock.call.staple('/$O/AppProduct-99.0.9999.99.dmg'),
mock.call.staple('/$O/AppProduct-99.0.9999.99.pkg'),
mock.call.shutil.rmtree('/$W_1'),
@@ -1647,10 +1574,6 @@
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = '2d7bf857-c2b2-4ebc-8b51-b2f43ecfc13e'
- kwargs['submit'].return_value = app_uuid
- kwargs['wait_for_results'].return_value = iter([app_uuid])
-
config = test_config.TestConfig()
pipeline.sign_all(self.paths, config, disable_packaging=True)
@@ -1667,7 +1590,6 @@
cwd='/$W_1/stable'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
mock.call._staple_chrome(
self.paths.replace_work('/$W_1/stable'), mock.ANY),
mock.call.shutil.rmtree('/$W_1'),
@@ -1679,62 +1601,11 @@
self.assertEqual(1, kwargs['_package_installer_tools'].call_count)
self.assertEqual(1, kwargs['run_command'].call_count)
- def test_sign_notarize_no_wait(self, **kwargs):
- manager = mock.Mock()
- for attr in kwargs:
- manager.attach_mock(kwargs[attr], attr)
-
- app_uuid = 'f38ee49c-c55b-4a10-a4f5-aaaa17636b76'
- dmg_uuid = '9f49067e-a13d-436a-8016-3a22a4f6ef92'
- kwargs['submit'].side_effect = [app_uuid, dmg_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]), iter([dmg_uuid])
- ]
- kwargs['_package_and_sign_dmg'].return_value = (
- '/$O/AppProduct-99.0.9999.99.dmg')
-
- config = test_config.TestConfig(
- notarize=model.NotarizeAndStapleLevel.NOWAIT)
- pipeline.sign_all(self.paths, config)
-
- self.assertEqual(1, kwargs['_package_installer_tools'].call_count)
-
- manager.assert_has_calls([
- # First customize the distribution and sign it.
- mock.call._customize_and_sign_chrome(mock.ANY, mock.ANY,
- '/$W_1/stable', mock.ANY),
-
- # Prepare the app for notarization.
- mock.call.run_command([
- 'zip', '--recurse-paths', '--symlinks', '--quiet',
- '/$W_1/AppProduct-99.0.9999.99.zip', 'App Product.app'
- ],
- cwd='/$W_1/stable'),
- mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
- mock.call.shutil.rmtree('/$W_2'),
-
- # Make the DMG.
- mock.call._package_and_sign_dmg(mock.ANY, mock.ANY),
-
- # Notarize the DMG.
- mock.call.submit('/$O/AppProduct-99.0.9999.99.dmg', mock.ANY),
- mock.call.shutil.rmtree('/$W_1'),
-
- # Package the installer tools.
- mock.call._package_installer_tools(mock.ANY, mock.ANY),
- ])
-
def test_sign_notarize_wait_no_staple(self, **kwargs):
manager = mock.Mock()
for attr in kwargs:
manager.attach_mock(kwargs[attr], attr)
- app_uuid = 'f38ee49c-c55b-4a10-a4f5-aaaa17636b76'
- dmg_uuid = '9f49067e-a13d-436a-8016-3a22a4f6ef92'
- kwargs['submit'].side_effect = [app_uuid, dmg_uuid]
- kwargs['wait_for_results'].side_effect = [
- iter([app_uuid]), iter([dmg_uuid])
- ]
kwargs[
'_package_and_sign_dmg'].return_value = '/$O/AppProduct-99.0.9999.99.dmg'
@@ -1757,7 +1628,6 @@
cwd='/$W_1/stable'),
mock.call.submit('/$W_1/AppProduct-99.0.9999.99.zip', mock.ANY),
mock.call.shutil.rmtree('/$W_2'),
- mock.call.wait_for_results({app_uuid: None}.keys(), mock.ANY),
# Make the DMG.
mock.call._package_and_sign_dmg(mock.ANY, mock.ANY),
@@ -1766,7 +1636,6 @@
mock.call.submit('/$O/AppProduct-99.0.9999.99.dmg', mock.ANY),
# Cleanup.
- mock.call.wait_for_results({dmg_uuid: None}.keys(), mock.ANY),
mock.call.shutil.rmtree('/$W_1'),
# Package the installer tools.
diff --git a/chrome/updater/mac/signing/pipeline.py b/chrome/updater/mac/signing/pipeline.py
index ae33230..a52ff5a 100644
--- a/chrome/updater/mac/signing/pipeline.py
+++ b/chrome/updater/mac/signing/pipeline.py
@@ -8,6 +8,7 @@
3. Signing the DMG.
"""
+import asyncio
import os.path
from signing import commands, model, notarize, parts, signing
@@ -167,7 +168,6 @@
"""
with commands.WorkDirectory(orig_paths) as notary_paths:
# First, sign and optionally submit the notarization requests.
- uuid = None
with commands.WorkDirectory(orig_paths) as paths:
dest_dir = os.path.join(notary_paths.work,
config.packaging_basename)
@@ -181,19 +181,15 @@
zip_file, config.app_dir
],
cwd=dest_dir)
- uuid = notarize.submit(zip_file, config)
+ uuid = asyncio.run(notarize.submit(zip_file, config))
- # Wait for the app notarization result to come back and staple.
- if config.notarize.should_wait():
- for _ in notarize.wait_for_results([uuid], config):
- pass # We are only waiting for a single notarization.
- if config.notarize.should_staple():
- notarize.staple_bundled_parts(
- # Only staple to the outermost app.
- parts.get_parts(config)[-1:],
- notary_paths.replace_work(
- os.path.join(notary_paths.work,
- config.packaging_basename)))
+ if config.notarize.should_staple():
+ notarize.staple_bundled_parts(
+ # Only staple to the outermost app.
+ parts.get_parts(config)[-1:],
+ notary_paths.replace_work(
+ os.path.join(notary_paths.work,
+ config.packaging_basename)))
# Package.
commands.move_file(
@@ -208,8 +204,7 @@
# Notarize the packages, then staple.
if config.notarize.should_notarize():
- uuid_to_path = {}
- uuid_to_path[notarize.submit(pkg_path, config)] = pkg_path
- uuid_to_path[notarize.submit(dmg_path, config)] = dmg_path
- for uuid in notarize.wait_for_results(uuid_to_path.keys(), config):
- notarize.staple(uuid_to_path[uuid])
+ asyncio.run(notarize.submit(pkg_path, config))
+ asyncio.run(notarize.submit(dmg_path, config))
+ notarize.staple(dmg_path)
+ notarize.staple(pkg_path)
