[toolchain-utils] Fix security issue.
Remove the chromeos-toolchain-credentials.json file, which should
not be in a publicly visible repo. Update the buildbot_test_*
scripts to take an argument for the directory containing the
credentials file. Update the buildbot_utils.py file to use the
credentials file argument, and to default to the location in our
role account.
BUG=782452
TEST=With some code commented out, did basic testing of flag/options
to make sure they work as expected.
Change-Id: I47a98a69b9ba8d2704d25abc37fe4447fa6ba343
Reviewed-on: https://chromium-review.googlesource.com/757901
Commit-Ready: Caroline Tice <cmtice@chromium.org>
Tested-by: Caroline Tice <cmtice@chromium.org>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
diff --git a/buildbot_test_llvm.py b/buildbot_test_llvm.py
index 0cc61e5..e1b515d 100755
--- a/buildbot_test_llvm.py
+++ b/buildbot_test_llvm.py
@@ -106,7 +106,7 @@
if ret_val != 0:
raise RuntimeError('chmod for testing_rsa failed')
- def DoAll(self):
+ def DoAll(self, crostc_dir):
"""Main function inside ToolchainComparator class.
Launch trybot, get image names, create crosperf experiment file, run
@@ -115,13 +115,23 @@
date_str = datetime.date.today()
description = 'master_%s_%s_%s' % (self._patches_string, self._build,
date_str)
- _ = buildbot_utils.GetTrybotImage(
- self._chromeos_root,
- self._build,
- self._patches,
- description,
- tryjob_flags=['--hwtest'],
- async=True)
+ if crostc_dir:
+ _ = buildbot_utils.GetTrybotImage(
+ self._chromeos_root,
+ self._build,
+ self._patches,
+ description,
+ tryjob_flags=['--hwtest'],
+ credentials_dir=crostc_dir,
+ async=True)
+ else:
+ _ = buildbot_utils.GetTrybotImage(
+ self._chromeos_root,
+ self._build,
+ self._patches,
+ description,
+ tryjob_flags=['--hwtest'],
+ async=True)
return 0
@@ -155,6 +165,12 @@
dest='compiler',
help='Which compiler (llvm, llvm-next or gcc) to use for '
'testing.')
+ parser.add_argument(
+ '--crostc_dir',
+ dest='crostc_dir',
+ help='Path to the directory containing the '
+ 'chromeos-toolchain-credentials.json file; normally in the '
+ 'crostc repo.')
options = parser.parse_args(argv[1:])
if not options.chromeos_root:
@@ -179,7 +195,7 @@
board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
fv = ToolchainVerifier(board, options.chromeos_root, options.weekday,
options.patches, options.compiler)
- fv.DoAll()
+ fv.DoAll(options.crostc_dir)
except SystemExit:
logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
with open(logfile, 'w') as f:
diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py
index 33d73a2..1889aa9 100755
--- a/buildbot_test_toolchains.py
+++ b/buildbot_test_toolchains.py
@@ -84,7 +84,8 @@
'%Y-%m-%d_%H:%M:%S')
self._reports_dir = os.path.join(
NIGHTLY_TESTS_DIR,
- '%s.%s' % (timestamp, board),)
+ '%s.%s' % (timestamp, board),
+ )
def _GetVanillaImageName(self, trybot_image):
"""Given a trybot artifact name, get latest vanilla image name.
@@ -248,7 +249,7 @@
(filename, MAIL_PROGRAM, email_title, self._board))
self._ce.RunCommand(command)
- def DoAll(self):
+ def DoAll(self, crostc_dir):
"""Main function inside ToolchainComparator class.
Launch trybot, get image names, create crosperf experiment file, run
@@ -257,13 +258,23 @@
date_str = datetime.date.today()
description = 'master_%s_%s_%s' % (self._patches_string, self._build,
date_str)
- build_id, trybot_image = buildbot_utils.GetTrybotImage(
- self._chromeos_root,
- self._build,
- self._patches,
- description,
- tryjob_flags=['--notests'],
- build_toolchain=True)
+ if crostc_dir:
+ build_id, trybot_image = buildbot_utils.GetTrybotImage(
+ self._chromeos_root,
+ self._build,
+ self._patches,
+ description,
+ tryjob_flags=['--notests'],
+ credentials_dir=crostc_dir,
+ build_toolchain=True)
+ else:
+ build_id, trybot_image = buildbot_utils.GetTrybotImage(
+ self._chromeos_root,
+ self._build,
+ self._patches,
+ description,
+ tryjob_flags=['--notests'],
+ build_toolchain=True)
print('trybot_url: \
https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/release/builds/%s'
@@ -318,6 +329,12 @@
action='store_true',
default=False,
help='Pass --noschedv2 to crosperf.')
+ parser.add_argument(
+ '--crostc_dir',
+ dest='crostc_dir',
+ help='Path to the directory containing the '
+ 'chromeos-toolchain-credentials.json file; normally in the '
+ 'crostc repo.')
options = parser.parse_args(argv[1:])
if not options.board:
@@ -332,7 +349,7 @@
fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
options.weekday, options.patches, options.noschedv2)
- return fc.DoAll()
+ return fc.DoAll(options.crostc_dir)
if __name__ == '__main__':
diff --git a/cros_utils/buildbot_utils.py b/cros_utils/buildbot_utils.py
index ae707ea..5184154 100644
--- a/cros_utils/buildbot_utils.py
+++ b/cros_utils/buildbot_utils.py
@@ -117,8 +117,12 @@
builder = 'llvm_next_toolchain'
sa_file = os.path.expanduser(
- os.path.join(file_dir, 'cros_utils',
- 'chromeos-toolchain-credentials.json'))
+ os.path.join(file_dir, 'chromeos-toolchain-credentials.json'))
+
+ if not os.path.exists(sa_file):
+ logger.GetLogger().LogFatal('ERROR: Unable to find %s; please check '
+ 'your script arguments.' % sa_file)
+
scopes = ['https://www.googleapis.com/auth/userinfo.email']
credentials = ServiceAccountCredentials.from_json_keyfile_name(
@@ -195,13 +199,15 @@
return trybot_image
-def GetTrybotImage(chromeos_root,
- buildbot_name,
- patch_list,
- build_tag,
- tryjob_flags=[],
- build_toolchain=False,
- async=False):
+def GetTrybotImage(
+ chromeos_root,
+ buildbot_name,
+ patch_list,
+ build_tag,
+ tryjob_flags=[],
+ build_toolchain=False,
+ credentials_dir='/usr/local/google/home/mobiletc-prebuild/sheriff_utils',
+ async=False):
"""Launch buildbot and get resulting trybot artifact name.
This function launches a buildbot with the appropriate flags to
@@ -221,10 +227,13 @@
build_tag is a (unique) string to be used to look up the buildbot results
from among all the build records.
+ tryjob_flags See cros tryjob --help for available options.
+
build_toolchain builds and uses the latest toolchain, rather than the
prebuilt one in SDK.
- tryjob_flags See cros tryjob --help for available options.
+ credentials_dir is the path to the chromeos-toolchain-credentials.json file,
+ which should be in the crostc repo.
"""
ce = command_executer.GetCommandExecuter()
base_dir = os.getcwd()
@@ -242,8 +251,7 @@
build = buildbot_name
description = build_tag
command = ('cros tryjob --yes --nochromesdk --remote-description %s'
- ' %s %s %s' %
- (description, tryjob_flags, patch_arg, build))
+ ' %s %s %s' % (description, tryjob_flags, patch_arg, build))
_, out, _ = ce.RunCommandWOutput(command)
if 'Tryjob submitted!' not in out:
logger.GetLogger().LogFatal('Error occurred while launching trybot job: '
@@ -269,7 +277,7 @@
long_slept = False
while not done:
done = True
- build_info = GetBuildInfo(base_dir, build)
+ build_info = GetBuildInfo(credentials_dir, build)
if not build_info:
if pending_time > TIME_OUT:
logger.GetLogger().LogFatal(
@@ -320,8 +328,8 @@
(pending_time / 60))
pending_time += SLEEP_TIME
else:
- logger.GetLogger().LogOutput(
- '{0} minutes passed.'.format(running_time / 60))
+ logger.GetLogger().LogOutput('{0} minutes passed.'.format(
+ running_time / 60))
logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
running_time += SLEEP_TIME
@@ -402,6 +410,6 @@
candidates = [[int(r) for r in m.group(1, 2, 3, 4)] for m in candidates if m]
candidates.sort(reverse=True)
for c in candidates:
- build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3])
- if DoesImageExist(chromeos_root, build):
- return build
+ build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3])
+ if DoesImageExist(chromeos_root, build):
+ return build
diff --git a/cros_utils/chromeos-toolchain-credentials.json b/cros_utils/chromeos-toolchain-credentials.json
deleted file mode 100644
index aed9a52..0000000
--- a/cros_utils/chromeos-toolchain-credentials.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "type": "service_account",
- "project_id": "chromeos-toolchain-u",
- "private_key_id": "d0efe593ad39aad4c685273ee80e4c24bb3f2e92",
- "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC5Rm6aqSjMNrRp\ntYNc++ec79L3QZ2MxLMYKyhlgGhppVt6p/wiSvLdI19nS5TAkKMjKv71rb9DbQlG\nfQVckiY+MlADJKi29lJrwqqNDCcRgB7CL6hgDfmhWMKonZn2MwvBVROD0gi7sY+A\nipIe92jVeqG8Gvp5kOgsBxCRV5YQok8j1FxE5fIsS2sg93VS1YAzH8uPYadWb/Z5\n9uwc8U7SL0mEPjXjsLEm8Y70zovGVjv7kOLqYBMUmROLvSouG/HrZWy9uTgxFOb5\njOhxKhDcDMPVM3g8lfc0EwPUB1NxXztoST9qBJVqdzQmHpPjeDxgru0A+zaQMkWA\ne8Sn5EQrAgMBAAECggEAUnhbe1SY6G3hWLyHQLiQnHbUPWNjselMnxF25deGoqAM\nXEiyHsGl4XGBYgVurVi0hU76NnmkjWrXmhzEajRT+ZODsiJ7RxXWEkmQiUBDk7Kn\n/mAgXsFZwMw1ucCNa93E+cXY7fBsGsAq1FjaOhZ+/6eanpSTsdEix5ZNdaS7E6Op\n9zIba9EjLIvSl435+eWq0C3aU9nd1RbbRwD6vGpgG8L/r957s+AAALTqdSZGWxJX\nEC9OKT07e76qvwAsq2BoBx5vW0xmeQdZgKrA10LLDWa7UjFbwSDJIBESYtd4rYMj\nAqg5eND0bC1RrgzI+RD/10l6Vj8bBFo/403s0P5LYQKBgQDiVGVFkrw5LSy82CGC\nvSraxPriivEweMfpkp6buMbD2Je0RMR4glc1vW5m0QUJmy+ymiIHVMCmE9xNBwbS\nRyCBnrs2+3FtdnruNdcaGh6sbTlY+qJI0rEZUdbb5OhlHZF47KW66hI6sWJ1YF8O\niLQTokW8ejybprCtl1HvEHhEbwKBgQDRkD/acZrvmcnqqmorqW6mgJEkrRF/i5Th\npDo3WegXA4irX0tNqh5w+wms8r41vUZSCZYvyi0Of9LMObVdB/gA/qVzETE0p5he\ns3Skp/VK8nF53pAUd+4dKlnCvD3TOEkIq+kxuEOs2iHJcvSjmKtMgqfMK/UtieB4\n7+MaOcbyBQKBgHOUndMVyEF6rGoUBaj6abQm++hNBDa4t360fYMQrZC+P1qz85GH\nHno3LvYar/Pj6EvRIqeTxH4LjmlXuUgRQqxvHzRI2/gGlWio3hxaUmfDr5GdDNsb\nnY1MmejZ0UQyAWQ7lbcKahzHEXzXpjOJ5ExShkJmOiVSzs8Xg6QOSRzJAoGAemYs\nRWQxQFysqJlcZaASdsGFBMzo+XwHOzt2nTmv6zEvNBj2nKgOG6MkZZVqL20bk3Lx\n+3u0kVFrR8k0+t9apQoWjHywJrb0setS55EKHfo4+RtbP/lEZFiGEM1ttt6bGat/\nCoE7VcwaC9VOufbDpm5xnzjVfQGY0EocdQbmAhkCgYB/isdqeDyafawr+38fcU1r\nX2+cK5JCrEzHIwg2QN3Z56cRrqrb+kK1H3w/F7ZfrlPSmS8XMwZV73QwieoGNIYL\nie9UZqRoZSG73FzIw5mXhWWA1adFz8HpGG5wRNshnPI2liOPwhnblfadJUfXb2br\n021vPgpsxamLjHSDSmSf6Q==\n-----END PRIVATE KEY-----\n",
- "client_email": "mobiletc-prebuild-2@chromeos-toolchain-u.iam.gserviceaccount.com",
- "client_id": "114495086044042319417",
- "auth_uri": "https://accounts.google.com/o/oauth2/auth",
- "token_uri": "https://accounts.google.com/o/oauth2/token",
- "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
- "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/mobiletc-prebuild-2%40chromeos-toolchain-u.iam.gserviceaccount.com"
-}
