Updating /analyze script to use logdog, improve code style
When the /analyze builder switched over to logdog the scripts for
retrieving the latest warnings and generating a report on the new ones
broke. This change updates the script to use the logdog tool to download
the results. It also moves the code to be much more inline with Google's
Python coding standards.
R=chengx@chromium.org
BUG=427616
Review-Url: https://codereview.chromium.org/2793453002
Cr-Original-Commit-Position: refs/heads/master@{#461552}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: e8b33cd8b431de2e385e174c7ee96e3aedd971ee
diff --git a/new_analyze_warnings/retrieve_warnings.py b/new_analyze_warnings/retrieve_warnings.py
index 33c5d40..7b3c75a 100644
--- a/new_analyze_warnings/retrieve_warnings.py
+++ b/new_analyze_warnings/retrieve_warnings.py
@@ -1,6 +1,7 @@
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Copyright (c) 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+
"""
This retrieves the latest warnings from the Chrome /analyze build machine, and
does a diff.
@@ -8,93 +9,90 @@
fills out the functionality.
"""
-import urllib
-import sys
import glob
import os
+import subprocess
+import sys
+import urllib
if len(sys.argv) < 2:
- print "Missing build number."
+ print 'Missing build number.'
sys.exit(10)
-buildNumber = int(sys.argv[1])
+build_number = int(sys.argv[1])
-baseURL = "http://build.chromium.org/p/chromium.fyi/builders/" + \
- "Chromium%20Windows%20Analyze/"
+base_url = 'http://build.chromium.org/p/chromium.fyi/builders/' + \
+ 'Chromium%20Windows%20Analyze/'
-print "Finding recent builds on %s" % baseURL
-baseData = urllib.urlopen(baseURL).read()
-recentOff = baseData.find("Recent Builds:")
-buildPattern = 'success</td> <td><a href="' + \
- '../../builders/Chromium%20Windows%20Analyze/builds/'
+print 'Finding recent builds on %s' % base_url
+base_data = urllib.urlopen(base_url).read()
+recent_off = base_data.find('Recent Builds:')
+build_marker = 'success</td> <td><a href="' + \
+ '../../builders/Chromium%20Windows%20Analyze/builds/'
# For some reason I couldn't get regular expressions to work on this data.
-latestBuildOff = baseData.find(buildPattern, recentOff) + len(buildPattern)
-if latestBuildOff < len(buildPattern):
- print "Couldn't find successful build."
+latest_build_off = base_data.find(build_marker, recent_off) + len(build_marker)
+if latest_build_off < len(build_marker):
+ print 'Couldn\'t find successful build.'
sys.exit(10)
-latestEndOff = baseData.find('"', latestBuildOff)
-latestBuildStr = baseData[latestBuildOff:latestEndOff]
-maxBuildNumber = int(latestBuildStr)
-if buildNumber > maxBuildNumber:
- print "Requested build number (%d) is too high. Maximum is %d." % \
- (buildNumber, maxBuildNumber)
+latest_end_off = base_data.find('"', latest_build_off)
+latest_build_str = base_data[latest_build_off:latest_end_off]
+max_build_number = int(latest_build_str)
+if build_number > max_build_number:
+ print 'Requested build number (%d) is too high. Maximum is %d.' % \
+ (build_number, max_build_number)
sys.exit(10)
# Treat negative numbers specially
if sys.argv[1][0] == '-':
- buildNumber = maxBuildNumber + buildNumber
- if buildNumber < 0:
- buildNumber = 0
- print "Retrieving build number %d of %d" % (buildNumber, maxBuildNumber)
+ build_number = max_build_number + build_number
+ if build_number < 0:
+ build_number = 0
+ print 'Retrieving build number %d of %d' % (build_number, max_build_number)
# Found the last summary results in the current directory
-results = glob.glob("analyze*_summary.txt")
+results = glob.glob('analyze*_summary.txt')
results.sort()
-previous = "%04d" % (buildNumber - 1)
+previous = '%04d' % (build_number - 1)
if results:
- possiblePrevious = results[-1][7:11]
- if int(possiblePrevious) == buildNumber:
+ possible_previous = results[-1][7:11]
+ if int(possible_previous) == build_number:
if len(results) > 1:
previous = results[-2][7:11]
else:
- previous = possiblePrevious
+ previous = possible_previous
-dataURL = baseURL + "builds/" + str(buildNumber) + "/steps/compile/logs/stdio"
-revisionURL = baseURL + "builds/" + str(buildNumber)
+data_descriptor = 'chromium/bb/chromium.fyi/Chromium_Windows_Analyze/' + \
+ '%d/+/recipes/steps/compile/0/stdout' % build_number
+revision_url = base_url + 'builds/' + str(build_number)
# Retrieve the revision
-revisionData = urllib.urlopen(revisionURL).read()
-key = "Got Revision</td><td>"
-Off = revisionData.find(key) + len(key)
-if Off > len(key):
- revision = revisionData[Off: Off + 40]
- print "Revision is '%s'" % revision
- print "Environment variables can be set with set_analyze_revision.bat"
- payload = "set ANALYZE_REVISION=%s\r\n" % revision
- payload += "set ANALYZE_BUILD_NUMBER=%04d\r\n" % buildNumber
- payload += "set ANALYZE_PREV_BUILD_NUMBER=%s\r\n" % previous
- open("set_analyze_revision.bat", "wt").write(payload)
+revisionData = urllib.urlopen(revision_url).read()
+key = 'Got Revision</td><td>'
+off = revisionData.find(key) + len(key)
+if off > len(key):
+ revision = revisionData[off: off + 40]
+ print 'Revision is "%s"' % revision
+ print 'Environment variables can be set with set_analyze_revision.bat'
+ payload = 'set ANALYZE_REVISION=%s\r\n' % revision
+ payload += 'set ANALYZE_BUILD_NUMBER=%04d\r\n' % build_number
+ payload += 'set ANALYZE_PREV_BUILD_NUMBER=%s\r\n' % previous
+ open('set_analyze_revision.bat', 'wt').write(payload)
# Retrieve the raw warning data
- print "Retrieving raw build results. Please wait."
- data = urllib.urlopen(dataURL).read()
- if data.count("status: SUCCESS") == 0:
- print "Build failed or is incomplete."
- else:
- # Fix up "'" and '"'
- data = data.replace("'", "'").replace(""", '"')
- # Fix up '<' and '>'
- data = data.replace("<", "<").replace(">", ">")
- # Fix up '&'
- data = data.replace("&", "&")
- # Fix up random spans
- data = data.replace('</span><span class="stdout">', '')
- # Fix up the source paths to match my local /analyze repo
- if "ANALYZE_REPO" in os.environ:
- sourcePath = r"e:\b\build\slave\chromium_windows_analyze\build\src"
- destPath = os.path.join(os.environ["ANALYZE_REPO"], "src")
- data = data.replace(sourcePath, destPath)
- outputName = "analyze%04d_full.txt" % buildNumber
- open(outputName, "w").write(data)
- print "Done. Data is in %s" % outputName
+ print 'Retrieving raw build results. Please wait.'
+ # Results are now retrieved using logdog. Instructions on how to install the
+ # logdog tool can be found here:
+ # https://bugs.chromium.org/p/chromium/issues/detail?id=698429#c1
+ # In particular, from c:\src\logdog_cipd_root:
+ # > depot_tools\cipd init
+ # > depot_tools\cipd install infra/tools/luci/logdog/logdog/windows-amd64
+ command = r'c:\src\logdog_cipd_root\logdog.exe cat %s' % data_descriptor
+ data = str(subprocess.check_output(command))
+ if 'ANALYZE_REPO' in os.environ:
+ source_path = r'e:\b\build\slave\chromium_windows_analyze\build\src'
+ dest_path = os.path.join(os.environ['ANALYZE_REPO'], 'src')
+ data = data.replace(source_path, dest_path)
+ output_name = 'analyze%04d_full.txt' % build_number
+ open(output_name, 'w').write(data)
+ print 'Done. Data is in %s' % output_name
else:
- print "No revision information found!"
+ print 'No revision information found!'
