| #!/usr/bin/env python |
| # Copyright 2014 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # Tool for seeing the real world impact of a patch. |
| # |
| # Layout Tests can tell you whether something has changed, but this can help |
| # you determine whether a subtle/controversial change is beneficial or not. |
| # |
| # It dumps the rendering of a large number of sites, both with and without a |
| # patch being evaluated, then sorts them by greatest difference in rendering, |
| # such that a human reviewer can quickly review the most impacted sites, |
| # rather than having to manually try sites to see if anything changes. |
| # |
| # In future it might be possible to extend this to other kinds of differences, |
| # e.g. page load times. |
| |
| from __future__ import print_function |
| |
| import argparse |
| from argparse import RawTextHelpFormatter |
| from contextlib import closing |
| import datetime |
| import errno |
| from distutils.spawn import find_executable |
| from operator import itemgetter |
| import multiprocessing |
| import os |
| import re |
| from cStringIO import StringIO |
| import subprocess |
| import sys |
| import textwrap |
| import time |
| from urllib2 import urlopen |
| from urlparse import urlparse |
| import webbrowser |
| from zipfile import ZipFile |
| |
| from nsfw_urls import nsfw_urls |
| |
| action = None |
| allow_js = False |
| additional_content_shell_flags = "" |
| chromium_src_root = "" |
| chromium_out_dir = "" |
| image_diff = "" |
| content_shell = "" |
| output_dir = "" |
| num_sites = 100 |
| urls = [] |
| print_lock = multiprocessing.Lock() |
| |
| |
| def MakeDirsIfNotExist(dir): |
| try: |
| os.makedirs(dir) |
| except OSError as e: |
| if e.errno != errno.EEXIST: |
| raise |
| |
| |
| def SetupPathsAndOut(): |
| global chromium_src_root, chromium_out_dir, output_dir |
| global image_diff, content_shell |
| chromium_src_root = os.path.abspath(os.path.join(os.path.dirname(__file__), |
| os.pardir, |
| os.pardir)) |
| # Find out directory (might be out_linux for users of cr). |
| for out_suffix in ["_linux", ""]: |
| out_dir = os.path.join(chromium_src_root, "out" + out_suffix) |
| if os.path.exists(out_dir): |
| chromium_out_dir = out_dir |
| break |
| if not chromium_out_dir: |
| return False |
| |
| this_script_name = "real_world_impact" |
| output_dir = os.path.join(chromium_out_dir, |
| "Release", |
| this_script_name) |
| MakeDirsIfNotExist(output_dir) |
| |
| image_diff = os.path.join(chromium_out_dir, "Release", "image_diff") |
| |
| if sys.platform == 'darwin': |
| content_shell = os.path.join(chromium_out_dir, "Release", |
| "Content Shell.app/Contents/MacOS/Content Shell") |
| elif sys.platform.startswith('linux'): |
| content_shell = os.path.join(chromium_out_dir, "Release", |
| "content_shell") |
| elif sys.platform.startswith('win'): |
| content_shell = os.path.join(chromium_out_dir, "Release", |
| "content_shell.exe") |
| return True |
| |
| |
| def CheckPrerequisites(): |
| if not find_executable("wget"): |
| print("wget not found! Install wget and re-run this.") |
| return False |
| if not os.path.exists(image_diff): |
| print("image_diff not found (%s)!" % image_diff) |
| print("Build the image_diff target and re-run this.") |
| return False |
| if not os.path.exists(content_shell): |
| print("Content shell not found (%s)!" % content_shell) |
| print("Build Release/content_shell and re-run this.") |
| return False |
| return True |
| |
| |
| def PickSampleUrls(): |
| global urls |
| data_dir = os.path.join(output_dir, "data") |
| MakeDirsIfNotExist(data_dir) |
| |
| # Download Alexa top 1,000,000 sites |
| # TODO(johnme): Should probably update this when it gets too stale... |
| csv_path = os.path.join(data_dir, "top-1m.csv") |
| if not os.path.exists(csv_path): |
| print("Downloading list of top 1,000,000 sites from Alexa...") |
| csv_url = "http://s3.amazonaws.com/alexa-static/top-1m.csv.zip" |
| with closing(urlopen(csv_url)) as stream: |
| ZipFile(StringIO(stream.read())).extract("top-1m.csv", data_dir) |
| |
| bad_urls_path = os.path.join(data_dir, "bad_urls.txt") |
| if os.path.exists(bad_urls_path): |
| with open(bad_urls_path) as f: |
| bad_urls = set(f.read().splitlines()) |
| else: |
| bad_urls = set() |
| |
| # See if we've already selected a sample of size num_sites (this way, if you |
| # call this script with arguments "before N" then "after N", where N is the |
| # same number, we'll use the same sample, as expected!). |
| urls_path = os.path.join(data_dir, "%06d_urls.txt" % num_sites) |
| if not os.path.exists(urls_path): |
| if action == 'compare': |
| print("Error: you must run 'before %d' and 'after %d' before " |
| "running 'compare %d'" % (num_sites, num_sites, num_sites)) |
| return False |
| print("Picking %d sample urls..." % num_sites) |
| |
| # TODO(johnme): For now this just gets the top num_sites entries. In future |
| # this should pick a weighted random sample. For example, it could fit a |
| # power-law distribution, which is a good model of website popularity |
| # (http://www.useit.com/alertbox/9704b.html). |
| urls = [] |
| remaining_num_sites = num_sites |
| with open(csv_path) as f: |
| for entry in f: |
| if remaining_num_sites <= 0: |
| break |
| remaining_num_sites -= 1 |
| hostname = entry.strip().split(',')[1] |
| if not '/' in hostname: # Skip Alexa 1,000,000 entries that have paths. |
| url = "http://%s/" % hostname |
| if not url in bad_urls: |
| urls.append(url) |
| # Don't write these to disk yet; we'll do that in SaveWorkingUrls below |
| # once we have tried to download them and seen which ones fail. |
| else: |
| with open(urls_path) as f: |
| urls = [u for u in f.read().splitlines() if not u in bad_urls] |
| return True |
| |
| |
| def SaveWorkingUrls(): |
| # TODO(johnme): Update the list if a url that used to work goes offline. |
| urls_path = os.path.join(output_dir, "data", "%06d_urls.txt" % num_sites) |
| if not os.path.exists(urls_path): |
| with open(urls_path, 'w') as f: |
| f.writelines(u + '\n' for u in urls) |
| |
| |
| def PrintElapsedTime(elapsed, detail=""): |
| elapsed = round(elapsed * 10) / 10.0 |
| m = elapsed / 60 |
| s = elapsed % 60 |
| print("Took %dm%.1fs" % (m, s), detail) |
| |
| |
| def DownloadStaticCopyTask(url): |
| url_parts = urlparse(url) |
| host_dir = os.path.join(output_dir, "data", url_parts.hostname) |
| # Use wget for now, as does a reasonable job of spidering page dependencies |
| # (e.g. CSS, JS, images). |
| success = True |
| try: |
| subprocess.check_call(["wget", |
| "--execute", "robots=off", |
| ("--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS " |
| "X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) C" |
| "hrome/32.0.1700.14 Safari/537.36"), |
| "--page-requisites", |
| "--span-hosts", |
| "--adjust-extension", |
| "--convert-links", |
| "--directory-prefix=" + host_dir, |
| "--force-directories", |
| "--default-page=index.html", |
| "--no-check-certificate", |
| "--timeout=5", # 5s timeout |
| "--tries=2", |
| "--quiet", |
| url]) |
| except KeyboardInterrupt: |
| success = False |
| except subprocess.CalledProcessError: |
| # Ignoring these for now, as some sites have issues with their subresources |
| # yet still produce a renderable index.html |
| pass #success = False |
| if success: |
| download_path = os.path.join(host_dir, url_parts.hostname, "index.html") |
| if not os.path.exists(download_path): |
| success = False |
| else: |
| with print_lock: |
| print("Downloaded:", url) |
| if not success: |
| with print_lock: |
| print("Failed to download:", url) |
| return False |
| return True |
| |
| |
| def DownloadStaticCopies(): |
| global urls |
| new_urls = [] |
| for url in urls: |
| url_parts = urlparse(url) |
| host_dir = os.path.join(output_dir, "data", url_parts.hostname) |
| download_path = os.path.join(host_dir, url_parts.hostname, "index.html") |
| if not os.path.exists(download_path): |
| new_urls.append(url) |
| |
| if new_urls: |
| print("Downloading static copies of %d sites..." % len(new_urls)) |
| start_time = time.time() |
| |
| results = multiprocessing.Pool(20).map(DownloadStaticCopyTask, new_urls) |
| failed_urls = [new_urls[i] for i,ret in enumerate(results) if not ret] |
| if failed_urls: |
| bad_urls_path = os.path.join(output_dir, "data", "bad_urls.txt") |
| with open(bad_urls_path, 'a') as f: |
| f.writelines(u + '\n' for u in failed_urls) |
| failed_urls_set = set(failed_urls) |
| urls = [u for u in urls if u not in failed_urls_set] |
| |
| PrintElapsedTime(time.time() - start_time) |
| |
| SaveWorkingUrls() |
| |
| |
| def RunDrtTask(url): |
| url_parts = urlparse(url) |
| host_dir = os.path.join(output_dir, "data", url_parts.hostname) |
| html_path = os.path.join(host_dir, url_parts.hostname, "index.html") |
| |
| if not allow_js: |
| nojs_path = os.path.join(host_dir, url_parts.hostname, "index-nojs.html") |
| if not os.path.exists(nojs_path): |
| with open(html_path) as f: |
| html = f.read() |
| if not html: |
| return False |
| # These aren't intended to be XSS safe :) |
| block_tags = (r'<\s*(script|object|video|audio|iframe|frameset|frame)' |
| r'\b.*?<\s*\/\s*\1\s*>') |
| block_attrs = r'\s(onload|onerror)\s*=\s*(\'[^\']*\'|"[^"]*|\S*)' |
| html = re.sub(block_tags, '', html, flags=re.I|re.S) |
| html = re.sub(block_attrs, '', html, flags=re.I) |
| with open(nojs_path, 'w') as f: |
| f.write(html) |
| html_path = nojs_path |
| |
| start_time = time.time() |
| |
| with open(os.devnull, "w") as fnull: |
| p = subprocess.Popen([content_shell, |
| "--run-web-tests", |
| additional_content_shell_flags, |
| html_path |
| ], |
| shell=False, |
| stdout=subprocess.PIPE, |
| stderr=fnull) |
| result = p.stdout.read() |
| PNG_START = b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A" |
| PNG_END = b"\x49\x45\x4E\x44\xAE\x42\x60\x82" |
| try: |
| start = result.index(PNG_START) |
| end = result.rindex(PNG_END) + 8 |
| except ValueError: |
| return False |
| |
| png_path = os.path.join(output_dir, action, url_parts.hostname + ".png") |
| MakeDirsIfNotExist(os.path.dirname(png_path)) |
| with open(png_path, 'wb') as f: |
| f.write(result[start:end]) |
| elapsed_time = (time.time() - start_time, url) |
| return elapsed_time |
| |
| |
| def RunDrt(): |
| print("Taking screenshots of %d pages..." % len(urls)) |
| start_time = time.time() |
| |
| results = multiprocessing.Pool().map(RunDrtTask, urls, 1) |
| |
| max_time, url = max(t for t in results if t) |
| elapsed_detail = "(slowest: %.2fs on %s)" % (max_time, url) |
| PrintElapsedTime(time.time() - start_time, elapsed_detail) |
| |
| |
| def CompareResultsTask(url): |
| url_parts = urlparse(url) |
| before_path = os.path.join(output_dir, "before", url_parts.hostname + ".png") |
| after_path = os.path.join(output_dir, "after", url_parts.hostname + ".png") |
| diff_path = os.path.join(output_dir, "diff", url_parts.hostname + ".png") |
| MakeDirsIfNotExist(os.path.join(output_dir, "diff")) |
| |
| # TODO(johnme): Don't hardcode "real_world_impact". |
| red_path = ("data:image/gif;base64,R0lGODlhAQABAPAAAP8AAP///yH5BAAAAAAALAAAAA" |
| "ABAAEAAAICRAEAOw==") |
| |
| before_exists = os.path.exists(before_path) |
| after_exists = os.path.exists(after_path) |
| if not before_exists and not after_exists: |
| # TODO(johnme): Make this more informative. |
| return (-100, url, red_path) |
| if before_exists != after_exists: |
| # TODO(johnme): Make this more informative. |
| return (200, url, red_path) |
| |
| # Get percentage difference. |
| p = subprocess.Popen([image_diff, "--histogram", |
| before_path, after_path], |
| shell=False, |
| stdout=subprocess.PIPE) |
| output,_ = p.communicate() |
| if p.returncode == 0: |
| return (0, url, before_path) |
| diff_match = re.match(r'histogram diff: (\d+\.\d{2})% (?:passed|failed)\n' |
| 'exact diff: (\d+\.\d{2})% (?:passed|failed)', output) |
| if not diff_match: |
| raise Exception("image_diff output format changed") |
| histogram_diff = float(diff_match.group(1)) |
| exact_diff = float(diff_match.group(2)) |
| combined_diff = max(histogram_diff + exact_diff / 8, 0.001) |
| |
| # Produce diff PNG. |
| subprocess.call([image_diff, "--diff", before_path, after_path, diff_path]) |
| return (combined_diff, url, diff_path) |
| |
| |
| def CompareResults(): |
| print("Running image_diff on %d pages..." % len(urls)) |
| start_time = time.time() |
| |
| results = multiprocessing.Pool().map(CompareResultsTask, urls) |
| results.sort(key=itemgetter(0), reverse=True) |
| |
| PrintElapsedTime(time.time() - start_time) |
| |
| now = datetime.datetime.today().strftime("%a %Y-%m-%d %H:%M") |
| html_start = textwrap.dedent("""\ |
| <!DOCTYPE html> |
| <html> |
| <head> |
| <title>Real World Impact report %s</title> |
| <script> |
| var togglingImg = null; |
| var toggleTimer = null; |
| |
| var before = true; |
| function toggle() { |
| var newFolder = before ? "before" : "after"; |
| togglingImg.src = togglingImg.src.replace(/before|after|diff/, newFolder); |
| before = !before; |
| toggleTimer = setTimeout(toggle, 300); |
| } |
| |
| function startToggle(img) { |
| before = true; |
| togglingImg = img; |
| if (!img.origSrc) |
| img.origSrc = img.src; |
| toggle(); |
| } |
| function stopToggle(img) { |
| clearTimeout(toggleTimer); |
| img.src = img.origSrc; |
| } |
| |
| document.onkeydown = function(e) { |
| e = e || window.event; |
| var keyCode = e.keyCode || e.which; |
| var newFolder; |
| switch (keyCode) { |
| case 49: //'1' |
| newFolder = "before"; break; |
| case 50: //'2' |
| newFolder = "after"; break; |
| case 51: //'3' |
| newFolder = "diff"; break; |
| default: |
| return; |
| } |
| var imgs = document.getElementsByTagName("img"); |
| for (var i = 0; i < imgs.length; i++) { |
| imgs[i].src = imgs[i].src.replace(/before|after|diff/, newFolder); |
| } |
| }; |
| </script> |
| <style> |
| h1 { |
| font-family: sans; |
| } |
| h2 { |
| font-family: monospace; |
| white-space: pre; |
| } |
| .nsfw-spacer { |
| height: 50vh; |
| } |
| .nsfw-warning { |
| background: yellow; |
| border: 10px solid red; |
| } |
| .info { |
| font-size: 1.2em; |
| font-style: italic; |
| } |
| body:not(.details-supported) details { |
| display: none; |
| } |
| </style> |
| </head> |
| <body> |
| <script> |
| if ('open' in document.createElement('details')) |
| document.body.className = "details-supported"; |
| </script> |
| <!--<div class="nsfw-spacer"></div>--> |
| <p class="nsfw-warning">Warning: sites below are taken from the Alexa top %d |
| and may be NSFW.</p> |
| <!--<div class="nsfw-spacer"></div>--> |
| <h1>Real World Impact report %s</h1> |
| <p class="info">Press 1, 2 and 3 to switch between before, after and diff |
| screenshots respectively; or hover over the images to rapidly alternate |
| between before and after.</p> |
| """ % (now, num_sites, now)) |
| |
| html_same_row = """\ |
| <h2>No difference on <a href="%s">%s</a>.</h2> |
| """ |
| |
| html_diff_row = """\ |
| <h2>%7.3f%% difference on <a href="%s">%s</a>:</h2> |
| <img src="%s" width="800" height="600" |
| onmouseover="startToggle(this)" onmouseout="stopToggle(this)"> |
| """ |
| |
| html_nsfw_diff_row = """\ |
| <h2>%7.3f%% difference on <a href="%s">%s</a>:</h2> |
| <details> |
| <summary>This site may be NSFW. Click to expand/collapse.</summary> |
| <img src="%s" width="800" height="600" |
| onmouseover="startToggle(this)" onmouseout="stopToggle(this)"> |
| </details> |
| """ |
| |
| html_end = textwrap.dedent("""\ |
| </body> |
| </html>""") |
| |
| html_path = os.path.join(output_dir, "diff.html") |
| with open(html_path, 'w') as f: |
| f.write(html_start) |
| for (diff_float, url, diff_path) in results: |
| diff_path = os.path.relpath(diff_path, output_dir) |
| if diff_float == 0: |
| f.write(html_same_row % (url, url)) |
| elif url in nsfw_urls: |
| f.write(html_nsfw_diff_row % (diff_float, url, url, diff_path)) |
| else: |
| f.write(html_diff_row % (diff_float, url, url, diff_path)) |
| f.write(html_end) |
| |
| webbrowser.open_new_tab("file://" + html_path) |
| |
| |
| def main(argv): |
| global num_sites, action, allow_js, additional_content_shell_flags |
| |
| parser = argparse.ArgumentParser( |
| formatter_class=RawTextHelpFormatter, |
| description="Compare the real world impact of a content shell change.", |
| epilog=textwrap.dedent("""\ |
| Example usage: |
| 1. Build content_shell in out/Release without any changes. |
| 2. Run: %s before [num sites to test (default %d)]. |
| 3. Either: |
| a. Apply your controversial patch and rebuild content_shell. |
| b. Pass --additional_flags="--enable_your_flag" in step 4. |
| 4. Run: %s after [num sites to test (default %d)]. |
| 5. Run: %s compare [num sites to test (default %d)]. |
| This will open the results in your web browser. |
| """ % (argv[0], num_sites, argv[0], num_sites, argv[0], num_sites))) |
| parser.add_argument("--allow_js", help="Don't disable Javascript", |
| action="store_true") |
| parser.add_argument("--additional_flags", |
| help="Additional flags to pass to content shell") |
| parser.add_argument("action", |
| help=textwrap.dedent("""\ |
| Action to perform. |
| download - Just download the sites. |
| before - Run content shell and record 'before' result. |
| after - Run content shell and record 'after' result. |
| compare - Compare before and after results. |
| """), |
| choices=["download", "before", "after", "compare"]) |
| parser.add_argument("num_sites", |
| help="Number of sites (default %s)" % num_sites, |
| type=int, default=num_sites, nargs='?') |
| args = parser.parse_args() |
| |
| action = args.action |
| |
| if (args.num_sites): |
| num_sites = args.num_sites |
| |
| if (args.allow_js): |
| allow_js = args.allow_js |
| |
| if (args.additional_flags): |
| additional_content_shell_flags = args.additional_flags |
| |
| if not SetupPathsAndOut() or not CheckPrerequisites() or not PickSampleUrls(): |
| return 1 |
| |
| if action == 'compare': |
| CompareResults() |
| else: |
| DownloadStaticCopies() |
| if action != 'download': |
| RunDrt() |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv)) |
