| # Copyright (c) 2011 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Component for automatically creating masks of changing areas of a website. |
| |
| Works by repeated invokation of a browser and scraping of the resulting page. |
| Areas that differ will be added to the auto-generated mask. The mask generator |
| considers the mask complete when further scrapes fail to produce any differences |
| in the mask. |
| """ |
| |
| import os # Functions for walking the directory tree |
| import tempfile # Get a temporary directory to hold intermediates |
| import time # Used for sleep() and naming masks by time |
| |
| import command_line |
| import drivers |
| from PIL import Image |
| from PIL import ImageChops |
| import scrapers |
| |
| |
| def CreateCommand(cmdline): |
| """Inserts the command and arguments into a command line for parsing.""" |
| cmd = cmdline.AddCommand( |
| ["maskmaker"], |
| "Automatically generates a mask from a list of URLs", |
| ValidateMaskmaker, |
| ExecuteMaskmaker) |
| |
| cmd.AddArgument( |
| ["-bp", "--browserpath"], "Full path to browser's executable", |
| type="readfile", metaname="PATH") |
| cmd.AddArgument( |
| ["-b", "--browser"], "Which browser to use", type="string", |
| default="chrome") |
| cmd.AddArgument( |
| ["-bv", "--browserver"], "Version of the browser", metaname="VERSION") |
| cmd.AddArgument( |
| ["-o", "--outdir"], "Directory to store generated masks", metaname="DIR", |
| required=True) |
| cmd.AddArgument( |
| ["-u", "--url"], "URL to compare") |
| cmd.AddArgument( |
| ["-l", "--list"], "List of URLs to compare", type="readfile") |
| cmd.AddMutualExclusion(["--url", "--list"]) |
| cmd.AddArgument( |
| ["-s", "--startline"], "First line of URL list", type="int") |
| cmd.AddArgument( |
| ["-e", "--endline"], "Last line of URL list (exclusive)", type="int") |
| cmd.AddArgument( |
| ["-c", "--count"], "Number of lines of URL file to use", type="int") |
| cmd.AddDependency("--startline", "--list") |
| cmd.AddRequiredGroup(["--url", "--list"]) |
| cmd.AddDependency("--endline", "--list") |
| cmd.AddDependency("--count", "--list") |
| cmd.AddMutualExclusion(["--count", "--endline"]) |
| cmd.AddDependency("--count", "--startline") |
| cmd.AddArgument( |
| ["-t", "--timeout"], "Amount of time (seconds) to wait for browser to " |
| "finish loading", |
| type="int", default=60) |
| cmd.AddArgument( |
| ["-w", "--wait"], |
| "Amount of time (in seconds) to wait between successive scrapes", |
| type="int", default=60) |
| cmd.AddArgument( |
| ["-sc", "--scrapes"], |
| "Number of successive scrapes which must result in no change to a mask " |
| "before mask creation is considered complete", type="int", default=10) |
| cmd.AddArgument( |
| ["-sz", "--size"], "Browser window size", default=(800, 600), type="coords") |
| cmd.AddArgument(["-sd", "--scrapedir"], "Directory to store scrapes") |
| cmd.AddArgument( |
| ["-gu", "--giveup"], |
| "Number of times to scrape before giving up", type="int", default=50) |
| cmd.AddArgument( |
| ["-th", "--threshhold"], |
| "Percentage of different pixels (0-100) above which the scrape will be" |
| "discarded and the mask not updated.", type="int", default=100) |
| cmd.AddArgument( |
| ["--er", "--errors"], |
| "Number of times a scrape can fail before giving up on the URL.", |
| type="int", default=1) |
| |
| |
| def ValidateMaskmaker(command): |
| """Validate the arguments to maskmaker. Raises ParseError if failed.""" |
| executables = [".exe", ".com", ".bat"] |
| if command["--browserpath"]: |
| if os.path.splitext(command["--browserpath"])[1].lower() not in executables: |
| raise command_line.ParseError("Browser filename must be an executable") |
| |
| |
| def ExecuteMaskmaker(command): |
| """Performs automatic mask generation.""" |
| |
| # Get the list of URLs to generate masks for |
| class MaskmakerURL(object): |
| """Helper class for holding information about a URL passed to maskmaker.""" |
| __slots__ = ['url', 'consecutive_successes', 'errors'] |
| def __init__(self, url): |
| self.url = url |
| self.consecutive_successes = 0 |
| self.errors = 0 |
| |
| if command["--url"]: |
| url_list = [MaskmakerURL(command["--url"])] |
| else: |
| startline = command["--startline"] |
| if command["--count"]: |
| endline = startline+command["--count"] |
| else: |
| endline = command["--endline"] |
| url_list = [MaskmakerURL(url.strip()) for url in |
| open(command["--list"], "r").readlines()[startline:endline]] |
| |
| complete_list = [] |
| error_list = [] |
| |
| outdir = command["--outdir"] |
| scrapes = command["--scrapes"] |
| errors = command["--errors"] |
| size = command["--size"] |
| scrape_pass = 0 |
| |
| scrapedir = command["--scrapedir"] |
| if not scrapedir: scrapedir = tempfile.gettempdir() |
| |
| # Get the scraper |
| scraper = scrapers.GetScraper((command["--browser"], command["--browserver"])) |
| |
| # Repeatedly iterate through the list of URLs until either every URL has |
| # a successful mask or too many errors, or we've exceeded the giveup limit |
| while url_list and scrape_pass < command["--giveup"]: |
| # Scrape each URL |
| for url in url_list: |
| print "Processing %r..." % url.url |
| mask_filename = drivers.windowing.URLtoFilename(url.url, outdir, ".bmp") |
| |
| # Load the existing mask. This is in a loop so we can try to recover |
| # from error conditions |
| while True: |
| try: |
| mask = Image.open(mask_filename) |
| if mask.size != size: |
| print " %r already exists and is the wrong size! (%r vs %r)" % ( |
| mask_filename, mask.size, size) |
| mask_filename = "%s_%r%s" % ( |
| mask_filename[:-4], size, mask_filename[-4:]) |
| print " Trying again as %r..." % mask_filename |
| continue |
| break |
| except IOError: |
| print " %r does not exist, creating" % mask_filename |
| mask = Image.new("1", size, 1) |
| mask.save(mask_filename) |
| |
| # Find the stored scrape path |
| mask_scrape_dir = os.path.join( |
| scrapedir, os.path.splitext(os.path.basename(mask_filename))[0]) |
| drivers.windowing.PreparePath(mask_scrape_dir) |
| |
| # Find the baseline image |
| mask_scrapes = os.listdir(mask_scrape_dir) |
| mask_scrapes.sort() |
| |
| if not mask_scrapes: |
| print " No baseline image found, mask will not be updated" |
| baseline = None |
| else: |
| baseline = Image.open(os.path.join(mask_scrape_dir, mask_scrapes[0])) |
| |
| mask_scrape_filename = os.path.join(mask_scrape_dir, |
| time.strftime("%y%m%d-%H%M%S.bmp")) |
| |
| # Do the scrape |
| result = scraper.Scrape( |
| [url.url], mask_scrape_dir, size, (0, 0), |
| command["--timeout"], path=command["--browserpath"], |
| filename=mask_scrape_filename) |
| |
| if result: |
| # Return value other than None means an error |
| print " Scrape failed with error '%r'" % result |
| url.errors += 1 |
| if url.errors >= errors: |
| print " ** Exceeded maximum error count for this URL, giving up" |
| continue |
| |
| # Load the new scrape |
| scrape = Image.open(mask_scrape_filename) |
| |
| # Calculate the difference between the new scrape and the baseline, |
| # subject to the current mask |
| if baseline: |
| diff = ImageChops.multiply(ImageChops.difference(scrape, baseline), |
| mask.convert(scrape.mode)) |
| |
| # If the difference is none, there's nothing to update |
| if max(diff.getextrema()) == (0, 0): |
| print " Scrape identical to baseline, no change in mask" |
| url.consecutive_successes += 1 |
| if url.consecutive_successes >= scrapes: |
| print " ** No change for %r scrapes, done!" % scrapes |
| else: |
| # convert the difference to black and white, then change all |
| # black pixels (where the scrape and the baseline were identical) |
| # to white, all others (where the scrape and the baseline differed) |
| # to black. |
| # |
| # Since the below command is a little unclear, here's how it works. |
| # 1. convert("L") converts the RGB image to grayscale |
| # 2. point() maps grayscale values (or the individual channels) |
| # of an RGB image) to different ones. Because it operates on |
| # individual channels, the grayscale conversion from step 1 |
| # is necessary. |
| # 3. The "1" second parameter to point() outputs the result as |
| # a monochrome bitmap. If the original RGB image were converted |
| # directly to monochrome, PIL would dither it. |
| diff = diff.convert("L").point([255]+[0]*255, "1") |
| |
| # count the number of different pixels |
| diff_pixels = diff.getcolors()[0][0] |
| |
| # is this too much? |
| diff_pixel_percent = diff_pixels * 100.0 / (mask.size[0]*mask.size[1]) |
| if diff_pixel_percent > command["--threshhold"]: |
| print (" Scrape differed from baseline by %.2f percent, ignoring" |
| % diff_pixel_percent) |
| else: |
| print " Scrape differed in %d pixels, updating mask" % diff_pixels |
| mask = ImageChops.multiply(mask, diff) |
| mask.save(mask_filename) |
| |
| # reset the number of consecutive "good" scrapes |
| url.consecutive_successes = 0 |
| |
| # Remove URLs whose mask is deemed done |
| complete_list.extend( |
| [url for url in url_list if url.consecutive_successes >= scrapes]) |
| error_list.extend( |
| [url for url in url_list if url.errors >= errors]) |
| url_list = [ |
| url for url in url_list if |
| url.consecutive_successes < scrapes and |
| url.errors < errors] |
| |
| scrape_pass += 1 |
| print "**Done with scrape pass %d\n" % scrape_pass |
| |
| if scrape_pass >= command["--giveup"]: |
| print "**Exceeded giveup threshhold. Giving up." |
| else: |
| print "Waiting %d seconds..." % command["--wait"] |
| time.sleep(command["--wait"]) |
| |
| print |
| print "*** MASKMAKER COMPLETE ***" |
| print "Summary report:" |
| print " %d masks successfully generated" % len(complete_list) |
| for url in complete_list: |
| print " ", url.url |
| print " %d masks failed with too many errors" % len(error_list) |
| for url in error_list: |
| print " ", url.url |
| if scrape_pass >= command["--giveup"]: |
| print (" %d masks were not completed before " |
| "reaching the giveup threshhold" % len(url_list)) |
| for url in url_list: |
| print " ", url.url |
