| #!/usr/bin/env python |
| |
| # This Source Code Form is subject to the terms of the Mozilla Public |
| # License, v. 2.0. If a copy of the MPL was not distributed with this |
| # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| |
| """Module to handle downloads for different types of Firefox and Thunderbird builds.""" |
| |
| |
| from datetime import datetime |
| from optparse import OptionParser, OptionGroup |
| import os |
| import re |
| import sys |
| import time |
| import urllib |
| import urllib2 |
| |
| import mozinfo |
| |
| from parser import DirectoryParser |
| from timezones import PacificTimezone |
| |
| |
| APPLICATIONS = ['b2g', 'firefox', 'thunderbird'] |
| |
| # Base URL for the path to all builds |
| BASE_URL = 'https://ftp.mozilla.org/pub/mozilla.org' |
| |
| PLATFORM_FRAGMENTS = {'linux': 'linux-i686', |
| 'linux64': 'linux-x86_64', |
| 'mac': 'mac', |
| 'mac64': 'mac64', |
| 'win32': 'win32', |
| 'win64': 'win64-x86_64'} |
| |
| DEFAULT_FILE_EXTENSIONS = {'linux': 'tar.bz2', |
| 'linux64': 'tar.bz2', |
| 'mac': 'dmg', |
| 'mac64': 'dmg', |
| 'win32': 'exe', |
| 'win64': 'exe'} |
| |
| class NotFoundException(Exception): |
| """Exception for a resource not being found (e.g. no logs)""" |
| def __init__(self, message, location): |
| self.location = location |
| Exception.__init__(self, ': '.join([message, location])) |
| |
| |
| class Scraper(object): |
| """Generic class to download an application from the Mozilla server""" |
| |
| def __init__(self, directory, version, platform=None, |
| application='firefox', locale='en-US', extension=None, |
| authentication=None, retry_attempts=3, retry_delay=10): |
| |
| # Private properties for caching |
| self._target = None |
| self._binary = None |
| |
| self.directory = directory |
| self.locale = locale |
| self.platform = platform or self.detect_platform() |
| self.version = version |
| self.extension = extension or DEFAULT_FILE_EXTENSIONS[self.platform] |
| self.authentication = authentication |
| self.retry_attempts = retry_attempts |
| self.retry_delay = retry_delay |
| |
| # build the base URL |
| self.application = application |
| self.base_url = '/'.join([BASE_URL, self.application]) |
| |
| |
| @property |
| def binary(self): |
| """Return the name of the build""" |
| |
| if self._binary is None: |
| # Retrieve all entries from the remote virtual folder |
| parser = DirectoryParser(self.path) |
| if not parser.entries: |
| raise NotFoundException('No entries found', self.path) |
| |
| # Download the first matched directory entry |
| pattern = re.compile(self.binary_regex, re.IGNORECASE) |
| for entry in parser.entries: |
| try: |
| self._binary = pattern.match(entry).group() |
| break |
| except: |
| # No match, continue with next entry |
| continue |
| |
| if self._binary is None: |
| raise NotFoundException("Binary not found in folder", self.path) |
| else: |
| return self._binary |
| |
| |
| @property |
| def binary_regex(self): |
| """Return the regex for the binary filename""" |
| |
| raise NotImplementedError(sys._getframe(0).f_code.co_name) |
| |
| |
| @property |
| def final_url(self): |
| """Return the final URL of the build""" |
| |
| return '/'.join([self.path, self.binary]) |
| |
| |
| @property |
| def path(self): |
| """Return the path to the build""" |
| |
| return '/'.join([self.base_url, self.path_regex]) |
| |
| |
| @property |
| def path_regex(self): |
| """Return the regex for the path to the build""" |
| |
| raise NotImplementedError(sys._getframe(0).f_code.co_name) |
| |
| |
| @property |
| def platform_regex(self): |
| """Return the platform fragment of the URL""" |
| |
| return PLATFORM_FRAGMENTS[self.platform]; |
| |
| |
| @property |
| def target(self): |
| """Return the target file name of the build""" |
| |
| if self._target is None: |
| self._target = os.path.join(self.directory, |
| self.build_filename(self.binary)) |
| return self._target |
| |
| |
| def build_filename(self, binary): |
| """Return the proposed filename with extension for the binary""" |
| |
| raise NotImplementedError(sys._getframe(0).f_code.co_name) |
| |
| |
| def detect_platform(self): |
| """Detect the current platform""" |
| |
| # For Mac and Linux 32bit we do not need the bits appended |
| if mozinfo.os == 'mac' or (mozinfo.os == 'linux' and mozinfo.bits == 32): |
| return mozinfo.os |
| else: |
| return "%s%d" % (mozinfo.os, mozinfo.bits) |
| |
| |
| def download(self): |
| """Download the specified file""" |
| |
| attempts = 0 |
| |
| if not os.path.isdir(self.directory): |
| os.makedirs(self.directory) |
| |
| # Don't re-download the file |
| if os.path.isfile(os.path.abspath(self.target)): |
| print "File has already been downloaded: %s" % (self.target) |
| return |
| |
| print 'Downloading from: %s' % (urllib.unquote(self.final_url)) |
| tmp_file = self.target + ".part" |
| |
| if self.authentication \ |
| and self.authentication['username'] \ |
| and self.authentication['password']: |
| password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() |
| password_mgr.add_password(None, |
| self.final_url, |
| self.authentication['username'], |
| self.authentication['password']) |
| handler = urllib2.HTTPBasicAuthHandler(password_mgr) |
| opener = urllib2.build_opener(urllib2.HTTPHandler, handler) |
| urllib2.install_opener(opener) |
| |
| while True: |
| attempts += 1 |
| try: |
| r = urllib2.urlopen(self.final_url) |
| CHUNK = 16 * 1024 |
| with open(tmp_file, 'wb') as f: |
| for chunk in iter(lambda: r.read(CHUNK), ''): |
| f.write(chunk) |
| break |
| except (urllib2.HTTPError, urllib2.URLError): |
| if tmp_file and os.path.isfile(tmp_file): |
| os.remove(tmp_file) |
| print 'Download failed! Retrying... (attempt %s)' % attempts |
| if attempts >= self.retry_attempts: |
| raise |
| time.sleep(self.retry_delay) |
| |
| os.rename(tmp_file, self.target) |
| |
| |
| class DailyScraper(Scraper): |
| """Class to download a daily build from the Mozilla server""" |
| |
| def __init__(self, branch='mozilla-central', build_id=None, date=None, |
| build_number=None, *args, **kwargs): |
| |
| Scraper.__init__(self, *args, **kwargs) |
| self.branch = branch |
| |
| # Internally we access builds via index |
| if build_number is not None: |
| self.build_index = int(build_number) - 1 |
| else: |
| self.build_index = None |
| |
| if build_id: |
| # A build id has been specified. Split up its components so the date |
| # and time can be extracted: '20111212042025' -> '2011-12-12 04:20:25' |
| self.date = datetime.strptime(build_id, '%Y%m%d%H%M%S') |
| self.builds, self.build_index = self.get_build_info_for_date(self.date, |
| has_time=True) |
| |
| elif date: |
| # A date (without time) has been specified. Use its value and the |
| # build index to find the requested build for that day. |
| self.date = datetime.strptime(date, '%Y-%m-%d') |
| self.builds, self.build_index = self.get_build_info_for_date(self.date, |
| build_index=self.build_index) |
| |
| else: |
| # If no build id nor date have been specified the lastest available |
| # build of the given branch has to be identified. We also have to |
| # retrieve the date of the build via its build id. |
| url = '%s/nightly/latest-%s/' % (self.base_url, self.branch) |
| |
| print 'Retrieving the build status file from %s' % url |
| parser = DirectoryParser(url) |
| parser.entries = parser.filter(r'.*%s\.txt' % self.platform_regex) |
| if not parser.entries: |
| message = 'Status file for %s build cannot be found' % self.platform_regex |
| raise NotFoundException(message, url) |
| |
| # Read status file for the platform, retrieve build id, and convert to a date |
| status_file = url + parser.entries[-1] |
| f = urllib.urlopen(status_file) |
| self.date = datetime.strptime(f.readline().strip(), '%Y%m%d%H%M%S') |
| self.builds, self.build_index = self.get_build_info_for_date(self.date, |
| has_time=True) |
| |
| |
| def get_build_info_for_date(self, date, has_time=False, build_index=None): |
| url = '/'.join([self.base_url, self.monthly_build_list_regex]) |
| |
| print 'Retrieving list of builds from %s' % url |
| parser = DirectoryParser(url) |
| regex = r'%(DATE)s-(\d+-)+%(BRANCH)s%(L10N)s$' % { |
| 'DATE': date.strftime('%Y-%m-%d'), |
| 'BRANCH': self.branch, |
| 'L10N': '' if self.locale == 'en-US' else '-l10n'} |
| parser.entries = parser.filter(regex) |
| if not parser.entries: |
| message = 'Folder for builds on %s has not been found' % self.date.strftime('%Y-%m-%d') |
| raise NotFoundException(message, url) |
| |
| if has_time: |
| # If a time is included in the date, use it to determine the build's index |
| regex = r'.*%s.*' % date.strftime('%H-%M-%S') |
| build_index = parser.entries.index(parser.filter(regex)[0]) |
| else: |
| # If no index has been given, set it to the last build of the day. |
| if build_index is None: |
| build_index = len(parser.entries) - 1 |
| |
| return (parser.entries, build_index) |
| |
| |
| @property |
| def binary_regex(self): |
| """Return the regex for the binary""" |
| |
| regex_base_name = r'^%(APP)s-.*\.%(LOCALE)s\.%(PLATFORM)s' |
| regex_suffix = {'linux': r'\.%(EXT)s$', |
| 'linux64': r'\.%(EXT)s$', |
| 'mac': r'\.%(EXT)s$', |
| 'mac64': r'\.%(EXT)s$', |
| 'win32': r'(\.installer)\.%(EXT)s$', |
| 'win64': r'(\.installer)\.%(EXT)s$'} |
| regex = regex_base_name + regex_suffix[self.platform] |
| |
| return regex % {'APP': self.application, |
| 'LOCALE': self.locale, |
| 'PLATFORM': self.platform_regex, |
| 'EXT': self.extension} |
| |
| |
| def build_filename(self, binary): |
| """Return the proposed filename with extension for the binary""" |
| |
| try: |
| # Get exact timestamp of the build to build the local file name |
| folder = self.builds[self.build_index] |
| timestamp = re.search('([\d\-]+)-\D.*', folder).group(1) |
| except: |
| # If it's not available use the build's date |
| timestamp = self.date.strftime('%Y-%m-%d') |
| |
| return '%(TIMESTAMP)s-%(BRANCH)s-%(NAME)s' % { |
| 'TIMESTAMP': timestamp, |
| 'BRANCH': self.branch, |
| 'NAME': binary} |
| |
| |
| @property |
| def monthly_build_list_regex(self): |
| """Return the regex for the folder which contains the builds of a month.""" |
| |
| # Regex for possible builds for the given date |
| return r'nightly/%(YEAR)s/%(MONTH)s/' % { |
| 'YEAR': self.date.year, |
| 'MONTH': str(self.date.month).zfill(2) } |
| |
| |
| @property |
| def path_regex(self): |
| """Return the regex for the path""" |
| |
| try: |
| return self.monthly_build_list_regex + self.builds[self.build_index] |
| except: |
| raise NotFoundException("Specified sub folder cannot be found", |
| self.base_url + self.monthly_build_list_regex) |
| |
| |
| class DirectScraper(Scraper): |
| """Class to download a file from a specified URL""" |
| |
| def __init__(self, url, *args, **kwargs): |
| Scraper.__init__(self, *args, **kwargs) |
| |
| self.url = url |
| |
| @property |
| def target(self): |
| return urllib.splitquery(self.final_url)[0].rpartition('/')[-1] |
| |
| @property |
| def final_url(self): |
| return self.url |
| |
| |
| class ReleaseScraper(Scraper): |
| """Class to download a release build from the Mozilla server""" |
| |
| def __init__(self, *args, **kwargs): |
| Scraper.__init__(self, *args, **kwargs) |
| |
| @property |
| def binary_regex(self): |
| """Return the regex for the binary""" |
| |
| regex = {'linux': r'^%(APP)s-.*\.%(EXT)s$', |
| 'linux64': r'^%(APP)s-.*\.%(EXT)s$', |
| 'mac': r'^%(APP)s.*\.%(EXT)s$', |
| 'mac64': r'^%(APP)s.*\.%(EXT)s$', |
| 'win32': r'^%(APP)s.*\.%(EXT)s$', |
| 'win64': r'^%(APP)s.*\.%(EXT)s$'} |
| return regex[self.platform] % {'APP': self.application, |
| 'EXT': self.extension} |
| |
| |
| @property |
| def path_regex(self): |
| """Return the regex for the path""" |
| |
| regex = r'releases/%(VERSION)s/%(PLATFORM)s/%(LOCALE)s' |
| return regex % {'LOCALE': self.locale, |
| 'PLATFORM': self.platform_regex, |
| 'VERSION': self.version} |
| |
| |
| def build_filename(self, binary): |
| """Return the proposed filename with extension for the binary""" |
| |
| template = '%(APP)s-%(VERSION)s.%(LOCALE)s.%(PLATFORM)s.%(EXT)s' |
| return template % {'APP': self.application, |
| 'VERSION': self.version, |
| 'LOCALE': self.locale, |
| 'PLATFORM': self.platform, |
| 'EXT': self.extension} |
| |
| |
| class ReleaseCandidateScraper(ReleaseScraper): |
| """Class to download a release candidate build from the Mozilla server""" |
| |
| def __init__(self, build_number=None, no_unsigned=False, *args, **kwargs): |
| Scraper.__init__(self, *args, **kwargs) |
| |
| # Internally we access builds via index |
| if build_number is not None: |
| self.build_index = int(build_number) - 1 |
| else: |
| self.build_index = None |
| |
| self.builds, self.build_index = self.get_build_info_for_version(self.version, self.build_index) |
| |
| self.no_unsigned = no_unsigned |
| self.unsigned = False |
| |
| |
| def get_build_info_for_version(self, version, build_index=None): |
| url = '/'.join([self.base_url, self.candidate_build_list_regex]) |
| |
| print 'Retrieving list of candidate builds from %s' % url |
| parser = DirectoryParser(url) |
| if not parser.entries: |
| message = 'Folder for specific candidate builds at has not been found' |
| raise NotFoundException(message, url) |
| |
| # If no index has been given, set it to the last build of the given version. |
| if build_index is None: |
| build_index = len(parser.entries) - 1 |
| |
| return (parser.entries, build_index) |
| |
| |
| @property |
| def candidate_build_list_regex(self): |
| """Return the regex for the folder which contains the builds of |
| a candidate build.""" |
| |
| # Regex for possible builds for the given date |
| return r'nightly/%(VERSION)s-candidates/' % { |
| 'VERSION': self.version } |
| |
| |
| @property |
| def path_regex(self): |
| """Return the regex for the path""" |
| |
| regex = r'%(PREFIX)s%(BUILD)s/%(UNSIGNED)s%(PLATFORM)s/%(LOCALE)s' |
| return regex % {'PREFIX': self.candidate_build_list_regex, |
| 'BUILD': self.builds[self.build_index], |
| 'LOCALE': self.locale, |
| 'PLATFORM': self.platform_regex, |
| 'UNSIGNED': "unsigned/" if self.unsigned else ""} |
| |
| |
| def build_filename(self, binary): |
| """Return the proposed filename with extension for the binary""" |
| |
| template = '%(APP)s-%(VERSION)s-build%(BUILD)s.%(LOCALE)s.%(PLATFORM)s.%(EXT)s' |
| return template % {'APP': self.application, |
| 'VERSION': self.version, |
| 'BUILD': self.builds[self.build_index], |
| 'LOCALE': self.locale, |
| 'PLATFORM': self.platform, |
| 'EXT': self.extension} |
| |
| |
| def download(self): |
| """Download the specified file""" |
| |
| try: |
| # Try to download the signed candidate build |
| Scraper.download(self) |
| except NotFoundException, e: |
| print str(e) |
| |
| # If the signed build cannot be downloaded and unsigned builds are |
| # allowed, try to download the unsigned build instead |
| if self.no_unsigned: |
| raise |
| else: |
| print "Signed build has not been found. Falling back to unsigned build." |
| self.unsigned = True |
| Scraper.download(self) |
| |
| |
| class TinderboxScraper(Scraper): |
| """Class to download a tinderbox build from the Mozilla server. |
| |
| There are two ways to specify a unique build: |
| 1. If the date (%Y-%m-%d) is given and build_number is given where |
| the build_number is the index of the build on the date |
| 2. If the build timestamp (UNIX) is given, and matches a specific build. |
| """ |
| |
| def __init__(self, branch='mozilla-central', build_number=None, date=None, |
| debug_build=False, *args, **kwargs): |
| Scraper.__init__(self, *args, **kwargs) |
| |
| self.branch = branch |
| self.debug_build = debug_build |
| self.locale_build = self.locale != 'en-US' |
| self.timestamp = None |
| |
| # Currently any time in RelEng is based on the Pacific time zone. |
| self.timezone = PacificTimezone(); |
| |
| # Internally we access builds via index |
| if build_number is not None: |
| self.build_index = int(build_number) - 1 |
| else: |
| self.build_index = None |
| |
| if date is not None: |
| try: |
| self.date = datetime.fromtimestamp(float(date), self.timezone) |
| self.timestamp = date |
| except: |
| self.date = datetime.strptime(date, '%Y-%m-%d') |
| else: |
| self.date = None |
| |
| # For localized builds we do not have to retrieve the list of builds |
| # because only the last build is available |
| if not self.locale_build: |
| self.builds, self.build_index = self.get_build_info(self.build_index) |
| |
| try: |
| self.timestamp = self.builds[self.build_index] |
| except: |
| raise NotFoundException("Specified sub folder cannot be found", |
| self.base_url + self.monthly_build_list_regex) |
| |
| |
| @property |
| def binary_regex(self): |
| """Return the regex for the binary""" |
| |
| regex_base_name = r'^%(APP)s-.*\.%(LOCALE)s\.' |
| regex_suffix = {'linux': r'.*\.%(EXT)s$', |
| 'linux64': r'.*\.%(EXT)s$', |
| 'mac': r'.*\.%(EXT)s$', |
| 'mac64': r'.*\.%(EXT)s$', |
| 'win32': r'.*(\.installer)\.%(EXT)s$', |
| 'win64': r'.*(\.installer)\.%(EXT)s$'} |
| |
| regex = regex_base_name + regex_suffix[self.platform] |
| |
| return regex % {'APP': self.application, |
| 'LOCALE': self.locale, |
| 'EXT': self.extension} |
| |
| |
| def build_filename(self, binary): |
| """Return the proposed filename with extension for the binary""" |
| |
| return '%(TIMESTAMP)s%(BRANCH)s%(DEBUG)s-%(NAME)s' % { |
| 'TIMESTAMP': self.timestamp + '-' if self.timestamp else '', |
| 'BRANCH': self.branch, |
| 'DEBUG': '-debug' if self.debug_build else '', |
| 'NAME': binary} |
| |
| |
| @property |
| def build_list_regex(self): |
| """Return the regex for the folder which contains the list of builds""" |
| |
| regex = 'tinderbox-builds/%(BRANCH)s-%(PLATFORM)s%(L10N)s%(DEBUG)s' |
| |
| return regex % {'BRANCH': self.branch, |
| 'PLATFORM': '' if self.locale_build else self.platform_regex, |
| 'L10N': 'l10n' if self.locale_build else '', |
| 'DEBUG': '-debug' if self.debug_build else ''} |
| |
| |
| def date_matches(self, timestamp): |
| """Determines whether the timestamp date is equal to the argument date""" |
| |
| if self.date is None: |
| return False |
| |
| timestamp = datetime.fromtimestamp(float(timestamp), self.timezone) |
| if self.date.date() == timestamp.date(): |
| return True |
| |
| return False |
| |
| |
| @property |
| def date_validation_regex(self): |
| """Return the regex for a valid date argument value""" |
| |
| return r'^\d{4}-\d{1,2}-\d{1,2}$|^\d+$' |
| |
| |
| def detect_platform(self): |
| """Detect the current platform""" |
| |
| platform = Scraper.detect_platform(self) |
| |
| # On OS X we have to special case the platform detection code and fallback |
| # to 64 bit builds for the en-US locale |
| if mozinfo.os == 'mac' and self.locale == 'en-US' and mozinfo.bits == 64: |
| platform = "%s%d" % (mozinfo.os, mozinfo.bits) |
| |
| return platform |
| |
| |
| def get_build_info(self, build_index=None): |
| url = '/'.join([self.base_url, self.build_list_regex]) |
| |
| print 'Retrieving list of builds from %s' % url |
| |
| # If a timestamp is given, retrieve just that build |
| regex = '^' + self.timestamp + '$' if self.timestamp else r'^\d+$' |
| |
| parser = DirectoryParser(url) |
| parser.entries = parser.filter(regex) |
| |
| # If date is given, retrieve the subset of builds on that date |
| if self.date is not None: |
| parser.entries = filter(self.date_matches, parser.entries) |
| |
| if not parser.entries: |
| message = 'No builds have been found' |
| raise NotFoundException(message, url) |
| |
| # If no index has been given, set it to the last build of the day. |
| if build_index is None: |
| build_index = len(parser.entries) - 1 |
| |
| return (parser.entries, build_index) |
| |
| |
| @property |
| def path_regex(self): |
| """Return the regex for the path""" |
| |
| if self.locale_build: |
| return self.build_list_regex |
| |
| return '/'.join([self.build_list_regex, self.builds[self.build_index]]) |
| |
| |
| @property |
| def platform_regex(self): |
| """Return the platform fragment of the URL""" |
| |
| PLATFORM_FRAGMENTS = {'linux': 'linux', |
| 'linux64': 'linux64', |
| 'mac': 'macosx', |
| 'mac64': 'macosx64', |
| 'win32': 'win32', |
| 'win64': 'win64'} |
| |
| return PLATFORM_FRAGMENTS[self.platform] |
| |
| |
| def cli(): |
| """Main function for the downloader""" |
| |
| BUILD_TYPES = {'release': ReleaseScraper, |
| 'candidate': ReleaseCandidateScraper, |
| 'daily': DailyScraper, |
| 'tinderbox': TinderboxScraper } |
| |
| usage = 'usage: %prog [options]' |
| parser = OptionParser(usage=usage, description=__doc__) |
| parser.add_option('--application', '-a', |
| dest='application', |
| choices=APPLICATIONS, |
| default='firefox', |
| metavar='APPLICATION', |
| help='The name of the application to download, ' |
| 'default: "%default"') |
| parser.add_option('--directory', '-d', |
| dest='directory', |
| default=os.getcwd(), |
| metavar='DIRECTORY', |
| help='Target directory for the download, default: ' |
| 'current working directory') |
| parser.add_option('--build-number', |
| dest='build_number', |
| default=None, |
| type="int", |
| metavar='BUILD_NUMBER', |
| help='Number of the build (for candidate, daily, ' |
| 'and tinderbox builds)') |
| parser.add_option('--locale', '-l', |
| dest='locale', |
| default='en-US', |
| metavar='LOCALE', |
| help='Locale of the application, default: "%default"') |
| parser.add_option('--platform', '-p', |
| dest='platform', |
| choices=PLATFORM_FRAGMENTS.keys(), |
| metavar='PLATFORM', |
| help='Platform of the application') |
| parser.add_option('--type', '-t', |
| dest='type', |
| choices=BUILD_TYPES.keys(), |
| default='release', |
| metavar='BUILD_TYPE', |
| help='Type of build to download, default: "%default"') |
| parser.add_option('--url', |
| dest='url', |
| default=None, |
| metavar='URL', |
| help='URL to download.') |
| parser.add_option('--version', '-v', |
| dest='version', |
| metavar='VERSION', |
| help='Version of the application to be used by release and\ |
| candidate builds, i.e. "3.6"') |
| parser.add_option('--extension', |
| dest='extension', |
| default=None, |
| metavar='EXTENSION', |
| help='File extension of the build (e.g. "zip"), default:\ |
| the standard build extension on the platform.') |
| parser.add_option('--username', |
| dest='username', |
| default=None, |
| metavar='USERNAME', |
| help='Username for basic HTTP authentication.') |
| parser.add_option('--password', |
| dest='password', |
| default=None, |
| metavar='PASSWORD', |
| help='Password for basic HTTP authentication.') |
| parser.add_option('--retry-attempts', |
| dest='retry_attempts', |
| default=3, |
| type=int, |
| metavar='RETRY_ATTEMPTS', |
| help='Number of times the download will be attempted in ' |
| 'the event of a failure, default: %default') |
| parser.add_option('--retry-delay', |
| dest='retry_delay', |
| default=10, |
| type=int, |
| metavar='RETRY_DELAY', |
| help='Amount of time (in seconds) to wait between retry ' |
| 'attempts, default: %default') |
| |
| # Option group for candidate builds |
| group = OptionGroup(parser, "Candidate builds", |
| "Extra options for candidate builds.") |
| group.add_option('--no-unsigned', |
| dest='no_unsigned', |
| action="store_true", |
| help="Don't allow to download unsigned builds if signed\ |
| builds are not available") |
| parser.add_option_group(group) |
| |
| # Option group for daily builds |
| group = OptionGroup(parser, "Daily builds", |
| "Extra options for daily builds.") |
| group.add_option('--branch', |
| dest='branch', |
| default='mozilla-central', |
| metavar='BRANCH', |
| help='Name of the branch, default: "%default"') |
| group.add_option('--build-id', |
| dest='build_id', |
| default=None, |
| metavar='BUILD_ID', |
| help='ID of the build to download') |
| group.add_option('--date', |
| dest='date', |
| default=None, |
| metavar='DATE', |
| help='Date of the build, default: latest build') |
| parser.add_option_group(group) |
| |
| # Option group for tinderbox builds |
| group = OptionGroup(parser, "Tinderbox builds", |
| "Extra options for tinderbox builds.") |
| group.add_option('--debug-build', |
| dest='debug_build', |
| action="store_true", |
| help="Download a debug build") |
| parser.add_option_group(group) |
| |
| # TODO: option group for nightly builds |
| (options, args) = parser.parse_args() |
| |
| # Check for required options and arguments |
| # Note: Will be optional when ini file support has been landed |
| if not options.url \ |
| and not options.type in ['daily', 'tinderbox'] \ |
| and not options.version: |
| parser.error('The version of the application to download has not been specified.') |
| |
| # Instantiate scraper and download the build |
| scraper_keywords = {'application': options.application, |
| 'locale': options.locale, |
| 'platform': options.platform, |
| 'version': options.version, |
| 'directory': options.directory, |
| 'extension': options.extension, |
| 'authentication': { |
| 'username': options.username, |
| 'password': options.password}, |
| 'retry_attempts': options.retry_attempts, |
| 'retry_delay': options.retry_delay} |
| scraper_options = {'candidate': { |
| 'build_number': options.build_number, |
| 'no_unsigned': options.no_unsigned}, |
| 'daily': { |
| 'branch': options.branch, |
| 'build_number': options.build_number, |
| 'build_id': options.build_id, |
| 'date': options.date}, |
| 'tinderbox': { |
| 'branch': options.branch, |
| 'build_number': options.build_number, |
| 'date': options.date, |
| 'debug_build': options.debug_build} |
| } |
| |
| kwargs = scraper_keywords.copy() |
| kwargs.update(scraper_options.get(options.type, {})) |
| |
| if options.url: |
| build = DirectScraper(options.url, **kwargs) |
| else: |
| build = BUILD_TYPES[options.type](**kwargs) |
| |
| build.download() |
| |
| if __name__ == "__main__": |
| cli() |
