| #!/usr/bin/env python |
| # Copyright (c) 2011 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Downloads web pages with fillable forms after parsing through a set of links. |
| |
| Used for collecting web pages with forms. Used as a standalone script. |
| This script assumes that it's run from within the same directory in which it's |
| checked into. If this script were to be run elsewhere then the path for |
| REGISTER_PAGE_DIR needs to be changed. |
| |
| This script assumes that third party modules are installed: |
| httplib2, lxml, pycurl. |
| |
| Usage: webforms_aggregator.py [options] [single url or file containing urls] |
| |
| Options: |
| -l LOG_LEVEL, --log_level LOG_LEVEL |
| LOG_LEVEL: debug, info, warning or error [default: error] |
| -h, --help show this help message and exit |
| """ |
| |
| import datetime |
| import errno |
| import logging |
| import optparse |
| import os |
| import re |
| # Needed in Linux so that PyCurl does not throw a segmentation fault. |
| import signal |
| import sys |
| import tempfile |
| import threading |
| import time |
| import urlparse |
| |
| import httplib2 |
| from lxml import html, etree |
| import pycurl |
| |
| REGISTER_PAGE_DIR = os.path.join(os.pardir, 'test', 'data', 'autofill', |
| 'heuristics', 'input') |
| NOT_FOUND_REG_PAGE_SITES_FILENAME = 'notFoundRegPageSites.txt' |
| |
| FORM_LOCATION_COMMENT = 'Form Location: %s' |
| HTML_FILE_PREFIX = 'grabber-' |
| |
| MAX_REDIRECTIONS = 10 |
| |
| # Strings in a webpage that are indicative of a registration link. |
| LINK_CLUES = ['regist', 'user', 'sign', 'login', 'account'] |
| |
| MAX_SAME_DOMAIN_URLS_NO = 30 |
| MAX_TOTAL_URLS_PER_DOMAIN = 300 |
| MAX_OPEN_FILES_NO = 500 |
| |
| # URLs are selected for downloading with the following rules from the link |
| # lists, giving more weight to the links that contain a link clue. |
| CLUE_SECURE_LINKS_NO = MAX_SAME_DOMAIN_URLS_NO * 3/10 |
| CLUE_GENERAL_LINKS_NO = MAX_SAME_DOMAIN_URLS_NO * 3/10 |
| SECURE_LINKS_NO = MAX_SAME_DOMAIN_URLS_NO * 2/10 |
| GENERAL_LINKS_NO = MAX_SAME_DOMAIN_URLS_NO * 2/10 |
| |
| MAX_ALLOWED_THREADS = MAX_OPEN_FILES_NO / MAX_SAME_DOMAIN_URLS_NO + 1 |
| |
| |
| class Retriever(object): |
| """Download, parse, and check if the web page contains a registration form. |
| |
| The objects of this class has a one to one relation with the web pages. For |
| each page that is downloaded and parsed an object of this class is created. |
| Each Retriever object creates a curl object. This object is added to the curl |
| multi object of the crawler object so that the corresponding pages gets |
| downloaded. |
| """ |
| logger = logging.getLogger(__name__) |
| |
| def __init__(self, url, domain, cookie_file): |
| """Initializes a Retriever object. |
| |
| Args: |
| url: url to download page from. |
| domain: only links with this domain will be retrieved. |
| cookie_file: the name of a cookie file, needed for pages that use session |
| cookies to change their contents. |
| """ |
| self._url = url |
| self._domain = domain |
| self._html_content = '' |
| |
| # Http links without clues from LINK_CLUES. |
| self._general_links = [] |
| # Http links that contain a clue from LINK_CLUES. |
| self._clues_general_links = [] |
| # Https links that do not contain any clues from LINK_CLUES. |
| self._secure_links = [] |
| # Https links that contain a clue from LINK_CLUES. |
| self._clues_secure_links = [] |
| self._cookie_file = cookie_file |
| self._curl_object = None |
| |
| def __del__(self): |
| """Cleans up before this object is destroyed. |
| |
| The function closes the corresponding curl object that does the downloading. |
| """ |
| if self._curl_object: |
| self._curl_object.close() |
| |
| def _AddLink(self, link): |
| """Adds url |link|, if not already present, to the appropriate list. |
| |
| The link only gets added to the single list that is appopriate for it: |
| _secure_links, _general_links, _clues_secure_links or _clues_general_links. |
| |
| Args: |
| link: the url that is inserted to the appropriate links list. |
| """ |
| # Handles sites with unicode URLs. |
| if isinstance(link, unicode): |
| # Encode in 'utf-8' to avoid the UnicodeEncodeError exception. |
| link = httplib2.iri2uri(link).encode('utf-8') |
| link_parsed = urlparse.urlparse(link) |
| link_lists = [self._clues_secure_links, self._secure_links, |
| self._clues_general_links, self._general_links] |
| # Checks that the registration page is within the domain. |
| if (self._domain in link_parsed[1] and |
| all(link not in x for x in link_lists)): |
| for clue in LINK_CLUES: |
| if clue in link.lower(): |
| if link_parsed[0].startswith('https'): |
| self._clues_secure_links.append(link) |
| return |
| else: |
| self._clues_general_links.append(link) |
| return |
| if link_parsed[0].startswith('https'): # No clues found in the link. |
| self._secure_links.append(link) |
| else: |
| self._general_links.append(link) |
| |
| def ParseAndGetLinks(self): |
| """Parses downloaded page and gets url link for non registration page. |
| |
| Checks if current page contains a registration page and if not it gets |
| the url links. If it is a registration page, it saves it in a file as |
| 'grabber-' + domain + '.html' after it has added the FORM_LOCATION_COMMENT |
| and it returns True. Otherwise it returns False. |
| |
| Returns: |
| True if current page contains a registration form, and False otherwise. |
| |
| Raises: |
| IOError: When can't write to the file. |
| """ |
| if not self._domain: |
| self.logger.error('Error: self._domain was not set') |
| sys.exit(1) |
| match_list = re.findall(r'(?P<quote>[\'\"])(?P<link>(?:https?:)?//.*?)\1', |
| self._html_content) |
| for group_list in match_list: |
| link = group_list[1] |
| if link.startswith('//'): |
| link = urlparse.urljoin(self._url, link) |
| self._AddLink(link) |
| try: |
| tree = html.fromstring(self._html_content, parser=html.HTMLParser()) |
| except etree.LxmlError: |
| self.logger.info('\t\tSkipping: not valid HTML code in this page <<< %s', |
| self._url) |
| return False |
| try: |
| body = tree.iter('body').next() |
| except StopIteration: |
| self.logger.info('\t\tSkipping: no "BODY" tag in this page <<< %s', |
| self._url) |
| return False |
| |
| # Get a list of all input elements with attribute type='password' |
| password_elements = list(body.iterfind('.//input[@type="password"]')) |
| # Check for multiple password elements to distinguish between a login form |
| # and a registration form (Password field and Confirm Password field). |
| if password_elements and len(password_elements) >= 2: |
| form_elements = [] |
| for password_elem in password_elements: |
| form_elem = password_elem.xpath('ancestor::form[1]') |
| if not form_elem: |
| continue |
| if not form_elem[0] in form_elements: |
| form_elements.append(form_elem[0]) |
| else: |
| # Confirms that the page contains a registration form if two passwords |
| # are contained in the same form for form_elem[0]. |
| if not os.path.isdir(REGISTER_PAGE_DIR): |
| os.makedirs(REGISTER_PAGE_DIR) |
| # Locate the HTML tag and insert the form location comment after it. |
| html_tag = tree.iter('html').next() |
| comment = etree.Comment(FORM_LOCATION_COMMENT % self._url) |
| html_tag.insert(0, comment) |
| # Create a new file and save the HTML registration page code. |
| f = open('%s/%s%s.html' % (REGISTER_PAGE_DIR, HTML_FILE_PREFIX, |
| self._domain), 'w') |
| try: |
| f.write(html.tostring(tree, pretty_print=True)) |
| except IOError as e: |
| self.logger.error('Error: %s', e) |
| raise |
| finally: |
| f.close() |
| return True # Registration page found. |
| # Indicates page is not a registration page and links must be parsed. |
| link_elements = list(body.iter('a')) |
| for link_elem in link_elements: |
| link = link_elem.get('href') |
| if not link or '#' == link[0]: |
| continue |
| link = urlparse.urljoin(self._url, link) |
| link_parsed = urlparse.urlparse(link) |
| if not link_parsed[0].startswith('http'): |
| continue |
| self._AddLink(link) |
| return False # Registration page not found. |
| |
| def InitRequestHead(self): |
| """Initializes curl object for a HEAD request. |
| |
| A HEAD request is initiated so that we can check from the headers if this is |
| a valid HTML file. If it is not a valid HTML file, then we do not initiate a |
| GET request, saving any unnecessary downloadings. |
| """ |
| self._curl_object = pycurl.Curl() |
| self._curl_object.setopt(pycurl.URL, self._url) |
| # The following line fixes the GnuTLS package error that pycurl depends |
| # on for getting https pages. |
| self._curl_object.setopt(pycurl.SSLVERSION, pycurl.SSLVERSION_SSLv3) |
| self._curl_object.setopt(pycurl.FOLLOWLOCATION, True) |
| self._curl_object.setopt(pycurl.NOBODY, True) |
| self._curl_object.setopt(pycurl.SSL_VERIFYPEER, False); |
| self._curl_object.setopt(pycurl.MAXREDIRS, MAX_REDIRECTIONS) |
| self._curl_object.setopt(pycurl.FAILONERROR, False) |
| self._curl_object.setopt(pycurl.COOKIEFILE, self._cookie_file) |
| self._curl_object.setopt(pycurl.COOKIEJAR, self._cookie_file) |
| self._curl_object.setopt(pycurl.CONNECTTIMEOUT, 30) |
| self._curl_object.setopt(pycurl.TIMEOUT, 300) |
| self._curl_object.setopt(pycurl.NOSIGNAL, 1) |
| |
| def InitRequestGet(self): |
| """Initializes curl object for a GET request. |
| |
| This is called only for valid HTML files. The Pycurl makes a GET request. |
| The page begins to download, but since not all the data of the pages comes |
| at once. When some of the data on the page is downloaded Pycurl will put |
| this data in the buffer. The data is appended to the end of the page until |
| everything is downloaded. |
| """ |
| self._curl_object.setopt(pycurl.NOBODY, False) |
| self._curl_object.setopt( |
| pycurl.WRITEFUNCTION, lambda buff: setattr( |
| self, '_html_content', self._html_content + buff)) |
| |
| def Download(self): |
| """Downloads the self._url page. |
| |
| It first does a HEAD request and then it proceeds to a GET request. |
| It uses a curl object for a single download. This function is called only |
| once for the initial url of a site when we still don't have more urls from a |
| domain. |
| |
| Returns: |
| True, if the downloaded page is valid HTML code, or False otherwise. |
| """ |
| self.InitRequestHead() |
| try: |
| self._curl_object.perform() |
| except pycurl.error as e: |
| self.logger.error('Error: %s, url: %s', e, self._url) |
| return False |
| self._url = urlparse.urljoin( |
| self._url, self._curl_object.getinfo(pycurl.EFFECTIVE_URL)) |
| content_type = self._curl_object.getinfo(pycurl.CONTENT_TYPE) |
| if content_type and ('text/html' in content_type.lower()): |
| self.InitRequestGet() |
| try: |
| self._curl_object.perform() |
| except pycurl.error as e: |
| self.logger.error('Error: %s, url: %s', e, self._url) |
| return False |
| return True |
| else: |
| self.logger.info('\tSkipping: Not an HTML page <<< %s', self._url) |
| return False |
| |
| def Run(self): |
| """Called only once for the initial url when we do not have more urls. |
| |
| Downloads the originally-specified site url, parses it and gets the links. |
| |
| Returns: |
| True, if a registration page is found, and False otherwise. |
| """ |
| if self.Download(): |
| if not self._domain: |
| url_parsed = urlparse.urlparse(self._url) |
| self._domain = url_parsed[1] |
| if self._domain.startswith('www'): |
| self._domain = '.'.join(self._domain.split('.')[1:]) |
| if self.ParseAndGetLinks(): |
| return True |
| return False |
| |
| |
| class Crawler(object): |
| """Crawls a site until a registration page is found or max level is reached. |
| |
| Creates, uses and destroys Retriever objects. Creates a cookie temp file |
| needed for session cookies. It keeps track of 'visited links' and |
| 'links to visit' of the site. To do this it uses the links discovered from |
| each Retriever object. Use Run() to crawl the site. |
| """ |
| try: |
| signal.signal(signal.SIGPIPE, signal.SIG_IGN) |
| except ImportError: |
| pass |
| logger = logging.getLogger(__name__) |
| |
| def __init__(self, url, logging_level=None): |
| """Init crawler URL, links lists, logger, and creates a cookie temp file. |
| |
| The cookie temp file is needed for session cookies. |
| |
| Args: |
| url: the initial "seed" url of the site. |
| logging_level: the desired verbosity level, default is None. |
| """ |
| if logging_level: |
| self.logger.setLevel(logging_level) |
| |
| self.url_error = False |
| url_parsed = urlparse.urlparse(url) |
| if not url_parsed[0].startswith('http'): |
| self.logger.error( |
| 'Error: "%s" does not begin with http:// or https://', url) |
| self.url_error = True |
| return |
| # Example: if url is 'http://www.example.com?name=john' then value [1] or |
| # network location is 'www.example.com'. |
| if not url_parsed[1]: |
| self.logger.error('Error: "%s" is not a valid url', url) |
| self.url_error = True |
| return |
| self._url = url |
| self._domain = '' |
| # Http links that contain a clue from LINK_CLUES. |
| self._clues_general_links = [] |
| # Http links that do not contain any clue from LINK_CLUES. |
| self._general_links = [] |
| # Https links that contain a clue from LINK_CLUES. |
| self._clues_secure_links = [] |
| # Https links that do not contain any clue from LINK_CLUES. |
| self._secure_links = [] |
| # All links downloaded and parsed so far. |
| self._links_visited = [] |
| self._retrievers_list = [] |
| self._cookie_file = tempfile.NamedTemporaryFile( |
| suffix='.cookie', delete=False) |
| self._cookie_file.close() |
| self._cookie_file = self._cookie_file.name # Keep only the filename. |
| |
| def __del__(self): |
| """Deletes cookie file when Crawler instances are destroyed.""" |
| if hasattr(self, '_cookie_file'): |
| self.logger.info('Deleting cookie file %s ...', self._cookie_file) |
| os.unlink(self._cookie_file) |
| |
| def _MultiPerform(self, curl_multi_object): |
| """Performs concurrent downloads using a CurlMulti object. |
| |
| Args: |
| curl_multi_object: a curl object that downloads multiple pages |
| concurrently. The class of this object is |pycurl.CurlMulti|. |
| """ |
| # Following code uses the example from section for the CurlMulti object |
| # at http://pycurl.sourceforge.net/doc/curlmultiobject.html. |
| while True: |
| ret, no_handles = curl_multi_object.perform() |
| if ret != pycurl.E_CALL_MULTI_PERFORM: |
| break |
| while no_handles: |
| curl_multi_object.select(1.0) |
| while True: |
| ret, no_handles = curl_multi_object.perform() |
| if ret != pycurl.E_CALL_MULTI_PERFORM: |
| break |
| |
| def _GetLinksPages(self, curl_multi_object): |
| """Downloads many pages concurrently using a CurlMulti Object. |
| |
| Creates many Retriever objects and adds them to a list. The constant |
| MAX_SAME_DOMAIN_URLS_NO defines the number of pages that can be downloaded |
| concurrently from the same domain using the pycurl multi object. It's |
| currently set to 30 URLs. These URLs are taken from the links lists, which |
| are from csl, gcl, sl, and gl. The rules define how many URLs are taken from |
| each list during each iteration. |
| |
| Example of the rules: |
| 3/10 from csl results in 9 URLs |
| 3/10 from cgl results in 9 URLs |
| 2/10 from sl results in 6 URLs |
| 2/10 from gl results in 6 URLs |
| |
| Adding up the above URLs gives 30 URLs that can be downloaded concurrently. |
| If these lists have fewer items than the defined rules, such as if a site |
| does not contain any secure links, then csl and sl lists will be of 0 length |
| and only 15 pages would be downloaded concurrently from the same domain. |
| |
| Since 30 URLs can be handled concurrently, the number of links taken from |
| other lists can be increased. This means that we can take 24 links from the |
| cgl list so that 24 from gfl + 6 from gl = 30 URLs. If the cgl list has less |
| than 24 links, e.g. there are only 21 links, then only 9 links may be taken |
| from gl so ) + 21 + 0 + 9 = 30. |
| |
| Args: |
| curl_multi_object: Each Retriever object has a curl object which is |
| added to the CurlMulti Object. |
| """ |
| self._retrievers_list = [] |
| |
| csl_no = min(CLUE_SECURE_LINKS_NO, len(self._clues_secure_links)) |
| cgl_no = min(CLUE_GENERAL_LINKS_NO, len(self._clues_general_links)) |
| sl_no = min(SECURE_LINKS_NO, len(self._secure_links)) |
| gl_no = min(GENERAL_LINKS_NO, len(self._general_links)) |
| |
| # If some links within the list have fewer items than needed, the missing |
| # links will be taken by the following priority: csl, cgl, sl, gl. |
| # c: clues, s: secure, g: general, l: list. |
| spare_links = MAX_SAME_DOMAIN_URLS_NO - (csl_no + sl_no + cgl_no + gl_no) |
| if spare_links > 0: |
| csl_no = min(csl_no + spare_links, len(self._clues_secure_links)) |
| spare_links = MAX_SAME_DOMAIN_URLS_NO - (csl_no + sl_no + cgl_no + gl_no) |
| if spare_links > 0: |
| cgl_no = min(cgl_no + spare_links, len(self._clues_general_links)) |
| spare_links = MAX_SAME_DOMAIN_URLS_NO - (csl_no + sl_no + cgl_no + gl_no) |
| if spare_links > 0: |
| sl_no = min(sl_no + spare_links, len(self._secure_links)) |
| spare_links = MAX_SAME_DOMAIN_URLS_NO - (csl_no + sl_no + cgl_no + gl_no) |
| if spare_links > 0: |
| gl_no = min(gl_no + spare_links, len(self._general_links)) |
| |
| for no_of_links, links in [ |
| (csl_no, self._clues_secure_links), |
| (sl_no, self._secure_links), |
| (cgl_no, self._clues_general_links), |
| (gl_no, self._general_links)]: |
| for i in xrange(no_of_links): |
| if not links: |
| break |
| url = links.pop(0) |
| self._links_visited.append(url) |
| r = Retriever(url, self._domain, self._cookie_file) |
| r.InitRequestHead() |
| curl_multi_object.add_handle(r._curl_object) |
| self._retrievers_list.append(r) |
| |
| if self._retrievers_list: |
| try: |
| self._MultiPerform(curl_multi_object) |
| except pycurl.error as e: |
| self.logger.error('Error: %s, url: %s', e, self._url) |
| finally: |
| for r in self._retrievers_list: |
| curl_multi_object.remove_handle(r._curl_object) |
| # |_retrievers_list[:]| is a copy of |_retrievers_list| to avoid removing |
| # items from the iterated list. |
| for r in self._retrievers_list[:]: |
| r._url = urlparse.urljoin(r._url, r._curl_object.getinfo( |
| pycurl.EFFECTIVE_URL)) |
| content_type = r._curl_object.getinfo(pycurl.CONTENT_TYPE) |
| if content_type and ('text/html' in content_type.lower()): |
| r.InitRequestGet() |
| curl_multi_object.add_handle(r._curl_object) |
| else: |
| self._retrievers_list.remove(r) |
| self.logger.info('\tSkipping: Not an HTML page <<< %s', r._url) |
| if self._retrievers_list: |
| try: |
| self._MultiPerform(curl_multi_object) |
| except pycurl.error as e: |
| self.logger.error('Error: %s, url: %s', e, self._url) |
| finally: |
| for r in self._retrievers_list: |
| curl_multi_object.remove_handle(r._curl_object) |
| self.logger.info('Downloaded: %s', r._url) |
| |
| def _LogRegPageFound(self, retriever): |
| """Display logging for registration page found. |
| |
| Args: |
| retriever: The object that has retrieved the page. |
| """ |
| self.logger.info('\t##############################################') |
| self.logger.info('\t### %s ###', retriever._domain) |
| self.logger.info('\t##############################################') |
| self.logger.info('\t!!!!!!!!! registration page FOUND !!!!!!!!!!!') |
| self.logger.info('\t%s', retriever._url) |
| self.logger.info('\t##############################################') |
| |
| def _GetNewLinks(self, retriever): |
| """Appends new links discovered by each retriever to the appropriate lists. |
| |
| Links are copied to the links list of the crawler object, which holds all |
| the links found from all retrievers that the crawler object created. The |
| Crawler object exists as far as a specific site is examined and the |
| Retriever object exists as far as a page of this site is examined. |
| |
| Args: |
| retriever: a temporary object that downloads a specific page, parses the |
| content and gets the page's href link. |
| """ |
| for link in retriever._clues_secure_links: |
| if (not link in self._clues_secure_links and |
| not link in self._links_visited): |
| self._clues_secure_links.append(link) |
| for link in retriever._secure_links: |
| if (not link in self._secure_links and |
| not link in self._links_visited): |
| self._secure_links.append(link) |
| for link in retriever._clues_general_links: |
| if (not link in self._clues_general_links and |
| not link in self._links_visited): |
| self._clues_general_links.append(link) |
| for link in retriever._general_links: |
| if (not link in self._general_links and |
| not link in self._links_visited): |
| self._general_links.append(link) |
| |
| def Run(self): |
| """Runs the Crawler. |
| |
| Creates a Retriever object and calls its run method to get the first links, |
| and then uses CurlMulti object and creates many Retriever objects to get |
| the subsequent pages. |
| |
| The number of pages (=Retriever objs) created each time is restricted by |
| MAX_SAME_DOMAIN_URLS_NO. After this number of Retriever objects download |
| and parse their pages, we do the same again. The number of total pages |
| visited is kept in urls_visited. |
| If no registration page is found, the Crawler object will give up its try |
| after MAX_TOTAL_URLS_PER_DOMAIN is reached. |
| |
| Returns: |
| True is returned if registration page is found, or False otherwise. |
| """ |
| reg_page_found = False |
| if self.url_error: |
| return False |
| r = Retriever(self._url, self._domain, self._cookie_file) |
| if r.Run(): |
| self._LogRegPageFound(r) |
| reg_page_found = True |
| else: |
| self._url = r._url |
| self._domain = r._domain |
| self.logger.info('url to crawl: %s', self._url) |
| self.logger.info('domain: %s', self._domain) |
| self._links_visited.append(r._url) |
| self._GetNewLinks(r) |
| urls_visited = 1 |
| while True: |
| if (not (self._clues_secure_links or self._secure_links or |
| self._clues_general_links or self._general_links) or |
| urls_visited >= MAX_TOTAL_URLS_PER_DOMAIN): |
| break # Registration page not found. |
| m = pycurl.CurlMulti() |
| self._GetLinksPages(m) |
| urls_visited += len(self._retrievers_list) |
| self.logger.info('\t<----- URLs visited for domain "%s": %d ----->', |
| self._domain, urls_visited) |
| for r in self._retrievers_list: |
| if r.ParseAndGetLinks(): |
| self._LogRegPageFound(r) |
| reg_page_found = True |
| break |
| else: |
| self.logger.info('parsed: %s', r._url) |
| self._GetNewLinks(r) |
| m.close() |
| if reg_page_found: |
| break |
| while self._retrievers_list: |
| r = self._retrievers_list.pop() |
| return reg_page_found |
| |
| |
| class WorkerThread(threading.Thread): |
| """Creates a new thread of execution.""" |
| def __init__(self, url): |
| """Creates _url and page_found attri to populate urls_with_no_reg_page file. |
| |
| Used after thread's termination for the creation of a file with a list of |
| the urls for which a registration page wasn't found. |
| |
| Args: |
| url: will be used as an argument to create a Crawler object later. |
| """ |
| threading.Thread.__init__(self) |
| self._url = url |
| self.page_found = False |
| |
| def run(self): |
| """Execution of thread creates a Crawler object and runs it. |
| |
| Caution: this function name should not be changed to 'Run' or any other |
| names because it is overriding the 'run' method of the 'threading.Thread' |
| class. Otherwise it will never be called. |
| """ |
| self.page_found = Crawler(self._url).Run() |
| |
| |
| class ThreadedCrawler(object): |
| """Calls the Run function of WorkerThread which creates & runs a Crawler obj. |
| |
| The crawler object runs concurrently, examining one site each. |
| """ |
| logger = logging.getLogger(__name__) |
| |
| def __init__(self, urls_file, logging_level=None): |
| """Creates threaded Crawler objects. |
| |
| Args: |
| urls_file: a text file containing a URL in each line. |
| logging_level: verbosity level, default is None. |
| |
| Raises: |
| IOError: If cannot find URLs from the list. |
| """ |
| if logging_level: |
| self.logger.setLevel(logging_level) |
| |
| self._urls_list = [] |
| f = open(urls_file) |
| try: |
| for url in f.readlines(): |
| url = url.strip() |
| if not urlparse.urlparse(url)[0].startswith('http'): |
| self.logger.info( |
| '%s: skipping this (does not begin with "http://")', url) |
| continue |
| self._urls_list.append(url) |
| except IOError as e: |
| self.logger.error('Error: %s', e) |
| raise |
| finally: |
| f.close() |
| if not self._urls_list: |
| error_msg = 'No URLs were found.' |
| self.logger.error('ERROR: %s', error_msg) |
| raise IOError(error_msg) |
| |
| def Run(self): |
| """Runs Crawler objects using python threads. |
| |
| Number of concurrent threads is restricted to MAX_ALLOWED_THREADS. |
| |
| Returns: |
| The number of registration pages found. -1 if no URLs are given. |
| |
| Raises: |
| OSError: When creating the same directory that already exists. |
| """ |
| if self._urls_list: |
| allThreads = [] |
| # originalNumThreads is the number of threads just before the |
| # ThreadedCrawler starts creating new threads. As a standalone script it |
| # will be 1. |
| originalNumThreads = threading.active_count() |
| for url in self._urls_list: |
| self.logger.info('URL fed to a crawler thread: %s', url) |
| t = WorkerThread(url) |
| t.start() |
| allThreads.append(t) |
| while threading.active_count() >= ( |
| MAX_ALLOWED_THREADS + originalNumThreads): |
| time.sleep(.4) |
| while threading.active_count() > originalNumThreads: |
| time.sleep(.4) |
| self.logger.info('----------------') |
| self.logger.info('--- FINISHED ---') |
| self.logger.info('----------------') |
| urls_no = 0 |
| urls_not_found_no = 0 |
| not_file_name = os.path.join( |
| REGISTER_PAGE_DIR, NOT_FOUND_REG_PAGE_SITES_FILENAME) |
| not_file_dir = os.path.dirname(not_file_name) |
| try: |
| os.makedirs(not_file_dir) |
| except OSError as e: |
| if e.errno != errno.EEXIST: |
| raise |
| fnot = open(not_file_name, 'wb') |
| try: |
| for t in sorted(allThreads, key=lambda t: t._url): |
| urls_no += 1 |
| if not t.page_found: |
| urls_not_found_no += 1 |
| fnot.write('%s' % t._url) |
| fnot.write(os.linesep) |
| except IOError as e: |
| self.logger.error('Error: %s', e) |
| finally: |
| fnot.close() |
| self.logger.info('Total number of URLs given: %d\n', urls_no) |
| self.logger.info( |
| 'Registration pages found: %d\n', (urls_no - urls_not_found_no)) |
| self.logger.info( |
| 'URLs that did not return a registration page: %d\n', |
| urls_not_found_no) |
| return urls_no - urls_not_found_no |
| else: |
| self.logger.error('Error: no URLs were found.') |
| return -1 |
| |
| |
| def main(): |
| usage = 'usage: %prog [options] single_url_or_urls_filename' |
| parser = optparse.OptionParser(usage) |
| parser.add_option( |
| '-l', '--log_level', metavar='LOG_LEVEL', default='error', |
| help='LOG_LEVEL: debug, info, warning or error [default: %default]') |
| |
| (options, args) = parser.parse_args() |
| options.log_level = options.log_level.upper() |
| if options.log_level not in ['DEBUG', 'INFO', 'WARNING', 'ERROR']: |
| print 'Wrong log_level argument.' |
| parser.print_help() |
| return 1 |
| options.log_level = getattr(logging, options.log_level) |
| |
| if len(args) != 1: |
| parser.error('Wrong number of arguments.') |
| |
| logger = logging.getLogger(__name__) |
| if options.log_level: |
| console = logging.StreamHandler() |
| logger.addHandler(console) |
| logger.setLevel(options.log_level) |
| |
| arg_is_a_file = os.path.isfile(args[0]) |
| if arg_is_a_file: |
| CrawlerClass = ThreadedCrawler |
| else: |
| CrawlerClass = Crawler |
| t0 = datetime.datetime.now() |
| c = CrawlerClass(args[0], options.log_level) |
| c.Run() |
| if not arg_is_a_file and c.url_error: |
| logger.error( |
| 'ERROR: "%s" is neither a valid filename nor a valid URL' % args[0]) |
| t1 = datetime.datetime.now() |
| delta_t = t1 - t0 |
| logger.info('Started at: %s\n', t0) |
| logger.info('Ended at: %s\n', t1) |
| logger.info('Total execution time: %s\n', delta_t) |
| return 0 |
| |
| |
| if __name__ == "__main__": |
| sys.exit(main()) |
