Minijack: Add Google App Engine support for frontend
Frontend now works on Google App Engine, backed by BigQuery as database.
Library added: apiclient/, gflags.py, gflags_validators.py, httplib2/,
oauth2client/, uritemplate/
dump_db.py would convert data in sqlite database to json format, that
can be imported to BigQuery. There should be 4 tables named Device,
Test, Event, Component, and the schema of the tables should be as in
schemas/device.json, schemas/test.json, schemas/event.json,
schemas/component.json. The script to progressively add data to BigQuery
is not available yet.
BUG=None
TEST=Manual
Start minijack frontend by manage.py, verify everything still works.
Dump data to BigQuery, add privatekey.pem, settings_bigquery.py according
to README, deploy minijack to Google App Engine, verify
everything works as in local server.
Change-Id: I93323279cd1e1dee13694dd6ec9c0faa7c2e396f
Reviewed-on: https://chromium-review.googlesource.com/65829
Reviewed-by: Tom Wai-Hong Tam <waihong@chromium.org>
Commit-Queue: Pi-Hsun Shih <pihsun@chromium.org>
Tested-by: Pi-Hsun Shih <pihsun@chromium.org>
diff --git a/py/minijack/.gitignore b/py/minijack/.gitignore
new file mode 100644
index 0000000..a645c01
--- /dev/null
+++ b/py/minijack/.gitignore
@@ -0,0 +1,4 @@
+*.p12
+*.pem
+settings_bigquery.py
+app.yaml
diff --git a/py/minijack/README b/py/minijack/README
new file mode 100644
index 0000000..58dee1b
--- /dev/null
+++ b/py/minijack/README
@@ -0,0 +1,18 @@
+* Instructions for deploying on Google App Engine:
+We need a Google API Service Account's PEM key on GAE.
+
+Generate and download the PKCS12 format private key from API console, and
+execute `openssl pkcs12 -in privatekey.p12 -nodes -nocerts > privatekey.pem`
+to convert the key from PKCS12 to PEM. Then manually delete anything before
+"-----BEGIN PRIVATE KEY-----" from privatekey.pem.
+
+Use dump_db.py to dump all datas to json files, and upload them with schemas
+in the directory schemas/ to Google BigQuery tables. The tables must be named
+Device, Test, Event, Component (case sensitive)
+
+Copy settings_bigquery.py.TEMPLATE to settings_bigquery.py, and set the settings
+accordingly.
+
+Copy app.yaml.TEMPLATE to app.yaml, and change the application name to the real
+application name deploying to.
+
diff --git a/py/minijack/apiclient/__init__.py b/py/minijack/apiclient/__init__.py
new file mode 100644
index 0000000..f901408
--- /dev/null
+++ b/py/minijack/apiclient/__init__.py
@@ -0,0 +1 @@
+__version__ = "1.1"
diff --git a/py/minijack/apiclient/discovery.py b/py/minijack/apiclient/discovery.py
new file mode 100644
index 0000000..4c6cb60
--- /dev/null
+++ b/py/minijack/apiclient/discovery.py
@@ -0,0 +1,959 @@
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Client for discovery based APIs.
+
+A client library for Google's discovery based APIs.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+__all__ = [
+ 'build',
+ 'build_from_document',
+ 'fix_method_name',
+ 'key2param',
+ ]
+
+
+# Standard library imports
+import copy
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+import keyword
+import logging
+import mimetypes
+import os
+import re
+import urllib
+import urlparse
+
+try:
+ from urlparse import parse_qsl
+except ImportError:
+ from cgi import parse_qsl
+
+# Third-party imports
+import httplib2
+import mimeparse
+import uritemplate
+
+# Local imports
+from apiclient.errors import HttpError
+from apiclient.errors import InvalidJsonError
+from apiclient.errors import MediaUploadSizeError
+from apiclient.errors import UnacceptableMimeTypeError
+from apiclient.errors import UnknownApiNameOrVersion
+from apiclient.errors import UnknownFileType
+from apiclient.http import HttpRequest
+from apiclient.http import MediaFileUpload
+from apiclient.http import MediaUpload
+from apiclient.model import JsonModel
+from apiclient.model import MediaModel
+from apiclient.model import RawModel
+from apiclient.schema import Schemas
+from oauth2client.anyjson import simplejson
+from oauth2client.util import _add_query_parameter
+from oauth2client.util import positional
+
+
+# The client library requires a version of httplib2 that supports RETRIES.
+httplib2.RETRIES = 1
+
+logger = logging.getLogger(__name__)
+
+URITEMPLATE = re.compile('{[^}]*}')
+VARNAME = re.compile('[a-zA-Z0-9_-]+')
+DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
+ '{api}/{apiVersion}/rest')
+DEFAULT_METHOD_DOC = 'A description of how to use this function'
+HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH'])
+_MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40}
+BODY_PARAMETER_DEFAULT_VALUE = {
+ 'description': 'The request body.',
+ 'type': 'object',
+ 'required': True,
+}
+MEDIA_BODY_PARAMETER_DEFAULT_VALUE = {
+ 'description': ('The filename of the media request body, or an instance '
+ 'of a MediaUpload object.'),
+ 'type': 'string',
+ 'required': False,
+}
+
+# Parameters accepted by the stack, but not visible via discovery.
+# TODO(dhermes): Remove 'userip' in 'v2'.
+STACK_QUERY_PARAMETERS = frozenset(['trace', 'pp', 'userip', 'strict'])
+STACK_QUERY_PARAMETER_DEFAULT_VALUE = {'type': 'string', 'location': 'query'}
+
+# Library-specific reserved words beyond Python keywords.
+RESERVED_WORDS = frozenset(['body'])
+
+
+def fix_method_name(name):
+ """Fix method names to avoid reserved word conflicts.
+
+ Args:
+ name: string, method name.
+
+ Returns:
+ The name with a '_' prefixed if the name is a reserved word.
+ """
+ if keyword.iskeyword(name) or name in RESERVED_WORDS:
+ return name + '_'
+ else:
+ return name
+
+
+def key2param(key):
+ """Converts key names into parameter names.
+
+ For example, converting "max-results" -> "max_results"
+
+ Args:
+ key: string, the method key name.
+
+ Returns:
+ A safe method name based on the key name.
+ """
+ result = []
+ key = list(key)
+ if not key[0].isalpha():
+ result.append('x')
+ for c in key:
+ if c.isalnum():
+ result.append(c)
+ else:
+ result.append('_')
+
+ return ''.join(result)
+
+
+@positional(2)
+def build(serviceName,
+ version,
+ http=None,
+ discoveryServiceUrl=DISCOVERY_URI,
+ developerKey=None,
+ model=None,
+ requestBuilder=HttpRequest):
+ """Construct a Resource for interacting with an API.
+
+ Construct a Resource object for interacting with an API. The serviceName and
+ version are the names from the Discovery service.
+
+ Args:
+ serviceName: string, name of the service.
+ version: string, the version of the service.
+ http: httplib2.Http, An instance of httplib2.Http or something that acts
+ like it that HTTP requests will be made through.
+ discoveryServiceUrl: string, a URI Template that points to the location of
+ the discovery service. It should have two parameters {api} and
+ {apiVersion} that when filled in produce an absolute URI to the discovery
+ document for that service.
+ developerKey: string, key obtained from
+ https://code.google.com/apis/console.
+ model: apiclient.Model, converts to and from the wire format.
+ requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP
+ request.
+
+ Returns:
+ A Resource object with methods for interacting with the service.
+ """
+ params = {
+ 'api': serviceName,
+ 'apiVersion': version
+ }
+
+ if http is None:
+ http = httplib2.Http()
+
+ requested_url = uritemplate.expand(discoveryServiceUrl, params)
+
+ # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
+ # variable that contains the network address of the client sending the
+ # request. If it exists then add that to the request for the discovery
+ # document to avoid exceeding the quota on discovery requests.
+ if 'REMOTE_ADDR' in os.environ:
+ requested_url = _add_query_parameter(requested_url, 'userIp',
+ os.environ['REMOTE_ADDR'])
+ logger.info('URL being requested: %s' % requested_url)
+
+ resp, content = http.request(requested_url)
+
+ if resp.status == 404:
+ raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
+ version))
+ if resp.status >= 400:
+ raise HttpError(resp, content, uri=requested_url)
+
+ try:
+ service = simplejson.loads(content)
+ except ValueError, e:
+ logger.error('Failed to parse as JSON: ' + content)
+ raise InvalidJsonError()
+
+ return build_from_document(content, base=discoveryServiceUrl, http=http,
+ developerKey=developerKey, model=model, requestBuilder=requestBuilder)
+
+
+@positional(1)
+def build_from_document(
+ service,
+ base=None,
+ future=None,
+ http=None,
+ developerKey=None,
+ model=None,
+ requestBuilder=HttpRequest):
+ """Create a Resource for interacting with an API.
+
+ Same as `build()`, but constructs the Resource object from a discovery
+ document that is it given, as opposed to retrieving one over HTTP.
+
+ Args:
+ service: string or object, the JSON discovery document describing the API.
+ The value passed in may either be the JSON string or the deserialized
+ JSON.
+ base: string, base URI for all HTTP requests, usually the discovery URI.
+ This parameter is no longer used as rootUrl and servicePath are included
+ within the discovery document. (deprecated)
+ future: string, discovery document with future capabilities (deprecated).
+ http: httplib2.Http, An instance of httplib2.Http or something that acts
+ like it that HTTP requests will be made through.
+ developerKey: string, Key for controlling API usage, generated
+ from the API Console.
+ model: Model class instance that serializes and de-serializes requests and
+ responses.
+ requestBuilder: Takes an http request and packages it up to be executed.
+
+ Returns:
+ A Resource object with methods for interacting with the service.
+ """
+
+ # future is no longer used.
+ future = {}
+
+ if isinstance(service, basestring):
+ service = simplejson.loads(service)
+ base = urlparse.urljoin(service['rootUrl'], service['servicePath'])
+ schema = Schemas(service)
+
+ if model is None:
+ features = service.get('features', [])
+ model = JsonModel('dataWrapper' in features)
+ return Resource(http=http, baseUrl=base, model=model,
+ developerKey=developerKey, requestBuilder=requestBuilder,
+ resourceDesc=service, rootDesc=service, schema=schema)
+
+
+def _cast(value, schema_type):
+ """Convert value to a string based on JSON Schema type.
+
+ See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
+ JSON Schema.
+
+ Args:
+ value: any, the value to convert
+ schema_type: string, the type that value should be interpreted as
+
+ Returns:
+ A string representation of 'value' based on the schema_type.
+ """
+ if schema_type == 'string':
+ if type(value) == type('') or type(value) == type(u''):
+ return value
+ else:
+ return str(value)
+ elif schema_type == 'integer':
+ return str(int(value))
+ elif schema_type == 'number':
+ return str(float(value))
+ elif schema_type == 'boolean':
+ return str(bool(value)).lower()
+ else:
+ if type(value) == type('') or type(value) == type(u''):
+ return value
+ else:
+ return str(value)
+
+
+def _media_size_to_long(maxSize):
+ """Convert a string media size, such as 10GB or 3TB into an integer.
+
+ Args:
+ maxSize: string, size as a string, such as 2MB or 7GB.
+
+ Returns:
+ The size as an integer value.
+ """
+ if len(maxSize) < 2:
+ return 0L
+ units = maxSize[-2:].upper()
+ bit_shift = _MEDIA_SIZE_BIT_SHIFTS.get(units)
+ if bit_shift is not None:
+ return long(maxSize[:-2]) << bit_shift
+ else:
+ return long(maxSize)
+
+
+def _media_path_url_from_info(root_desc, path_url):
+ """Creates an absolute media path URL.
+
+ Constructed using the API root URI and service path from the discovery
+ document and the relative path for the API method.
+
+ Args:
+ root_desc: Dictionary; the entire original deserialized discovery document.
+ path_url: String; the relative URL for the API method. Relative to the API
+ root, which is specified in the discovery document.
+
+ Returns:
+ String; the absolute URI for media upload for the API method.
+ """
+ return '%(root)supload/%(service_path)s%(path)s' % {
+ 'root': root_desc['rootUrl'],
+ 'service_path': root_desc['servicePath'],
+ 'path': path_url,
+ }
+
+
+def _fix_up_parameters(method_desc, root_desc, http_method):
+ """Updates parameters of an API method with values specific to this library.
+
+ Specifically, adds whatever global parameters are specified by the API to the
+ parameters for the individual method. Also adds parameters which don't
+ appear in the discovery document, but are available to all discovery based
+ APIs (these are listed in STACK_QUERY_PARAMETERS).
+
+ SIDE EFFECTS: This updates the parameters dictionary object in the method
+ description.
+
+ Args:
+ method_desc: Dictionary with metadata describing an API method. Value comes
+ from the dictionary of methods stored in the 'methods' key in the
+ deserialized discovery document.
+ root_desc: Dictionary; the entire original deserialized discovery document.
+ http_method: String; the HTTP method used to call the API method described
+ in method_desc.
+
+ Returns:
+ The updated Dictionary stored in the 'parameters' key of the method
+ description dictionary.
+ """
+ parameters = method_desc.setdefault('parameters', {})
+
+ # Add in the parameters common to all methods.
+ for name, description in root_desc.get('parameters', {}).iteritems():
+ parameters[name] = description
+
+ # Add in undocumented query parameters.
+ for name in STACK_QUERY_PARAMETERS:
+ parameters[name] = STACK_QUERY_PARAMETER_DEFAULT_VALUE.copy()
+
+ # Add 'body' (our own reserved word) to parameters if the method supports
+ # a request payload.
+ if http_method in HTTP_PAYLOAD_METHODS and 'request' in method_desc:
+ body = BODY_PARAMETER_DEFAULT_VALUE.copy()
+ body.update(method_desc['request'])
+ parameters['body'] = body
+
+ return parameters
+
+
+def _fix_up_media_upload(method_desc, root_desc, path_url, parameters):
+ """Updates parameters of API by adding 'media_body' if supported by method.
+
+ SIDE EFFECTS: If the method supports media upload and has a required body,
+ sets body to be optional (required=False) instead. Also, if there is a
+ 'mediaUpload' in the method description, adds 'media_upload' key to
+ parameters.
+
+ Args:
+ method_desc: Dictionary with metadata describing an API method. Value comes
+ from the dictionary of methods stored in the 'methods' key in the
+ deserialized discovery document.
+ root_desc: Dictionary; the entire original deserialized discovery document.
+ path_url: String; the relative URL for the API method. Relative to the API
+ root, which is specified in the discovery document.
+ parameters: A dictionary describing method parameters for method described
+ in method_desc.
+
+ Returns:
+ Triple (accept, max_size, media_path_url) where:
+ - accept is a list of strings representing what content types are
+ accepted for media upload. Defaults to empty list if not in the
+ discovery document.
+ - max_size is a long representing the max size in bytes allowed for a
+ media upload. Defaults to 0L if not in the discovery document.
+ - media_path_url is a String; the absolute URI for media upload for the
+ API method. Constructed using the API root URI and service path from
+ the discovery document and the relative path for the API method. If
+ media upload is not supported, this is None.
+ """
+ media_upload = method_desc.get('mediaUpload', {})
+ accept = media_upload.get('accept', [])
+ max_size = _media_size_to_long(media_upload.get('maxSize', ''))
+ media_path_url = None
+
+ if media_upload:
+ media_path_url = _media_path_url_from_info(root_desc, path_url)
+ parameters['media_body'] = MEDIA_BODY_PARAMETER_DEFAULT_VALUE.copy()
+ if 'body' in parameters:
+ parameters['body']['required'] = False
+
+ return accept, max_size, media_path_url
+
+
+def _fix_up_method_description(method_desc, root_desc):
+ """Updates a method description in a discovery document.
+
+ SIDE EFFECTS: Changes the parameters dictionary in the method description with
+ extra parameters which are used locally.
+
+ Args:
+ method_desc: Dictionary with metadata describing an API method. Value comes
+ from the dictionary of methods stored in the 'methods' key in the
+ deserialized discovery document.
+ root_desc: Dictionary; the entire original deserialized discovery document.
+
+ Returns:
+ Tuple (path_url, http_method, method_id, accept, max_size, media_path_url)
+ where:
+ - path_url is a String; the relative URL for the API method. Relative to
+ the API root, which is specified in the discovery document.
+ - http_method is a String; the HTTP method used to call the API method
+ described in the method description.
+ - method_id is a String; the name of the RPC method associated with the
+ API method, and is in the method description in the 'id' key.
+ - accept is a list of strings representing what content types are
+ accepted for media upload. Defaults to empty list if not in the
+ discovery document.
+ - max_size is a long representing the max size in bytes allowed for a
+ media upload. Defaults to 0L if not in the discovery document.
+ - media_path_url is a String; the absolute URI for media upload for the
+ API method. Constructed using the API root URI and service path from
+ the discovery document and the relative path for the API method. If
+ media upload is not supported, this is None.
+ """
+ path_url = method_desc['path']
+ http_method = method_desc['httpMethod']
+ method_id = method_desc['id']
+
+ parameters = _fix_up_parameters(method_desc, root_desc, http_method)
+ # Order is important. `_fix_up_media_upload` needs `method_desc` to have a
+ # 'parameters' key and needs to know if there is a 'body' parameter because it
+ # also sets a 'media_body' parameter.
+ accept, max_size, media_path_url = _fix_up_media_upload(
+ method_desc, root_desc, path_url, parameters)
+
+ return path_url, http_method, method_id, accept, max_size, media_path_url
+
+
+# TODO(dhermes): Convert this class to ResourceMethod and make it callable
+class ResourceMethodParameters(object):
+ """Represents the parameters associated with a method.
+
+ Attributes:
+ argmap: Map from method parameter name (string) to query parameter name
+ (string).
+ required_params: List of required parameters (represented by parameter
+ name as string).
+ repeated_params: List of repeated parameters (represented by parameter
+ name as string).
+ pattern_params: Map from method parameter name (string) to regular
+ expression (as a string). If the pattern is set for a parameter, the
+ value for that parameter must match the regular expression.
+ query_params: List of parameters (represented by parameter name as string)
+ that will be used in the query string.
+ path_params: Set of parameters (represented by parameter name as string)
+ that will be used in the base URL path.
+ param_types: Map from method parameter name (string) to parameter type. Type
+ can be any valid JSON schema type; valid values are 'any', 'array',
+ 'boolean', 'integer', 'number', 'object', or 'string'. Reference:
+ http://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.1
+ enum_params: Map from method parameter name (string) to list of strings,
+ where each list of strings is the list of acceptable enum values.
+ """
+
+ def __init__(self, method_desc):
+ """Constructor for ResourceMethodParameters.
+
+ Sets default values and defers to set_parameters to populate.
+
+ Args:
+ method_desc: Dictionary with metadata describing an API method. Value
+ comes from the dictionary of methods stored in the 'methods' key in
+ the deserialized discovery document.
+ """
+ self.argmap = {}
+ self.required_params = []
+ self.repeated_params = []
+ self.pattern_params = {}
+ self.query_params = []
+ # TODO(dhermes): Change path_params to a list if the extra URITEMPLATE
+ # parsing is gotten rid of.
+ self.path_params = set()
+ self.param_types = {}
+ self.enum_params = {}
+
+ self.set_parameters(method_desc)
+
+ def set_parameters(self, method_desc):
+ """Populates maps and lists based on method description.
+
+ Iterates through each parameter for the method and parses the values from
+ the parameter dictionary.
+
+ Args:
+ method_desc: Dictionary with metadata describing an API method. Value
+ comes from the dictionary of methods stored in the 'methods' key in
+ the deserialized discovery document.
+ """
+ for arg, desc in method_desc.get('parameters', {}).iteritems():
+ param = key2param(arg)
+ self.argmap[param] = arg
+
+ if desc.get('pattern'):
+ self.pattern_params[param] = desc['pattern']
+ if desc.get('enum'):
+ self.enum_params[param] = desc['enum']
+ if desc.get('required'):
+ self.required_params.append(param)
+ if desc.get('repeated'):
+ self.repeated_params.append(param)
+ if desc.get('location') == 'query':
+ self.query_params.append(param)
+ if desc.get('location') == 'path':
+ self.path_params.add(param)
+ self.param_types[param] = desc.get('type', 'string')
+
+ # TODO(dhermes): Determine if this is still necessary. Discovery based APIs
+ # should have all path parameters already marked with
+ # 'location: path'.
+ for match in URITEMPLATE.finditer(method_desc['path']):
+ for namematch in VARNAME.finditer(match.group(0)):
+ name = key2param(namematch.group(0))
+ self.path_params.add(name)
+ if name in self.query_params:
+ self.query_params.remove(name)
+
+
+def createMethod(methodName, methodDesc, rootDesc, schema):
+ """Creates a method for attaching to a Resource.
+
+ Args:
+ methodName: string, name of the method to use.
+ methodDesc: object, fragment of deserialized discovery document that
+ describes the method.
+ rootDesc: object, the entire deserialized discovery document.
+ schema: object, mapping of schema names to schema descriptions.
+ """
+ methodName = fix_method_name(methodName)
+ (pathUrl, httpMethod, methodId, accept,
+ maxSize, mediaPathUrl) = _fix_up_method_description(methodDesc, rootDesc)
+
+ parameters = ResourceMethodParameters(methodDesc)
+
+ def method(self, **kwargs):
+ # Don't bother with doc string, it will be over-written by createMethod.
+
+ for name in kwargs.iterkeys():
+ if name not in parameters.argmap:
+ raise TypeError('Got an unexpected keyword argument "%s"' % name)
+
+ # Remove args that have a value of None.
+ keys = kwargs.keys()
+ for name in keys:
+ if kwargs[name] is None:
+ del kwargs[name]
+
+ for name in parameters.required_params:
+ if name not in kwargs:
+ raise TypeError('Missing required parameter "%s"' % name)
+
+ for name, regex in parameters.pattern_params.iteritems():
+ if name in kwargs:
+ if isinstance(kwargs[name], basestring):
+ pvalues = [kwargs[name]]
+ else:
+ pvalues = kwargs[name]
+ for pvalue in pvalues:
+ if re.match(regex, pvalue) is None:
+ raise TypeError(
+ 'Parameter "%s" value "%s" does not match the pattern "%s"' %
+ (name, pvalue, regex))
+
+ for name, enums in parameters.enum_params.iteritems():
+ if name in kwargs:
+ # We need to handle the case of a repeated enum
+ # name differently, since we want to handle both
+ # arg='value' and arg=['value1', 'value2']
+ if (name in parameters.repeated_params and
+ not isinstance(kwargs[name], basestring)):
+ values = kwargs[name]
+ else:
+ values = [kwargs[name]]
+ for value in values:
+ if value not in enums:
+ raise TypeError(
+ 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
+ (name, value, str(enums)))
+
+ actual_query_params = {}
+ actual_path_params = {}
+ for key, value in kwargs.iteritems():
+ to_type = parameters.param_types.get(key, 'string')
+ # For repeated parameters we cast each member of the list.
+ if key in parameters.repeated_params and type(value) == type([]):
+ cast_value = [_cast(x, to_type) for x in value]
+ else:
+ cast_value = _cast(value, to_type)
+ if key in parameters.query_params:
+ actual_query_params[parameters.argmap[key]] = cast_value
+ if key in parameters.path_params:
+ actual_path_params[parameters.argmap[key]] = cast_value
+ body_value = kwargs.get('body', None)
+ media_filename = kwargs.get('media_body', None)
+
+ if self._developerKey:
+ actual_query_params['key'] = self._developerKey
+
+ model = self._model
+ if methodName.endswith('_media'):
+ model = MediaModel()
+ elif 'response' not in methodDesc:
+ model = RawModel()
+
+ headers = {}
+ headers, params, query, body = model.request(headers,
+ actual_path_params, actual_query_params, body_value)
+
+ expanded_url = uritemplate.expand(pathUrl, params)
+ url = urlparse.urljoin(self._baseUrl, expanded_url + query)
+
+ resumable = None
+ multipart_boundary = ''
+
+ if media_filename:
+ # Ensure we end up with a valid MediaUpload object.
+ if isinstance(media_filename, basestring):
+ (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
+ if media_mime_type is None:
+ raise UnknownFileType(media_filename)
+ if not mimeparse.best_match([media_mime_type], ','.join(accept)):
+ raise UnacceptableMimeTypeError(media_mime_type)
+ media_upload = MediaFileUpload(media_filename,
+ mimetype=media_mime_type)
+ elif isinstance(media_filename, MediaUpload):
+ media_upload = media_filename
+ else:
+ raise TypeError('media_filename must be str or MediaUpload.')
+
+ # Check the maxSize
+ if maxSize > 0 and media_upload.size() > maxSize:
+ raise MediaUploadSizeError("Media larger than: %s" % maxSize)
+
+ # Use the media path uri for media uploads
+ expanded_url = uritemplate.expand(mediaPathUrl, params)
+ url = urlparse.urljoin(self._baseUrl, expanded_url + query)
+ if media_upload.resumable():
+ url = _add_query_parameter(url, 'uploadType', 'resumable')
+
+ if media_upload.resumable():
+ # This is all we need to do for resumable, if the body exists it gets
+ # sent in the first request, otherwise an empty body is sent.
+ resumable = media_upload
+ else:
+ # A non-resumable upload
+ if body is None:
+ # This is a simple media upload
+ headers['content-type'] = media_upload.mimetype()
+ body = media_upload.getbytes(0, media_upload.size())
+ url = _add_query_parameter(url, 'uploadType', 'media')
+ else:
+ # This is a multipart/related upload.
+ msgRoot = MIMEMultipart('related')
+ # msgRoot should not write out it's own headers
+ setattr(msgRoot, '_write_headers', lambda self: None)
+
+ # attach the body as one part
+ msg = MIMENonMultipart(*headers['content-type'].split('/'))
+ msg.set_payload(body)
+ msgRoot.attach(msg)
+
+ # attach the media as the second part
+ msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
+ msg['Content-Transfer-Encoding'] = 'binary'
+
+ payload = media_upload.getbytes(0, media_upload.size())
+ msg.set_payload(payload)
+ msgRoot.attach(msg)
+ body = msgRoot.as_string()
+
+ multipart_boundary = msgRoot.get_boundary()
+ headers['content-type'] = ('multipart/related; '
+ 'boundary="%s"') % multipart_boundary
+ url = _add_query_parameter(url, 'uploadType', 'multipart')
+
+ logger.info('URL being requested: %s' % url)
+ return self._requestBuilder(self._http,
+ model.response,
+ url,
+ method=httpMethod,
+ body=body,
+ headers=headers,
+ methodId=methodId,
+ resumable=resumable)
+
+ docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
+ if len(parameters.argmap) > 0:
+ docs.append('Args:\n')
+
+ # Skip undocumented params and params common to all methods.
+ skip_parameters = rootDesc.get('parameters', {}).keys()
+ skip_parameters.extend(STACK_QUERY_PARAMETERS)
+
+ all_args = parameters.argmap.keys()
+ args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
+
+ # Move body to the front of the line.
+ if 'body' in all_args:
+ args_ordered.append('body')
+
+ for name in all_args:
+ if name not in args_ordered:
+ args_ordered.append(name)
+
+ for arg in args_ordered:
+ if arg in skip_parameters:
+ continue
+
+ repeated = ''
+ if arg in parameters.repeated_params:
+ repeated = ' (repeated)'
+ required = ''
+ if arg in parameters.required_params:
+ required = ' (required)'
+ paramdesc = methodDesc['parameters'][parameters.argmap[arg]]
+ paramdoc = paramdesc.get('description', 'A parameter')
+ if '$ref' in paramdesc:
+ docs.append(
+ (' %s: object, %s%s%s\n The object takes the'
+ ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
+ schema.prettyPrintByName(paramdesc['$ref'])))
+ else:
+ paramtype = paramdesc.get('type', 'string')
+ docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
+ repeated))
+ enum = paramdesc.get('enum', [])
+ enumDesc = paramdesc.get('enumDescriptions', [])
+ if enum and enumDesc:
+ docs.append(' Allowed values\n')
+ for (name, desc) in zip(enum, enumDesc):
+ docs.append(' %s - %s\n' % (name, desc))
+ if 'response' in methodDesc:
+ if methodName.endswith('_media'):
+ docs.append('\nReturns:\n The media object as a string.\n\n ')
+ else:
+ docs.append('\nReturns:\n An object of the form:\n\n ')
+ docs.append(schema.prettyPrintSchema(methodDesc['response']))
+
+ setattr(method, '__doc__', ''.join(docs))
+ return (methodName, method)
+
+
+def createNextMethod(methodName):
+ """Creates any _next methods for attaching to a Resource.
+
+ The _next methods allow for easy iteration through list() responses.
+
+ Args:
+ methodName: string, name of the method to use.
+ """
+ methodName = fix_method_name(methodName)
+
+ def methodNext(self, previous_request, previous_response):
+ """Retrieves the next page of results.
+
+Args:
+ previous_request: The request for the previous page. (required)
+ previous_response: The response from the request for the previous page. (required)
+
+Returns:
+ A request object that you can call 'execute()' on to request the next
+ page. Returns None if there are no more items in the collection.
+ """
+ # Retrieve nextPageToken from previous_response
+ # Use as pageToken in previous_request to create new request.
+
+ if 'nextPageToken' not in previous_response:
+ return None
+
+ request = copy.copy(previous_request)
+
+ pageToken = previous_response['nextPageToken']
+ parsed = list(urlparse.urlparse(request.uri))
+ q = parse_qsl(parsed[4])
+
+ # Find and remove old 'pageToken' value from URI
+ newq = [(key, value) for (key, value) in q if key != 'pageToken']
+ newq.append(('pageToken', pageToken))
+ parsed[4] = urllib.urlencode(newq)
+ uri = urlparse.urlunparse(parsed)
+
+ request.uri = uri
+
+ logger.info('URL being requested: %s' % uri)
+
+ return request
+
+ return (methodName, methodNext)
+
+
+class Resource(object):
+ """A class for interacting with a resource."""
+
+ def __init__(self, http, baseUrl, model, requestBuilder, developerKey,
+ resourceDesc, rootDesc, schema):
+ """Build a Resource from the API description.
+
+ Args:
+ http: httplib2.Http, Object to make http requests with.
+ baseUrl: string, base URL for the API. All requests are relative to this
+ URI.
+ model: apiclient.Model, converts to and from the wire format.
+ requestBuilder: class or callable that instantiates an
+ apiclient.HttpRequest object.
+ developerKey: string, key obtained from
+ https://code.google.com/apis/console
+ resourceDesc: object, section of deserialized discovery document that
+ describes a resource. Note that the top level discovery document
+ is considered a resource.
+ rootDesc: object, the entire deserialized discovery document.
+ schema: object, mapping of schema names to schema descriptions.
+ """
+ self._dynamic_attrs = []
+
+ self._http = http
+ self._baseUrl = baseUrl
+ self._model = model
+ self._developerKey = developerKey
+ self._requestBuilder = requestBuilder
+ self._resourceDesc = resourceDesc
+ self._rootDesc = rootDesc
+ self._schema = schema
+
+ self._set_service_methods()
+
+ def _set_dynamic_attr(self, attr_name, value):
+ """Sets an instance attribute and tracks it in a list of dynamic attributes.
+
+ Args:
+ attr_name: string; The name of the attribute to be set
+ value: The value being set on the object and tracked in the dynamic cache.
+ """
+ self._dynamic_attrs.append(attr_name)
+ self.__dict__[attr_name] = value
+
+ def __getstate__(self):
+ """Trim the state down to something that can be pickled.
+
+ Uses the fact that the instance variable _dynamic_attrs holds attrs that
+ will be wiped and restored on pickle serialization.
+ """
+ state_dict = copy.copy(self.__dict__)
+ for dynamic_attr in self._dynamic_attrs:
+ del state_dict[dynamic_attr]
+ del state_dict['_dynamic_attrs']
+ return state_dict
+
+ def __setstate__(self, state):
+ """Reconstitute the state of the object from being pickled.
+
+ Uses the fact that the instance variable _dynamic_attrs holds attrs that
+ will be wiped and restored on pickle serialization.
+ """
+ self.__dict__.update(state)
+ self._dynamic_attrs = []
+ self._set_service_methods()
+
+ def _set_service_methods(self):
+ self._add_basic_methods(self._resourceDesc, self._rootDesc, self._schema)
+ self._add_nested_resources(self._resourceDesc, self._rootDesc, self._schema)
+ self._add_next_methods(self._resourceDesc, self._schema)
+
+ def _add_basic_methods(self, resourceDesc, rootDesc, schema):
+ # Add basic methods to Resource
+ if 'methods' in resourceDesc:
+ for methodName, methodDesc in resourceDesc['methods'].iteritems():
+ fixedMethodName, method = createMethod(
+ methodName, methodDesc, rootDesc, schema)
+ self._set_dynamic_attr(fixedMethodName,
+ method.__get__(self, self.__class__))
+ # Add in _media methods. The functionality of the attached method will
+ # change when it sees that the method name ends in _media.
+ if methodDesc.get('supportsMediaDownload', False):
+ fixedMethodName, method = createMethod(
+ methodName + '_media', methodDesc, rootDesc, schema)
+ self._set_dynamic_attr(fixedMethodName,
+ method.__get__(self, self.__class__))
+
+ def _add_nested_resources(self, resourceDesc, rootDesc, schema):
+ # Add in nested resources
+ if 'resources' in resourceDesc:
+
+ def createResourceMethod(methodName, methodDesc):
+ """Create a method on the Resource to access a nested Resource.
+
+ Args:
+ methodName: string, name of the method to use.
+ methodDesc: object, fragment of deserialized discovery document that
+ describes the method.
+ """
+ methodName = fix_method_name(methodName)
+
+ def methodResource(self):
+ return Resource(http=self._http, baseUrl=self._baseUrl,
+ model=self._model, developerKey=self._developerKey,
+ requestBuilder=self._requestBuilder,
+ resourceDesc=methodDesc, rootDesc=rootDesc,
+ schema=schema)
+
+ setattr(methodResource, '__doc__', 'A collection resource.')
+ setattr(methodResource, '__is_resource__', True)
+
+ return (methodName, methodResource)
+
+ for methodName, methodDesc in resourceDesc['resources'].iteritems():
+ fixedMethodName, method = createResourceMethod(methodName, methodDesc)
+ self._set_dynamic_attr(fixedMethodName,
+ method.__get__(self, self.__class__))
+
+ def _add_next_methods(self, resourceDesc, schema):
+ # Add _next() methods
+ # Look for response bodies in schema that contain nextPageToken, and methods
+ # that take a pageToken parameter.
+ if 'methods' in resourceDesc:
+ for methodName, methodDesc in resourceDesc['methods'].iteritems():
+ if 'response' in methodDesc:
+ responseSchema = methodDesc['response']
+ if '$ref' in responseSchema:
+ responseSchema = schema.get(responseSchema['$ref'])
+ hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
+ {})
+ hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
+ if hasNextPageToken and hasPageToken:
+ fixedMethodName, method = createNextMethod(methodName + '_next')
+ self._set_dynamic_attr(fixedMethodName,
+ method.__get__(self, self.__class__))
diff --git a/py/minijack/apiclient/errors.py b/py/minijack/apiclient/errors.py
new file mode 100644
index 0000000..2bf9149
--- /dev/null
+++ b/py/minijack/apiclient/errors.py
@@ -0,0 +1,137 @@
+#!/usr/bin/python2.4
+#
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Errors for the library.
+
+All exceptions defined by the library
+should be defined in this file.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+
+from oauth2client import util
+from oauth2client.anyjson import simplejson
+
+
+class Error(Exception):
+ """Base error for this module."""
+ pass
+
+
+class HttpError(Error):
+ """HTTP data was invalid or unexpected."""
+
+ @util.positional(3)
+ def __init__(self, resp, content, uri=None):
+ self.resp = resp
+ self.content = content
+ self.uri = uri
+
+ def _get_reason(self):
+ """Calculate the reason for the error from the response content."""
+ reason = self.resp.reason
+ try:
+ data = simplejson.loads(self.content)
+ reason = data['error']['message']
+ except (ValueError, KeyError):
+ pass
+ if reason is None:
+ reason = ''
+ return reason
+
+ def __repr__(self):
+ if self.uri:
+ return '<HttpError %s when requesting %s returned "%s">' % (
+ self.resp.status, self.uri, self._get_reason().strip())
+ else:
+ return '<HttpError %s "%s">' % (self.resp.status, self._get_reason())
+
+ __str__ = __repr__
+
+
+class InvalidJsonError(Error):
+ """The JSON returned could not be parsed."""
+ pass
+
+
+class UnknownFileType(Error):
+ """File type unknown or unexpected."""
+ pass
+
+
+class UnknownLinkType(Error):
+ """Link type unknown or unexpected."""
+ pass
+
+
+class UnknownApiNameOrVersion(Error):
+ """No API with that name and version exists."""
+ pass
+
+
+class UnacceptableMimeTypeError(Error):
+ """That is an unacceptable mimetype for this operation."""
+ pass
+
+
+class MediaUploadSizeError(Error):
+ """Media is larger than the method can accept."""
+ pass
+
+
+class ResumableUploadError(HttpError):
+ """Error occured during resumable upload."""
+ pass
+
+
+class InvalidChunkSizeError(Error):
+ """The given chunksize is not valid."""
+ pass
+
+
+class BatchError(HttpError):
+ """Error occured during batch operations."""
+
+ @util.positional(2)
+ def __init__(self, reason, resp=None, content=None):
+ self.resp = resp
+ self.content = content
+ self.reason = reason
+
+ def __repr__(self):
+ return '<BatchError %s "%s">' % (self.resp.status, self.reason)
+
+ __str__ = __repr__
+
+
+class UnexpectedMethodError(Error):
+ """Exception raised by RequestMockBuilder on unexpected calls."""
+
+ @util.positional(1)
+ def __init__(self, methodId=None):
+ """Constructor for an UnexpectedMethodError."""
+ super(UnexpectedMethodError, self).__init__(
+ 'Received unexpected call %s' % methodId)
+
+
+class UnexpectedBodyError(Error):
+ """Exception raised by RequestMockBuilder on unexpected bodies."""
+
+ def __init__(self, expected, provided):
+ """Constructor for an UnexpectedMethodError."""
+ super(UnexpectedBodyError, self).__init__(
+ 'Expected: [%s] - Provided: [%s]' % (expected, provided))
diff --git a/py/minijack/apiclient/http.py b/py/minijack/apiclient/http.py
new file mode 100644
index 0000000..a956477
--- /dev/null
+++ b/py/minijack/apiclient/http.py
@@ -0,0 +1,1536 @@
+# Copyright (C) 2012 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Classes to encapsulate a single HTTP request.
+
+The classes implement a command pattern, with every
+object supporting an execute() method that does the
+actuall HTTP request.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+import StringIO
+import base64
+import copy
+import gzip
+import httplib2
+import mimeparse
+import mimetypes
+import os
+import sys
+import urllib
+import urlparse
+import uuid
+
+from email.generator import Generator
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.parser import FeedParser
+from errors import BatchError
+from errors import HttpError
+from errors import InvalidChunkSizeError
+from errors import ResumableUploadError
+from errors import UnexpectedBodyError
+from errors import UnexpectedMethodError
+from model import JsonModel
+from oauth2client import util
+from oauth2client.anyjson import simplejson
+
+
+DEFAULT_CHUNK_SIZE = 512*1024
+
+MAX_URI_LENGTH = 2048
+
+
+class MediaUploadProgress(object):
+ """Status of a resumable upload."""
+
+ def __init__(self, resumable_progress, total_size):
+ """Constructor.
+
+ Args:
+ resumable_progress: int, bytes sent so far.
+ total_size: int, total bytes in complete upload, or None if the total
+ upload size isn't known ahead of time.
+ """
+ self.resumable_progress = resumable_progress
+ self.total_size = total_size
+
+ def progress(self):
+ """Percent of upload completed, as a float.
+
+ Returns:
+ the percentage complete as a float, returning 0.0 if the total size of
+ the upload is unknown.
+ """
+ if self.total_size is not None:
+ return float(self.resumable_progress) / float(self.total_size)
+ else:
+ return 0.0
+
+
+class MediaDownloadProgress(object):
+ """Status of a resumable download."""
+
+ def __init__(self, resumable_progress, total_size):
+ """Constructor.
+
+ Args:
+ resumable_progress: int, bytes received so far.
+ total_size: int, total bytes in complete download.
+ """
+ self.resumable_progress = resumable_progress
+ self.total_size = total_size
+
+ def progress(self):
+ """Percent of download completed, as a float.
+
+ Returns:
+ the percentage complete as a float, returning 0.0 if the total size of
+ the download is unknown.
+ """
+ if self.total_size is not None:
+ return float(self.resumable_progress) / float(self.total_size)
+ else:
+ return 0.0
+
+
+class MediaUpload(object):
+ """Describes a media object to upload.
+
+ Base class that defines the interface of MediaUpload subclasses.
+
+ Note that subclasses of MediaUpload may allow you to control the chunksize
+ when uploading a media object. It is important to keep the size of the chunk
+ as large as possible to keep the upload efficient. Other factors may influence
+ the size of the chunk you use, particularly if you are working in an
+ environment where individual HTTP requests may have a hardcoded time limit,
+ such as under certain classes of requests under Google App Engine.
+
+ Streams are io.Base compatible objects that support seek(). Some MediaUpload
+ subclasses support using streams directly to upload data. Support for
+ streaming may be indicated by a MediaUpload sub-class and if appropriate for a
+ platform that stream will be used for uploading the media object. The support
+ for streaming is indicated by has_stream() returning True. The stream() method
+ should return an io.Base object that supports seek(). On platforms where the
+ underlying httplib module supports streaming, for example Python 2.6 and
+ later, the stream will be passed into the http library which will result in
+ less memory being used and possibly faster uploads.
+
+ If you need to upload media that can't be uploaded using any of the existing
+ MediaUpload sub-class then you can sub-class MediaUpload for your particular
+ needs.
+ """
+
+ def chunksize(self):
+ """Chunk size for resumable uploads.
+
+ Returns:
+ Chunk size in bytes.
+ """
+ raise NotImplementedError()
+
+ def mimetype(self):
+ """Mime type of the body.
+
+ Returns:
+ Mime type.
+ """
+ return 'application/octet-stream'
+
+ def size(self):
+ """Size of upload.
+
+ Returns:
+ Size of the body, or None of the size is unknown.
+ """
+ return None
+
+ def resumable(self):
+ """Whether this upload is resumable.
+
+ Returns:
+ True if resumable upload or False.
+ """
+ return False
+
+ def getbytes(self, begin, end):
+ """Get bytes from the media.
+
+ Args:
+ begin: int, offset from beginning of file.
+ length: int, number of bytes to read, starting at begin.
+
+ Returns:
+ A string of bytes read. May be shorter than length if EOF was reached
+ first.
+ """
+ raise NotImplementedError()
+
+ def has_stream(self):
+ """Does the underlying upload support a streaming interface.
+
+ Streaming means it is an io.IOBase subclass that supports seek, i.e.
+ seekable() returns True.
+
+ Returns:
+ True if the call to stream() will return an instance of a seekable io.Base
+ subclass.
+ """
+ return False
+
+ def stream(self):
+ """A stream interface to the data being uploaded.
+
+ Returns:
+ The returned value is an io.IOBase subclass that supports seek, i.e.
+ seekable() returns True.
+ """
+ raise NotImplementedError()
+
+ @util.positional(1)
+ def _to_json(self, strip=None):
+ """Utility function for creating a JSON representation of a MediaUpload.
+
+ Args:
+ strip: array, An array of names of members to not include in the JSON.
+
+ Returns:
+ string, a JSON representation of this instance, suitable to pass to
+ from_json().
+ """
+ t = type(self)
+ d = copy.copy(self.__dict__)
+ if strip is not None:
+ for member in strip:
+ del d[member]
+ d['_class'] = t.__name__
+ d['_module'] = t.__module__
+ return simplejson.dumps(d)
+
+ def to_json(self):
+ """Create a JSON representation of an instance of MediaUpload.
+
+ Returns:
+ string, a JSON representation of this instance, suitable to pass to
+ from_json().
+ """
+ return self._to_json()
+
+ @classmethod
+ def new_from_json(cls, s):
+ """Utility class method to instantiate a MediaUpload subclass from a JSON
+ representation produced by to_json().
+
+ Args:
+ s: string, JSON from to_json().
+
+ Returns:
+ An instance of the subclass of MediaUpload that was serialized with
+ to_json().
+ """
+ data = simplejson.loads(s)
+ # Find and call the right classmethod from_json() to restore the object.
+ module = data['_module']
+ m = __import__(module, fromlist=module.split('.')[:-1])
+ kls = getattr(m, data['_class'])
+ from_json = getattr(kls, 'from_json')
+ return from_json(s)
+
+
+class MediaIoBaseUpload(MediaUpload):
+ """A MediaUpload for a io.Base objects.
+
+ Note that the Python file object is compatible with io.Base and can be used
+ with this class also.
+
+ fh = io.BytesIO('...Some data to upload...')
+ media = MediaIoBaseUpload(fh, mimetype='image/png',
+ chunksize=1024*1024, resumable=True)
+ farm.animals().insert(
+ id='cow',
+ name='cow.png',
+ media_body=media).execute()
+
+ Depending on the platform you are working on, you may pass -1 as the
+ chunksize, which indicates that the entire file should be uploaded in a single
+ request. If the underlying platform supports streams, such as Python 2.6 or
+ later, then this can be very efficient as it avoids multiple connections, and
+ also avoids loading the entire file into memory before sending it. Note that
+ Google App Engine has a 5MB limit on request size, so you should never set
+ your chunksize larger than 5MB, or to -1.
+ """
+
+ @util.positional(3)
+ def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE,
+ resumable=False):
+ """Constructor.
+
+ Args:
+ fd: io.Base or file object, The source of the bytes to upload. MUST be
+ opened in blocking mode, do not use streams opened in non-blocking mode.
+ The given stream must be seekable, that is, it must be able to call
+ seek() on fd.
+ mimetype: string, Mime-type of the file.
+ chunksize: int, File will be uploaded in chunks of this many bytes. Only
+ used if resumable=True. Pass in a value of -1 if the file is to be
+ uploaded as a single chunk. Note that Google App Engine has a 5MB limit
+ on request size, so you should never set your chunksize larger than 5MB,
+ or to -1.
+ resumable: bool, True if this is a resumable upload. False means upload
+ in a single request.
+ """
+ super(MediaIoBaseUpload, self).__init__()
+ self._fd = fd
+ self._mimetype = mimetype
+ if not (chunksize == -1 or chunksize > 0):
+ raise InvalidChunkSizeError()
+ self._chunksize = chunksize
+ self._resumable = resumable
+
+ self._fd.seek(0, os.SEEK_END)
+ self._size = self._fd.tell()
+
+ def chunksize(self):
+ """Chunk size for resumable uploads.
+
+ Returns:
+ Chunk size in bytes.
+ """
+ return self._chunksize
+
+ def mimetype(self):
+ """Mime type of the body.
+
+ Returns:
+ Mime type.
+ """
+ return self._mimetype
+
+ def size(self):
+ """Size of upload.
+
+ Returns:
+ Size of the body, or None of the size is unknown.
+ """
+ return self._size
+
+ def resumable(self):
+ """Whether this upload is resumable.
+
+ Returns:
+ True if resumable upload or False.
+ """
+ return self._resumable
+
+ def getbytes(self, begin, length):
+ """Get bytes from the media.
+
+ Args:
+ begin: int, offset from beginning of file.
+ length: int, number of bytes to read, starting at begin.
+
+ Returns:
+ A string of bytes read. May be shorted than length if EOF was reached
+ first.
+ """
+ self._fd.seek(begin)
+ return self._fd.read(length)
+
+ def has_stream(self):
+ """Does the underlying upload support a streaming interface.
+
+ Streaming means it is an io.IOBase subclass that supports seek, i.e.
+ seekable() returns True.
+
+ Returns:
+ True if the call to stream() will return an instance of a seekable io.Base
+ subclass.
+ """
+ return True
+
+ def stream(self):
+ """A stream interface to the data being uploaded.
+
+ Returns:
+ The returned value is an io.IOBase subclass that supports seek, i.e.
+ seekable() returns True.
+ """
+ return self._fd
+
+ def to_json(self):
+ """This upload type is not serializable."""
+ raise NotImplementedError('MediaIoBaseUpload is not serializable.')
+
+
+class MediaFileUpload(MediaIoBaseUpload):
+ """A MediaUpload for a file.
+
+ Construct a MediaFileUpload and pass as the media_body parameter of the
+ method. For example, if we had a service that allowed uploading images:
+
+
+ media = MediaFileUpload('cow.png', mimetype='image/png',
+ chunksize=1024*1024, resumable=True)
+ farm.animals().insert(
+ id='cow',
+ name='cow.png',
+ media_body=media).execute()
+
+ Depending on the platform you are working on, you may pass -1 as the
+ chunksize, which indicates that the entire file should be uploaded in a single
+ request. If the underlying platform supports streams, such as Python 2.6 or
+ later, then this can be very efficient as it avoids multiple connections, and
+ also avoids loading the entire file into memory before sending it. Note that
+ Google App Engine has a 5MB limit on request size, so you should never set
+ your chunksize larger than 5MB, or to -1.
+ """
+
+ @util.positional(2)
+ def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE,
+ resumable=False):
+ """Constructor.
+
+ Args:
+ filename: string, Name of the file.
+ mimetype: string, Mime-type of the file. If None then a mime-type will be
+ guessed from the file extension.
+ chunksize: int, File will be uploaded in chunks of this many bytes. Only
+ used if resumable=True. Pass in a value of -1 if the file is to be
+ uploaded in a single chunk. Note that Google App Engine has a 5MB limit
+ on request size, so you should never set your chunksize larger than 5MB,
+ or to -1.
+ resumable: bool, True if this is a resumable upload. False means upload
+ in a single request.
+ """
+ self._filename = filename
+ fd = open(self._filename, 'rb')
+ if mimetype is None:
+ (mimetype, encoding) = mimetypes.guess_type(filename)
+ super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize,
+ resumable=resumable)
+
+ def to_json(self):
+ """Creating a JSON representation of an instance of MediaFileUpload.
+
+ Returns:
+ string, a JSON representation of this instance, suitable to pass to
+ from_json().
+ """
+ return self._to_json(strip=['_fd'])
+
+ @staticmethod
+ def from_json(s):
+ d = simplejson.loads(s)
+ return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'],
+ chunksize=d['_chunksize'], resumable=d['_resumable'])
+
+
+class MediaInMemoryUpload(MediaIoBaseUpload):
+ """MediaUpload for a chunk of bytes.
+
+ DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
+ the stream.
+ """
+
+ @util.positional(2)
+ def __init__(self, body, mimetype='application/octet-stream',
+ chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
+ """Create a new MediaInMemoryUpload.
+
+ DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
+ the stream.
+
+ Args:
+ body: string, Bytes of body content.
+ mimetype: string, Mime-type of the file or default of
+ 'application/octet-stream'.
+ chunksize: int, File will be uploaded in chunks of this many bytes. Only
+ used if resumable=True.
+ resumable: bool, True if this is a resumable upload. False means upload
+ in a single request.
+ """
+ fd = StringIO.StringIO(body)
+ super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize,
+ resumable=resumable)
+
+
+class MediaIoBaseDownload(object):
+ """"Download media resources.
+
+ Note that the Python file object is compatible with io.Base and can be used
+ with this class also.
+
+
+ Example:
+ request = farms.animals().get_media(id='cow')
+ fh = io.FileIO('cow.png', mode='wb')
+ downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024)
+
+ done = False
+ while done is False:
+ status, done = downloader.next_chunk()
+ if status:
+ print "Download %d%%." % int(status.progress() * 100)
+ print "Download Complete!"
+ """
+
+ @util.positional(3)
+ def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
+ """Constructor.
+
+ Args:
+ fd: io.Base or file object, The stream in which to write the downloaded
+ bytes.
+ request: apiclient.http.HttpRequest, the media request to perform in
+ chunks.
+ chunksize: int, File will be downloaded in chunks of this many bytes.
+ """
+ self._fd = fd
+ self._request = request
+ self._uri = request.uri
+ self._chunksize = chunksize
+ self._progress = 0
+ self._total_size = None
+ self._done = False
+
+ def next_chunk(self):
+ """Get the next chunk of the download.
+
+ Returns:
+ (status, done): (MediaDownloadStatus, boolean)
+ The value of 'done' will be True when the media has been fully
+ downloaded.
+
+ Raises:
+ apiclient.errors.HttpError if the response was not a 2xx.
+ httplib2.HttpLib2Error if a transport error has occured.
+ """
+ headers = {
+ 'range': 'bytes=%d-%d' % (
+ self._progress, self._progress + self._chunksize)
+ }
+ http = self._request.http
+ http.follow_redirects = False
+
+ resp, content = http.request(self._uri, headers=headers)
+ if resp.status in [301, 302, 303, 307, 308] and 'location' in resp:
+ self._uri = resp['location']
+ resp, content = http.request(self._uri, headers=headers)
+ if resp.status in [200, 206]:
+ self._progress += len(content)
+ self._fd.write(content)
+
+ if 'content-range' in resp:
+ content_range = resp['content-range']
+ length = content_range.rsplit('/', 1)[1]
+ self._total_size = int(length)
+
+ if self._progress == self._total_size:
+ self._done = True
+ return MediaDownloadProgress(self._progress, self._total_size), self._done
+ else:
+ raise HttpError(resp, content, uri=self._uri)
+
+
+class _StreamSlice(object):
+ """Truncated stream.
+
+ Takes a stream and presents a stream that is a slice of the original stream.
+ This is used when uploading media in chunks. In later versions of Python a
+ stream can be passed to httplib in place of the string of data to send. The
+ problem is that httplib just blindly reads to the end of the stream. This
+ wrapper presents a virtual stream that only reads to the end of the chunk.
+ """
+
+ def __init__(self, stream, begin, chunksize):
+ """Constructor.
+
+ Args:
+ stream: (io.Base, file object), the stream to wrap.
+ begin: int, the seek position the chunk begins at.
+ chunksize: int, the size of the chunk.
+ """
+ self._stream = stream
+ self._begin = begin
+ self._chunksize = chunksize
+ self._stream.seek(begin)
+
+ def read(self, n=-1):
+ """Read n bytes.
+
+ Args:
+ n, int, the number of bytes to read.
+
+ Returns:
+ A string of length 'n', or less if EOF is reached.
+ """
+ # The data left available to read sits in [cur, end)
+ cur = self._stream.tell()
+ end = self._begin + self._chunksize
+ if n == -1 or cur + n > end:
+ n = end - cur
+ return self._stream.read(n)
+
+
+class HttpRequest(object):
+ """Encapsulates a single HTTP request."""
+
+ @util.positional(4)
+ def __init__(self, http, postproc, uri,
+ method='GET',
+ body=None,
+ headers=None,
+ methodId=None,
+ resumable=None):
+ """Constructor for an HttpRequest.
+
+ Args:
+ http: httplib2.Http, the transport object to use to make a request
+ postproc: callable, called on the HTTP response and content to transform
+ it into a data object before returning, or raising an exception
+ on an error.
+ uri: string, the absolute URI to send the request to
+ method: string, the HTTP method to use
+ body: string, the request body of the HTTP request,
+ headers: dict, the HTTP request headers
+ methodId: string, a unique identifier for the API method being called.
+ resumable: MediaUpload, None if this is not a resumbale request.
+ """
+ self.uri = uri
+ self.method = method
+ self.body = body
+ self.headers = headers or {}
+ self.methodId = methodId
+ self.http = http
+ self.postproc = postproc
+ self.resumable = resumable
+ self.response_callbacks = []
+ self._in_error_state = False
+
+ # Pull the multipart boundary out of the content-type header.
+ major, minor, params = mimeparse.parse_mime_type(
+ headers.get('content-type', 'application/json'))
+
+ # The size of the non-media part of the request.
+ self.body_size = len(self.body or '')
+
+ # The resumable URI to send chunks to.
+ self.resumable_uri = None
+
+ # The bytes that have been uploaded.
+ self.resumable_progress = 0
+
+ @util.positional(1)
+ def execute(self, http=None):
+ """Execute the request.
+
+ Args:
+ http: httplib2.Http, an http object to be used in place of the
+ one the HttpRequest request object was constructed with.
+
+ Returns:
+ A deserialized object model of the response body as determined
+ by the postproc.
+
+ Raises:
+ apiclient.errors.HttpError if the response was not a 2xx.
+ httplib2.HttpLib2Error if a transport error has occured.
+ """
+ if http is None:
+ http = self.http
+ if self.resumable:
+ body = None
+ while body is None:
+ _, body = self.next_chunk(http=http)
+ return body
+ else:
+ if 'content-length' not in self.headers:
+ self.headers['content-length'] = str(self.body_size)
+ # If the request URI is too long then turn it into a POST request.
+ if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
+ self.method = 'POST'
+ self.headers['x-http-method-override'] = 'GET'
+ self.headers['content-type'] = 'application/x-www-form-urlencoded'
+ parsed = urlparse.urlparse(self.uri)
+ self.uri = urlparse.urlunparse(
+ (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
+ None)
+ )
+ self.body = parsed.query
+ self.headers['content-length'] = str(len(self.body))
+
+ resp, content = http.request(str(self.uri), method=str(self.method),
+ body=self.body, headers=self.headers)
+ for callback in self.response_callbacks:
+ callback(resp)
+ if resp.status >= 300:
+ raise HttpError(resp, content, uri=self.uri)
+ return self.postproc(resp, content)
+
+ @util.positional(2)
+ def add_response_callback(self, cb):
+ """add_response_headers_callback
+
+ Args:
+ cb: Callback to be called on receiving the response headers, of signature:
+
+ def cb(resp):
+ # Where resp is an instance of httplib2.Response
+ """
+ self.response_callbacks.append(cb)
+
+ @util.positional(1)
+ def next_chunk(self, http=None):
+ """Execute the next step of a resumable upload.
+
+ Can only be used if the method being executed supports media uploads and
+ the MediaUpload object passed in was flagged as using resumable upload.
+
+ Example:
+
+ media = MediaFileUpload('cow.png', mimetype='image/png',
+ chunksize=1000, resumable=True)
+ request = farm.animals().insert(
+ id='cow',
+ name='cow.png',
+ media_body=media)
+
+ response = None
+ while response is None:
+ status, response = request.next_chunk()
+ if status:
+ print "Upload %d%% complete." % int(status.progress() * 100)
+
+
+ Returns:
+ (status, body): (ResumableMediaStatus, object)
+ The body will be None until the resumable media is fully uploaded.
+
+ Raises:
+ apiclient.errors.HttpError if the response was not a 2xx.
+ httplib2.HttpLib2Error if a transport error has occured.
+ """
+ if http is None:
+ http = self.http
+
+ if self.resumable.size() is None:
+ size = '*'
+ else:
+ size = str(self.resumable.size())
+
+ if self.resumable_uri is None:
+ start_headers = copy.copy(self.headers)
+ start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
+ if size != '*':
+ start_headers['X-Upload-Content-Length'] = size
+ start_headers['content-length'] = str(self.body_size)
+
+ resp, content = http.request(self.uri, self.method,
+ body=self.body,
+ headers=start_headers)
+ if resp.status == 200 and 'location' in resp:
+ self.resumable_uri = resp['location']
+ else:
+ raise ResumableUploadError(resp, content)
+ elif self._in_error_state:
+ # If we are in an error state then query the server for current state of
+ # the upload by sending an empty PUT and reading the 'range' header in
+ # the response.
+ headers = {
+ 'Content-Range': 'bytes */%s' % size,
+ 'content-length': '0'
+ }
+ resp, content = http.request(self.resumable_uri, 'PUT',
+ headers=headers)
+ status, body = self._process_response(resp, content)
+ if body:
+ # The upload was complete.
+ return (status, body)
+
+ # The httplib.request method can take streams for the body parameter, but
+ # only in Python 2.6 or later. If a stream is available under those
+ # conditions then use it as the body argument.
+ if self.resumable.has_stream() and sys.version_info[1] >= 6:
+ data = self.resumable.stream()
+ if self.resumable.chunksize() == -1:
+ data.seek(self.resumable_progress)
+ chunk_end = self.resumable.size() - self.resumable_progress - 1
+ else:
+ # Doing chunking with a stream, so wrap a slice of the stream.
+ data = _StreamSlice(data, self.resumable_progress,
+ self.resumable.chunksize())
+ chunk_end = min(
+ self.resumable_progress + self.resumable.chunksize() - 1,
+ self.resumable.size() - 1)
+ else:
+ data = self.resumable.getbytes(
+ self.resumable_progress, self.resumable.chunksize())
+
+ # A short read implies that we are at EOF, so finish the upload.
+ if len(data) < self.resumable.chunksize():
+ size = str(self.resumable_progress + len(data))
+
+ chunk_end = self.resumable_progress + len(data) - 1
+
+ headers = {
+ 'Content-Range': 'bytes %d-%d/%s' % (
+ self.resumable_progress, chunk_end, size),
+ # Must set the content-length header here because httplib can't
+ # calculate the size when working with _StreamSlice.
+ 'Content-Length': str(chunk_end - self.resumable_progress + 1)
+ }
+ try:
+ resp, content = http.request(self.resumable_uri, 'PUT',
+ body=data,
+ headers=headers)
+ except:
+ self._in_error_state = True
+ raise
+
+ return self._process_response(resp, content)
+
+ def _process_response(self, resp, content):
+ """Process the response from a single chunk upload.
+
+ Args:
+ resp: httplib2.Response, the response object.
+ content: string, the content of the response.
+
+ Returns:
+ (status, body): (ResumableMediaStatus, object)
+ The body will be None until the resumable media is fully uploaded.
+
+ Raises:
+ apiclient.errors.HttpError if the response was not a 2xx or a 308.
+ """
+ if resp.status in [200, 201]:
+ self._in_error_state = False
+ return None, self.postproc(resp, content)
+ elif resp.status == 308:
+ self._in_error_state = False
+ # A "308 Resume Incomplete" indicates we are not done.
+ self.resumable_progress = int(resp['range'].split('-')[1]) + 1
+ if 'location' in resp:
+ self.resumable_uri = resp['location']
+ else:
+ self._in_error_state = True
+ raise HttpError(resp, content, uri=self.uri)
+
+ return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
+ None)
+
+ def to_json(self):
+ """Returns a JSON representation of the HttpRequest."""
+ d = copy.copy(self.__dict__)
+ if d['resumable'] is not None:
+ d['resumable'] = self.resumable.to_json()
+ del d['http']
+ del d['postproc']
+
+ return simplejson.dumps(d)
+
+ @staticmethod
+ def from_json(s, http, postproc):
+ """Returns an HttpRequest populated with info from a JSON object."""
+ d = simplejson.loads(s)
+ if d['resumable'] is not None:
+ d['resumable'] = MediaUpload.new_from_json(d['resumable'])
+ return HttpRequest(
+ http,
+ postproc,
+ uri=d['uri'],
+ method=d['method'],
+ body=d['body'],
+ headers=d['headers'],
+ methodId=d['methodId'],
+ resumable=d['resumable'])
+
+
+class BatchHttpRequest(object):
+ """Batches multiple HttpRequest objects into a single HTTP request.
+
+ Example:
+ from apiclient.http import BatchHttpRequest
+
+ def list_animals(request_id, response, exception):
+ \"\"\"Do something with the animals list response.\"\"\"
+ if exception is not None:
+ # Do something with the exception.
+ pass
+ else:
+ # Do something with the response.
+ pass
+
+ def list_farmers(request_id, response, exception):
+ \"\"\"Do something with the farmers list response.\"\"\"
+ if exception is not None:
+ # Do something with the exception.
+ pass
+ else:
+ # Do something with the response.
+ pass
+
+ service = build('farm', 'v2')
+
+ batch = BatchHttpRequest()
+
+ batch.add(service.animals().list(), list_animals)
+ batch.add(service.farmers().list(), list_farmers)
+ batch.execute(http=http)
+ """
+
+ @util.positional(1)
+ def __init__(self, callback=None, batch_uri=None):
+ """Constructor for a BatchHttpRequest.
+
+ Args:
+ callback: callable, A callback to be called for each response, of the
+ form callback(id, response, exception). The first parameter is the
+ request id, and the second is the deserialized response object. The
+ third is an apiclient.errors.HttpError exception object if an HTTP error
+ occurred while processing the request, or None if no error occurred.
+ batch_uri: string, URI to send batch requests to.
+ """
+ if batch_uri is None:
+ batch_uri = 'https://www.googleapis.com/batch'
+ self._batch_uri = batch_uri
+
+ # Global callback to be called for each individual response in the batch.
+ self._callback = callback
+
+ # A map from id to request.
+ self._requests = {}
+
+ # A map from id to callback.
+ self._callbacks = {}
+
+ # List of request ids, in the order in which they were added.
+ self._order = []
+
+ # The last auto generated id.
+ self._last_auto_id = 0
+
+ # Unique ID on which to base the Content-ID headers.
+ self._base_id = None
+
+ # A map from request id to (httplib2.Response, content) response pairs
+ self._responses = {}
+
+ # A map of id(Credentials) that have been refreshed.
+ self._refreshed_credentials = {}
+
+ def _refresh_and_apply_credentials(self, request, http):
+ """Refresh the credentials and apply to the request.
+
+ Args:
+ request: HttpRequest, the request.
+ http: httplib2.Http, the global http object for the batch.
+ """
+ # For the credentials to refresh, but only once per refresh_token
+ # If there is no http per the request then refresh the http passed in
+ # via execute()
+ creds = None
+ if request.http is not None and hasattr(request.http.request,
+ 'credentials'):
+ creds = request.http.request.credentials
+ elif http is not None and hasattr(http.request, 'credentials'):
+ creds = http.request.credentials
+ if creds is not None:
+ if id(creds) not in self._refreshed_credentials:
+ creds.refresh(http)
+ self._refreshed_credentials[id(creds)] = 1
+
+ # Only apply the credentials if we are using the http object passed in,
+ # otherwise apply() will get called during _serialize_request().
+ if request.http is None or not hasattr(request.http.request,
+ 'credentials'):
+ creds.apply(request.headers)
+
+ def _id_to_header(self, id_):
+ """Convert an id to a Content-ID header value.
+
+ Args:
+ id_: string, identifier of individual request.
+
+ Returns:
+ A Content-ID header with the id_ encoded into it. A UUID is prepended to
+ the value because Content-ID headers are supposed to be universally
+ unique.
+ """
+ if self._base_id is None:
+ self._base_id = uuid.uuid4()
+
+ return '<%s+%s>' % (self._base_id, urllib.quote(id_))
+
+ def _header_to_id(self, header):
+ """Convert a Content-ID header value to an id.
+
+ Presumes the Content-ID header conforms to the format that _id_to_header()
+ returns.
+
+ Args:
+ header: string, Content-ID header value.
+
+ Returns:
+ The extracted id value.
+
+ Raises:
+ BatchError if the header is not in the expected format.
+ """
+ if header[0] != '<' or header[-1] != '>':
+ raise BatchError("Invalid value for Content-ID: %s" % header)
+ if '+' not in header:
+ raise BatchError("Invalid value for Content-ID: %s" % header)
+ base, id_ = header[1:-1].rsplit('+', 1)
+
+ return urllib.unquote(id_)
+
+ def _serialize_request(self, request):
+ """Convert an HttpRequest object into a string.
+
+ Args:
+ request: HttpRequest, the request to serialize.
+
+ Returns:
+ The request as a string in application/http format.
+ """
+ # Construct status line
+ parsed = urlparse.urlparse(request.uri)
+ request_line = urlparse.urlunparse(
+ (None, None, parsed.path, parsed.params, parsed.query, None)
+ )
+ status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
+ major, minor = request.headers.get('content-type', 'application/json').split('/')
+ msg = MIMENonMultipart(major, minor)
+ headers = request.headers.copy()
+
+ if request.http is not None and hasattr(request.http.request,
+ 'credentials'):
+ request.http.request.credentials.apply(headers)
+
+ # MIMENonMultipart adds its own Content-Type header.
+ if 'content-type' in headers:
+ del headers['content-type']
+
+ for key, value in headers.iteritems():
+ msg[key] = value
+ msg['Host'] = parsed.netloc
+ msg.set_unixfrom(None)
+
+ if request.body is not None:
+ msg.set_payload(request.body)
+ msg['content-length'] = str(len(request.body))
+
+ # Serialize the mime message.
+ fp = StringIO.StringIO()
+ # maxheaderlen=0 means don't line wrap headers.
+ g = Generator(fp, maxheaderlen=0)
+ g.flatten(msg, unixfrom=False)
+ body = fp.getvalue()
+
+ # Strip off the \n\n that the MIME lib tacks onto the end of the payload.
+ if request.body is None:
+ body = body[:-2]
+
+ return status_line.encode('utf-8') + body
+
+ def _deserialize_response(self, payload):
+ """Convert string into httplib2 response and content.
+
+ Args:
+ payload: string, headers and body as a string.
+
+ Returns:
+ A pair (resp, content), such as would be returned from httplib2.request.
+ """
+ # Strip off the status line
+ status_line, payload = payload.split('\n', 1)
+ protocol, status, reason = status_line.split(' ', 2)
+
+ # Parse the rest of the response
+ parser = FeedParser()
+ parser.feed(payload)
+ msg = parser.close()
+ msg['status'] = status
+
+ # Create httplib2.Response from the parsed headers.
+ resp = httplib2.Response(msg)
+ resp.reason = reason
+ resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
+
+ content = payload.split('\r\n\r\n', 1)[1]
+
+ return resp, content
+
+ def _new_id(self):
+ """Create a new id.
+
+ Auto incrementing number that avoids conflicts with ids already used.
+
+ Returns:
+ string, a new unique id.
+ """
+ self._last_auto_id += 1
+ while str(self._last_auto_id) in self._requests:
+ self._last_auto_id += 1
+ return str(self._last_auto_id)
+
+ @util.positional(2)
+ def add(self, request, callback=None, request_id=None):
+ """Add a new request.
+
+ Every callback added will be paired with a unique id, the request_id. That
+ unique id will be passed back to the callback when the response comes back
+ from the server. The default behavior is to have the library generate it's
+ own unique id. If the caller passes in a request_id then they must ensure
+ uniqueness for each request_id, and if they are not an exception is
+ raised. Callers should either supply all request_ids or nevery supply a
+ request id, to avoid such an error.
+
+ Args:
+ request: HttpRequest, Request to add to the batch.
+ callback: callable, A callback to be called for this response, of the
+ form callback(id, response, exception). The first parameter is the
+ request id, and the second is the deserialized response object. The
+ third is an apiclient.errors.HttpError exception object if an HTTP error
+ occurred while processing the request, or None if no errors occurred.
+ request_id: string, A unique id for the request. The id will be passed to
+ the callback with the response.
+
+ Returns:
+ None
+
+ Raises:
+ BatchError if a media request is added to a batch.
+ KeyError is the request_id is not unique.
+ """
+ if request_id is None:
+ request_id = self._new_id()
+ if request.resumable is not None:
+ raise BatchError("Media requests cannot be used in a batch request.")
+ if request_id in self._requests:
+ raise KeyError("A request with this ID already exists: %s" % request_id)
+ self._requests[request_id] = request
+ self._callbacks[request_id] = callback
+ self._order.append(request_id)
+
+ def _execute(self, http, order, requests):
+ """Serialize batch request, send to server, process response.
+
+ Args:
+ http: httplib2.Http, an http object to be used to make the request with.
+ order: list, list of request ids in the order they were added to the
+ batch.
+ request: list, list of request objects to send.
+
+ Raises:
+ httplib2.HttpLib2Error if a transport error has occured.
+ apiclient.errors.BatchError if the response is the wrong format.
+ """
+ message = MIMEMultipart('mixed')
+ # Message should not write out it's own headers.
+ setattr(message, '_write_headers', lambda self: None)
+
+ # Add all the individual requests.
+ for request_id in order:
+ request = requests[request_id]
+
+ msg = MIMENonMultipart('application', 'http')
+ msg['Content-Transfer-Encoding'] = 'binary'
+ msg['Content-ID'] = self._id_to_header(request_id)
+
+ body = self._serialize_request(request)
+ msg.set_payload(body)
+ message.attach(msg)
+
+ body = message.as_string()
+
+ headers = {}
+ headers['content-type'] = ('multipart/mixed; '
+ 'boundary="%s"') % message.get_boundary()
+
+ resp, content = http.request(self._batch_uri, 'POST', body=body,
+ headers=headers)
+
+ if resp.status >= 300:
+ raise HttpError(resp, content, uri=self._batch_uri)
+
+ # Now break out the individual responses and store each one.
+ boundary, _ = content.split(None, 1)
+
+ # Prepend with a content-type header so FeedParser can handle it.
+ header = 'content-type: %s\r\n\r\n' % resp['content-type']
+ for_parser = header + content
+
+ parser = FeedParser()
+ parser.feed(for_parser)
+ mime_response = parser.close()
+
+ if not mime_response.is_multipart():
+ raise BatchError("Response not in multipart/mixed format.", resp=resp,
+ content=content)
+
+ for part in mime_response.get_payload():
+ request_id = self._header_to_id(part['Content-ID'])
+ response, content = self._deserialize_response(part.get_payload())
+ self._responses[request_id] = (response, content)
+
+ @util.positional(1)
+ def execute(self, http=None):
+ """Execute all the requests as a single batched HTTP request.
+
+ Args:
+ http: httplib2.Http, an http object to be used in place of the one the
+ HttpRequest request object was constructed with. If one isn't supplied
+ then use a http object from the requests in this batch.
+
+ Returns:
+ None
+
+ Raises:
+ httplib2.HttpLib2Error if a transport error has occured.
+ apiclient.errors.BatchError if the response is the wrong format.
+ """
+
+ # If http is not supplied use the first valid one given in the requests.
+ if http is None:
+ for request_id in self._order:
+ request = self._requests[request_id]
+ if request is not None:
+ http = request.http
+ break
+
+ if http is None:
+ raise ValueError("Missing a valid http object.")
+
+ self._execute(http, self._order, self._requests)
+
+ # Loop over all the requests and check for 401s. For each 401 request the
+ # credentials should be refreshed and then sent again in a separate batch.
+ redo_requests = {}
+ redo_order = []
+
+ for request_id in self._order:
+ resp, content = self._responses[request_id]
+ if resp['status'] == '401':
+ redo_order.append(request_id)
+ request = self._requests[request_id]
+ self._refresh_and_apply_credentials(request, http)
+ redo_requests[request_id] = request
+
+ if redo_requests:
+ self._execute(http, redo_order, redo_requests)
+
+ # Now process all callbacks that are erroring, and raise an exception for
+ # ones that return a non-2xx response? Or add extra parameter to callback
+ # that contains an HttpError?
+
+ for request_id in self._order:
+ resp, content = self._responses[request_id]
+
+ request = self._requests[request_id]
+ callback = self._callbacks[request_id]
+
+ response = None
+ exception = None
+ try:
+ if resp.status >= 300:
+ raise HttpError(resp, content, uri=request.uri)
+ response = request.postproc(resp, content)
+ except HttpError, e:
+ exception = e
+
+ if callback is not None:
+ callback(request_id, response, exception)
+ if self._callback is not None:
+ self._callback(request_id, response, exception)
+
+
+class HttpRequestMock(object):
+ """Mock of HttpRequest.
+
+ Do not construct directly, instead use RequestMockBuilder.
+ """
+
+ def __init__(self, resp, content, postproc):
+ """Constructor for HttpRequestMock
+
+ Args:
+ resp: httplib2.Response, the response to emulate coming from the request
+ content: string, the response body
+ postproc: callable, the post processing function usually supplied by
+ the model class. See model.JsonModel.response() as an example.
+ """
+ self.resp = resp
+ self.content = content
+ self.postproc = postproc
+ if resp is None:
+ self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
+ if 'reason' in self.resp:
+ self.resp.reason = self.resp['reason']
+
+ def execute(self, http=None):
+ """Execute the request.
+
+ Same behavior as HttpRequest.execute(), but the response is
+ mocked and not really from an HTTP request/response.
+ """
+ return self.postproc(self.resp, self.content)
+
+
+class RequestMockBuilder(object):
+ """A simple mock of HttpRequest
+
+ Pass in a dictionary to the constructor that maps request methodIds to
+ tuples of (httplib2.Response, content, opt_expected_body) that should be
+ returned when that method is called. None may also be passed in for the
+ httplib2.Response, in which case a 200 OK response will be generated.
+ If an opt_expected_body (str or dict) is provided, it will be compared to
+ the body and UnexpectedBodyError will be raised on inequality.
+
+ Example:
+ response = '{"data": {"id": "tag:google.c...'
+ requestBuilder = RequestMockBuilder(
+ {
+ 'plus.activities.get': (None, response),
+ }
+ )
+ apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
+
+ Methods that you do not supply a response for will return a
+ 200 OK with an empty string as the response content or raise an excpetion
+ if check_unexpected is set to True. The methodId is taken from the rpcName
+ in the discovery document.
+
+ For more details see the project wiki.
+ """
+
+ def __init__(self, responses, check_unexpected=False):
+ """Constructor for RequestMockBuilder
+
+ The constructed object should be a callable object
+ that can replace the class HttpResponse.
+
+ responses - A dictionary that maps methodIds into tuples
+ of (httplib2.Response, content). The methodId
+ comes from the 'rpcName' field in the discovery
+ document.
+ check_unexpected - A boolean setting whether or not UnexpectedMethodError
+ should be raised on unsupplied method.
+ """
+ self.responses = responses
+ self.check_unexpected = check_unexpected
+
+ def __call__(self, http, postproc, uri, method='GET', body=None,
+ headers=None, methodId=None, resumable=None):
+ """Implements the callable interface that discovery.build() expects
+ of requestBuilder, which is to build an object compatible with
+ HttpRequest.execute(). See that method for the description of the
+ parameters and the expected response.
+ """
+ if methodId in self.responses:
+ response = self.responses[methodId]
+ resp, content = response[:2]
+ if len(response) > 2:
+ # Test the body against the supplied expected_body.
+ expected_body = response[2]
+ if bool(expected_body) != bool(body):
+ # Not expecting a body and provided one
+ # or expecting a body and not provided one.
+ raise UnexpectedBodyError(expected_body, body)
+ if isinstance(expected_body, str):
+ expected_body = simplejson.loads(expected_body)
+ body = simplejson.loads(body)
+ if body != expected_body:
+ raise UnexpectedBodyError(expected_body, body)
+ return HttpRequestMock(resp, content, postproc)
+ elif self.check_unexpected:
+ raise UnexpectedMethodError(methodId=methodId)
+ else:
+ model = JsonModel(False)
+ return HttpRequestMock(None, '{}', model.response)
+
+
+class HttpMock(object):
+ """Mock of httplib2.Http"""
+
+ def __init__(self, filename=None, headers=None):
+ """
+ Args:
+ filename: string, absolute filename to read response from
+ headers: dict, header to return with response
+ """
+ if headers is None:
+ headers = {'status': '200 OK'}
+ if filename:
+ f = file(filename, 'r')
+ self.data = f.read()
+ f.close()
+ else:
+ self.data = None
+ self.response_headers = headers
+ self.headers = None
+ self.uri = None
+ self.method = None
+ self.body = None
+ self.headers = None
+
+
+ def request(self, uri,
+ method='GET',
+ body=None,
+ headers=None,
+ redirections=1,
+ connection_type=None):
+ self.uri = uri
+ self.method = method
+ self.body = body
+ self.headers = headers
+ return httplib2.Response(self.response_headers), self.data
+
+
+class HttpMockSequence(object):
+ """Mock of httplib2.Http
+
+ Mocks a sequence of calls to request returning different responses for each
+ call. Create an instance initialized with the desired response headers
+ and content and then use as if an httplib2.Http instance.
+
+ http = HttpMockSequence([
+ ({'status': '401'}, ''),
+ ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
+ ({'status': '200'}, 'echo_request_headers'),
+ ])
+ resp, content = http.request("http://examples.com")
+
+ There are special values you can pass in for content to trigger
+ behavours that are helpful in testing.
+
+ 'echo_request_headers' means return the request headers in the response body
+ 'echo_request_headers_as_json' means return the request headers in
+ the response body
+ 'echo_request_body' means return the request body in the response body
+ 'echo_request_uri' means return the request uri in the response body
+ """
+
+ def __init__(self, iterable):
+ """
+ Args:
+ iterable: iterable, a sequence of pairs of (headers, body)
+ """
+ self._iterable = iterable
+ self.follow_redirects = True
+
+ def request(self, uri,
+ method='GET',
+ body=None,
+ headers=None,
+ redirections=1,
+ connection_type=None):
+ resp, content = self._iterable.pop(0)
+ if content == 'echo_request_headers':
+ content = headers
+ elif content == 'echo_request_headers_as_json':
+ content = simplejson.dumps(headers)
+ elif content == 'echo_request_body':
+ if hasattr(body, 'read'):
+ content = body.read()
+ else:
+ content = body
+ elif content == 'echo_request_uri':
+ content = uri
+ return httplib2.Response(resp), content
+
+
+def set_user_agent(http, user_agent):
+ """Set the user-agent on every request.
+
+ Args:
+ http - An instance of httplib2.Http
+ or something that acts like it.
+ user_agent: string, the value for the user-agent header.
+
+ Returns:
+ A modified instance of http that was passed in.
+
+ Example:
+
+ h = httplib2.Http()
+ h = set_user_agent(h, "my-app-name/6.0")
+
+ Most of the time the user-agent will be set doing auth, this is for the rare
+ cases where you are accessing an unauthenticated endpoint.
+ """
+ request_orig = http.request
+
+ # The closure that will replace 'httplib2.Http.request'.
+ def new_request(uri, method='GET', body=None, headers=None,
+ redirections=httplib2.DEFAULT_MAX_REDIRECTS,
+ connection_type=None):
+ """Modify the request headers to add the user-agent."""
+ if headers is None:
+ headers = {}
+ if 'user-agent' in headers:
+ headers['user-agent'] = user_agent + ' ' + headers['user-agent']
+ else:
+ headers['user-agent'] = user_agent
+ resp, content = request_orig(uri, method, body, headers,
+ redirections, connection_type)
+ return resp, content
+
+ http.request = new_request
+ return http
+
+
+def tunnel_patch(http):
+ """Tunnel PATCH requests over POST.
+ Args:
+ http - An instance of httplib2.Http
+ or something that acts like it.
+
+ Returns:
+ A modified instance of http that was passed in.
+
+ Example:
+
+ h = httplib2.Http()
+ h = tunnel_patch(h, "my-app-name/6.0")
+
+ Useful if you are running on a platform that doesn't support PATCH.
+ Apply this last if you are using OAuth 1.0, as changing the method
+ will result in a different signature.
+ """
+ request_orig = http.request
+
+ # The closure that will replace 'httplib2.Http.request'.
+ def new_request(uri, method='GET', body=None, headers=None,
+ redirections=httplib2.DEFAULT_MAX_REDIRECTS,
+ connection_type=None):
+ """Modify the request headers to add the user-agent."""
+ if headers is None:
+ headers = {}
+ if method == 'PATCH':
+ if 'oauth_token' in headers.get('authorization', ''):
+ logging.warning(
+ 'OAuth 1.0 request made with Credentials after tunnel_patch.')
+ headers['x-http-method-override'] = "PATCH"
+ method = 'POST'
+ resp, content = request_orig(uri, method, body, headers,
+ redirections, connection_type)
+ return resp, content
+
+ http.request = new_request
+ return http
diff --git a/py/minijack/apiclient/mimeparse.py b/py/minijack/apiclient/mimeparse.py
new file mode 100644
index 0000000..cbb9d07
--- /dev/null
+++ b/py/minijack/apiclient/mimeparse.py
@@ -0,0 +1,172 @@
+# Copyright (C) 2007 Joe Gregorio
+#
+# Licensed under the MIT License
+
+"""MIME-Type Parser
+
+This module provides basic functions for handling mime-types. It can handle
+matching mime-types against a list of media-ranges. See section 14.1 of the
+HTTP specification [RFC 2616] for a complete explanation.
+
+ http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1
+
+Contents:
+ - parse_mime_type(): Parses a mime-type into its component parts.
+ - parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q'
+ quality parameter.
+ - quality(): Determines the quality ('q') of a mime-type when
+ compared against a list of media-ranges.
+ - quality_parsed(): Just like quality() except the second parameter must be
+ pre-parsed.
+ - best_match(): Choose the mime-type with the highest quality ('q')
+ from a list of candidates.
+"""
+
+__version__ = '0.1.3'
+__author__ = 'Joe Gregorio'
+__email__ = 'joe@bitworking.org'
+__license__ = 'MIT License'
+__credits__ = ''
+
+
+def parse_mime_type(mime_type):
+ """Parses a mime-type into its component parts.
+
+ Carves up a mime-type and returns a tuple of the (type, subtype, params)
+ where 'params' is a dictionary of all the parameters for the media range.
+ For example, the media range 'application/xhtml;q=0.5' would get parsed
+ into:
+
+ ('application', 'xhtml', {'q', '0.5'})
+ """
+ parts = mime_type.split(';')
+ params = dict([tuple([s.strip() for s in param.split('=', 1)])\
+ for param in parts[1:]
+ ])
+ full_type = parts[0].strip()
+ # Java URLConnection class sends an Accept header that includes a
+ # single '*'. Turn it into a legal wildcard.
+ if full_type == '*':
+ full_type = '*/*'
+ (type, subtype) = full_type.split('/')
+
+ return (type.strip(), subtype.strip(), params)
+
+
+def parse_media_range(range):
+ """Parse a media-range into its component parts.
+
+ Carves up a media range and returns a tuple of the (type, subtype,
+ params) where 'params' is a dictionary of all the parameters for the media
+ range. For example, the media range 'application/*;q=0.5' would get parsed
+ into:
+
+ ('application', '*', {'q', '0.5'})
+
+ In addition this function also guarantees that there is a value for 'q'
+ in the params dictionary, filling it in with a proper default if
+ necessary.
+ """
+ (type, subtype, params) = parse_mime_type(range)
+ if not params.has_key('q') or not params['q'] or \
+ not float(params['q']) or float(params['q']) > 1\
+ or float(params['q']) < 0:
+ params['q'] = '1'
+
+ return (type, subtype, params)
+
+
+def fitness_and_quality_parsed(mime_type, parsed_ranges):
+ """Find the best match for a mime-type amongst parsed media-ranges.
+
+ Find the best match for a given mime-type against a list of media_ranges
+ that have already been parsed by parse_media_range(). Returns a tuple of
+ the fitness value and the value of the 'q' quality parameter of the best
+ match, or (-1, 0) if no match was found. Just as for quality_parsed(),
+ 'parsed_ranges' must be a list of parsed media ranges.
+ """
+ best_fitness = -1
+ best_fit_q = 0
+ (target_type, target_subtype, target_params) =\
+ parse_media_range(mime_type)
+ for (type, subtype, params) in parsed_ranges:
+ type_match = (type == target_type or\
+ type == '*' or\
+ target_type == '*')
+ subtype_match = (subtype == target_subtype or\
+ subtype == '*' or\
+ target_subtype == '*')
+ if type_match and subtype_match:
+ param_matches = reduce(lambda x, y: x + y, [1 for (key, value) in \
+ target_params.iteritems() if key != 'q' and \
+ params.has_key(key) and value == params[key]], 0)
+ fitness = (type == target_type) and 100 or 0
+ fitness += (subtype == target_subtype) and 10 or 0
+ fitness += param_matches
+ if fitness > best_fitness:
+ best_fitness = fitness
+ best_fit_q = params['q']
+
+ return best_fitness, float(best_fit_q)
+
+
+def quality_parsed(mime_type, parsed_ranges):
+ """Find the best match for a mime-type amongst parsed media-ranges.
+
+ Find the best match for a given mime-type against a list of media_ranges
+ that have already been parsed by parse_media_range(). Returns the 'q'
+ quality parameter of the best match, 0 if no match was found. This function
+ bahaves the same as quality() except that 'parsed_ranges' must be a list of
+ parsed media ranges.
+ """
+
+ return fitness_and_quality_parsed(mime_type, parsed_ranges)[1]
+
+
+def quality(mime_type, ranges):
+ """Return the quality ('q') of a mime-type against a list of media-ranges.
+
+ Returns the quality 'q' of a mime-type when compared against the
+ media-ranges in ranges. For example:
+
+ >>> quality('text/html','text/*;q=0.3, text/html;q=0.7,
+ text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5')
+ 0.7
+
+ """
+ parsed_ranges = [parse_media_range(r) for r in ranges.split(',')]
+
+ return quality_parsed(mime_type, parsed_ranges)
+
+
+def best_match(supported, header):
+ """Return mime-type with the highest quality ('q') from list of candidates.
+
+ Takes a list of supported mime-types and finds the best match for all the
+ media-ranges listed in header. The value of header must be a string that
+ conforms to the format of the HTTP Accept: header. The value of 'supported'
+ is a list of mime-types. The list of supported mime-types should be sorted
+ in order of increasing desirability, in case of a situation where there is
+ a tie.
+
+ >>> best_match(['application/xbel+xml', 'text/xml'],
+ 'text/*;q=0.5,*/*; q=0.1')
+ 'text/xml'
+ """
+ split_header = _filter_blank(header.split(','))
+ parsed_header = [parse_media_range(r) for r in split_header]
+ weighted_matches = []
+ pos = 0
+ for mime_type in supported:
+ weighted_matches.append((fitness_and_quality_parsed(mime_type,
+ parsed_header), pos, mime_type))
+ pos += 1
+ weighted_matches.sort()
+
+ return weighted_matches[-1][0][1] and weighted_matches[-1][2] or ''
+
+
+def _filter_blank(i):
+ for s in i:
+ if s.strip():
+ yield s
diff --git a/py/minijack/apiclient/model.py b/py/minijack/apiclient/model.py
new file mode 100644
index 0000000..12fcab6
--- /dev/null
+++ b/py/minijack/apiclient/model.py
@@ -0,0 +1,385 @@
+#!/usr/bin/python2.4
+#
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Model objects for requests and responses.
+
+Each API may support one or more serializations, such
+as JSON, Atom, etc. The model classes are responsible
+for converting between the wire format and the Python
+object representation.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+import gflags
+import logging
+import urllib
+
+from errors import HttpError
+from oauth2client.anyjson import simplejson
+
+FLAGS = gflags.FLAGS
+
+gflags.DEFINE_boolean('dump_request_response', False,
+ 'Dump all http server requests and responses. '
+ )
+
+
+def _abstract():
+ raise NotImplementedError('You need to override this function')
+
+
+class Model(object):
+ """Model base class.
+
+ All Model classes should implement this interface.
+ The Model serializes and de-serializes between a wire
+ format such as JSON and a Python object representation.
+ """
+
+ def request(self, headers, path_params, query_params, body_value):
+ """Updates outgoing requests with a serialized body.
+
+ Args:
+ headers: dict, request headers
+ path_params: dict, parameters that appear in the request path
+ query_params: dict, parameters that appear in the query
+ body_value: object, the request body as a Python object, which must be
+ serializable.
+ Returns:
+ A tuple of (headers, path_params, query, body)
+
+ headers: dict, request headers
+ path_params: dict, parameters that appear in the request path
+ query: string, query part of the request URI
+ body: string, the body serialized in the desired wire format.
+ """
+ _abstract()
+
+ def response(self, resp, content):
+ """Convert the response wire format into a Python object.
+
+ Args:
+ resp: httplib2.Response, the HTTP response headers and status
+ content: string, the body of the HTTP response
+
+ Returns:
+ The body de-serialized as a Python object.
+
+ Raises:
+ apiclient.errors.HttpError if a non 2xx response is received.
+ """
+ _abstract()
+
+
+class BaseModel(Model):
+ """Base model class.
+
+ Subclasses should provide implementations for the "serialize" and
+ "deserialize" methods, as well as values for the following class attributes.
+
+ Attributes:
+ accept: The value to use for the HTTP Accept header.
+ content_type: The value to use for the HTTP Content-type header.
+ no_content_response: The value to return when deserializing a 204 "No
+ Content" response.
+ alt_param: The value to supply as the "alt" query parameter for requests.
+ """
+
+ accept = None
+ content_type = None
+ no_content_response = None
+ alt_param = None
+
+ def _log_request(self, headers, path_params, query, body):
+ """Logs debugging information about the request if requested."""
+ if FLAGS.dump_request_response:
+ logging.info('--request-start--')
+ logging.info('-headers-start-')
+ for h, v in headers.iteritems():
+ logging.info('%s: %s', h, v)
+ logging.info('-headers-end-')
+ logging.info('-path-parameters-start-')
+ for h, v in path_params.iteritems():
+ logging.info('%s: %s', h, v)
+ logging.info('-path-parameters-end-')
+ logging.info('body: %s', body)
+ logging.info('query: %s', query)
+ logging.info('--request-end--')
+
+ def request(self, headers, path_params, query_params, body_value):
+ """Updates outgoing requests with a serialized body.
+
+ Args:
+ headers: dict, request headers
+ path_params: dict, parameters that appear in the request path
+ query_params: dict, parameters that appear in the query
+ body_value: object, the request body as a Python object, which must be
+ serializable by simplejson.
+ Returns:
+ A tuple of (headers, path_params, query, body)
+
+ headers: dict, request headers
+ path_params: dict, parameters that appear in the request path
+ query: string, query part of the request URI
+ body: string, the body serialized as JSON
+ """
+ query = self._build_query(query_params)
+ headers['accept'] = self.accept
+ headers['accept-encoding'] = 'gzip, deflate'
+ if 'user-agent' in headers:
+ headers['user-agent'] += ' '
+ else:
+ headers['user-agent'] = ''
+ headers['user-agent'] += 'google-api-python-client/1.0'
+
+ if body_value is not None:
+ headers['content-type'] = self.content_type
+ body_value = self.serialize(body_value)
+ self._log_request(headers, path_params, query, body_value)
+ return (headers, path_params, query, body_value)
+
+ def _build_query(self, params):
+ """Builds a query string.
+
+ Args:
+ params: dict, the query parameters
+
+ Returns:
+ The query parameters properly encoded into an HTTP URI query string.
+ """
+ if self.alt_param is not None:
+ params.update({'alt': self.alt_param})
+ astuples = []
+ for key, value in params.iteritems():
+ if type(value) == type([]):
+ for x in value:
+ x = x.encode('utf-8')
+ astuples.append((key, x))
+ else:
+ if getattr(value, 'encode', False) and callable(value.encode):
+ value = value.encode('utf-8')
+ astuples.append((key, value))
+ return '?' + urllib.urlencode(astuples)
+
+ def _log_response(self, resp, content):
+ """Logs debugging information about the response if requested."""
+ if FLAGS.dump_request_response:
+ logging.info('--response-start--')
+ for h, v in resp.iteritems():
+ logging.info('%s: %s', h, v)
+ if content:
+ logging.info(content)
+ logging.info('--response-end--')
+
+ def response(self, resp, content):
+ """Convert the response wire format into a Python object.
+
+ Args:
+ resp: httplib2.Response, the HTTP response headers and status
+ content: string, the body of the HTTP response
+
+ Returns:
+ The body de-serialized as a Python object.
+
+ Raises:
+ apiclient.errors.HttpError if a non 2xx response is received.
+ """
+ self._log_response(resp, content)
+ # Error handling is TBD, for example, do we retry
+ # for some operation/error combinations?
+ if resp.status < 300:
+ if resp.status == 204:
+ # A 204: No Content response should be treated differently
+ # to all the other success states
+ return self.no_content_response
+ return self.deserialize(content)
+ else:
+ logging.debug('Content from bad request was: %s' % content)
+ raise HttpError(resp, content)
+
+ def serialize(self, body_value):
+ """Perform the actual Python object serialization.
+
+ Args:
+ body_value: object, the request body as a Python object.
+
+ Returns:
+ string, the body in serialized form.
+ """
+ _abstract()
+
+ def deserialize(self, content):
+ """Perform the actual deserialization from response string to Python
+ object.
+
+ Args:
+ content: string, the body of the HTTP response
+
+ Returns:
+ The body de-serialized as a Python object.
+ """
+ _abstract()
+
+
+class JsonModel(BaseModel):
+ """Model class for JSON.
+
+ Serializes and de-serializes between JSON and the Python
+ object representation of HTTP request and response bodies.
+ """
+ accept = 'application/json'
+ content_type = 'application/json'
+ alt_param = 'json'
+
+ def __init__(self, data_wrapper=False):
+ """Construct a JsonModel.
+
+ Args:
+ data_wrapper: boolean, wrap requests and responses in a data wrapper
+ """
+ self._data_wrapper = data_wrapper
+
+ def serialize(self, body_value):
+ if (isinstance(body_value, dict) and 'data' not in body_value and
+ self._data_wrapper):
+ body_value = {'data': body_value}
+ return simplejson.dumps(body_value)
+
+ def deserialize(self, content):
+ body = simplejson.loads(content)
+ if self._data_wrapper and isinstance(body, dict) and 'data' in body:
+ body = body['data']
+ return body
+
+ @property
+ def no_content_response(self):
+ return {}
+
+
+class RawModel(JsonModel):
+ """Model class for requests that don't return JSON.
+
+ Serializes and de-serializes between JSON and the Python
+ object representation of HTTP request, and returns the raw bytes
+ of the response body.
+ """
+ accept = '*/*'
+ content_type = 'application/json'
+ alt_param = None
+
+ def deserialize(self, content):
+ return content
+
+ @property
+ def no_content_response(self):
+ return ''
+
+
+class MediaModel(JsonModel):
+ """Model class for requests that return Media.
+
+ Serializes and de-serializes between JSON and the Python
+ object representation of HTTP request, and returns the raw bytes
+ of the response body.
+ """
+ accept = '*/*'
+ content_type = 'application/json'
+ alt_param = 'media'
+
+ def deserialize(self, content):
+ return content
+
+ @property
+ def no_content_response(self):
+ return ''
+
+
+class ProtocolBufferModel(BaseModel):
+ """Model class for protocol buffers.
+
+ Serializes and de-serializes the binary protocol buffer sent in the HTTP
+ request and response bodies.
+ """
+ accept = 'application/x-protobuf'
+ content_type = 'application/x-protobuf'
+ alt_param = 'proto'
+
+ def __init__(self, protocol_buffer):
+ """Constructs a ProtocolBufferModel.
+
+ The serialzed protocol buffer returned in an HTTP response will be
+ de-serialized using the given protocol buffer class.
+
+ Args:
+ protocol_buffer: The protocol buffer class used to de-serialize a
+ response from the API.
+ """
+ self._protocol_buffer = protocol_buffer
+
+ def serialize(self, body_value):
+ return body_value.SerializeToString()
+
+ def deserialize(self, content):
+ return self._protocol_buffer.FromString(content)
+
+ @property
+ def no_content_response(self):
+ return self._protocol_buffer()
+
+
+def makepatch(original, modified):
+ """Create a patch object.
+
+ Some methods support PATCH, an efficient way to send updates to a resource.
+ This method allows the easy construction of patch bodies by looking at the
+ differences between a resource before and after it was modified.
+
+ Args:
+ original: object, the original deserialized resource
+ modified: object, the modified deserialized resource
+ Returns:
+ An object that contains only the changes from original to modified, in a
+ form suitable to pass to a PATCH method.
+
+ Example usage:
+ item = service.activities().get(postid=postid, userid=userid).execute()
+ original = copy.deepcopy(item)
+ item['object']['content'] = 'This is updated.'
+ service.activities.patch(postid=postid, userid=userid,
+ body=makepatch(original, item)).execute()
+ """
+ patch = {}
+ for key, original_value in original.iteritems():
+ modified_value = modified.get(key, None)
+ if modified_value is None:
+ # Use None to signal that the element is deleted
+ patch[key] = None
+ elif original_value != modified_value:
+ if type(original_value) == type({}):
+ # Recursively descend objects
+ patch[key] = makepatch(original_value, modified_value)
+ else:
+ # In the case of simple types or arrays we just replace
+ patch[key] = modified_value
+ else:
+ # Don't add anything to patch if there's no change
+ pass
+ for key in modified:
+ if key not in original:
+ patch[key] = modified[key]
+
+ return patch
diff --git a/py/minijack/apiclient/push.py b/py/minijack/apiclient/push.py
new file mode 100644
index 0000000..c520faf
--- /dev/null
+++ b/py/minijack/apiclient/push.py
@@ -0,0 +1,274 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Push notifications support.
+
+This code is based on experimental APIs and is subject to change.
+"""
+
+__author__ = 'afshar@google.com (Ali Afshar)'
+
+import binascii
+import collections
+import os
+import urllib
+
+SUBSCRIBE = 'X-GOOG-SUBSCRIBE'
+SUBSCRIPTION_ID = 'X-GOOG-SUBSCRIPTION-ID'
+TOPIC_ID = 'X-GOOG-TOPIC-ID'
+TOPIC_URI = 'X-GOOG-TOPIC-URI'
+CLIENT_TOKEN = 'X-GOOG-CLIENT-TOKEN'
+EVENT_TYPE = 'X-GOOG-EVENT-TYPE'
+UNSUBSCRIBE = 'X-GOOG-UNSUBSCRIBE'
+
+
+class InvalidSubscriptionRequestError(ValueError):
+ """The request cannot be subscribed."""
+
+
+def new_token():
+ """Gets a random token for use as a client_token in push notifications.
+
+ Returns:
+ str, a new random token.
+ """
+ return binascii.hexlify(os.urandom(32))
+
+
+class Channel(object):
+ """Base class for channel types."""
+
+ def __init__(self, channel_type, channel_args):
+ """Create a new Channel.
+
+ You probably won't need to create this channel manually, since there are
+ subclassed Channel for each specific type with a more customized set of
+ arguments to pass. However, you may wish to just create it manually here.
+
+ Args:
+ channel_type: str, the type of channel.
+ channel_args: dict, arguments to pass to the channel.
+ """
+ self.channel_type = channel_type
+ self.channel_args = channel_args
+
+ def as_header_value(self):
+ """Create the appropriate header for this channel.
+
+ Returns:
+ str encoded channel description suitable for use as a header.
+ """
+ return '%s?%s' % (self.channel_type, urllib.urlencode(self.channel_args))
+
+ def write_header(self, headers):
+ """Write the appropriate subscribe header to a headers dict.
+
+ Args:
+ headers: dict, headers to add subscribe header to.
+ """
+ headers[SUBSCRIBE] = self.as_header_value()
+
+
+class WebhookChannel(Channel):
+ """Channel for registering web hook notifications."""
+
+ def __init__(self, url, app_engine=False):
+ """Create a new WebhookChannel
+
+ Args:
+ url: str, URL to post notifications to.
+ app_engine: bool, default=False, whether the destination for the
+ notifications is an App Engine application.
+ """
+ super(WebhookChannel, self).__init__(
+ channel_type='web_hook',
+ channel_args={
+ 'url': url,
+ 'app_engine': app_engine and 'true' or 'false',
+ }
+ )
+
+
+class Headers(collections.defaultdict):
+ """Headers for managing subscriptions."""
+
+
+ ALL_HEADERS = set([SUBSCRIBE, SUBSCRIPTION_ID, TOPIC_ID, TOPIC_URI,
+ CLIENT_TOKEN, EVENT_TYPE, UNSUBSCRIBE])
+
+ def __init__(self):
+ """Create a new subscription configuration instance."""
+ collections.defaultdict.__init__(self, str)
+
+ def __setitem__(self, key, value):
+ """Set a header value, ensuring the key is an allowed value.
+
+ Args:
+ key: str, the header key.
+ value: str, the header value.
+ Raises:
+ ValueError if key is not one of the accepted headers.
+ """
+ normal_key = self._normalize_key(key)
+ if normal_key not in self.ALL_HEADERS:
+ raise ValueError('Header name must be one of %s.' % self.ALL_HEADERS)
+ else:
+ return collections.defaultdict.__setitem__(self, normal_key, value)
+
+ def __getitem__(self, key):
+ """Get a header value, normalizing the key case.
+
+ Args:
+ key: str, the header key.
+ Returns:
+ String header value.
+ Raises:
+ KeyError if the key is not one of the accepted headers.
+ """
+ normal_key = self._normalize_key(key)
+ if normal_key not in self.ALL_HEADERS:
+ raise ValueError('Header name must be one of %s.' % self.ALL_HEADERS)
+ else:
+ return collections.defaultdict.__getitem__(self, normal_key)
+
+ def _normalize_key(self, key):
+ """Normalize a header name for use as a key."""
+ return key.upper()
+
+ def items(self):
+ """Generator for each header."""
+ for header in self.ALL_HEADERS:
+ value = self[header]
+ if value:
+ yield header, value
+
+ def write(self, headers):
+ """Applies the subscription headers.
+
+ Args:
+ headers: dict of headers to insert values into.
+ """
+ for header, value in self.items():
+ headers[header.lower()] = value
+
+ def read(self, headers):
+ """Read from headers.
+
+ Args:
+ headers: dict of headers to read from.
+ """
+ for header in self.ALL_HEADERS:
+ if header.lower() in headers:
+ self[header] = headers[header.lower()]
+
+
+class Subscription(object):
+ """Information about a subscription."""
+
+ def __init__(self):
+ """Create a new Subscription."""
+ self.headers = Headers()
+
+ @classmethod
+ def for_request(cls, request, channel, client_token=None):
+ """Creates a subscription and attaches it to a request.
+
+ Args:
+ request: An http.HttpRequest to modify for making a subscription.
+ channel: A apiclient.push.Channel describing the subscription to
+ create.
+ client_token: (optional) client token to verify the notification.
+
+ Returns:
+ New subscription object.
+ """
+ subscription = cls.for_channel(channel=channel, client_token=client_token)
+ subscription.headers.write(request.headers)
+ if request.method != 'GET':
+ raise InvalidSubscriptionRequestError(
+ 'Can only subscribe to requests which are GET.')
+ request.method = 'POST'
+
+ def _on_response(response, subscription=subscription):
+ """Called with the response headers. Reads the subscription headers."""
+ subscription.headers.read(response)
+
+ request.add_response_callback(_on_response)
+ return subscription
+
+ @classmethod
+ def for_channel(cls, channel, client_token=None):
+ """Alternate constructor to create a subscription from a channel.
+
+ Args:
+ channel: A apiclient.push.Channel describing the subscription to
+ create.
+ client_token: (optional) client token to verify the notification.
+
+ Returns:
+ New subscription object.
+ """
+ subscription = cls()
+ channel.write_header(subscription.headers)
+ if client_token is None:
+ client_token = new_token()
+ subscription.headers[SUBSCRIPTION_ID] = new_token()
+ subscription.headers[CLIENT_TOKEN] = client_token
+ return subscription
+
+ def verify(self, headers):
+ """Verifies that a webhook notification has the correct client_token.
+
+ Args:
+ headers: dict of request headers for a push notification.
+
+ Returns:
+ Boolean value indicating whether the notification is verified.
+ """
+ new_subscription = Subscription()
+ new_subscription.headers.read(headers)
+ return new_subscription.client_token == self.client_token
+
+ @property
+ def subscribe(self):
+ """Subscribe header value."""
+ return self.headers[SUBSCRIBE]
+
+ @property
+ def subscription_id(self):
+ """Subscription ID header value."""
+ return self.headers[SUBSCRIPTION_ID]
+
+ @property
+ def topic_id(self):
+ """Topic ID header value."""
+ return self.headers[TOPIC_ID]
+
+ @property
+ def topic_uri(self):
+ """Topic URI header value."""
+ return self.headers[TOPIC_URI]
+
+ @property
+ def client_token(self):
+ """Client Token header value."""
+ return self.headers[CLIENT_TOKEN]
+
+ @property
+ def event_type(self):
+ """Event Type header value."""
+ return self.headers[EVENT_TYPE]
+
+ @property
+ def unsubscribe(self):
+ """Unsuscribe header value."""
+ return self.headers[UNSUBSCRIBE]
diff --git a/py/minijack/apiclient/schema.py b/py/minijack/apiclient/schema.py
new file mode 100644
index 0000000..d076a86
--- /dev/null
+++ b/py/minijack/apiclient/schema.py
@@ -0,0 +1,312 @@
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Schema processing for discovery based APIs
+
+Schemas holds an APIs discovery schemas. It can return those schema as
+deserialized JSON objects, or pretty print them as prototype objects that
+conform to the schema.
+
+For example, given the schema:
+
+ schema = \"\"\"{
+ "Foo": {
+ "type": "object",
+ "properties": {
+ "etag": {
+ "type": "string",
+ "description": "ETag of the collection."
+ },
+ "kind": {
+ "type": "string",
+ "description": "Type of the collection ('calendar#acl').",
+ "default": "calendar#acl"
+ },
+ "nextPageToken": {
+ "type": "string",
+ "description": "Token used to access the next
+ page of this result. Omitted if no further results are available."
+ }
+ }
+ }
+ }\"\"\"
+
+ s = Schemas(schema)
+ print s.prettyPrintByName('Foo')
+
+ Produces the following output:
+
+ {
+ "nextPageToken": "A String", # Token used to access the
+ # next page of this result. Omitted if no further results are available.
+ "kind": "A String", # Type of the collection ('calendar#acl').
+ "etag": "A String", # ETag of the collection.
+ },
+
+The constructor takes a discovery document in which to look up named schema.
+"""
+
+# TODO(jcgregorio) support format, enum, minimum, maximum
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+import copy
+
+from oauth2client import util
+from oauth2client.anyjson import simplejson
+
+
+class Schemas(object):
+ """Schemas for an API."""
+
+ def __init__(self, discovery):
+ """Constructor.
+
+ Args:
+ discovery: object, Deserialized discovery document from which we pull
+ out the named schema.
+ """
+ self.schemas = discovery.get('schemas', {})
+
+ # Cache of pretty printed schemas.
+ self.pretty = {}
+
+ @util.positional(2)
+ def _prettyPrintByName(self, name, seen=None, dent=0):
+ """Get pretty printed object prototype from the schema name.
+
+ Args:
+ name: string, Name of schema in the discovery document.
+ seen: list of string, Names of schema already seen. Used to handle
+ recursive definitions.
+
+ Returns:
+ string, A string that contains a prototype object with
+ comments that conforms to the given schema.
+ """
+ if seen is None:
+ seen = []
+
+ if name in seen:
+ # Do not fall into an infinite loop over recursive definitions.
+ return '# Object with schema name: %s' % name
+ seen.append(name)
+
+ if name not in self.pretty:
+ self.pretty[name] = _SchemaToStruct(self.schemas[name],
+ seen, dent=dent).to_str(self._prettyPrintByName)
+
+ seen.pop()
+
+ return self.pretty[name]
+
+ def prettyPrintByName(self, name):
+ """Get pretty printed object prototype from the schema name.
+
+ Args:
+ name: string, Name of schema in the discovery document.
+
+ Returns:
+ string, A string that contains a prototype object with
+ comments that conforms to the given schema.
+ """
+ # Return with trailing comma and newline removed.
+ return self._prettyPrintByName(name, seen=[], dent=1)[:-2]
+
+ @util.positional(2)
+ def _prettyPrintSchema(self, schema, seen=None, dent=0):
+ """Get pretty printed object prototype of schema.
+
+ Args:
+ schema: object, Parsed JSON schema.
+ seen: list of string, Names of schema already seen. Used to handle
+ recursive definitions.
+
+ Returns:
+ string, A string that contains a prototype object with
+ comments that conforms to the given schema.
+ """
+ if seen is None:
+ seen = []
+
+ return _SchemaToStruct(schema, seen, dent=dent).to_str(self._prettyPrintByName)
+
+ def prettyPrintSchema(self, schema):
+ """Get pretty printed object prototype of schema.
+
+ Args:
+ schema: object, Parsed JSON schema.
+
+ Returns:
+ string, A string that contains a prototype object with
+ comments that conforms to the given schema.
+ """
+ # Return with trailing comma and newline removed.
+ return self._prettyPrintSchema(schema, dent=1)[:-2]
+
+ def get(self, name):
+ """Get deserialized JSON schema from the schema name.
+
+ Args:
+ name: string, Schema name.
+ """
+ return self.schemas[name]
+
+
+class _SchemaToStruct(object):
+ """Convert schema to a prototype object."""
+
+ @util.positional(3)
+ def __init__(self, schema, seen, dent=0):
+ """Constructor.
+
+ Args:
+ schema: object, Parsed JSON schema.
+ seen: list, List of names of schema already seen while parsing. Used to
+ handle recursive definitions.
+ dent: int, Initial indentation depth.
+ """
+ # The result of this parsing kept as list of strings.
+ self.value = []
+
+ # The final value of the parsing.
+ self.string = None
+
+ # The parsed JSON schema.
+ self.schema = schema
+
+ # Indentation level.
+ self.dent = dent
+
+ # Method that when called returns a prototype object for the schema with
+ # the given name.
+ self.from_cache = None
+
+ # List of names of schema already seen while parsing.
+ self.seen = seen
+
+ def emit(self, text):
+ """Add text as a line to the output.
+
+ Args:
+ text: string, Text to output.
+ """
+ self.value.extend([" " * self.dent, text, '\n'])
+
+ def emitBegin(self, text):
+ """Add text to the output, but with no line terminator.
+
+ Args:
+ text: string, Text to output.
+ """
+ self.value.extend([" " * self.dent, text])
+
+ def emitEnd(self, text, comment):
+ """Add text and comment to the output with line terminator.
+
+ Args:
+ text: string, Text to output.
+ comment: string, Python comment.
+ """
+ if comment:
+ divider = '\n' + ' ' * (self.dent + 2) + '# '
+ lines = comment.splitlines()
+ lines = [x.rstrip() for x in lines]
+ comment = divider.join(lines)
+ self.value.extend([text, ' # ', comment, '\n'])
+ else:
+ self.value.extend([text, '\n'])
+
+ def indent(self):
+ """Increase indentation level."""
+ self.dent += 1
+
+ def undent(self):
+ """Decrease indentation level."""
+ self.dent -= 1
+
+ def _to_str_impl(self, schema):
+ """Prototype object based on the schema, in Python code with comments.
+
+ Args:
+ schema: object, Parsed JSON schema file.
+
+ Returns:
+ Prototype object based on the schema, in Python code with comments.
+ """
+ stype = schema.get('type')
+ if stype == 'object':
+ self.emitEnd('{', schema.get('description', ''))
+ self.indent()
+ if 'properties' in schema:
+ for pname, pschema in schema.get('properties', {}).iteritems():
+ self.emitBegin('"%s": ' % pname)
+ self._to_str_impl(pschema)
+ elif 'additionalProperties' in schema:
+ self.emitBegin('"a_key": ')
+ self._to_str_impl(schema['additionalProperties'])
+ self.undent()
+ self.emit('},')
+ elif '$ref' in schema:
+ schemaName = schema['$ref']
+ description = schema.get('description', '')
+ s = self.from_cache(schemaName, seen=self.seen)
+ parts = s.splitlines()
+ self.emitEnd(parts[0], description)
+ for line in parts[1:]:
+ self.emit(line.rstrip())
+ elif stype == 'boolean':
+ value = schema.get('default', 'True or False')
+ self.emitEnd('%s,' % str(value), schema.get('description', ''))
+ elif stype == 'string':
+ value = schema.get('default', 'A String')
+ self.emitEnd('"%s",' % str(value), schema.get('description', ''))
+ elif stype == 'integer':
+ value = schema.get('default', '42')
+ self.emitEnd('%s,' % str(value), schema.get('description', ''))
+ elif stype == 'number':
+ value = schema.get('default', '3.14')
+ self.emitEnd('%s,' % str(value), schema.get('description', ''))
+ elif stype == 'null':
+ self.emitEnd('None,', schema.get('description', ''))
+ elif stype == 'any':
+ self.emitEnd('"",', schema.get('description', ''))
+ elif stype == 'array':
+ self.emitEnd('[', schema.get('description'))
+ self.indent()
+ self.emitBegin('')
+ self._to_str_impl(schema['items'])
+ self.undent()
+ self.emit('],')
+ else:
+ self.emit('Unknown type! %s' % stype)
+ self.emitEnd('', '')
+
+ self.string = ''.join(self.value)
+ return self.string
+
+ def to_str(self, from_cache):
+ """Prototype object based on the schema, in Python code with comments.
+
+ Args:
+ from_cache: callable(name, seen), Callable that retrieves an object
+ prototype for a schema with the given name. Seen is a list of schema
+ names already seen as we recursively descend the schema definition.
+
+ Returns:
+ Prototype object based on the schema, in Python code with comments.
+ The lines of the code will all be properly indented.
+ """
+ self.from_cache = from_cache
+ return self._to_str_impl(self.schema)
diff --git a/py/minijack/app.yaml.TEMPLATE b/py/minijack/app.yaml.TEMPLATE
new file mode 100644
index 0000000..cfbda21
--- /dev/null
+++ b/py/minijack/app.yaml.TEMPLATE
@@ -0,0 +1,23 @@
+application: # TODO: change this.
+version: 1
+runtime: python27
+api_version: 1
+threadsafe: true
+
+builtins:
+- django_wsgi: on
+
+libraries:
+- name: django
+ version: "1.5"
+- name: numpy
+ version: latest
+- name: pycrypto
+ version: latest
+- name: ssl
+ version: latest
+
+handlers:
+- url: /static
+ static_dir: frontend/static/
+ expiration: '365d'
diff --git a/py/minijack/db/__init__.py b/py/minijack/db/__init__.py
index 1074dfd..98eab0c 100644
--- a/py/minijack/db/__init__.py
+++ b/py/minijack/db/__init__.py
@@ -173,7 +173,7 @@
executor = self._executor_factory.NewExecutor()
executor.Execute(sql_cmd, args)
if iter_all:
- return iter(lambda: executor.FetchOne(model=condition), None)
+ return executor.IterateAll(model=condition)
elif one_row:
return executor.FetchOne(model=condition)
else:
@@ -559,7 +559,7 @@
def GetOne(self):
"""Gets the first model which matches the given condition."""
sql_query, args = self.BuildQuery()
- sql_query += 'LIMIT 1'
+ sql_query += ' LIMIT 1'
executor = self._database.GetExecutorFactory().NewExecutor()
executor.Execute(sql_query, args)
if self._return_type == 'model':
@@ -604,4 +604,7 @@
yield v
-from db.sqlite import Executor, ExecutorFactory, Database, IntegrityError
+if settings.IS_APPENGINE:
+ from db.bigquery import Executor, ExecutorFactory, Database
+else:
+ from db.sqlite import Executor, ExecutorFactory, Database
diff --git a/py/minijack/db/bigquery.py b/py/minijack/db/bigquery.py
new file mode 100644
index 0000000..b174799
--- /dev/null
+++ b/py/minijack/db/bigquery.py
@@ -0,0 +1,409 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import httplib2
+
+from apiclient.discovery import build
+from apiclient.errors import HttpError
+
+from oauth2client.client import SignedJwtAssertionCredentials
+
+from db import models
+from db import DatabaseException, Table
+
+import db.base
+import settings
+
+
+def _EscapeArgument(arg):
+ """Escapes the argument used in SQL statement of BigQuery"""
+ if isinstance(arg, str) or isinstance(arg, unicode):
+ ret = '"'
+ dangerous_chars = set('%?,;\\\"\'&')
+ for c in arg:
+ if ord(c) < 32 or c in dangerous_chars:
+ ret += "\\x%02x" % ord(c)
+ else:
+ ret += c
+ ret += '"'
+ elif isinstance(arg, int) or isinstance(arg, float):
+ ret = str(arg)
+ else:
+ raise ValueError("Unknown type of argument in BigQuery's EscapeArgument")
+ return ret
+
+
+def _BuildQuery(sql_cmd, args):
+ """Builds the real SQL query statement using original query and arguments.
+
+ Since Google BigQuery API doesn't support query with arguments, we have to
+ manually escape arguments and embed them into the SQL query statement.
+
+ Args:
+ sql_cmd: The original SQL query, with '?' as placeholder for arguments.
+ args: The arguments in the query.
+ """
+ safe_args = [_EscapeArgument(arg) for arg in args]
+ idx = 0
+ ret = ""
+ for c in sql_cmd:
+ if c == '?':
+ if idx == len(safe_args):
+ raise DatabaseException(
+ 'Too many ? in query %s, %s' % (sql_cmd, args))
+ ret += safe_args[idx]
+ idx += 1
+ else:
+ ret += c
+ if idx != len(safe_args):
+ raise DatabaseException(
+ 'Too many arguments in query %s, %s' % (sql_cmd, args))
+ return ret
+
+
+class Executor(db.base.BaseExecutor):
+ """A database executor.
+
+ It abstracts the underlying database execution behaviors, like executing
+ an SQL query, fetching results, etc.
+
+ Properties:
+ _service: The service object for Google BigQuery API client.
+ _page_token: The pageToken retrived from BigQuery API.
+ _job_id: The job id of last job.
+ _column_names: The column names of the last query.
+ """
+ def __init__(self, service):
+ super(Executor, self).__init__()
+ self._service = service
+ self._page_token = None
+ self._job_id = None
+ self._column_names = []
+
+ def Execute(self, sql_cmd, args=None, dummy_commit=False, many=False):
+ """Executes an SQL command.
+
+ Args:
+ sql_cmd: The SQL command.
+ args: The arguments passed to the SQL command, a tuple or a dict.
+ commit: True to commit the transaction, used when modifying the content.
+ many: Do multiple execution. If True, the args argument should be a list.
+ """
+ logging.debug('Execute SQL command: %s, %s;', sql_cmd, args)
+ if not args:
+ args = tuple()
+ if isinstance(args, dict):
+ raise NotImplementedError('Dictionary query arguments not implemented.')
+ # TODO(pihsun): Implement these
+ elif not isinstance(args, tuple) and not isinstance(args, list):
+ raise ValueError('Unknown type for args %s' % args)
+ if many:
+ raise NotImplementedError('Multiple execution not implemented.')
+ else:
+ self._page_token = None
+ try:
+ query = _BuildQuery(sql_cmd, args)
+ query_result = self._service.jobs().query(
+ projectId=settings.PROJECT_ID,
+ body={
+ 'kind': 'bigquery#queryRequest',
+ 'defaultDataset': {
+ 'datasetId': settings.DATASET_ID,
+ },
+ 'query': query,
+ 'maxResults': 1,
+ 'preserveNulls': True,
+ }).execute()
+ except HttpError as e:
+ self._page_token = None
+ self._job_id = None
+ self._column_names = []
+ raise e
+ else:
+ if not query_result['jobComplete']:
+ raise DatabaseException('Job not completed within time limit')
+ # TODO(pihsun): We have the first row here, cache it for further use?
+ self._job_id = query_result['jobReference']['jobId']
+ self._column_names = [f['name']
+ for f in query_result['schema']['fields']]
+
+ def GetDescription(self):
+ """Gets the column names of the last query.
+
+ Returns:
+ A list of the columns names. Empty list if not a valid query.
+ """
+ return self._column_names
+
+ def FetchOne(self, model=None):
+ """Fetches one row of the previous query.
+
+ Args:
+ model: The model instance or class, describing the schema. The return
+ value is the same type of this model class. None to return a
+ raw tuple.
+
+ Returns:
+ A model instance if the argument model is valid; otherwise, a raw tuple.
+ None when no more data is available.
+ """
+ query_result = self._service.jobs().getQueryResults(
+ projectId=settings.PROJECT_ID,
+ jobId=self._job_id,
+ pageToken=self._page_token,
+ maxResults=1).execute()
+ if 'pageToken' in query_result:
+ self._page_token = query_result['pageToken']
+ if not 'rows' in query_result:
+ # No result
+ return None
+ result = tuple(c['v'] for c in query_result['rows'][0]['f'])
+ if result and model:
+ model = models.ToModelSubclass(model)
+ return model(result)
+ else:
+ return result
+
+ def FetchAll(self, model=None):
+ """Fetches all rows of the previous query.
+
+ Args:
+ model: The model instance or class, describing the schema. The return
+ value is the same type of this model class. None to return a
+ raw tuple.
+
+ Returns:
+ A list of model instances if the argument model is valid; otherwise, a
+ list of raw tuples.
+ """
+ query_result = self._service.jobs().getQueryResults(
+ projectId=settings.PROJECT_ID,
+ jobId=self._job_id,
+ pageToken=self._page_token).execute()
+ if 'pageToken' in query_result:
+ self._page_token = query_result['pageToken']
+ if not 'rows' in query_result:
+ # No result
+ return []
+ results = [tuple(c['v'] for c in r['f']) for r in query_result['rows']]
+ if results and model:
+ model = models.ToModelSubclass(model)
+ return [model(result) for result in results]
+ else:
+ return results
+
+ def IterateAll(self, model=None):
+ """Iterates through all row of the previous query.
+
+ Args:
+ model: The model instance or class, describing the schema. The return
+ value is the same type of this model class. None to return a
+ raw tuple.
+
+ Returns:
+ A iterator to model instances if the argument model is valid;
+ otherwise, a iterator to raw tuples.
+ """
+ return iter(self.FetchAll(model))
+
+
+class ExecutorFactory(db.base.BaseExecutorFactory):
+ """A factory to generate Executor objects.
+
+ Properties:
+ _service: The service object for Google BigQuery API client.
+ """
+ def __init__(self, service):
+ super(ExecutorFactory, self).__init__()
+ self._service = service
+
+ def NewExecutor(self):
+ """Generates a new Executor object."""
+ return Executor(self._service)
+
+
+class Database(db.base.BaseDatabase):
+ """A database to store Minijack results.
+
+ It abstracts the underlying database.
+ It uses BigQuery's dataset as an implementation.
+
+ Properties:
+ _service: The service object for Google BigQuery API client.
+ _tables: A dict of the created tables.
+ _executor_factory: A factory of executor objects.
+ _table_names: All table names in the dataset, cache when first fetched.
+ """
+ def __init__(self):
+ super(Database, self).__init__()
+ credential = SignedJwtAssertionCredentials(
+ settings.GOOGLE_API_ID,
+ settings.GOOGLE_API_PRIVATE_KEY,
+ scope=[
+ 'https://www.googleapis.com/auth/bigquery',
+ ])
+ http = httplib2.Http(timeout=60)
+ http = credential.authorize(http)
+
+ self._service = build('bigquery', 'v2', http=http)
+ self._tables = {}
+ self._executor_factory = ExecutorFactory(self._service)
+ self._table_names = None
+
+ def DoesTableExist(self, model):
+ """Checks the table with the given model schema exists or not.
+
+ Args:
+ model: A model class or a model instance.
+
+ Returns:
+ True if exists; otherwise, False.
+ """
+ if self._table_names is None:
+ result = self._service.tables().list(
+ projectId=settings.PROJECT_ID,
+ datasetId=settings.DATASET_ID).execute()
+ self._table_names = set(t['tableReference']['tableId']
+ for t in result['tables'])
+ return model.GetModelName() in self._table_names
+
+ def DoIndexesExist(self, model):
+ """Checks the indexes with the given model schema exist or not.
+
+ Args:
+ model: A model class or a model instance.
+
+ Returns:
+ True if exist; otherwise, False.
+ """
+ # There's no index in BigQuery.
+ # Since if this is False, we would attempt to execute the 'CREATE INDEX'
+ # SQL in BigQuery, and would result in an error.
+ return True
+
+ def GetExecutorFactory(self):
+ """Gets the executor factory."""
+ return self._executor_factory
+
+ def VerifySchema(self, model):
+ """Verifies the table in the database has the same given model schema.
+
+ Args:
+ model: A model class or a model instance.
+
+ Returns:
+ True if the same schema; otherwise, False.
+ """
+ # TODO(pihsun): Implement the check.
+ return True
+
+ def GetOrCreateTable(self, model):
+ """Gets or creates a table using the schema of the given model.
+
+ Args:
+ model: A string, a model class, or a model instance.
+
+ Returns:
+ The table instance.
+ """
+ if isinstance(model, str):
+ table_name = model
+ else:
+ table_name = model.GetModelName()
+ if table_name not in self._tables:
+ if not isinstance(model, str):
+ table = Table(self._executor_factory)
+ table.Init(model)
+ if self.DoesTableExist(model):
+ if not self.VerifySchema(model):
+ raise DatabaseException('Different schema in table %s' % table_name)
+ else:
+ raise DatabaseException(
+ 'Table %s doesn\'t exist in BigQuery' % table_name)
+ self._tables[table_name] = table
+ else:
+ raise DatabaseException('Table %s not initialized.' % table_name)
+ return self._tables[table_name]
+
+ def Close(self):
+ """Closes the database."""
+ pass
+
+ def Update(self, model):
+ """Updates the model in the database."""
+ raise DatabaseException('Can\'t do update on BigQuery')
+
+ def DeleteAll(self, condition):
+ """Deletes all the models which match the given condition."""
+ raise DatabaseException('Can\'t do delete on BigQuery')
+
+ def UpdateOrInsert(self, model):
+ """Updates the model or insert it if not exists."""
+ raise DatabaseException('Can\'t do update on BigQuery')
+
+ _operator_dict = dict([
+ ('exact', '%(key)s = %(val)s'),
+ ('gt', '%(key)s > %(val)s'),
+ ('lt', '%(key)s < %(val)s'),
+ ('gte', '%(key)s >= %(val)s'),
+ ('lte', '%(key)s <= %(val)s'),
+ ('regex', 'REGEXP_MATCH(%(key)s, %(val)s)'),
+ ('in', '%(key)s IN %(val)s'),
+ ('contains', '%(key)s CONTAINS %(val)s'),
+ ('startswith', 'LEFT(%(key), LENGTH(%(val))) = %(val)'),
+ ('endswith', 'RIGHT(%(key), LENGTH(%(val))) = %(val)'),
+ ])
+
+ @staticmethod
+ def EscapeColumnName(name, table=None):
+ """Escape column name so some keyword can be used as column name"""
+ if table:
+ return '[%s.%s]' % (table, name)
+ else:
+ return '[%s]' % name
+
+ _database = None
+
+ @classmethod
+ def Connect(cls):
+ """Connects to the database if necessary, and returns a Database."""
+ if cls._database is None:
+ cls._database = Database()
+ return cls._database
+
+ # TODO(pihsun): This only works when there is exactly one primary key field
+ # for parent model, so it won't work on ComponentDetail now.
+ def GetRelated(self, child_type, parents):
+ """Gets all related child_type objects of parent.
+
+ For example, if model B is nested in model A, calling with child_type = B
+ and parents = list of instance of A would retrieve all B that is nested
+ inside one of parents.
+
+ Args:
+ child_type: The type of the object to be retrived.
+ parents: A list of reference parent objects
+
+ Returns:
+ The related objects
+ """
+ if not isinstance(parents, list):
+ parents = [parents]
+ pk = parents[0].GetPrimaryKey()[0]
+ fields = [self.EscapeColumnName(f) if f == pk
+ else self.EscapeColumnName(f, child_type.nested_name)
+ + ' AS ' + self.EscapeColumnName(f)
+ for f in child_type.GetFieldNames()]
+ sql_where = (self.EscapeColumnName(pk) + ' IN (' +
+ ','.join(['?'] * len(parents)) + ')')
+ sql_cmd = ('SELECT %s FROM %s WHERE %s' %
+ (','.join(f for f in fields),
+ self.EscapeColumnName(parents[0].GetModelName()),
+ sql_where))
+ field_values = [getattr(p, pk) for p in parents]
+ executor = self._executor_factory.NewExecutor()
+ executor.Execute(sql_cmd, field_values)
+ return executor.FetchAll(model=child_type)
+
diff --git a/py/minijack/dump_db.py b/py/minijack/dump_db.py
new file mode 100755
index 0000000..72eb4e4
--- /dev/null
+++ b/py/minijack/dump_db.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This program dumps all datas in a local sqlite database into device.json,
+test.json, event.json and component.json that are suitable to be imported to
+Google BigQuery using the schemas in schemas/.
+
+To use it, invoke as a standalone program:
+ ./dump_db.py [options]
+"""
+
+import json
+import logging
+
+import minijack_common # pylint: disable=W0611
+import db
+from models import ComponentDetail
+from models import Device, Test, Component
+from models import Event, Attr
+
+
+def Main():
+ logging.basicConfig(level=logging.INFO)
+ database = db.Database.Connect()
+
+ logging.info('Dumping Device.')
+ with open('device.json', 'w') as f:
+ for d in database(Device).Values():
+ json.dump(d, f, separators=(',', ':'), sort_keys=True)
+ f.write('\n')
+
+ logging.info('Dumping Test.')
+ with open('test.json', 'w') as f:
+ for t in database(Test).Values():
+ json.dump(t, f, separators=(',', ':'), sort_keys=True)
+ f.write('\n')
+
+ ATTR_LENGTH_LIMIT = 100000
+ logging.info('Dumping Event.')
+ with open('event.json', 'w') as f:
+ for e in database(Event).Values():
+ e['attrs'] = []
+ for a in database(Attr).Filter(event_id=e['event_id']).Values():
+ e['attrs'].append(dict((k, v[:ATTR_LENGTH_LIMIT])
+ for k, v in a.iteritems()
+ if k != 'event_id'))
+ json.dump(e, f, separators=(',', ':'), sort_keys=True)
+ f.write('\n')
+
+ logging.info('Dumping Component.')
+ with open('component.json', 'w') as f:
+ for c in database(Component).Values():
+ c['details'] = []
+ for cd in database(ComponentDetail).Filter(
+ device_id=c['device_id'],
+ component_class=c['component_class']).Values():
+ c['details'].append(dict((k, v[:ATTR_LENGTH_LIMIT])
+ for k, v in cd.iteritems()
+ if k != 'device_id' and k != 'component_class'))
+ json.dump(c, f, separators=(',', ':'), sort_keys=True)
+ f.write('\n')
+
+
+if __name__ == "__main__":
+ Main()
diff --git a/py/minijack/frontend/views.py b/py/minijack/frontend/views.py
index 0308189..9c261c9 100644
--- a/py/minijack/frontend/views.py
+++ b/py/minijack/frontend/views.py
@@ -4,8 +4,6 @@
import copy
import operator
-import subprocess
-import tempfile
import itertools
from datetime import datetime
@@ -17,6 +15,13 @@
from db import Database
from frontend import test_renderers, data
+import settings
+
+
+if not settings.IS_APPENGINE:
+ import subprocess
+ import tempfile
+
DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
@@ -209,6 +214,7 @@
duration_list = [float(t['duration']) for t in test_list
if float(t['duration']) != 0.0]
+ # TODO(pihsun): Can do statistics in BigQuery.
duration_stats = data.GetStatistic(duration_list)
try_list = [len(list(g)) for _, g in
@@ -244,6 +250,10 @@
def GetScreenshotImage(dummy_request, ip_address):
+ if settings.IS_APPENGINE:
+ return HttpResponse(
+ "Screenshot feature is disabled on App Engine" +
+ " (can't create subprocess)")
remote_url = 'root@' + ip_address
remote_filename = '/tmp/screenshot.png'
capture_cmd = (
diff --git a/py/minijack/gflags.py b/py/minijack/gflags.py
new file mode 100644
index 0000000..822256a
--- /dev/null
+++ b/py/minijack/gflags.py
@@ -0,0 +1,2862 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2002, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# ---
+# Author: Chad Lester
+# Design and style contributions by:
+# Amit Patel, Bogdan Cocosel, Daniel Dulitz, Eric Tiedemann,
+# Eric Veach, Laurence Gonsalves, Matthew Springer
+# Code reorganized a bit by Craig Silverstein
+
+"""This module is used to define and parse command line flags.
+
+This module defines a *distributed* flag-definition policy: rather than
+an application having to define all flags in or near main(), each python
+module defines flags that are useful to it. When one python module
+imports another, it gains access to the other's flags. (This is
+implemented by having all modules share a common, global registry object
+containing all the flag information.)
+
+Flags are defined through the use of one of the DEFINE_xxx functions.
+The specific function used determines how the flag is parsed, checked,
+and optionally type-converted, when it's seen on the command line.
+
+
+IMPLEMENTATION: DEFINE_* creates a 'Flag' object and registers it with a
+'FlagValues' object (typically the global FlagValues FLAGS, defined
+here). The 'FlagValues' object can scan the command line arguments and
+pass flag arguments to the corresponding 'Flag' objects for
+value-checking and type conversion. The converted flag values are
+available as attributes of the 'FlagValues' object.
+
+Code can access the flag through a FlagValues object, for instance
+gflags.FLAGS.myflag. Typically, the __main__ module passes the command
+line arguments to gflags.FLAGS for parsing.
+
+At bottom, this module calls getopt(), so getopt functionality is
+supported, including short- and long-style flags, and the use of -- to
+terminate flags.
+
+Methods defined by the flag module will throw 'FlagsError' exceptions.
+The exception argument will be a human-readable string.
+
+
+FLAG TYPES: This is a list of the DEFINE_*'s that you can do. All flags
+take a name, default value, help-string, and optional 'short' name
+(one-letter name). Some flags have other arguments, which are described
+with the flag.
+
+DEFINE_string: takes any input, and interprets it as a string.
+
+DEFINE_bool or
+DEFINE_boolean: typically does not take an argument: say --myflag to
+ set FLAGS.myflag to true, or --nomyflag to set
+ FLAGS.myflag to false. Alternately, you can say
+ --myflag=true or --myflag=t or --myflag=1 or
+ --myflag=false or --myflag=f or --myflag=0
+
+DEFINE_float: takes an input and interprets it as a floating point
+ number. Takes optional args lower_bound and upper_bound;
+ if the number specified on the command line is out of
+ range, it will raise a FlagError.
+
+DEFINE_integer: takes an input and interprets it as an integer. Takes
+ optional args lower_bound and upper_bound as for floats.
+
+DEFINE_enum: takes a list of strings which represents legal values. If
+ the command-line value is not in this list, raise a flag
+ error. Otherwise, assign to FLAGS.flag as a string.
+
+DEFINE_list: Takes a comma-separated list of strings on the commandline.
+ Stores them in a python list object.
+
+DEFINE_spaceseplist: Takes a space-separated list of strings on the
+ commandline. Stores them in a python list object.
+ Example: --myspacesepflag "foo bar baz"
+
+DEFINE_multistring: The same as DEFINE_string, except the flag can be
+ specified more than once on the commandline. The
+ result is a python list object (list of strings),
+ even if the flag is only on the command line once.
+
+DEFINE_multi_int: The same as DEFINE_integer, except the flag can be
+ specified more than once on the commandline. The
+ result is a python list object (list of ints), even if
+ the flag is only on the command line once.
+
+
+SPECIAL FLAGS: There are a few flags that have special meaning:
+ --help prints a list of all the flags in a human-readable fashion
+ --helpshort prints a list of all key flags (see below).
+ --helpxml prints a list of all flags, in XML format. DO NOT parse
+ the output of --help and --helpshort. Instead, parse
+ the output of --helpxml. For more info, see
+ "OUTPUT FOR --helpxml" below.
+ --flagfile=foo read flags from file foo.
+ --undefok=f1,f2 ignore unrecognized option errors for f1,f2.
+ For boolean flags, you should use --undefok=boolflag, and
+ --boolflag and --noboolflag will be accepted. Do not use
+ --undefok=noboolflag.
+ -- as in getopt(), terminates flag-processing
+
+
+FLAGS VALIDATORS: If your program:
+ - requires flag X to be specified
+ - needs flag Y to match a regular expression
+ - or requires any more general constraint to be satisfied
+then validators are for you!
+
+Each validator represents a constraint over one flag, which is enforced
+starting from the initial parsing of the flags and until the program
+terminates.
+
+Also, lower_bound and upper_bound for numerical flags are enforced using flag
+validators.
+
+Howto:
+If you want to enforce a constraint over one flag, use
+
+gflags.RegisterValidator(flag_name,
+ checker,
+ message='Flag validation failed',
+ flag_values=FLAGS)
+
+After flag values are initially parsed, and after any change to the specified
+flag, method checker(flag_value) will be executed. If constraint is not
+satisfied, an IllegalFlagValue exception will be raised. See
+RegisterValidator's docstring for a detailed explanation on how to construct
+your own checker.
+
+
+EXAMPLE USAGE:
+
+FLAGS = gflags.FLAGS
+
+gflags.DEFINE_integer('my_version', 0, 'Version number.')
+gflags.DEFINE_string('filename', None, 'Input file name', short_name='f')
+
+gflags.RegisterValidator('my_version',
+ lambda value: value % 2 == 0,
+ message='--my_version must be divisible by 2')
+gflags.MarkFlagAsRequired('filename')
+
+
+NOTE ON --flagfile:
+
+Flags may be loaded from text files in addition to being specified on
+the commandline.
+
+Any flags you don't feel like typing, throw them in a file, one flag per
+line, for instance:
+ --myflag=myvalue
+ --nomyboolean_flag
+You then specify your file with the special flag '--flagfile=somefile'.
+You CAN recursively nest flagfile= tokens OR use multiple files on the
+command line. Lines beginning with a single hash '#' or a double slash
+'//' are comments in your flagfile.
+
+Any flagfile=<file> will be interpreted as having a relative path from
+the current working directory rather than from the place the file was
+included from:
+ myPythonScript.py --flagfile=config/somefile.cfg
+
+If somefile.cfg includes further --flagfile= directives, these will be
+referenced relative to the original CWD, not from the directory the
+including flagfile was found in!
+
+The caveat applies to people who are including a series of nested files
+in a different dir than they are executing out of. Relative path names
+are always from CWD, not from the directory of the parent include
+flagfile. We do now support '~' expanded directory names.
+
+Absolute path names ALWAYS work!
+
+
+EXAMPLE USAGE:
+
+
+ FLAGS = gflags.FLAGS
+
+ # Flag names are globally defined! So in general, we need to be
+ # careful to pick names that are unlikely to be used by other libraries.
+ # If there is a conflict, we'll get an error at import time.
+ gflags.DEFINE_string('name', 'Mr. President', 'your name')
+ gflags.DEFINE_integer('age', None, 'your age in years', lower_bound=0)
+ gflags.DEFINE_boolean('debug', False, 'produces debugging output')
+ gflags.DEFINE_enum('gender', 'male', ['male', 'female'], 'your gender')
+
+ def main(argv):
+ try:
+ argv = FLAGS(argv) # parse flags
+ except gflags.FlagsError, e:
+ print '%s\\nUsage: %s ARGS\\n%s' % (e, sys.argv[0], FLAGS)
+ sys.exit(1)
+ if FLAGS.debug: print 'non-flag arguments:', argv
+ print 'Happy Birthday', FLAGS.name
+ if FLAGS.age is not None:
+ print 'You are a %d year old %s' % (FLAGS.age, FLAGS.gender)
+
+ if __name__ == '__main__':
+ main(sys.argv)
+
+
+KEY FLAGS:
+
+As we already explained, each module gains access to all flags defined
+by all the other modules it transitively imports. In the case of
+non-trivial scripts, this means a lot of flags ... For documentation
+purposes, it is good to identify the flags that are key (i.e., really
+important) to a module. Clearly, the concept of "key flag" is a
+subjective one. When trying to determine whether a flag is key to a
+module or not, assume that you are trying to explain your module to a
+potential user: which flags would you really like to mention first?
+
+We'll describe shortly how to declare which flags are key to a module.
+For the moment, assume we know the set of key flags for each module.
+Then, if you use the app.py module, you can use the --helpshort flag to
+print only the help for the flags that are key to the main module, in a
+human-readable format.
+
+NOTE: If you need to parse the flag help, do NOT use the output of
+--help / --helpshort. That output is meant for human consumption, and
+may be changed in the future. Instead, use --helpxml; flags that are
+key for the main module are marked there with a <key>yes</key> element.
+
+The set of key flags for a module M is composed of:
+
+1. Flags defined by module M by calling a DEFINE_* function.
+
+2. Flags that module M explictly declares as key by using the function
+
+ DECLARE_key_flag(<flag_name>)
+
+3. Key flags of other modules that M specifies by using the function
+
+ ADOPT_module_key_flags(<other_module>)
+
+ This is a "bulk" declaration of key flags: each flag that is key for
+ <other_module> becomes key for the current module too.
+
+Notice that if you do not use the functions described at points 2 and 3
+above, then --helpshort prints information only about the flags defined
+by the main module of our script. In many cases, this behavior is good
+enough. But if you move part of the main module code (together with the
+related flags) into a different module, then it is nice to use
+DECLARE_key_flag / ADOPT_module_key_flags and make sure --helpshort
+lists all relevant flags (otherwise, your code refactoring may confuse
+your users).
+
+Note: each of DECLARE_key_flag / ADOPT_module_key_flags has its own
+pluses and minuses: DECLARE_key_flag is more targeted and may lead a
+more focused --helpshort documentation. ADOPT_module_key_flags is good
+for cases when an entire module is considered key to the current script.
+Also, it does not require updates to client scripts when a new flag is
+added to the module.
+
+
+EXAMPLE USAGE 2 (WITH KEY FLAGS):
+
+Consider an application that contains the following three files (two
+auxiliary modules and a main module)
+
+File libfoo.py:
+
+ import gflags
+
+ gflags.DEFINE_integer('num_replicas', 3, 'Number of replicas to start')
+ gflags.DEFINE_boolean('rpc2', True, 'Turn on the usage of RPC2.')
+
+ ... some code ...
+
+File libbar.py:
+
+ import gflags
+
+ gflags.DEFINE_string('bar_gfs_path', '/gfs/path',
+ 'Path to the GFS files for libbar.')
+ gflags.DEFINE_string('email_for_bar_errors', 'bar-team@google.com',
+ 'Email address for bug reports about module libbar.')
+ gflags.DEFINE_boolean('bar_risky_hack', False,
+ 'Turn on an experimental and buggy optimization.')
+
+ ... some code ...
+
+File myscript.py:
+
+ import gflags
+ import libfoo
+ import libbar
+
+ gflags.DEFINE_integer('num_iterations', 0, 'Number of iterations.')
+
+ # Declare that all flags that are key for libfoo are
+ # key for this module too.
+ gflags.ADOPT_module_key_flags(libfoo)
+
+ # Declare that the flag --bar_gfs_path (defined in libbar) is key
+ # for this module.
+ gflags.DECLARE_key_flag('bar_gfs_path')
+
+ ... some code ...
+
+When myscript is invoked with the flag --helpshort, the resulted help
+message lists information about all the key flags for myscript:
+--num_iterations, --num_replicas, --rpc2, and --bar_gfs_path.
+
+Of course, myscript uses all the flags declared by it (in this case,
+just --num_replicas) or by any of the modules it transitively imports
+(e.g., the modules libfoo, libbar). E.g., it can access the value of
+FLAGS.bar_risky_hack, even if --bar_risky_hack is not declared as a key
+flag for myscript.
+
+
+OUTPUT FOR --helpxml:
+
+The --helpxml flag generates output with the following structure:
+
+<?xml version="1.0"?>
+<AllFlags>
+ <program>PROGRAM_BASENAME</program>
+ <usage>MAIN_MODULE_DOCSTRING</usage>
+ (<flag>
+ [<key>yes</key>]
+ <file>DECLARING_MODULE</file>
+ <name>FLAG_NAME</name>
+ <meaning>FLAG_HELP_MESSAGE</meaning>
+ <default>DEFAULT_FLAG_VALUE</default>
+ <current>CURRENT_FLAG_VALUE</current>
+ <type>FLAG_TYPE</type>
+ [OPTIONAL_ELEMENTS]
+ </flag>)*
+</AllFlags>
+
+Notes:
+
+1. The output is intentionally similar to the output generated by the
+C++ command-line flag library. The few differences are due to the
+Python flags that do not have a C++ equivalent (at least not yet),
+e.g., DEFINE_list.
+
+2. New XML elements may be added in the future.
+
+3. DEFAULT_FLAG_VALUE is in serialized form, i.e., the string you can
+pass for this flag on the command-line. E.g., for a flag defined
+using DEFINE_list, this field may be foo,bar, not ['foo', 'bar'].
+
+4. CURRENT_FLAG_VALUE is produced using str(). This means that the
+string 'false' will be represented in the same way as the boolean
+False. Using repr() would have removed this ambiguity and simplified
+parsing, but would have broken the compatibility with the C++
+command-line flags.
+
+5. OPTIONAL_ELEMENTS describe elements relevant for certain kinds of
+flags: lower_bound, upper_bound (for flags that specify bounds),
+enum_value (for enum flags), list_separator (for flags that consist of
+a list of values, separated by a special token).
+
+6. We do not provide any example here: please use --helpxml instead.
+
+This module requires at least python 2.2.1 to run.
+"""
+
+import cgi
+import getopt
+import os
+import re
+import string
+import struct
+import sys
+# pylint: disable-msg=C6204
+try:
+ import fcntl
+except ImportError:
+ fcntl = None
+try:
+ # Importing termios will fail on non-unix platforms.
+ import termios
+except ImportError:
+ termios = None
+
+import gflags_validators
+# pylint: enable-msg=C6204
+
+
+# Are we running under pychecker?
+_RUNNING_PYCHECKER = 'pychecker.python' in sys.modules
+
+
+def _GetCallingModuleObjectAndName():
+ """Returns the module that's calling into this module.
+
+ We generally use this function to get the name of the module calling a
+ DEFINE_foo... function.
+ """
+ # Walk down the stack to find the first globals dict that's not ours.
+ for depth in range(1, sys.getrecursionlimit()):
+ if not sys._getframe(depth).f_globals is globals():
+ globals_for_frame = sys._getframe(depth).f_globals
+ module, module_name = _GetModuleObjectAndName(globals_for_frame)
+ if module_name is not None:
+ return module, module_name
+ raise AssertionError("No module was found")
+
+
+def _GetCallingModule():
+ """Returns the name of the module that's calling into this module."""
+ return _GetCallingModuleObjectAndName()[1]
+
+
+def _GetThisModuleObjectAndName():
+ """Returns: (module object, module name) for this module."""
+ return _GetModuleObjectAndName(globals())
+
+
+# module exceptions:
+class FlagsError(Exception):
+ """The base class for all flags errors."""
+ pass
+
+
+class DuplicateFlag(FlagsError):
+ """Raised if there is a flag naming conflict."""
+ pass
+
+class CantOpenFlagFileError(FlagsError):
+ """Raised if flagfile fails to open: doesn't exist, wrong permissions, etc."""
+ pass
+
+
+class DuplicateFlagCannotPropagateNoneToSwig(DuplicateFlag):
+ """Special case of DuplicateFlag -- SWIG flag value can't be set to None.
+
+ This can be raised when a duplicate flag is created. Even if allow_override is
+ True, we still abort if the new value is None, because it's currently
+ impossible to pass None default value back to SWIG. See FlagValues.SetDefault
+ for details.
+ """
+ pass
+
+
+class DuplicateFlagError(DuplicateFlag):
+ """A DuplicateFlag whose message cites the conflicting definitions.
+
+ A DuplicateFlagError conveys more information than a DuplicateFlag,
+ namely the modules where the conflicting definitions occur. This
+ class was created to avoid breaking external modules which depend on
+ the existing DuplicateFlags interface.
+ """
+
+ def __init__(self, flagname, flag_values, other_flag_values=None):
+ """Create a DuplicateFlagError.
+
+ Args:
+ flagname: Name of the flag being redefined.
+ flag_values: FlagValues object containing the first definition of
+ flagname.
+ other_flag_values: If this argument is not None, it should be the
+ FlagValues object where the second definition of flagname occurs.
+ If it is None, we assume that we're being called when attempting
+ to create the flag a second time, and we use the module calling
+ this one as the source of the second definition.
+ """
+ self.flagname = flagname
+ first_module = flag_values.FindModuleDefiningFlag(
+ flagname, default='<unknown>')
+ if other_flag_values is None:
+ second_module = _GetCallingModule()
+ else:
+ second_module = other_flag_values.FindModuleDefiningFlag(
+ flagname, default='<unknown>')
+ msg = "The flag '%s' is defined twice. First from %s, Second from %s" % (
+ self.flagname, first_module, second_module)
+ DuplicateFlag.__init__(self, msg)
+
+
+class IllegalFlagValue(FlagsError):
+ """The flag command line argument is illegal."""
+ pass
+
+
+class UnrecognizedFlag(FlagsError):
+ """Raised if a flag is unrecognized."""
+ pass
+
+
+# An UnrecognizedFlagError conveys more information than an UnrecognizedFlag.
+# Since there are external modules that create DuplicateFlags, the interface to
+# DuplicateFlag shouldn't change. The flagvalue will be assigned the full value
+# of the flag and its argument, if any, allowing handling of unrecognized flags
+# in an exception handler.
+# If flagvalue is the empty string, then this exception is an due to a
+# reference to a flag that was not already defined.
+class UnrecognizedFlagError(UnrecognizedFlag):
+ def __init__(self, flagname, flagvalue=''):
+ self.flagname = flagname
+ self.flagvalue = flagvalue
+ UnrecognizedFlag.__init__(
+ self, "Unknown command line flag '%s'" % flagname)
+
+# Global variable used by expvar
+_exported_flags = {}
+_help_width = 80 # width of help output
+
+
+def GetHelpWidth():
+ """Returns: an integer, the width of help lines that is used in TextWrap."""
+ if (not sys.stdout.isatty()) or (termios is None) or (fcntl is None):
+ return _help_width
+ try:
+ data = fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, '1234')
+ columns = struct.unpack('hh', data)[1]
+ # Emacs mode returns 0.
+ # Here we assume that any value below 40 is unreasonable
+ if columns >= 40:
+ return columns
+ # Returning an int as default is fine, int(int) just return the int.
+ return int(os.getenv('COLUMNS', _help_width))
+
+ except (TypeError, IOError, struct.error):
+ return _help_width
+
+
+def CutCommonSpacePrefix(text):
+ """Removes a common space prefix from the lines of a multiline text.
+
+ If the first line does not start with a space, it is left as it is and
+ only in the remaining lines a common space prefix is being searched
+ for. That means the first line will stay untouched. This is especially
+ useful to turn doc strings into help texts. This is because some
+ people prefer to have the doc comment start already after the
+ apostrophe and then align the following lines while others have the
+ apostrophes on a separate line.
+
+ The function also drops trailing empty lines and ignores empty lines
+ following the initial content line while calculating the initial
+ common whitespace.
+
+ Args:
+ text: text to work on
+
+ Returns:
+ the resulting text
+ """
+ text_lines = text.splitlines()
+ # Drop trailing empty lines
+ while text_lines and not text_lines[-1]:
+ text_lines = text_lines[:-1]
+ if text_lines:
+ # We got some content, is the first line starting with a space?
+ if text_lines[0] and text_lines[0][0].isspace():
+ text_first_line = []
+ else:
+ text_first_line = [text_lines.pop(0)]
+ # Calculate length of common leading whitespace (only over content lines)
+ common_prefix = os.path.commonprefix([line for line in text_lines if line])
+ space_prefix_len = len(common_prefix) - len(common_prefix.lstrip())
+ # If we have a common space prefix, drop it from all lines
+ if space_prefix_len:
+ for index in xrange(len(text_lines)):
+ if text_lines[index]:
+ text_lines[index] = text_lines[index][space_prefix_len:]
+ return '\n'.join(text_first_line + text_lines)
+ return ''
+
+
+def TextWrap(text, length=None, indent='', firstline_indent=None, tabs=' '):
+ """Wraps a given text to a maximum line length and returns it.
+
+ We turn lines that only contain whitespace into empty lines. We keep
+ new lines and tabs (e.g., we do not treat tabs as spaces).
+
+ Args:
+ text: text to wrap
+ length: maximum length of a line, includes indentation
+ if this is None then use GetHelpWidth()
+ indent: indent for all but first line
+ firstline_indent: indent for first line; if None, fall back to indent
+ tabs: replacement for tabs
+
+ Returns:
+ wrapped text
+
+ Raises:
+ FlagsError: if indent not shorter than length
+ FlagsError: if firstline_indent not shorter than length
+ """
+ # Get defaults where callee used None
+ if length is None:
+ length = GetHelpWidth()
+ if indent is None:
+ indent = ''
+ if len(indent) >= length:
+ raise FlagsError('Indent must be shorter than length')
+ # In line we will be holding the current line which is to be started
+ # with indent (or firstline_indent if available) and then appended
+ # with words.
+ if firstline_indent is None:
+ firstline_indent = ''
+ line = indent
+ else:
+ line = firstline_indent
+ if len(firstline_indent) >= length:
+ raise FlagsError('First line indent must be shorter than length')
+
+ # If the callee does not care about tabs we simply convert them to
+ # spaces If callee wanted tabs to be single space then we do that
+ # already here.
+ if not tabs or tabs == ' ':
+ text = text.replace('\t', ' ')
+ else:
+ tabs_are_whitespace = not tabs.strip()
+
+ line_regex = re.compile('([ ]*)(\t*)([^ \t]+)', re.MULTILINE)
+
+ # Split the text into lines and the lines with the regex above. The
+ # resulting lines are collected in result[]. For each split we get the
+ # spaces, the tabs and the next non white space (e.g. next word).
+ result = []
+ for text_line in text.splitlines():
+ # Store result length so we can find out whether processing the next
+ # line gave any new content
+ old_result_len = len(result)
+ # Process next line with line_regex. For optimization we do an rstrip().
+ # - process tabs (changes either line or word, see below)
+ # - process word (first try to squeeze on line, then wrap or force wrap)
+ # Spaces found on the line are ignored, they get added while wrapping as
+ # needed.
+ for spaces, current_tabs, word in line_regex.findall(text_line.rstrip()):
+ # If tabs weren't converted to spaces, handle them now
+ if current_tabs:
+ # If the last thing we added was a space anyway then drop
+ # it. But let's not get rid of the indentation.
+ if (((result and line != indent) or
+ (not result and line != firstline_indent)) and line[-1] == ' '):
+ line = line[:-1]
+ # Add the tabs, if that means adding whitespace, just add it at
+ # the line, the rstrip() code while shorten the line down if
+ # necessary
+ if tabs_are_whitespace:
+ line += tabs * len(current_tabs)
+ else:
+ # if not all tab replacement is whitespace we prepend it to the word
+ word = tabs * len(current_tabs) + word
+ # Handle the case where word cannot be squeezed onto current last line
+ if len(line) + len(word) > length and len(indent) + len(word) <= length:
+ result.append(line.rstrip())
+ line = indent + word
+ word = ''
+ # No space left on line or can we append a space?
+ if len(line) + 1 >= length:
+ result.append(line.rstrip())
+ line = indent
+ else:
+ line += ' '
+ # Add word and shorten it up to allowed line length. Restart next
+ # line with indent and repeat, or add a space if we're done (word
+ # finished) This deals with words that cannot fit on one line
+ # (e.g. indent + word longer than allowed line length).
+ while len(line) + len(word) >= length:
+ line += word
+ result.append(line[:length])
+ word = line[length:]
+ line = indent
+ # Default case, simply append the word and a space
+ if word:
+ line += word + ' '
+ # End of input line. If we have content we finish the line. If the
+ # current line is just the indent but we had content in during this
+ # original line then we need to add an empty line.
+ if (result and line != indent) or (not result and line != firstline_indent):
+ result.append(line.rstrip())
+ elif len(result) == old_result_len:
+ result.append('')
+ line = indent
+
+ return '\n'.join(result)
+
+
+def DocToHelp(doc):
+ """Takes a __doc__ string and reformats it as help."""
+
+ # Get rid of starting and ending white space. Using lstrip() or even
+ # strip() could drop more than maximum of first line and right space
+ # of last line.
+ doc = doc.strip()
+
+ # Get rid of all empty lines
+ whitespace_only_line = re.compile('^[ \t]+$', re.M)
+ doc = whitespace_only_line.sub('', doc)
+
+ # Cut out common space at line beginnings
+ doc = CutCommonSpacePrefix(doc)
+
+ # Just like this module's comment, comments tend to be aligned somehow.
+ # In other words they all start with the same amount of white space
+ # 1) keep double new lines
+ # 2) keep ws after new lines if not empty line
+ # 3) all other new lines shall be changed to a space
+ # Solution: Match new lines between non white space and replace with space.
+ doc = re.sub('(?<=\S)\n(?=\S)', ' ', doc, re.M)
+
+ return doc
+
+
+def _GetModuleObjectAndName(globals_dict):
+ """Returns the module that defines a global environment, and its name.
+
+ Args:
+ globals_dict: A dictionary that should correspond to an environment
+ providing the values of the globals.
+
+ Returns:
+ A pair consisting of (1) module object and (2) module name (a
+ string). Returns (None, None) if the module could not be
+ identified.
+ """
+ # The use of .items() (instead of .iteritems()) is NOT a mistake: if
+ # a parallel thread imports a module while we iterate over
+ # .iteritems() (not nice, but possible), we get a RuntimeError ...
+ # Hence, we use the slightly slower but safer .items().
+ for name, module in sys.modules.items():
+ if getattr(module, '__dict__', None) is globals_dict:
+ if name == '__main__':
+ # Pick a more informative name for the main module.
+ name = sys.argv[0]
+ return (module, name)
+ return (None, None)
+
+
+def _GetMainModule():
+ """Returns: string, name of the module from which execution started."""
+ # First, try to use the same logic used by _GetCallingModuleObjectAndName(),
+ # i.e., call _GetModuleObjectAndName(). For that we first need to
+ # find the dictionary that the main module uses to store the
+ # globals.
+ #
+ # That's (normally) the same dictionary object that the deepest
+ # (oldest) stack frame is using for globals.
+ deepest_frame = sys._getframe(0)
+ while deepest_frame.f_back is not None:
+ deepest_frame = deepest_frame.f_back
+ globals_for_main_module = deepest_frame.f_globals
+ main_module_name = _GetModuleObjectAndName(globals_for_main_module)[1]
+ # The above strategy fails in some cases (e.g., tools that compute
+ # code coverage by redefining, among other things, the main module).
+ # If so, just use sys.argv[0]. We can probably always do this, but
+ # it's safest to try to use the same logic as _GetCallingModuleObjectAndName()
+ if main_module_name is None:
+ main_module_name = sys.argv[0]
+ return main_module_name
+
+
+class FlagValues:
+ """Registry of 'Flag' objects.
+
+ A 'FlagValues' can then scan command line arguments, passing flag
+ arguments through to the 'Flag' objects that it owns. It also
+ provides easy access to the flag values. Typically only one
+ 'FlagValues' object is needed by an application: gflags.FLAGS
+
+ This class is heavily overloaded:
+
+ 'Flag' objects are registered via __setitem__:
+ FLAGS['longname'] = x # register a new flag
+
+ The .value attribute of the registered 'Flag' objects can be accessed
+ as attributes of this 'FlagValues' object, through __getattr__. Both
+ the long and short name of the original 'Flag' objects can be used to
+ access its value:
+ FLAGS.longname # parsed flag value
+ FLAGS.x # parsed flag value (short name)
+
+ Command line arguments are scanned and passed to the registered 'Flag'
+ objects through the __call__ method. Unparsed arguments, including
+ argv[0] (e.g. the program name) are returned.
+ argv = FLAGS(sys.argv) # scan command line arguments
+
+ The original registered Flag objects can be retrieved through the use
+ of the dictionary-like operator, __getitem__:
+ x = FLAGS['longname'] # access the registered Flag object
+
+ The str() operator of a 'FlagValues' object provides help for all of
+ the registered 'Flag' objects.
+ """
+
+ def __init__(self):
+ # Since everything in this class is so heavily overloaded, the only
+ # way of defining and using fields is to access __dict__ directly.
+
+ # Dictionary: flag name (string) -> Flag object.
+ self.__dict__['__flags'] = {}
+ # Dictionary: module name (string) -> list of Flag objects that are defined
+ # by that module.
+ self.__dict__['__flags_by_module'] = {}
+ # Dictionary: module id (int) -> list of Flag objects that are defined by
+ # that module.
+ self.__dict__['__flags_by_module_id'] = {}
+ # Dictionary: module name (string) -> list of Flag objects that are
+ # key for that module.
+ self.__dict__['__key_flags_by_module'] = {}
+
+ # Set if we should use new style gnu_getopt rather than getopt when parsing
+ # the args. Only possible with Python 2.3+
+ self.UseGnuGetOpt(False)
+
+ def UseGnuGetOpt(self, use_gnu_getopt=True):
+ """Use GNU-style scanning. Allows mixing of flag and non-flag arguments.
+
+ See http://docs.python.org/library/getopt.html#getopt.gnu_getopt
+
+ Args:
+ use_gnu_getopt: wether or not to use GNU style scanning.
+ """
+ self.__dict__['__use_gnu_getopt'] = use_gnu_getopt
+
+ def IsGnuGetOpt(self):
+ return self.__dict__['__use_gnu_getopt']
+
+ def FlagDict(self):
+ return self.__dict__['__flags']
+
+ def FlagsByModuleDict(self):
+ """Returns the dictionary of module_name -> list of defined flags.
+
+ Returns:
+ A dictionary. Its keys are module names (strings). Its values
+ are lists of Flag objects.
+ """
+ return self.__dict__['__flags_by_module']
+
+ def FlagsByModuleIdDict(self):
+ """Returns the dictionary of module_id -> list of defined flags.
+
+ Returns:
+ A dictionary. Its keys are module IDs (ints). Its values
+ are lists of Flag objects.
+ """
+ return self.__dict__['__flags_by_module_id']
+
+ def KeyFlagsByModuleDict(self):
+ """Returns the dictionary of module_name -> list of key flags.
+
+ Returns:
+ A dictionary. Its keys are module names (strings). Its values
+ are lists of Flag objects.
+ """
+ return self.__dict__['__key_flags_by_module']
+
+ def _RegisterFlagByModule(self, module_name, flag):
+ """Records the module that defines a specific flag.
+
+ We keep track of which flag is defined by which module so that we
+ can later sort the flags by module.
+
+ Args:
+ module_name: A string, the name of a Python module.
+ flag: A Flag object, a flag that is key to the module.
+ """
+ flags_by_module = self.FlagsByModuleDict()
+ flags_by_module.setdefault(module_name, []).append(flag)
+
+ def _RegisterFlagByModuleId(self, module_id, flag):
+ """Records the module that defines a specific flag.
+
+ Args:
+ module_id: An int, the ID of the Python module.
+ flag: A Flag object, a flag that is key to the module.
+ """
+ flags_by_module_id = self.FlagsByModuleIdDict()
+ flags_by_module_id.setdefault(module_id, []).append(flag)
+
+ def _RegisterKeyFlagForModule(self, module_name, flag):
+ """Specifies that a flag is a key flag for a module.
+
+ Args:
+ module_name: A string, the name of a Python module.
+ flag: A Flag object, a flag that is key to the module.
+ """
+ key_flags_by_module = self.KeyFlagsByModuleDict()
+ # The list of key flags for the module named module_name.
+ key_flags = key_flags_by_module.setdefault(module_name, [])
+ # Add flag, but avoid duplicates.
+ if flag not in key_flags:
+ key_flags.append(flag)
+
+ def _GetFlagsDefinedByModule(self, module):
+ """Returns the list of flags defined by a module.
+
+ Args:
+ module: A module object or a module name (a string).
+
+ Returns:
+ A new list of Flag objects. Caller may update this list as he
+ wishes: none of those changes will affect the internals of this
+ FlagValue object.
+ """
+ if not isinstance(module, str):
+ module = module.__name__
+
+ return list(self.FlagsByModuleDict().get(module, []))
+
+ def _GetKeyFlagsForModule(self, module):
+ """Returns the list of key flags for a module.
+
+ Args:
+ module: A module object or a module name (a string)
+
+ Returns:
+ A new list of Flag objects. Caller may update this list as he
+ wishes: none of those changes will affect the internals of this
+ FlagValue object.
+ """
+ if not isinstance(module, str):
+ module = module.__name__
+
+ # Any flag is a key flag for the module that defined it. NOTE:
+ # key_flags is a fresh list: we can update it without affecting the
+ # internals of this FlagValues object.
+ key_flags = self._GetFlagsDefinedByModule(module)
+
+ # Take into account flags explicitly declared as key for a module.
+ for flag in self.KeyFlagsByModuleDict().get(module, []):
+ if flag not in key_flags:
+ key_flags.append(flag)
+ return key_flags
+
+ def FindModuleDefiningFlag(self, flagname, default=None):
+ """Return the name of the module defining this flag, or default.
+
+ Args:
+ flagname: Name of the flag to lookup.
+ default: Value to return if flagname is not defined. Defaults
+ to None.
+
+ Returns:
+ The name of the module which registered the flag with this name.
+ If no such module exists (i.e. no flag with this name exists),
+ we return default.
+ """
+ for module, flags in self.FlagsByModuleDict().iteritems():
+ for flag in flags:
+ if flag.name == flagname or flag.short_name == flagname:
+ return module
+ return default
+
+ def FindModuleIdDefiningFlag(self, flagname, default=None):
+ """Return the ID of the module defining this flag, or default.
+
+ Args:
+ flagname: Name of the flag to lookup.
+ default: Value to return if flagname is not defined. Defaults
+ to None.
+
+ Returns:
+ The ID of the module which registered the flag with this name.
+ If no such module exists (i.e. no flag with this name exists),
+ we return default.
+ """
+ for module_id, flags in self.FlagsByModuleIdDict().iteritems():
+ for flag in flags:
+ if flag.name == flagname or flag.short_name == flagname:
+ return module_id
+ return default
+
+ def AppendFlagValues(self, flag_values):
+ """Appends flags registered in another FlagValues instance.
+
+ Args:
+ flag_values: registry to copy from
+ """
+ for flag_name, flag in flag_values.FlagDict().iteritems():
+ # Each flags with shortname appears here twice (once under its
+ # normal name, and again with its short name). To prevent
+ # problems (DuplicateFlagError) with double flag registration, we
+ # perform a check to make sure that the entry we're looking at is
+ # for its normal name.
+ if flag_name == flag.name:
+ try:
+ self[flag_name] = flag
+ except DuplicateFlagError:
+ raise DuplicateFlagError(flag_name, self,
+ other_flag_values=flag_values)
+
+ def RemoveFlagValues(self, flag_values):
+ """Remove flags that were previously appended from another FlagValues.
+
+ Args:
+ flag_values: registry containing flags to remove.
+ """
+ for flag_name in flag_values.FlagDict():
+ self.__delattr__(flag_name)
+
+ def __setitem__(self, name, flag):
+ """Registers a new flag variable."""
+ fl = self.FlagDict()
+ if not isinstance(flag, Flag):
+ raise IllegalFlagValue(flag)
+ if not isinstance(name, type("")):
+ raise FlagsError("Flag name must be a string")
+ if len(name) == 0:
+ raise FlagsError("Flag name cannot be empty")
+ # If running under pychecker, duplicate keys are likely to be
+ # defined. Disable check for duplicate keys when pycheck'ing.
+ if (name in fl and not flag.allow_override and
+ not fl[name].allow_override and not _RUNNING_PYCHECKER):
+ module, module_name = _GetCallingModuleObjectAndName()
+ if (self.FindModuleDefiningFlag(name) == module_name and
+ id(module) != self.FindModuleIdDefiningFlag(name)):
+ # If the flag has already been defined by a module with the same name,
+ # but a different ID, we can stop here because it indicates that the
+ # module is simply being imported a subsequent time.
+ return
+ raise DuplicateFlagError(name, self)
+ short_name = flag.short_name
+ if short_name is not None:
+ if (short_name in fl and not flag.allow_override and
+ not fl[short_name].allow_override and not _RUNNING_PYCHECKER):
+ raise DuplicateFlagError(short_name, self)
+ fl[short_name] = flag
+ fl[name] = flag
+ global _exported_flags
+ _exported_flags[name] = flag
+
+ def __getitem__(self, name):
+ """Retrieves the Flag object for the flag --name."""
+ return self.FlagDict()[name]
+
+ def __getattr__(self, name):
+ """Retrieves the 'value' attribute of the flag --name."""
+ fl = self.FlagDict()
+ if name not in fl:
+ raise AttributeError(name)
+ return fl[name].value
+
+ def __setattr__(self, name, value):
+ """Sets the 'value' attribute of the flag --name."""
+ fl = self.FlagDict()
+ fl[name].value = value
+ self._AssertValidators(fl[name].validators)
+ return value
+
+ def _AssertAllValidators(self):
+ all_validators = set()
+ for flag in self.FlagDict().itervalues():
+ for validator in flag.validators:
+ all_validators.add(validator)
+ self._AssertValidators(all_validators)
+
+ def _AssertValidators(self, validators):
+ """Assert if all validators in the list are satisfied.
+
+ Asserts validators in the order they were created.
+ Args:
+ validators: Iterable(gflags_validators.Validator), validators to be
+ verified
+ Raises:
+ AttributeError: if validators work with a non-existing flag.
+ IllegalFlagValue: if validation fails for at least one validator
+ """
+ for validator in sorted(
+ validators, key=lambda validator: validator.insertion_index):
+ try:
+ validator.Verify(self)
+ except gflags_validators.Error, e:
+ message = validator.PrintFlagsWithValues(self)
+ raise IllegalFlagValue('%s: %s' % (message, str(e)))
+
+ def _FlagIsRegistered(self, flag_obj):
+ """Checks whether a Flag object is registered under some name.
+
+ Note: this is non trivial: in addition to its normal name, a flag
+ may have a short name too. In self.FlagDict(), both the normal and
+ the short name are mapped to the same flag object. E.g., calling
+ only "del FLAGS.short_name" is not unregistering the corresponding
+ Flag object (it is still registered under the longer name).
+
+ Args:
+ flag_obj: A Flag object.
+
+ Returns:
+ A boolean: True iff flag_obj is registered under some name.
+ """
+ flag_dict = self.FlagDict()
+ # Check whether flag_obj is registered under its long name.
+ name = flag_obj.name
+ if flag_dict.get(name, None) == flag_obj:
+ return True
+ # Check whether flag_obj is registered under its short name.
+ short_name = flag_obj.short_name
+ if (short_name is not None and
+ flag_dict.get(short_name, None) == flag_obj):
+ return True
+ # The flag cannot be registered under any other name, so we do not
+ # need to do a full search through the values of self.FlagDict().
+ return False
+
+ def __delattr__(self, flag_name):
+ """Deletes a previously-defined flag from a flag object.
+
+ This method makes sure we can delete a flag by using
+
+ del flag_values_object.<flag_name>
+
+ E.g.,
+
+ gflags.DEFINE_integer('foo', 1, 'Integer flag.')
+ del gflags.FLAGS.foo
+
+ Args:
+ flag_name: A string, the name of the flag to be deleted.
+
+ Raises:
+ AttributeError: When there is no registered flag named flag_name.
+ """
+ fl = self.FlagDict()
+ if flag_name not in fl:
+ raise AttributeError(flag_name)
+
+ flag_obj = fl[flag_name]
+ del fl[flag_name]
+
+ if not self._FlagIsRegistered(flag_obj):
+ # If the Flag object indicated by flag_name is no longer
+ # registered (please see the docstring of _FlagIsRegistered), then
+ # we delete the occurrences of the flag object in all our internal
+ # dictionaries.
+ self.__RemoveFlagFromDictByModule(self.FlagsByModuleDict(), flag_obj)
+ self.__RemoveFlagFromDictByModule(self.FlagsByModuleIdDict(), flag_obj)
+ self.__RemoveFlagFromDictByModule(self.KeyFlagsByModuleDict(), flag_obj)
+
+ def __RemoveFlagFromDictByModule(self, flags_by_module_dict, flag_obj):
+ """Removes a flag object from a module -> list of flags dictionary.
+
+ Args:
+ flags_by_module_dict: A dictionary that maps module names to lists of
+ flags.
+ flag_obj: A flag object.
+ """
+ for unused_module, flags_in_module in flags_by_module_dict.iteritems():
+ # while (as opposed to if) takes care of multiple occurrences of a
+ # flag in the list for the same module.
+ while flag_obj in flags_in_module:
+ flags_in_module.remove(flag_obj)
+
+ def SetDefault(self, name, value):
+ """Changes the default value of the named flag object."""
+ fl = self.FlagDict()
+ if name not in fl:
+ raise AttributeError(name)
+ fl[name].SetDefault(value)
+ self._AssertValidators(fl[name].validators)
+
+ def __contains__(self, name):
+ """Returns True if name is a value (flag) in the dict."""
+ return name in self.FlagDict()
+
+ has_key = __contains__ # a synonym for __contains__()
+
+ def __iter__(self):
+ return iter(self.FlagDict())
+
+ def __call__(self, argv):
+ """Parses flags from argv; stores parsed flags into this FlagValues object.
+
+ All unparsed arguments are returned. Flags are parsed using the GNU
+ Program Argument Syntax Conventions, using getopt:
+
+ http://www.gnu.org/software/libc/manual/html_mono/libc.html#Getopt
+
+ Args:
+ argv: argument list. Can be of any type that may be converted to a list.
+
+ Returns:
+ The list of arguments not parsed as options, including argv[0]
+
+ Raises:
+ FlagsError: on any parsing error
+ """
+ # Support any sequence type that can be converted to a list
+ argv = list(argv)
+
+ shortopts = ""
+ longopts = []
+
+ fl = self.FlagDict()
+
+ # This pre parses the argv list for --flagfile=<> options.
+ argv = argv[:1] + self.ReadFlagsFromFiles(argv[1:], force_gnu=False)
+
+ # Correct the argv to support the google style of passing boolean
+ # parameters. Boolean parameters may be passed by using --mybool,
+ # --nomybool, --mybool=(true|false|1|0). getopt does not support
+ # having options that may or may not have a parameter. We replace
+ # instances of the short form --mybool and --nomybool with their
+ # full forms: --mybool=(true|false).
+ original_argv = list(argv) # list() makes a copy
+ shortest_matches = None
+ for name, flag in fl.items():
+ if not flag.boolean:
+ continue
+ if shortest_matches is None:
+ # Determine the smallest allowable prefix for all flag names
+ shortest_matches = self.ShortestUniquePrefixes(fl)
+ no_name = 'no' + name
+ prefix = shortest_matches[name]
+ no_prefix = shortest_matches[no_name]
+
+ # Replace all occurrences of this boolean with extended forms
+ for arg_idx in range(1, len(argv)):
+ arg = argv[arg_idx]
+ if arg.find('=') >= 0: continue
+ if arg.startswith('--'+prefix) and ('--'+name).startswith(arg):
+ argv[arg_idx] = ('--%s=true' % name)
+ elif arg.startswith('--'+no_prefix) and ('--'+no_name).startswith(arg):
+ argv[arg_idx] = ('--%s=false' % name)
+
+ # Loop over all of the flags, building up the lists of short options
+ # and long options that will be passed to getopt. Short options are
+ # specified as a string of letters, each letter followed by a colon
+ # if it takes an argument. Long options are stored in an array of
+ # strings. Each string ends with an '=' if it takes an argument.
+ for name, flag in fl.items():
+ longopts.append(name + "=")
+ if len(name) == 1: # one-letter option: allow short flag type also
+ shortopts += name
+ if not flag.boolean:
+ shortopts += ":"
+
+ longopts.append('undefok=')
+ undefok_flags = []
+
+ # In case --undefok is specified, loop to pick up unrecognized
+ # options one by one.
+ unrecognized_opts = []
+ args = argv[1:]
+ while True:
+ try:
+ if self.__dict__['__use_gnu_getopt']:
+ optlist, unparsed_args = getopt.gnu_getopt(args, shortopts, longopts)
+ else:
+ optlist, unparsed_args = getopt.getopt(args, shortopts, longopts)
+ break
+ except getopt.GetoptError, e:
+ if not e.opt or e.opt in fl:
+ # Not an unrecognized option, re-raise the exception as a FlagsError
+ raise FlagsError(e)
+ # Remove offender from args and try again
+ for arg_index in range(len(args)):
+ if ((args[arg_index] == '--' + e.opt) or
+ (args[arg_index] == '-' + e.opt) or
+ (args[arg_index].startswith('--' + e.opt + '='))):
+ unrecognized_opts.append((e.opt, args[arg_index]))
+ args = args[0:arg_index] + args[arg_index+1:]
+ break
+ else:
+ # We should have found the option, so we don't expect to get
+ # here. We could assert, but raising the original exception
+ # might work better.
+ raise FlagsError(e)
+
+ for name, arg in optlist:
+ if name == '--undefok':
+ flag_names = arg.split(',')
+ undefok_flags.extend(flag_names)
+ # For boolean flags, if --undefok=boolflag is specified, then we should
+ # also accept --noboolflag, in addition to --boolflag.
+ # Since we don't know the type of the undefok'd flag, this will affect
+ # non-boolean flags as well.
+ # NOTE: You shouldn't use --undefok=noboolflag, because then we will
+ # accept --nonoboolflag here. We are choosing not to do the conversion
+ # from noboolflag -> boolflag because of the ambiguity that flag names
+ # can start with 'no'.
+ undefok_flags.extend('no' + name for name in flag_names)
+ continue
+ if name.startswith('--'):
+ # long option
+ name = name[2:]
+ short_option = 0
+ else:
+ # short option
+ name = name[1:]
+ short_option = 1
+ if name in fl:
+ flag = fl[name]
+ if flag.boolean and short_option: arg = 1
+ flag.Parse(arg)
+
+ # If there were unrecognized options, raise an exception unless
+ # the options were named via --undefok.
+ for opt, value in unrecognized_opts:
+ if opt not in undefok_flags:
+ raise UnrecognizedFlagError(opt, value)
+
+ if unparsed_args:
+ if self.__dict__['__use_gnu_getopt']:
+ # if using gnu_getopt just return the program name + remainder of argv.
+ ret_val = argv[:1] + unparsed_args
+ else:
+ # unparsed_args becomes the first non-flag detected by getopt to
+ # the end of argv. Because argv may have been modified above,
+ # return original_argv for this region.
+ ret_val = argv[:1] + original_argv[-len(unparsed_args):]
+ else:
+ ret_val = argv[:1]
+
+ self._AssertAllValidators()
+ return ret_val
+
+ def Reset(self):
+ """Resets the values to the point before FLAGS(argv) was called."""
+ for f in self.FlagDict().values():
+ f.Unparse()
+
+ def RegisteredFlags(self):
+ """Returns: a list of the names and short names of all registered flags."""
+ return list(self.FlagDict())
+
+ def FlagValuesDict(self):
+ """Returns: a dictionary that maps flag names to flag values."""
+ flag_values = {}
+
+ for flag_name in self.RegisteredFlags():
+ flag = self.FlagDict()[flag_name]
+ flag_values[flag_name] = flag.value
+
+ return flag_values
+
+ def __str__(self):
+ """Generates a help string for all known flags."""
+ return self.GetHelp()
+
+ def GetHelp(self, prefix=''):
+ """Generates a help string for all known flags."""
+ helplist = []
+
+ flags_by_module = self.FlagsByModuleDict()
+ if flags_by_module:
+
+ modules = sorted(flags_by_module)
+
+ # Print the help for the main module first, if possible.
+ main_module = _GetMainModule()
+ if main_module in modules:
+ modules.remove(main_module)
+ modules = [main_module] + modules
+
+ for module in modules:
+ self.__RenderOurModuleFlags(module, helplist)
+
+ self.__RenderModuleFlags('gflags',
+ _SPECIAL_FLAGS.FlagDict().values(),
+ helplist)
+
+ else:
+ # Just print one long list of flags.
+ self.__RenderFlagList(
+ self.FlagDict().values() + _SPECIAL_FLAGS.FlagDict().values(),
+ helplist, prefix)
+
+ return '\n'.join(helplist)
+
+ def __RenderModuleFlags(self, module, flags, output_lines, prefix=""):
+ """Generates a help string for a given module."""
+ if not isinstance(module, str):
+ module = module.__name__
+ output_lines.append('\n%s%s:' % (prefix, module))
+ self.__RenderFlagList(flags, output_lines, prefix + " ")
+
+ def __RenderOurModuleFlags(self, module, output_lines, prefix=""):
+ """Generates a help string for a given module."""
+ flags = self._GetFlagsDefinedByModule(module)
+ if flags:
+ self.__RenderModuleFlags(module, flags, output_lines, prefix)
+
+ def __RenderOurModuleKeyFlags(self, module, output_lines, prefix=""):
+ """Generates a help string for the key flags of a given module.
+
+ Args:
+ module: A module object or a module name (a string).
+ output_lines: A list of strings. The generated help message
+ lines will be appended to this list.
+ prefix: A string that is prepended to each generated help line.
+ """
+ key_flags = self._GetKeyFlagsForModule(module)
+ if key_flags:
+ self.__RenderModuleFlags(module, key_flags, output_lines, prefix)
+
+ def ModuleHelp(self, module):
+ """Describe the key flags of a module.
+
+ Args:
+ module: A module object or a module name (a string).
+
+ Returns:
+ string describing the key flags of a module.
+ """
+ helplist = []
+ self.__RenderOurModuleKeyFlags(module, helplist)
+ return '\n'.join(helplist)
+
+ def MainModuleHelp(self):
+ """Describe the key flags of the main module.
+
+ Returns:
+ string describing the key flags of a module.
+ """
+ return self.ModuleHelp(_GetMainModule())
+
+ def __RenderFlagList(self, flaglist, output_lines, prefix=" "):
+ fl = self.FlagDict()
+ special_fl = _SPECIAL_FLAGS.FlagDict()
+ flaglist = [(flag.name, flag) for flag in flaglist]
+ flaglist.sort()
+ flagset = {}
+ for (name, flag) in flaglist:
+ # It's possible this flag got deleted or overridden since being
+ # registered in the per-module flaglist. Check now against the
+ # canonical source of current flag information, the FlagDict.
+ if fl.get(name, None) != flag and special_fl.get(name, None) != flag:
+ # a different flag is using this name now
+ continue
+ # only print help once
+ if flag in flagset: continue
+ flagset[flag] = 1
+ flaghelp = ""
+ if flag.short_name: flaghelp += "-%s," % flag.short_name
+ if flag.boolean:
+ flaghelp += "--[no]%s" % flag.name + ":"
+ else:
+ flaghelp += "--%s" % flag.name + ":"
+ flaghelp += " "
+ if flag.help:
+ flaghelp += flag.help
+ flaghelp = TextWrap(flaghelp, indent=prefix+" ",
+ firstline_indent=prefix)
+ if flag.default_as_str:
+ flaghelp += "\n"
+ flaghelp += TextWrap("(default: %s)" % flag.default_as_str,
+ indent=prefix+" ")
+ if flag.parser.syntactic_help:
+ flaghelp += "\n"
+ flaghelp += TextWrap("(%s)" % flag.parser.syntactic_help,
+ indent=prefix+" ")
+ output_lines.append(flaghelp)
+
+ def get(self, name, default):
+ """Returns the value of a flag (if not None) or a default value.
+
+ Args:
+ name: A string, the name of a flag.
+ default: Default value to use if the flag value is None.
+ """
+
+ value = self.__getattr__(name)
+ if value is not None: # Can't do if not value, b/c value might be '0' or ""
+ return value
+ else:
+ return default
+
+ def ShortestUniquePrefixes(self, fl):
+ """Returns: dictionary; maps flag names to their shortest unique prefix."""
+ # Sort the list of flag names
+ sorted_flags = []
+ for name, flag in fl.items():
+ sorted_flags.append(name)
+ if flag.boolean:
+ sorted_flags.append('no%s' % name)
+ sorted_flags.sort()
+
+ # For each name in the sorted list, determine the shortest unique
+ # prefix by comparing itself to the next name and to the previous
+ # name (the latter check uses cached info from the previous loop).
+ shortest_matches = {}
+ prev_idx = 0
+ for flag_idx in range(len(sorted_flags)):
+ curr = sorted_flags[flag_idx]
+ if flag_idx == (len(sorted_flags) - 1):
+ next = None
+ else:
+ next = sorted_flags[flag_idx+1]
+ next_len = len(next)
+ for curr_idx in range(len(curr)):
+ if (next is None
+ or curr_idx >= next_len
+ or curr[curr_idx] != next[curr_idx]):
+ # curr longer than next or no more chars in common
+ shortest_matches[curr] = curr[:max(prev_idx, curr_idx) + 1]
+ prev_idx = curr_idx
+ break
+ else:
+ # curr shorter than (or equal to) next
+ shortest_matches[curr] = curr
+ prev_idx = curr_idx + 1 # next will need at least one more char
+ return shortest_matches
+
+ def __IsFlagFileDirective(self, flag_string):
+ """Checks whether flag_string contain a --flagfile=<foo> directive."""
+ if isinstance(flag_string, type("")):
+ if flag_string.startswith('--flagfile='):
+ return 1
+ elif flag_string == '--flagfile':
+ return 1
+ elif flag_string.startswith('-flagfile='):
+ return 1
+ elif flag_string == '-flagfile':
+ return 1
+ else:
+ return 0
+ return 0
+
+ def ExtractFilename(self, flagfile_str):
+ """Returns filename from a flagfile_str of form -[-]flagfile=filename.
+
+ The cases of --flagfile foo and -flagfile foo shouldn't be hitting
+ this function, as they are dealt with in the level above this
+ function.
+ """
+ if flagfile_str.startswith('--flagfile='):
+ return os.path.expanduser((flagfile_str[(len('--flagfile=')):]).strip())
+ elif flagfile_str.startswith('-flagfile='):
+ return os.path.expanduser((flagfile_str[(len('-flagfile=')):]).strip())
+ else:
+ raise FlagsError('Hit illegal --flagfile type: %s' % flagfile_str)
+
+ def __GetFlagFileLines(self, filename, parsed_file_list):
+ """Returns the useful (!=comments, etc) lines from a file with flags.
+
+ Args:
+ filename: A string, the name of the flag file.
+ parsed_file_list: A list of the names of the files we have
+ already read. MUTATED BY THIS FUNCTION.
+
+ Returns:
+ List of strings. See the note below.
+
+ NOTE(springer): This function checks for a nested --flagfile=<foo>
+ tag and handles the lower file recursively. It returns a list of
+ all the lines that _could_ contain command flags. This is
+ EVERYTHING except whitespace lines and comments (lines starting
+ with '#' or '//').
+ """
+ line_list = [] # All line from flagfile.
+ flag_line_list = [] # Subset of lines w/o comments, blanks, flagfile= tags.
+ try:
+ file_obj = open(filename, 'r')
+ except IOError, e_msg:
+ raise CantOpenFlagFileError('ERROR:: Unable to open flagfile: %s' % e_msg)
+
+ line_list = file_obj.readlines()
+ file_obj.close()
+ parsed_file_list.append(filename)
+
+ # This is where we check each line in the file we just read.
+ for line in line_list:
+ if line.isspace():
+ pass
+ # Checks for comment (a line that starts with '#').
+ elif line.startswith('#') or line.startswith('//'):
+ pass
+ # Checks for a nested "--flagfile=<bar>" flag in the current file.
+ # If we find one, recursively parse down into that file.
+ elif self.__IsFlagFileDirective(line):
+ sub_filename = self.ExtractFilename(line)
+ # We do a little safety check for reparsing a file we've already done.
+ if not sub_filename in parsed_file_list:
+ included_flags = self.__GetFlagFileLines(sub_filename,
+ parsed_file_list)
+ flag_line_list.extend(included_flags)
+ else: # Case of hitting a circularly included file.
+ sys.stderr.write('Warning: Hit circular flagfile dependency: %s\n' %
+ (sub_filename,))
+ else:
+ # Any line that's not a comment or a nested flagfile should get
+ # copied into 2nd position. This leaves earlier arguments
+ # further back in the list, thus giving them higher priority.
+ flag_line_list.append(line.strip())
+ return flag_line_list
+
+ def ReadFlagsFromFiles(self, argv, force_gnu=True):
+ """Processes command line args, but also allow args to be read from file.
+
+ Args:
+ argv: A list of strings, usually sys.argv[1:], which may contain one or
+ more flagfile directives of the form --flagfile="./filename".
+ Note that the name of the program (sys.argv[0]) should be omitted.
+ force_gnu: If False, --flagfile parsing obeys normal flag semantics.
+ If True, --flagfile parsing instead follows gnu_getopt semantics.
+ *** WARNING *** force_gnu=False may become the future default!
+
+ Returns:
+
+ A new list which has the original list combined with what we read
+ from any flagfile(s).
+
+ References: Global gflags.FLAG class instance.
+
+ This function should be called before the normal FLAGS(argv) call.
+ This function scans the input list for a flag that looks like:
+ --flagfile=<somefile>. Then it opens <somefile>, reads all valid key
+ and value pairs and inserts them into the input list between the
+ first item of the list and any subsequent items in the list.
+
+ Note that your application's flags are still defined the usual way
+ using gflags DEFINE_flag() type functions.
+
+ Notes (assuming we're getting a commandline of some sort as our input):
+ --> Flags from the command line argv _should_ always take precedence!
+ --> A further "--flagfile=<otherfile.cfg>" CAN be nested in a flagfile.
+ It will be processed after the parent flag file is done.
+ --> For duplicate flags, first one we hit should "win".
+ --> In a flagfile, a line beginning with # or // is a comment.
+ --> Entirely blank lines _should_ be ignored.
+ """
+ parsed_file_list = []
+ rest_of_args = argv
+ new_argv = []
+ while rest_of_args:
+ current_arg = rest_of_args[0]
+ rest_of_args = rest_of_args[1:]
+ if self.__IsFlagFileDirective(current_arg):
+ # This handles the case of -(-)flagfile foo. In this case the
+ # next arg really is part of this one.
+ if current_arg == '--flagfile' or current_arg == '-flagfile':
+ if not rest_of_args:
+ raise IllegalFlagValue('--flagfile with no argument')
+ flag_filename = os.path.expanduser(rest_of_args[0])
+ rest_of_args = rest_of_args[1:]
+ else:
+ # This handles the case of (-)-flagfile=foo.
+ flag_filename = self.ExtractFilename(current_arg)
+ new_argv.extend(
+ self.__GetFlagFileLines(flag_filename, parsed_file_list))
+ else:
+ new_argv.append(current_arg)
+ # Stop parsing after '--', like getopt and gnu_getopt.
+ if current_arg == '--':
+ break
+ # Stop parsing after a non-flag, like getopt.
+ if not current_arg.startswith('-'):
+ if not force_gnu and not self.__dict__['__use_gnu_getopt']:
+ break
+
+ if rest_of_args:
+ new_argv.extend(rest_of_args)
+
+ return new_argv
+
+ def FlagsIntoString(self):
+ """Returns a string with the flags assignments from this FlagValues object.
+
+ This function ignores flags whose value is None. Each flag
+ assignment is separated by a newline.
+
+ NOTE: MUST mirror the behavior of the C++ CommandlineFlagsIntoString
+ from http://code.google.com/p/google-gflags
+ """
+ s = ''
+ for flag in self.FlagDict().values():
+ if flag.value is not None:
+ s += flag.Serialize() + '\n'
+ return s
+
+ def AppendFlagsIntoFile(self, filename):
+ """Appends all flags assignments from this FlagInfo object to a file.
+
+ Output will be in the format of a flagfile.
+
+ NOTE: MUST mirror the behavior of the C++ AppendFlagsIntoFile
+ from http://code.google.com/p/google-gflags
+ """
+ out_file = open(filename, 'a')
+ out_file.write(self.FlagsIntoString())
+ out_file.close()
+
+ def WriteHelpInXMLFormat(self, outfile=None):
+ """Outputs flag documentation in XML format.
+
+ NOTE: We use element names that are consistent with those used by
+ the C++ command-line flag library, from
+ http://code.google.com/p/google-gflags
+ We also use a few new elements (e.g., <key>), but we do not
+ interfere / overlap with existing XML elements used by the C++
+ library. Please maintain this consistency.
+
+ Args:
+ outfile: File object we write to. Default None means sys.stdout.
+ """
+ outfile = outfile or sys.stdout
+
+ outfile.write('<?xml version=\"1.0\"?>\n')
+ outfile.write('<AllFlags>\n')
+ indent = ' '
+ _WriteSimpleXMLElement(outfile, 'program', os.path.basename(sys.argv[0]),
+ indent)
+
+ usage_doc = sys.modules['__main__'].__doc__
+ if not usage_doc:
+ usage_doc = '\nUSAGE: %s [flags]\n' % sys.argv[0]
+ else:
+ usage_doc = usage_doc.replace('%s', sys.argv[0])
+ _WriteSimpleXMLElement(outfile, 'usage', usage_doc, indent)
+
+ # Get list of key flags for the main module.
+ key_flags = self._GetKeyFlagsForModule(_GetMainModule())
+
+ # Sort flags by declaring module name and next by flag name.
+ flags_by_module = self.FlagsByModuleDict()
+ all_module_names = list(flags_by_module.keys())
+ all_module_names.sort()
+ for module_name in all_module_names:
+ flag_list = [(f.name, f) for f in flags_by_module[module_name]]
+ flag_list.sort()
+ for unused_flag_name, flag in flag_list:
+ is_key = flag in key_flags
+ flag.WriteInfoInXMLFormat(outfile, module_name,
+ is_key=is_key, indent=indent)
+
+ outfile.write('</AllFlags>\n')
+ outfile.flush()
+
+ def AddValidator(self, validator):
+ """Register new flags validator to be checked.
+
+ Args:
+ validator: gflags_validators.Validator
+ Raises:
+ AttributeError: if validators work with a non-existing flag.
+ """
+ for flag_name in validator.GetFlagsNames():
+ flag = self.FlagDict()[flag_name]
+ flag.validators.append(validator)
+
+# end of FlagValues definition
+
+
+# The global FlagValues instance
+FLAGS = FlagValues()
+
+
+def _StrOrUnicode(value):
+ """Converts value to a python string or, if necessary, unicode-string."""
+ try:
+ return str(value)
+ except UnicodeEncodeError:
+ return unicode(value)
+
+
+def _MakeXMLSafe(s):
+ """Escapes <, >, and & from s, and removes XML 1.0-illegal chars."""
+ s = cgi.escape(s) # Escape <, >, and &
+ # Remove characters that cannot appear in an XML 1.0 document
+ # (http://www.w3.org/TR/REC-xml/#charsets).
+ #
+ # NOTE: if there are problems with current solution, one may move to
+ # XML 1.1, which allows such chars, if they're entity-escaped (&#xHH;).
+ s = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f]', '', s)
+ # Convert non-ascii characters to entities. Note: requires python >=2.3
+ s = s.encode('ascii', 'xmlcharrefreplace') # u'\xce\x88' -> 'uΈ'
+ return s
+
+
+def _WriteSimpleXMLElement(outfile, name, value, indent):
+ """Writes a simple XML element.
+
+ Args:
+ outfile: File object we write the XML element to.
+ name: A string, the name of XML element.
+ value: A Python object, whose string representation will be used
+ as the value of the XML element.
+ indent: A string, prepended to each line of generated output.
+ """
+ value_str = _StrOrUnicode(value)
+ if isinstance(value, bool):
+ # Display boolean values as the C++ flag library does: no caps.
+ value_str = value_str.lower()
+ safe_value_str = _MakeXMLSafe(value_str)
+ outfile.write('%s<%s>%s</%s>\n' % (indent, name, safe_value_str, name))
+
+
+class Flag:
+ """Information about a command-line flag.
+
+ 'Flag' objects define the following fields:
+ .name - the name for this flag
+ .default - the default value for this flag
+ .default_as_str - default value as repr'd string, e.g., "'true'" (or None)
+ .value - the most recent parsed value of this flag; set by Parse()
+ .help - a help string or None if no help is available
+ .short_name - the single letter alias for this flag (or None)
+ .boolean - if 'true', this flag does not accept arguments
+ .present - true if this flag was parsed from command line flags.
+ .parser - an ArgumentParser object
+ .serializer - an ArgumentSerializer object
+ .allow_override - the flag may be redefined without raising an error
+
+ The only public method of a 'Flag' object is Parse(), but it is
+ typically only called by a 'FlagValues' object. The Parse() method is
+ a thin wrapper around the 'ArgumentParser' Parse() method. The parsed
+ value is saved in .value, and the .present attribute is updated. If
+ this flag was already present, a FlagsError is raised.
+
+ Parse() is also called during __init__ to parse the default value and
+ initialize the .value attribute. This enables other python modules to
+ safely use flags even if the __main__ module neglects to parse the
+ command line arguments. The .present attribute is cleared after
+ __init__ parsing. If the default value is set to None, then the
+ __init__ parsing step is skipped and the .value attribute is
+ initialized to None.
+
+ Note: The default value is also presented to the user in the help
+ string, so it is important that it be a legal value for this flag.
+ """
+
+ def __init__(self, parser, serializer, name, default, help_string,
+ short_name=None, boolean=0, allow_override=0):
+ self.name = name
+
+ if not help_string:
+ help_string = '(no help available)'
+
+ self.help = help_string
+ self.short_name = short_name
+ self.boolean = boolean
+ self.present = 0
+ self.parser = parser
+ self.serializer = serializer
+ self.allow_override = allow_override
+ self.value = None
+ self.validators = []
+
+ self.SetDefault(default)
+
+ def __hash__(self):
+ return hash(id(self))
+
+ def __eq__(self, other):
+ return self is other
+
+ def __lt__(self, other):
+ if isinstance(other, Flag):
+ return id(self) < id(other)
+ return NotImplemented
+
+ def __GetParsedValueAsString(self, value):
+ if value is None:
+ return None
+ if self.serializer:
+ return repr(self.serializer.Serialize(value))
+ if self.boolean:
+ if value:
+ return repr('true')
+ else:
+ return repr('false')
+ return repr(_StrOrUnicode(value))
+
+ def Parse(self, argument):
+ try:
+ self.value = self.parser.Parse(argument)
+ except ValueError, e: # recast ValueError as IllegalFlagValue
+ raise IllegalFlagValue("flag --%s=%s: %s" % (self.name, argument, e))
+ self.present += 1
+
+ def Unparse(self):
+ if self.default is None:
+ self.value = None
+ else:
+ self.Parse(self.default)
+ self.present = 0
+
+ def Serialize(self):
+ if self.value is None:
+ return ''
+ if self.boolean:
+ if self.value:
+ return "--%s" % self.name
+ else:
+ return "--no%s" % self.name
+ else:
+ if not self.serializer:
+ raise FlagsError("Serializer not present for flag %s" % self.name)
+ return "--%s=%s" % (self.name, self.serializer.Serialize(self.value))
+
+ def SetDefault(self, value):
+ """Changes the default value (and current value too) for this Flag."""
+ # We can't allow a None override because it may end up not being
+ # passed to C++ code when we're overriding C++ flags. So we
+ # cowardly bail out until someone fixes the semantics of trying to
+ # pass None to a C++ flag. See swig_flags.Init() for details on
+ # this behavior.
+ # TODO(olexiy): Users can directly call this method, bypassing all flags
+ # validators (we don't have FlagValues here, so we can not check
+ # validators).
+ # The simplest solution I see is to make this method private.
+ # Another approach would be to store reference to the corresponding
+ # FlagValues with each flag, but this seems to be an overkill.
+ if value is None and self.allow_override:
+ raise DuplicateFlagCannotPropagateNoneToSwig(self.name)
+
+ self.default = value
+ self.Unparse()
+ self.default_as_str = self.__GetParsedValueAsString(self.value)
+
+ def Type(self):
+ """Returns: a string that describes the type of this Flag."""
+ # NOTE: we use strings, and not the types.*Type constants because
+ # our flags can have more exotic types, e.g., 'comma separated list
+ # of strings', 'whitespace separated list of strings', etc.
+ return self.parser.Type()
+
+ def WriteInfoInXMLFormat(self, outfile, module_name, is_key=False, indent=''):
+ """Writes common info about this flag, in XML format.
+
+ This is information that is relevant to all flags (e.g., name,
+ meaning, etc.). If you defined a flag that has some other pieces of
+ info, then please override _WriteCustomInfoInXMLFormat.
+
+ Please do NOT override this method.
+
+ Args:
+ outfile: File object we write to.
+ module_name: A string, the name of the module that defines this flag.
+ is_key: A boolean, True iff this flag is key for main module.
+ indent: A string that is prepended to each generated line.
+ """
+ outfile.write(indent + '<flag>\n')
+ inner_indent = indent + ' '
+ if is_key:
+ _WriteSimpleXMLElement(outfile, 'key', 'yes', inner_indent)
+ _WriteSimpleXMLElement(outfile, 'file', module_name, inner_indent)
+ # Print flag features that are relevant for all flags.
+ _WriteSimpleXMLElement(outfile, 'name', self.name, inner_indent)
+ if self.short_name:
+ _WriteSimpleXMLElement(outfile, 'short_name', self.short_name,
+ inner_indent)
+ if self.help:
+ _WriteSimpleXMLElement(outfile, 'meaning', self.help, inner_indent)
+ # The default flag value can either be represented as a string like on the
+ # command line, or as a Python object. We serialize this value in the
+ # latter case in order to remain consistent.
+ if self.serializer and not isinstance(self.default, str):
+ default_serialized = self.serializer.Serialize(self.default)
+ else:
+ default_serialized = self.default
+ _WriteSimpleXMLElement(outfile, 'default', default_serialized, inner_indent)
+ _WriteSimpleXMLElement(outfile, 'current', self.value, inner_indent)
+ _WriteSimpleXMLElement(outfile, 'type', self.Type(), inner_indent)
+ # Print extra flag features this flag may have.
+ self._WriteCustomInfoInXMLFormat(outfile, inner_indent)
+ outfile.write(indent + '</flag>\n')
+
+ def _WriteCustomInfoInXMLFormat(self, outfile, indent):
+ """Writes extra info about this flag, in XML format.
+
+ "Extra" means "not already printed by WriteInfoInXMLFormat above."
+
+ Args:
+ outfile: File object we write to.
+ indent: A string that is prepended to each generated line.
+ """
+ # Usually, the parser knows the extra details about the flag, so
+ # we just forward the call to it.
+ self.parser.WriteCustomInfoInXMLFormat(outfile, indent)
+# End of Flag definition
+
+
+class _ArgumentParserCache(type):
+ """Metaclass used to cache and share argument parsers among flags."""
+
+ _instances = {}
+
+ def __call__(mcs, *args, **kwargs):
+ """Returns an instance of the argument parser cls.
+
+ This method overrides behavior of the __new__ methods in
+ all subclasses of ArgumentParser (inclusive). If an instance
+ for mcs with the same set of arguments exists, this instance is
+ returned, otherwise a new instance is created.
+
+ If any keyword arguments are defined, or the values in args
+ are not hashable, this method always returns a new instance of
+ cls.
+
+ Args:
+ args: Positional initializer arguments.
+ kwargs: Initializer keyword arguments.
+
+ Returns:
+ An instance of cls, shared or new.
+ """
+ if kwargs:
+ return type.__call__(mcs, *args, **kwargs)
+ else:
+ instances = mcs._instances
+ key = (mcs,) + tuple(args)
+ try:
+ return instances[key]
+ except KeyError:
+ # No cache entry for key exists, create a new one.
+ return instances.setdefault(key, type.__call__(mcs, *args))
+ except TypeError:
+ # An object in args cannot be hashed, always return
+ # a new instance.
+ return type.__call__(mcs, *args)
+
+
+class ArgumentParser(object):
+ """Base class used to parse and convert arguments.
+
+ The Parse() method checks to make sure that the string argument is a
+ legal value and convert it to a native type. If the value cannot be
+ converted, it should throw a 'ValueError' exception with a human
+ readable explanation of why the value is illegal.
+
+ Subclasses should also define a syntactic_help string which may be
+ presented to the user to describe the form of the legal values.
+
+ Argument parser classes must be stateless, since instances are cached
+ and shared between flags. Initializer arguments are allowed, but all
+ member variables must be derived from initializer arguments only.
+ """
+ __metaclass__ = _ArgumentParserCache
+
+ syntactic_help = ""
+
+ def Parse(self, argument):
+ """Default implementation: always returns its argument unmodified."""
+ return argument
+
+ def Type(self):
+ return 'string'
+
+ def WriteCustomInfoInXMLFormat(self, outfile, indent):
+ pass
+
+
+class ArgumentSerializer:
+ """Base class for generating string representations of a flag value."""
+
+ def Serialize(self, value):
+ return _StrOrUnicode(value)
+
+
+class ListSerializer(ArgumentSerializer):
+
+ def __init__(self, list_sep):
+ self.list_sep = list_sep
+
+ def Serialize(self, value):
+ return self.list_sep.join([_StrOrUnicode(x) for x in value])
+
+
+# Flags validators
+
+
+def RegisterValidator(flag_name,
+ checker,
+ message='Flag validation failed',
+ flag_values=FLAGS):
+ """Adds a constraint, which will be enforced during program execution.
+
+ The constraint is validated when flags are initially parsed, and after each
+ change of the corresponding flag's value.
+ Args:
+ flag_name: string, name of the flag to be checked.
+ checker: method to validate the flag.
+ input - value of the corresponding flag (string, boolean, etc.
+ This value will be passed to checker by the library). See file's
+ docstring for examples.
+ output - Boolean.
+ Must return True if validator constraint is satisfied.
+ If constraint is not satisfied, it should either return False or
+ raise gflags_validators.Error(desired_error_message).
+ message: error text to be shown to the user if checker returns False.
+ If checker raises gflags_validators.Error, message from the raised
+ Error will be shown.
+ flag_values: FlagValues
+ Raises:
+ AttributeError: if flag_name is not registered as a valid flag name.
+ """
+ flag_values.AddValidator(gflags_validators.SimpleValidator(flag_name,
+ checker,
+ message))
+
+
+def MarkFlagAsRequired(flag_name, flag_values=FLAGS):
+ """Ensure that flag is not None during program execution.
+
+ Registers a flag validator, which will follow usual validator
+ rules.
+ Args:
+ flag_name: string, name of the flag
+ flag_values: FlagValues
+ Raises:
+ AttributeError: if flag_name is not registered as a valid flag name.
+ """
+ RegisterValidator(flag_name,
+ lambda value: value is not None,
+ message='Flag --%s must be specified.' % flag_name,
+ flag_values=flag_values)
+
+
+def _RegisterBoundsValidatorIfNeeded(parser, name, flag_values):
+ """Enforce lower and upper bounds for numeric flags.
+
+ Args:
+ parser: NumericParser (either FloatParser or IntegerParser). Provides lower
+ and upper bounds, and help text to display.
+ name: string, name of the flag
+ flag_values: FlagValues
+ """
+ if parser.lower_bound is not None or parser.upper_bound is not None:
+
+ def Checker(value):
+ if value is not None and parser.IsOutsideBounds(value):
+ message = '%s is not %s' % (value, parser.syntactic_help)
+ raise gflags_validators.Error(message)
+ return True
+
+ RegisterValidator(name,
+ Checker,
+ flag_values=flag_values)
+
+
+# The DEFINE functions are explained in mode details in the module doc string.
+
+
+def DEFINE(parser, name, default, help, flag_values=FLAGS, serializer=None,
+ **args):
+ """Registers a generic Flag object.
+
+ NOTE: in the docstrings of all DEFINE* functions, "registers" is short
+ for "creates a new flag and registers it".
+
+ Auxiliary function: clients should use the specialized DEFINE_<type>
+ function instead.
+
+ Args:
+ parser: ArgumentParser that is used to parse the flag arguments.
+ name: A string, the flag name.
+ default: The default value of the flag.
+ help: A help string.
+ flag_values: FlagValues object the flag will be registered with.
+ serializer: ArgumentSerializer that serializes the flag value.
+ args: Dictionary with extra keyword args that are passes to the
+ Flag __init__.
+ """
+ DEFINE_flag(Flag(parser, serializer, name, default, help, **args),
+ flag_values)
+
+
+def DEFINE_flag(flag, flag_values=FLAGS):
+ """Registers a 'Flag' object with a 'FlagValues' object.
+
+ By default, the global FLAGS 'FlagValue' object is used.
+
+ Typical users will use one of the more specialized DEFINE_xxx
+ functions, such as DEFINE_string or DEFINE_integer. But developers
+ who need to create Flag objects themselves should use this function
+ to register their flags.
+ """
+ # copying the reference to flag_values prevents pychecker warnings
+ fv = flag_values
+ fv[flag.name] = flag
+ # Tell flag_values who's defining the flag.
+ if isinstance(flag_values, FlagValues):
+ # Regarding the above isinstance test: some users pass funny
+ # values of flag_values (e.g., {}) in order to avoid the flag
+ # registration (in the past, there used to be a flag_values ==
+ # FLAGS test here) and redefine flags with the same name (e.g.,
+ # debug). To avoid breaking their code, we perform the
+ # registration only if flag_values is a real FlagValues object.
+ module, module_name = _GetCallingModuleObjectAndName()
+ flag_values._RegisterFlagByModule(module_name, flag)
+ flag_values._RegisterFlagByModuleId(id(module), flag)
+
+
+def _InternalDeclareKeyFlags(flag_names,
+ flag_values=FLAGS, key_flag_values=None):
+ """Declares a flag as key for the calling module.
+
+ Internal function. User code should call DECLARE_key_flag or
+ ADOPT_module_key_flags instead.
+
+ Args:
+ flag_names: A list of strings that are names of already-registered
+ Flag objects.
+ flag_values: A FlagValues object that the flags listed in
+ flag_names have registered with (the value of the flag_values
+ argument from the DEFINE_* calls that defined those flags).
+ This should almost never need to be overridden.
+ key_flag_values: A FlagValues object that (among possibly many
+ other things) keeps track of the key flags for each module.
+ Default None means "same as flag_values". This should almost
+ never need to be overridden.
+
+ Raises:
+ UnrecognizedFlagError: when we refer to a flag that was not
+ defined yet.
+ """
+ key_flag_values = key_flag_values or flag_values
+
+ module = _GetCallingModule()
+
+ for flag_name in flag_names:
+ if flag_name not in flag_values:
+ raise UnrecognizedFlagError(flag_name)
+ flag = flag_values.FlagDict()[flag_name]
+ key_flag_values._RegisterKeyFlagForModule(module, flag)
+
+
+def DECLARE_key_flag(flag_name, flag_values=FLAGS):
+ """Declares one flag as key to the current module.
+
+ Key flags are flags that are deemed really important for a module.
+ They are important when listing help messages; e.g., if the
+ --helpshort command-line flag is used, then only the key flags of the
+ main module are listed (instead of all flags, as in the case of
+ --help).
+
+ Sample usage:
+
+ gflags.DECLARED_key_flag('flag_1')
+
+ Args:
+ flag_name: A string, the name of an already declared flag.
+ (Redeclaring flags as key, including flags implicitly key
+ because they were declared in this module, is a no-op.)
+ flag_values: A FlagValues object. This should almost never
+ need to be overridden.
+ """
+ if flag_name in _SPECIAL_FLAGS:
+ # Take care of the special flags, e.g., --flagfile, --undefok.
+ # These flags are defined in _SPECIAL_FLAGS, and are treated
+ # specially during flag parsing, taking precedence over the
+ # user-defined flags.
+ _InternalDeclareKeyFlags([flag_name],
+ flag_values=_SPECIAL_FLAGS,
+ key_flag_values=flag_values)
+ return
+ _InternalDeclareKeyFlags([flag_name], flag_values=flag_values)
+
+
+def ADOPT_module_key_flags(module, flag_values=FLAGS):
+ """Declares that all flags key to a module are key to the current module.
+
+ Args:
+ module: A module object.
+ flag_values: A FlagValues object. This should almost never need
+ to be overridden.
+
+ Raises:
+ FlagsError: When given an argument that is a module name (a
+ string), instead of a module object.
+ """
+ # NOTE(salcianu): an even better test would be if not
+ # isinstance(module, types.ModuleType) but I didn't want to import
+ # types for such a tiny use.
+ if isinstance(module, str):
+ raise FlagsError('Received module name %s; expected a module object.'
+ % module)
+ _InternalDeclareKeyFlags(
+ [f.name for f in flag_values._GetKeyFlagsForModule(module.__name__)],
+ flag_values=flag_values)
+ # If module is this flag module, take _SPECIAL_FLAGS into account.
+ if module == _GetThisModuleObjectAndName()[0]:
+ _InternalDeclareKeyFlags(
+ # As we associate flags with _GetCallingModuleObjectAndName(), the
+ # special flags defined in this module are incorrectly registered with
+ # a different module. So, we can't use _GetKeyFlagsForModule.
+ # Instead, we take all flags from _SPECIAL_FLAGS (a private
+ # FlagValues, where no other module should register flags).
+ [f.name for f in _SPECIAL_FLAGS.FlagDict().values()],
+ flag_values=_SPECIAL_FLAGS,
+ key_flag_values=flag_values)
+
+
+#
+# STRING FLAGS
+#
+
+
+def DEFINE_string(name, default, help, flag_values=FLAGS, **args):
+ """Registers a flag whose value can be any string."""
+ parser = ArgumentParser()
+ serializer = ArgumentSerializer()
+ DEFINE(parser, name, default, help, flag_values, serializer, **args)
+
+
+#
+# BOOLEAN FLAGS
+#
+
+
+class BooleanParser(ArgumentParser):
+ """Parser of boolean values."""
+
+ def Convert(self, argument):
+ """Converts the argument to a boolean; raise ValueError on errors."""
+ if type(argument) == str:
+ if argument.lower() in ['true', 't', '1']:
+ return True
+ elif argument.lower() in ['false', 'f', '0']:
+ return False
+
+ bool_argument = bool(argument)
+ if argument == bool_argument:
+ # The argument is a valid boolean (True, False, 0, or 1), and not just
+ # something that always converts to bool (list, string, int, etc.).
+ return bool_argument
+
+ raise ValueError('Non-boolean argument to boolean flag', argument)
+
+ def Parse(self, argument):
+ val = self.Convert(argument)
+ return val
+
+ def Type(self):
+ return 'bool'
+
+
+class BooleanFlag(Flag):
+ """Basic boolean flag.
+
+ Boolean flags do not take any arguments, and their value is either
+ True (1) or False (0). The false value is specified on the command
+ line by prepending the word 'no' to either the long or the short flag
+ name.
+
+ For example, if a Boolean flag was created whose long name was
+ 'update' and whose short name was 'x', then this flag could be
+ explicitly unset through either --noupdate or --nox.
+ """
+
+ def __init__(self, name, default, help, short_name=None, **args):
+ p = BooleanParser()
+ Flag.__init__(self, p, None, name, default, help, short_name, 1, **args)
+ if not self.help: self.help = "a boolean value"
+
+
+def DEFINE_boolean(name, default, help, flag_values=FLAGS, **args):
+ """Registers a boolean flag.
+
+ Such a boolean flag does not take an argument. If a user wants to
+ specify a false value explicitly, the long option beginning with 'no'
+ must be used: i.e. --noflag
+
+ This flag will have a value of None, True or False. None is possible
+ if default=None and the user does not specify the flag on the command
+ line.
+ """
+ DEFINE_flag(BooleanFlag(name, default, help, **args), flag_values)
+
+
+# Match C++ API to unconfuse C++ people.
+DEFINE_bool = DEFINE_boolean
+
+
+class HelpFlag(BooleanFlag):
+ """
+ HelpFlag is a special boolean flag that prints usage information and
+ raises a SystemExit exception if it is ever found in the command
+ line arguments. Note this is called with allow_override=1, so other
+ apps can define their own --help flag, replacing this one, if they want.
+ """
+ def __init__(self):
+ BooleanFlag.__init__(self, "help", 0, "show this help",
+ short_name="?", allow_override=1)
+ def Parse(self, arg):
+ if arg:
+ doc = sys.modules["__main__"].__doc__
+ flags = str(FLAGS)
+ print doc or ("\nUSAGE: %s [flags]\n" % sys.argv[0])
+ if flags:
+ print "flags:"
+ print flags
+ sys.exit(1)
+class HelpXMLFlag(BooleanFlag):
+ """Similar to HelpFlag, but generates output in XML format."""
+ def __init__(self):
+ BooleanFlag.__init__(self, 'helpxml', False,
+ 'like --help, but generates XML output',
+ allow_override=1)
+ def Parse(self, arg):
+ if arg:
+ FLAGS.WriteHelpInXMLFormat(sys.stdout)
+ sys.exit(1)
+class HelpshortFlag(BooleanFlag):
+ """
+ HelpshortFlag is a special boolean flag that prints usage
+ information for the "main" module, and rasies a SystemExit exception
+ if it is ever found in the command line arguments. Note this is
+ called with allow_override=1, so other apps can define their own
+ --helpshort flag, replacing this one, if they want.
+ """
+ def __init__(self):
+ BooleanFlag.__init__(self, "helpshort", 0,
+ "show usage only for this module", allow_override=1)
+ def Parse(self, arg):
+ if arg:
+ doc = sys.modules["__main__"].__doc__
+ flags = FLAGS.MainModuleHelp()
+ print doc or ("\nUSAGE: %s [flags]\n" % sys.argv[0])
+ if flags:
+ print "flags:"
+ print flags
+ sys.exit(1)
+
+#
+# Numeric parser - base class for Integer and Float parsers
+#
+
+
+class NumericParser(ArgumentParser):
+ """Parser of numeric values.
+
+ Parsed value may be bounded to a given upper and lower bound.
+ """
+
+ def IsOutsideBounds(self, val):
+ return ((self.lower_bound is not None and val < self.lower_bound) or
+ (self.upper_bound is not None and val > self.upper_bound))
+
+ def Parse(self, argument):
+ val = self.Convert(argument)
+ if self.IsOutsideBounds(val):
+ raise ValueError("%s is not %s" % (val, self.syntactic_help))
+ return val
+
+ def WriteCustomInfoInXMLFormat(self, outfile, indent):
+ if self.lower_bound is not None:
+ _WriteSimpleXMLElement(outfile, 'lower_bound', self.lower_bound, indent)
+ if self.upper_bound is not None:
+ _WriteSimpleXMLElement(outfile, 'upper_bound', self.upper_bound, indent)
+
+ def Convert(self, argument):
+ """Default implementation: always returns its argument unmodified."""
+ return argument
+
+# End of Numeric Parser
+
+#
+# FLOAT FLAGS
+#
+
+
+class FloatParser(NumericParser):
+ """Parser of floating point values.
+
+ Parsed value may be bounded to a given upper and lower bound.
+ """
+ number_article = "a"
+ number_name = "number"
+ syntactic_help = " ".join((number_article, number_name))
+
+ def __init__(self, lower_bound=None, upper_bound=None):
+ super(FloatParser, self).__init__()
+ self.lower_bound = lower_bound
+ self.upper_bound = upper_bound
+ sh = self.syntactic_help
+ if lower_bound is not None and upper_bound is not None:
+ sh = ("%s in the range [%s, %s]" % (sh, lower_bound, upper_bound))
+ elif lower_bound == 0:
+ sh = "a non-negative %s" % self.number_name
+ elif upper_bound == 0:
+ sh = "a non-positive %s" % self.number_name
+ elif upper_bound is not None:
+ sh = "%s <= %s" % (self.number_name, upper_bound)
+ elif lower_bound is not None:
+ sh = "%s >= %s" % (self.number_name, lower_bound)
+ self.syntactic_help = sh
+
+ def Convert(self, argument):
+ """Converts argument to a float; raises ValueError on errors."""
+ return float(argument)
+
+ def Type(self):
+ return 'float'
+# End of FloatParser
+
+
+def DEFINE_float(name, default, help, lower_bound=None, upper_bound=None,
+ flag_values=FLAGS, **args):
+ """Registers a flag whose value must be a float.
+
+ If lower_bound or upper_bound are set, then this flag must be
+ within the given range.
+ """
+ parser = FloatParser(lower_bound, upper_bound)
+ serializer = ArgumentSerializer()
+ DEFINE(parser, name, default, help, flag_values, serializer, **args)
+ _RegisterBoundsValidatorIfNeeded(parser, name, flag_values=flag_values)
+
+#
+# INTEGER FLAGS
+#
+
+
+class IntegerParser(NumericParser):
+ """Parser of an integer value.
+
+ Parsed value may be bounded to a given upper and lower bound.
+ """
+ number_article = "an"
+ number_name = "integer"
+ syntactic_help = " ".join((number_article, number_name))
+
+ def __init__(self, lower_bound=None, upper_bound=None):
+ super(IntegerParser, self).__init__()
+ self.lower_bound = lower_bound
+ self.upper_bound = upper_bound
+ sh = self.syntactic_help
+ if lower_bound is not None and upper_bound is not None:
+ sh = ("%s in the range [%s, %s]" % (sh, lower_bound, upper_bound))
+ elif lower_bound == 1:
+ sh = "a positive %s" % self.number_name
+ elif upper_bound == -1:
+ sh = "a negative %s" % self.number_name
+ elif lower_bound == 0:
+ sh = "a non-negative %s" % self.number_name
+ elif upper_bound == 0:
+ sh = "a non-positive %s" % self.number_name
+ elif upper_bound is not None:
+ sh = "%s <= %s" % (self.number_name, upper_bound)
+ elif lower_bound is not None:
+ sh = "%s >= %s" % (self.number_name, lower_bound)
+ self.syntactic_help = sh
+
+ def Convert(self, argument):
+ __pychecker__ = 'no-returnvalues'
+ if type(argument) == str:
+ base = 10
+ if len(argument) > 2 and argument[0] == "0" and argument[1] == "x":
+ base = 16
+ return int(argument, base)
+ else:
+ return int(argument)
+
+ def Type(self):
+ return 'int'
+
+
+def DEFINE_integer(name, default, help, lower_bound=None, upper_bound=None,
+ flag_values=FLAGS, **args):
+ """Registers a flag whose value must be an integer.
+
+ If lower_bound, or upper_bound are set, then this flag must be
+ within the given range.
+ """
+ parser = IntegerParser(lower_bound, upper_bound)
+ serializer = ArgumentSerializer()
+ DEFINE(parser, name, default, help, flag_values, serializer, **args)
+ _RegisterBoundsValidatorIfNeeded(parser, name, flag_values=flag_values)
+
+
+#
+# ENUM FLAGS
+#
+
+
+class EnumParser(ArgumentParser):
+ """Parser of a string enum value (a string value from a given set).
+
+ If enum_values (see below) is not specified, any string is allowed.
+ """
+
+ def __init__(self, enum_values=None):
+ super(EnumParser, self).__init__()
+ self.enum_values = enum_values
+
+ def Parse(self, argument):
+ if self.enum_values and argument not in self.enum_values:
+ raise ValueError("value should be one of <%s>" %
+ "|".join(self.enum_values))
+ return argument
+
+ def Type(self):
+ return 'string enum'
+
+
+class EnumFlag(Flag):
+ """Basic enum flag; its value can be any string from list of enum_values."""
+
+ def __init__(self, name, default, help, enum_values=None,
+ short_name=None, **args):
+ enum_values = enum_values or []
+ p = EnumParser(enum_values)
+ g = ArgumentSerializer()
+ Flag.__init__(self, p, g, name, default, help, short_name, **args)
+ if not self.help: self.help = "an enum string"
+ self.help = "<%s>: %s" % ("|".join(enum_values), self.help)
+
+ def _WriteCustomInfoInXMLFormat(self, outfile, indent):
+ for enum_value in self.parser.enum_values:
+ _WriteSimpleXMLElement(outfile, 'enum_value', enum_value, indent)
+
+
+def DEFINE_enum(name, default, enum_values, help, flag_values=FLAGS,
+ **args):
+ """Registers a flag whose value can be any string from enum_values."""
+ DEFINE_flag(EnumFlag(name, default, help, enum_values, ** args),
+ flag_values)
+
+
+#
+# LIST FLAGS
+#
+
+
+class BaseListParser(ArgumentParser):
+ """Base class for a parser of lists of strings.
+
+ To extend, inherit from this class; from the subclass __init__, call
+
+ BaseListParser.__init__(self, token, name)
+
+ where token is a character used to tokenize, and name is a description
+ of the separator.
+ """
+
+ def __init__(self, token=None, name=None):
+ assert name
+ super(BaseListParser, self).__init__()
+ self._token = token
+ self._name = name
+ self.syntactic_help = "a %s separated list" % self._name
+
+ def Parse(self, argument):
+ if isinstance(argument, list):
+ return argument
+ elif argument == '':
+ return []
+ else:
+ return [s.strip() for s in argument.split(self._token)]
+
+ def Type(self):
+ return '%s separated list of strings' % self._name
+
+
+class ListParser(BaseListParser):
+ """Parser for a comma-separated list of strings."""
+
+ def __init__(self):
+ BaseListParser.__init__(self, ',', 'comma')
+
+ def WriteCustomInfoInXMLFormat(self, outfile, indent):
+ BaseListParser.WriteCustomInfoInXMLFormat(self, outfile, indent)
+ _WriteSimpleXMLElement(outfile, 'list_separator', repr(','), indent)
+
+
+class WhitespaceSeparatedListParser(BaseListParser):
+ """Parser for a whitespace-separated list of strings."""
+
+ def __init__(self):
+ BaseListParser.__init__(self, None, 'whitespace')
+
+ def WriteCustomInfoInXMLFormat(self, outfile, indent):
+ BaseListParser.WriteCustomInfoInXMLFormat(self, outfile, indent)
+ separators = list(string.whitespace)
+ separators.sort()
+ for ws_char in string.whitespace:
+ _WriteSimpleXMLElement(outfile, 'list_separator', repr(ws_char), indent)
+
+
+def DEFINE_list(name, default, help, flag_values=FLAGS, **args):
+ """Registers a flag whose value is a comma-separated list of strings."""
+ parser = ListParser()
+ serializer = ListSerializer(',')
+ DEFINE(parser, name, default, help, flag_values, serializer, **args)
+
+
+def DEFINE_spaceseplist(name, default, help, flag_values=FLAGS, **args):
+ """Registers a flag whose value is a whitespace-separated list of strings.
+
+ Any whitespace can be used as a separator.
+ """
+ parser = WhitespaceSeparatedListParser()
+ serializer = ListSerializer(' ')
+ DEFINE(parser, name, default, help, flag_values, serializer, **args)
+
+
+#
+# MULTI FLAGS
+#
+
+
+class MultiFlag(Flag):
+ """A flag that can appear multiple time on the command-line.
+
+ The value of such a flag is a list that contains the individual values
+ from all the appearances of that flag on the command-line.
+
+ See the __doc__ for Flag for most behavior of this class. Only
+ differences in behavior are described here:
+
+ * The default value may be either a single value or a list of values.
+ A single value is interpreted as the [value] singleton list.
+
+ * The value of the flag is always a list, even if the option was
+ only supplied once, and even if the default value is a single
+ value
+ """
+
+ def __init__(self, *args, **kwargs):
+ Flag.__init__(self, *args, **kwargs)
+ self.help += ';\n repeat this option to specify a list of values'
+
+ def Parse(self, arguments):
+ """Parses one or more arguments with the installed parser.
+
+ Args:
+ arguments: a single argument or a list of arguments (typically a
+ list of default values); a single argument is converted
+ internally into a list containing one item.
+ """
+ if not isinstance(arguments, list):
+ # Default value may be a list of values. Most other arguments
+ # will not be, so convert them into a single-item list to make
+ # processing simpler below.
+ arguments = [arguments]
+
+ if self.present:
+ # keep a backup reference to list of previously supplied option values
+ values = self.value
+ else:
+ # "erase" the defaults with an empty list
+ values = []
+
+ for item in arguments:
+ # have Flag superclass parse argument, overwriting self.value reference
+ Flag.Parse(self, item) # also increments self.present
+ values.append(self.value)
+
+ # put list of option values back in the 'value' attribute
+ self.value = values
+
+ def Serialize(self):
+ if not self.serializer:
+ raise FlagsError("Serializer not present for flag %s" % self.name)
+ if self.value is None:
+ return ''
+
+ s = ''
+
+ multi_value = self.value
+
+ for self.value in multi_value:
+ if s: s += ' '
+ s += Flag.Serialize(self)
+
+ self.value = multi_value
+
+ return s
+
+ def Type(self):
+ return 'multi ' + self.parser.Type()
+
+
+def DEFINE_multi(parser, serializer, name, default, help, flag_values=FLAGS,
+ **args):
+ """Registers a generic MultiFlag that parses its args with a given parser.
+
+ Auxiliary function. Normal users should NOT use it directly.
+
+ Developers who need to create their own 'Parser' classes for options
+ which can appear multiple times can call this module function to
+ register their flags.
+ """
+ DEFINE_flag(MultiFlag(parser, serializer, name, default, help, **args),
+ flag_values)
+
+
+def DEFINE_multistring(name, default, help, flag_values=FLAGS, **args):
+ """Registers a flag whose value can be a list of any strings.
+
+ Use the flag on the command line multiple times to place multiple
+ string values into the list. The 'default' may be a single string
+ (which will be converted into a single-element list) or a list of
+ strings.
+ """
+ parser = ArgumentParser()
+ serializer = ArgumentSerializer()
+ DEFINE_multi(parser, serializer, name, default, help, flag_values, **args)
+
+
+def DEFINE_multi_int(name, default, help, lower_bound=None, upper_bound=None,
+ flag_values=FLAGS, **args):
+ """Registers a flag whose value can be a list of arbitrary integers.
+
+ Use the flag on the command line multiple times to place multiple
+ integer values into the list. The 'default' may be a single integer
+ (which will be converted into a single-element list) or a list of
+ integers.
+ """
+ parser = IntegerParser(lower_bound, upper_bound)
+ serializer = ArgumentSerializer()
+ DEFINE_multi(parser, serializer, name, default, help, flag_values, **args)
+
+
+def DEFINE_multi_float(name, default, help, lower_bound=None, upper_bound=None,
+ flag_values=FLAGS, **args):
+ """Registers a flag whose value can be a list of arbitrary floats.
+
+ Use the flag on the command line multiple times to place multiple
+ float values into the list. The 'default' may be a single float
+ (which will be converted into a single-element list) or a list of
+ floats.
+ """
+ parser = FloatParser(lower_bound, upper_bound)
+ serializer = ArgumentSerializer()
+ DEFINE_multi(parser, serializer, name, default, help, flag_values, **args)
+
+
+# Now register the flags that we want to exist in all applications.
+# These are all defined with allow_override=1, so user-apps can use
+# these flagnames for their own purposes, if they want.
+DEFINE_flag(HelpFlag())
+DEFINE_flag(HelpshortFlag())
+DEFINE_flag(HelpXMLFlag())
+
+# Define special flags here so that help may be generated for them.
+# NOTE: Please do NOT use _SPECIAL_FLAGS from outside this module.
+_SPECIAL_FLAGS = FlagValues()
+
+
+DEFINE_string(
+ 'flagfile', "",
+ "Insert flag definitions from the given file into the command line.",
+ _SPECIAL_FLAGS)
+
+DEFINE_string(
+ 'undefok', "",
+ "comma-separated list of flag names that it is okay to specify "
+ "on the command line even if the program does not define a flag "
+ "with that name. IMPORTANT: flags in this list that have "
+ "arguments MUST use the --flag=value format.", _SPECIAL_FLAGS)
diff --git a/py/minijack/gflags_validators.py b/py/minijack/gflags_validators.py
new file mode 100644
index 0000000..d83058d
--- /dev/null
+++ b/py/minijack/gflags_validators.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Module to enforce different constraints on flags.
+
+A validator represents an invariant, enforced over a one or more flags.
+See 'FLAGS VALIDATORS' in gflags.py's docstring for a usage manual.
+"""
+
+__author__ = 'olexiy@google.com (Olexiy Oryeshko)'
+
+
+class Error(Exception):
+ """Thrown If validator constraint is not satisfied."""
+
+
+class Validator(object):
+ """Base class for flags validators.
+
+ Users should NOT overload these classes, and use gflags.Register...
+ methods instead.
+ """
+
+ # Used to assign each validator an unique insertion_index
+ validators_count = 0
+
+ def __init__(self, checker, message):
+ """Constructor to create all validators.
+
+ Args:
+ checker: function to verify the constraint.
+ Input of this method varies, see SimpleValidator and
+ DictionaryValidator for a detailed description.
+ message: string, error message to be shown to the user
+ """
+ self.checker = checker
+ self.message = message
+ Validator.validators_count += 1
+ # Used to assert validators in the order they were registered (CL/18694236)
+ self.insertion_index = Validator.validators_count
+
+ def Verify(self, flag_values):
+ """Verify that constraint is satisfied.
+
+ flags library calls this method to verify Validator's constraint.
+ Args:
+ flag_values: gflags.FlagValues, containing all flags
+ Raises:
+ Error: if constraint is not satisfied.
+ """
+ param = self._GetInputToCheckerFunction(flag_values)
+ if not self.checker(param):
+ raise Error(self.message)
+
+ def GetFlagsNames(self):
+ """Return the names of the flags checked by this validator.
+
+ Returns:
+ [string], names of the flags
+ """
+ raise NotImplementedError('This method should be overloaded')
+
+ def PrintFlagsWithValues(self, flag_values):
+ raise NotImplementedError('This method should be overloaded')
+
+ def _GetInputToCheckerFunction(self, flag_values):
+ """Given flag values, construct the input to be given to checker.
+
+ Args:
+ flag_values: gflags.FlagValues, containing all flags.
+ Returns:
+ Return type depends on the specific validator.
+ """
+ raise NotImplementedError('This method should be overloaded')
+
+
+class SimpleValidator(Validator):
+ """Validator behind RegisterValidator() method.
+
+ Validates that a single flag passes its checker function. The checker function
+ takes the flag value and returns True (if value looks fine) or, if flag value
+ is not valid, either returns False or raises an Exception."""
+ def __init__(self, flag_name, checker, message):
+ """Constructor.
+
+ Args:
+ flag_name: string, name of the flag.
+ checker: function to verify the validator.
+ input - value of the corresponding flag (string, boolean, etc).
+ output - Boolean. Must return True if validator constraint is satisfied.
+ If constraint is not satisfied, it should either return False or
+ raise Error.
+ message: string, error message to be shown to the user if validator's
+ condition is not satisfied
+ """
+ super(SimpleValidator, self).__init__(checker, message)
+ self.flag_name = flag_name
+
+ def GetFlagsNames(self):
+ return [self.flag_name]
+
+ def PrintFlagsWithValues(self, flag_values):
+ return 'flag --%s=%s' % (self.flag_name, flag_values[self.flag_name].value)
+
+ def _GetInputToCheckerFunction(self, flag_values):
+ """Given flag values, construct the input to be given to checker.
+
+ Args:
+ flag_values: gflags.FlagValues
+ Returns:
+ value of the corresponding flag.
+ """
+ return flag_values[self.flag_name].value
+
+
+class DictionaryValidator(Validator):
+ """Validator behind RegisterDictionaryValidator method.
+
+ Validates that flag values pass their common checker function. The checker
+ function takes flag values and returns True (if values look fine) or,
+ if values are not valid, either returns False or raises an Exception.
+ """
+ def __init__(self, flag_names, checker, message):
+ """Constructor.
+
+ Args:
+ flag_names: [string], containing names of the flags used by checker.
+ checker: function to verify the validator.
+ input - dictionary, with keys() being flag_names, and value for each
+ key being the value of the corresponding flag (string, boolean, etc).
+ output - Boolean. Must return True if validator constraint is satisfied.
+ If constraint is not satisfied, it should either return False or
+ raise Error.
+ message: string, error message to be shown to the user if validator's
+ condition is not satisfied
+ """
+ super(DictionaryValidator, self).__init__(checker, message)
+ self.flag_names = flag_names
+
+ def _GetInputToCheckerFunction(self, flag_values):
+ """Given flag values, construct the input to be given to checker.
+
+ Args:
+ flag_values: gflags.FlagValues
+ Returns:
+ dictionary, with keys() being self.lag_names, and value for each key
+ being the value of the corresponding flag (string, boolean, etc).
+ """
+ return dict([key, flag_values[key].value] for key in self.flag_names)
+
+ def PrintFlagsWithValues(self, flag_values):
+ prefix = 'flags '
+ flags_with_values = []
+ for key in self.flag_names:
+ flags_with_values.append('%s=%s' % (key, flag_values[key].value))
+ return prefix + ', '.join(flags_with_values)
+
+ def GetFlagsNames(self):
+ return self.flag_names
diff --git a/py/minijack/httplib2/__init__.py b/py/minijack/httplib2/__init__.py
new file mode 100644
index 0000000..9780d4e
--- /dev/null
+++ b/py/minijack/httplib2/__init__.py
@@ -0,0 +1,1657 @@
+from __future__ import generators
+"""
+httplib2
+
+A caching http interface that supports ETags and gzip
+to conserve bandwidth.
+
+Requires Python 2.3 or later
+
+Changelog:
+2007-08-18, Rick: Modified so it's able to use a socks proxy if needed.
+
+"""
+
+__author__ = "Joe Gregorio (joe@bitworking.org)"
+__copyright__ = "Copyright 2006, Joe Gregorio"
+__contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)",
+ "James Antill",
+ "Xavier Verges Farrero",
+ "Jonathan Feinberg",
+ "Blair Zajac",
+ "Sam Ruby",
+ "Louis Nyffenegger"]
+__license__ = "MIT"
+__version__ = "0.8"
+
+import re
+import sys
+import email
+import email.Utils
+import email.Message
+import email.FeedParser
+import StringIO
+import gzip
+import zlib
+import httplib
+import urlparse
+import urllib
+import base64
+import os
+import copy
+import calendar
+import time
+import random
+import errno
+try:
+ from hashlib import sha1 as _sha, md5 as _md5
+except ImportError:
+ # prior to Python 2.5, these were separate modules
+ import sha
+ import md5
+ _sha = sha.new
+ _md5 = md5.new
+import hmac
+from gettext import gettext as _
+import socket
+
+try:
+ from httplib2 import socks
+except ImportError:
+ try:
+ import socks
+ except (ImportError, AttributeError):
+ socks = None
+
+# Build the appropriate socket wrapper for ssl
+try:
+ import ssl # python 2.6
+ ssl_SSLError = ssl.SSLError
+ def _ssl_wrap_socket(sock, key_file, cert_file,
+ disable_validation, ca_certs):
+ if disable_validation:
+ cert_reqs = ssl.CERT_NONE
+ else:
+ cert_reqs = ssl.CERT_REQUIRED
+ # We should be specifying SSL version 3 or TLS v1, but the ssl module
+ # doesn't expose the necessary knobs. So we need to go with the default
+ # of SSLv23.
+ return ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file,
+ cert_reqs=cert_reqs, ca_certs=ca_certs)
+except (AttributeError, ImportError):
+ ssl_SSLError = None
+ def _ssl_wrap_socket(sock, key_file, cert_file,
+ disable_validation, ca_certs):
+ if not disable_validation:
+ raise CertificateValidationUnsupported(
+ "SSL certificate validation is not supported without "
+ "the ssl module installed. To avoid this error, install "
+ "the ssl module, or explicity disable validation.")
+ ssl_sock = socket.ssl(sock, key_file, cert_file)
+ return httplib.FakeSocket(sock, ssl_sock)
+
+
+if sys.version_info >= (2,3):
+ from iri2uri import iri2uri
+else:
+ def iri2uri(uri):
+ return uri
+
+def has_timeout(timeout): # python 2.6
+ if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'):
+ return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT)
+ return (timeout is not None)
+
+__all__ = [
+ 'Http', 'Response', 'ProxyInfo', 'HttpLib2Error', 'RedirectMissingLocation',
+ 'RedirectLimit', 'FailedToDecompressContent',
+ 'UnimplementedDigestAuthOptionError',
+ 'UnimplementedHmacDigestAuthOptionError',
+ 'debuglevel', 'ProxiesUnavailableError']
+
+
+# The httplib debug level, set to a non-zero value to get debug output
+debuglevel = 0
+
+# A request will be tried 'RETRIES' times if it fails at the socket/connection level.
+RETRIES = 2
+
+# Python 2.3 support
+if sys.version_info < (2,4):
+ def sorted(seq):
+ seq.sort()
+ return seq
+
+# Python 2.3 support
+def HTTPResponse__getheaders(self):
+ """Return list of (header, value) tuples."""
+ if self.msg is None:
+ raise httplib.ResponseNotReady()
+ return self.msg.items()
+
+if not hasattr(httplib.HTTPResponse, 'getheaders'):
+ httplib.HTTPResponse.getheaders = HTTPResponse__getheaders
+
+# All exceptions raised here derive from HttpLib2Error
+class HttpLib2Error(Exception): pass
+
+# Some exceptions can be caught and optionally
+# be turned back into responses.
+class HttpLib2ErrorWithResponse(HttpLib2Error):
+ def __init__(self, desc, response, content):
+ self.response = response
+ self.content = content
+ HttpLib2Error.__init__(self, desc)
+
+class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass
+class RedirectLimit(HttpLib2ErrorWithResponse): pass
+class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass
+class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass
+class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass
+
+class MalformedHeader(HttpLib2Error): pass
+class RelativeURIError(HttpLib2Error): pass
+class ServerNotFoundError(HttpLib2Error): pass
+class ProxiesUnavailableError(HttpLib2Error): pass
+class CertificateValidationUnsupported(HttpLib2Error): pass
+class SSLHandshakeError(HttpLib2Error): pass
+class NotSupportedOnThisPlatform(HttpLib2Error): pass
+class CertificateHostnameMismatch(SSLHandshakeError):
+ def __init__(self, desc, host, cert):
+ HttpLib2Error.__init__(self, desc)
+ self.host = host
+ self.cert = cert
+
+# Open Items:
+# -----------
+# Proxy support
+
+# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?)
+
+# Pluggable cache storage (supports storing the cache in
+# flat files by default. We need a plug-in architecture
+# that can support Berkeley DB and Squid)
+
+# == Known Issues ==
+# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator.
+# Does not handle Cache-Control: max-stale
+# Does not use Age: headers when calculating cache freshness.
+
+
+# The number of redirections to follow before giving up.
+# Note that only GET redirects are automatically followed.
+# Will also honor 301 requests by saving that info and never
+# requesting that URI again.
+DEFAULT_MAX_REDIRECTS = 5
+
+try:
+ # Users can optionally provide a module that tells us where the CA_CERTS
+ # are located.
+ import ca_certs_locater
+ CA_CERTS = ca_certs_locater.get()
+except ImportError:
+ # Default CA certificates file bundled with httplib2.
+ CA_CERTS = os.path.join(
+ os.path.dirname(os.path.abspath(__file__ )), "cacerts.txt")
+
+# Which headers are hop-by-hop headers by default
+HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade']
+
+def _get_end2end_headers(response):
+ hopbyhop = list(HOP_BY_HOP)
+ hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')])
+ return [header for header in response.keys() if header not in hopbyhop]
+
+URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
+
+def parse_uri(uri):
+ """Parses a URI using the regex given in Appendix B of RFC 3986.
+
+ (scheme, authority, path, query, fragment) = parse_uri(uri)
+ """
+ groups = URI.match(uri).groups()
+ return (groups[1], groups[3], groups[4], groups[6], groups[8])
+
+def urlnorm(uri):
+ (scheme, authority, path, query, fragment) = parse_uri(uri)
+ if not scheme or not authority:
+ raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri)
+ authority = authority.lower()
+ scheme = scheme.lower()
+ if not path:
+ path = "/"
+ # Could do syntax based normalization of the URI before
+ # computing the digest. See Section 6.2.2 of Std 66.
+ request_uri = query and "?".join([path, query]) or path
+ scheme = scheme.lower()
+ defrag_uri = scheme + "://" + authority + request_uri
+ return scheme, authority, request_uri, defrag_uri
+
+
+# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/)
+re_url_scheme = re.compile(r'^\w+://')
+re_slash = re.compile(r'[?/:|]+')
+
+def safename(filename):
+ """Return a filename suitable for the cache.
+
+ Strips dangerous and common characters to create a filename we
+ can use to store the cache in.
+ """
+
+ try:
+ if re_url_scheme.match(filename):
+ if isinstance(filename,str):
+ filename = filename.decode('utf-8')
+ filename = filename.encode('idna')
+ else:
+ filename = filename.encode('idna')
+ except UnicodeError:
+ pass
+ if isinstance(filename,unicode):
+ filename=filename.encode('utf-8')
+ filemd5 = _md5(filename).hexdigest()
+ filename = re_url_scheme.sub("", filename)
+ filename = re_slash.sub(",", filename)
+
+ # limit length of filename
+ if len(filename)>200:
+ filename=filename[:200]
+ return ",".join((filename, filemd5))
+
+NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+')
+def _normalize_headers(headers):
+ return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()])
+
+def _parse_cache_control(headers):
+ retval = {}
+ if headers.has_key('cache-control'):
+ parts = headers['cache-control'].split(',')
+ parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")]
+ parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")]
+ retval = dict(parts_with_args + parts_wo_args)
+ return retval
+
+# Whether to use a strict mode to parse WWW-Authenticate headers
+# Might lead to bad results in case of ill-formed header value,
+# so disabled by default, falling back to relaxed parsing.
+# Set to true to turn on, usefull for testing servers.
+USE_WWW_AUTH_STRICT_PARSING = 0
+
+# In regex below:
+# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP
+# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space
+# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both:
+# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x08\x0A-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?
+WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$")
+WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(?<!\")[^ \t\r\n,]+(?!\"))\"?)(.*)$")
+UNQUOTE_PAIRS = re.compile(r'\\(.)')
+def _parse_www_authenticate(headers, headername='www-authenticate'):
+ """Returns a dictionary of dictionaries, one dict
+ per auth_scheme."""
+ retval = {}
+ if headers.has_key(headername):
+ try:
+
+ authenticate = headers[headername].strip()
+ www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED
+ while authenticate:
+ # Break off the scheme at the beginning of the line
+ if headername == 'authentication-info':
+ (auth_scheme, the_rest) = ('digest', authenticate)
+ else:
+ (auth_scheme, the_rest) = authenticate.split(" ", 1)
+ # Now loop over all the key value pairs that come after the scheme,
+ # being careful not to roll into the next scheme
+ match = www_auth.search(the_rest)
+ auth_params = {}
+ while match:
+ if match and len(match.groups()) == 3:
+ (key, value, the_rest) = match.groups()
+ auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')])
+ match = www_auth.search(the_rest)
+ retval[auth_scheme.lower()] = auth_params
+ authenticate = the_rest.strip()
+
+ except ValueError:
+ raise MalformedHeader("WWW-Authenticate")
+ return retval
+
+
+def _entry_disposition(response_headers, request_headers):
+ """Determine freshness from the Date, Expires and Cache-Control headers.
+
+ We don't handle the following:
+
+ 1. Cache-Control: max-stale
+ 2. Age: headers are not used in the calculations.
+
+ Not that this algorithm is simpler than you might think
+ because we are operating as a private (non-shared) cache.
+ This lets us ignore 's-maxage'. We can also ignore
+ 'proxy-invalidate' since we aren't a proxy.
+ We will never return a stale document as
+ fresh as a design decision, and thus the non-implementation
+ of 'max-stale'. This also lets us safely ignore 'must-revalidate'
+ since we operate as if every server has sent 'must-revalidate'.
+ Since we are private we get to ignore both 'public' and
+ 'private' parameters. We also ignore 'no-transform' since
+ we don't do any transformations.
+ The 'no-store' parameter is handled at a higher level.
+ So the only Cache-Control parameters we look at are:
+
+ no-cache
+ only-if-cached
+ max-age
+ min-fresh
+ """
+
+ retval = "STALE"
+ cc = _parse_cache_control(request_headers)
+ cc_response = _parse_cache_control(response_headers)
+
+ if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1:
+ retval = "TRANSPARENT"
+ if 'cache-control' not in request_headers:
+ request_headers['cache-control'] = 'no-cache'
+ elif cc.has_key('no-cache'):
+ retval = "TRANSPARENT"
+ elif cc_response.has_key('no-cache'):
+ retval = "STALE"
+ elif cc.has_key('only-if-cached'):
+ retval = "FRESH"
+ elif response_headers.has_key('date'):
+ date = calendar.timegm(email.Utils.parsedate_tz(response_headers['date']))
+ now = time.time()
+ current_age = max(0, now - date)
+ if cc_response.has_key('max-age'):
+ try:
+ freshness_lifetime = int(cc_response['max-age'])
+ except ValueError:
+ freshness_lifetime = 0
+ elif response_headers.has_key('expires'):
+ expires = email.Utils.parsedate_tz(response_headers['expires'])
+ if None == expires:
+ freshness_lifetime = 0
+ else:
+ freshness_lifetime = max(0, calendar.timegm(expires) - date)
+ else:
+ freshness_lifetime = 0
+ if cc.has_key('max-age'):
+ try:
+ freshness_lifetime = int(cc['max-age'])
+ except ValueError:
+ freshness_lifetime = 0
+ if cc.has_key('min-fresh'):
+ try:
+ min_fresh = int(cc['min-fresh'])
+ except ValueError:
+ min_fresh = 0
+ current_age += min_fresh
+ if freshness_lifetime > current_age:
+ retval = "FRESH"
+ return retval
+
+def _decompressContent(response, new_content):
+ content = new_content
+ try:
+ encoding = response.get('content-encoding', None)
+ if encoding in ['gzip', 'deflate']:
+ if encoding == 'gzip':
+ content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read()
+ if encoding == 'deflate':
+ content = zlib.decompress(content)
+ response['content-length'] = str(len(content))
+ # Record the historical presence of the encoding in a way the won't interfere.
+ response['-content-encoding'] = response['content-encoding']
+ del response['content-encoding']
+ except IOError:
+ content = ""
+ raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content)
+ return content
+
+def _updateCache(request_headers, response_headers, content, cache, cachekey):
+ if cachekey:
+ cc = _parse_cache_control(request_headers)
+ cc_response = _parse_cache_control(response_headers)
+ if cc.has_key('no-store') or cc_response.has_key('no-store'):
+ cache.delete(cachekey)
+ else:
+ info = email.Message.Message()
+ for key, value in response_headers.iteritems():
+ if key not in ['status','content-encoding','transfer-encoding']:
+ info[key] = value
+
+ # Add annotations to the cache to indicate what headers
+ # are variant for this request.
+ vary = response_headers.get('vary', None)
+ if vary:
+ vary_headers = vary.lower().replace(' ', '').split(',')
+ for header in vary_headers:
+ key = '-varied-%s' % header
+ try:
+ info[key] = request_headers[header]
+ except KeyError:
+ pass
+
+ status = response_headers.status
+ if status == 304:
+ status = 200
+
+ status_header = 'status: %d\r\n' % status
+
+ header_str = info.as_string()
+
+ header_str = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", header_str)
+ text = "".join([status_header, header_str, content])
+
+ cache.set(cachekey, text)
+
+def _cnonce():
+ dig = _md5("%s:%s" % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)])).hexdigest()
+ return dig[:16]
+
+def _wsse_username_token(cnonce, iso_now, password):
+ return base64.b64encode(_sha("%s%s%s" % (cnonce, iso_now, password)).digest()).strip()
+
+
+# For credentials we need two things, first
+# a pool of credential to try (not necesarily tied to BAsic, Digest, etc.)
+# Then we also need a list of URIs that have already demanded authentication
+# That list is tricky since sub-URIs can take the same auth, or the
+# auth scheme may change as you descend the tree.
+# So we also need each Auth instance to be able to tell us
+# how close to the 'top' it is.
+
+class Authentication(object):
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ (scheme, authority, path, query, fragment) = parse_uri(request_uri)
+ self.path = path
+ self.host = host
+ self.credentials = credentials
+ self.http = http
+
+ def depth(self, request_uri):
+ (scheme, authority, path, query, fragment) = parse_uri(request_uri)
+ return request_uri[len(self.path):].count("/")
+
+ def inscope(self, host, request_uri):
+ # XXX Should we normalize the request_uri?
+ (scheme, authority, path, query, fragment) = parse_uri(request_uri)
+ return (host == self.host) and path.startswith(self.path)
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers to add the appropriate
+ Authorization header. Over-ride this in sub-classes."""
+ pass
+
+ def response(self, response, content):
+ """Gives us a chance to update with new nonces
+ or such returned from the last authorized response.
+ Over-rise this in sub-classes if necessary.
+
+ Return TRUE is the request is to be retried, for
+ example Digest may return stale=true.
+ """
+ return False
+
+
+
+class BasicAuthentication(Authentication):
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers to add the appropriate
+ Authorization header."""
+ headers['authorization'] = 'Basic ' + base64.b64encode("%s:%s" % self.credentials).strip()
+
+
+class DigestAuthentication(Authentication):
+ """Only do qop='auth' and MD5, since that
+ is all Apache currently implements"""
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+ challenge = _parse_www_authenticate(response, 'www-authenticate')
+ self.challenge = challenge['digest']
+ qop = self.challenge.get('qop', 'auth')
+ self.challenge['qop'] = ('auth' in [x.strip() for x in qop.split()]) and 'auth' or None
+ if self.challenge['qop'] is None:
+ raise UnimplementedDigestAuthOptionError( _("Unsupported value for qop: %s." % qop))
+ self.challenge['algorithm'] = self.challenge.get('algorithm', 'MD5').upper()
+ if self.challenge['algorithm'] != 'MD5':
+ raise UnimplementedDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm']))
+ self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]])
+ self.challenge['nc'] = 1
+
+ def request(self, method, request_uri, headers, content, cnonce = None):
+ """Modify the request headers"""
+ H = lambda x: _md5(x).hexdigest()
+ KD = lambda s, d: H("%s:%s" % (s, d))
+ A2 = "".join([method, ":", request_uri])
+ self.challenge['cnonce'] = cnonce or _cnonce()
+ request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % (
+ self.challenge['nonce'],
+ '%08x' % self.challenge['nc'],
+ self.challenge['cnonce'],
+ self.challenge['qop'], H(A2)))
+ headers['authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % (
+ self.credentials[0],
+ self.challenge['realm'],
+ self.challenge['nonce'],
+ request_uri,
+ self.challenge['algorithm'],
+ request_digest,
+ self.challenge['qop'],
+ self.challenge['nc'],
+ self.challenge['cnonce'])
+ if self.challenge.get('opaque'):
+ headers['authorization'] += ', opaque="%s"' % self.challenge['opaque']
+ self.challenge['nc'] += 1
+
+ def response(self, response, content):
+ if not response.has_key('authentication-info'):
+ challenge = _parse_www_authenticate(response, 'www-authenticate').get('digest', {})
+ if 'true' == challenge.get('stale'):
+ self.challenge['nonce'] = challenge['nonce']
+ self.challenge['nc'] = 1
+ return True
+ else:
+ updated_challenge = _parse_www_authenticate(response, 'authentication-info').get('digest', {})
+
+ if updated_challenge.has_key('nextnonce'):
+ self.challenge['nonce'] = updated_challenge['nextnonce']
+ self.challenge['nc'] = 1
+ return False
+
+
+class HmacDigestAuthentication(Authentication):
+ """Adapted from Robert Sayre's code and DigestAuthentication above."""
+ __author__ = "Thomas Broyer (t.broyer@ltgt.net)"
+
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+ challenge = _parse_www_authenticate(response, 'www-authenticate')
+ self.challenge = challenge['hmacdigest']
+ # TODO: self.challenge['domain']
+ self.challenge['reason'] = self.challenge.get('reason', 'unauthorized')
+ if self.challenge['reason'] not in ['unauthorized', 'integrity']:
+ self.challenge['reason'] = 'unauthorized'
+ self.challenge['salt'] = self.challenge.get('salt', '')
+ if not self.challenge.get('snonce'):
+ raise UnimplementedHmacDigestAuthOptionError( _("The challenge doesn't contain a server nonce, or this one is empty."))
+ self.challenge['algorithm'] = self.challenge.get('algorithm', 'HMAC-SHA-1')
+ if self.challenge['algorithm'] not in ['HMAC-SHA-1', 'HMAC-MD5']:
+ raise UnimplementedHmacDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm']))
+ self.challenge['pw-algorithm'] = self.challenge.get('pw-algorithm', 'SHA-1')
+ if self.challenge['pw-algorithm'] not in ['SHA-1', 'MD5']:
+ raise UnimplementedHmacDigestAuthOptionError( _("Unsupported value for pw-algorithm: %s." % self.challenge['pw-algorithm']))
+ if self.challenge['algorithm'] == 'HMAC-MD5':
+ self.hashmod = _md5
+ else:
+ self.hashmod = _sha
+ if self.challenge['pw-algorithm'] == 'MD5':
+ self.pwhashmod = _md5
+ else:
+ self.pwhashmod = _sha
+ self.key = "".join([self.credentials[0], ":",
+ self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(),
+ ":", self.challenge['realm']])
+ self.key = self.pwhashmod.new(self.key).hexdigest().lower()
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers"""
+ keys = _get_end2end_headers(headers)
+ keylist = "".join(["%s " % k for k in keys])
+ headers_val = "".join([headers[k] for k in keys])
+ created = time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())
+ cnonce = _cnonce()
+ request_digest = "%s:%s:%s:%s:%s" % (method, request_uri, cnonce, self.challenge['snonce'], headers_val)
+ request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower()
+ headers['authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % (
+ self.credentials[0],
+ self.challenge['realm'],
+ self.challenge['snonce'],
+ cnonce,
+ request_uri,
+ created,
+ request_digest,
+ keylist)
+
+ def response(self, response, content):
+ challenge = _parse_www_authenticate(response, 'www-authenticate').get('hmacdigest', {})
+ if challenge.get('reason') in ['integrity', 'stale']:
+ return True
+ return False
+
+
+class WsseAuthentication(Authentication):
+ """This is thinly tested and should not be relied upon.
+ At this time there isn't any third party server to test against.
+ Blogger and TypePad implemented this algorithm at one point
+ but Blogger has since switched to Basic over HTTPS and
+ TypePad has implemented it wrong, by never issuing a 401
+ challenge but instead requiring your client to telepathically know that
+ their endpoint is expecting WSSE profile="UsernameToken"."""
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers to add the appropriate
+ Authorization header."""
+ headers['authorization'] = 'WSSE profile="UsernameToken"'
+ iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+ cnonce = _cnonce()
+ password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1])
+ headers['X-WSSE'] = 'UsernameToken Username="%s", PasswordDigest="%s", Nonce="%s", Created="%s"' % (
+ self.credentials[0],
+ password_digest,
+ cnonce,
+ iso_now)
+
+class GoogleLoginAuthentication(Authentication):
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ from urllib import urlencode
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+ challenge = _parse_www_authenticate(response, 'www-authenticate')
+ service = challenge['googlelogin'].get('service', 'xapi')
+ # Bloggger actually returns the service in the challenge
+ # For the rest we guess based on the URI
+ if service == 'xapi' and request_uri.find("calendar") > 0:
+ service = "cl"
+ # No point in guessing Base or Spreadsheet
+ #elif request_uri.find("spreadsheets") > 0:
+ # service = "wise"
+
+ auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent'])
+ resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'})
+ lines = content.split('\n')
+ d = dict([tuple(line.split("=", 1)) for line in lines if line])
+ if resp.status == 403:
+ self.Auth = ""
+ else:
+ self.Auth = d['Auth']
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers to add the appropriate
+ Authorization header."""
+ headers['authorization'] = 'GoogleLogin Auth=' + self.Auth
+
+
+AUTH_SCHEME_CLASSES = {
+ "basic": BasicAuthentication,
+ "wsse": WsseAuthentication,
+ "digest": DigestAuthentication,
+ "hmacdigest": HmacDigestAuthentication,
+ "googlelogin": GoogleLoginAuthentication
+}
+
+AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"]
+
+class FileCache(object):
+ """Uses a local directory as a store for cached files.
+ Not really safe to use if multiple threads or processes are going to
+ be running on the same cache.
+ """
+ def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior
+ self.cache = cache
+ self.safe = safe
+ if not os.path.exists(cache):
+ os.makedirs(self.cache)
+
+ def get(self, key):
+ retval = None
+ cacheFullPath = os.path.join(self.cache, self.safe(key))
+ try:
+ f = file(cacheFullPath, "rb")
+ retval = f.read()
+ f.close()
+ except IOError:
+ pass
+ return retval
+
+ def set(self, key, value):
+ cacheFullPath = os.path.join(self.cache, self.safe(key))
+ f = file(cacheFullPath, "wb")
+ f.write(value)
+ f.close()
+
+ def delete(self, key):
+ cacheFullPath = os.path.join(self.cache, self.safe(key))
+ if os.path.exists(cacheFullPath):
+ os.remove(cacheFullPath)
+
+class Credentials(object):
+ def __init__(self):
+ self.credentials = []
+
+ def add(self, name, password, domain=""):
+ self.credentials.append((domain.lower(), name, password))
+
+ def clear(self):
+ self.credentials = []
+
+ def iter(self, domain):
+ for (cdomain, name, password) in self.credentials:
+ if cdomain == "" or domain == cdomain:
+ yield (name, password)
+
+class KeyCerts(Credentials):
+ """Identical to Credentials except that
+ name/password are mapped to key/cert."""
+ pass
+
+class AllHosts(object):
+ pass
+
+class ProxyInfo(object):
+ """Collect information required to use a proxy."""
+ bypass_hosts = ()
+
+ def __init__(self, proxy_type, proxy_host, proxy_port,
+ proxy_rdns=None, proxy_user=None, proxy_pass=None):
+ """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX
+ constants. For example:
+
+ p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP,
+ proxy_host='localhost', proxy_port=8000)
+ """
+ self.proxy_type = proxy_type
+ self.proxy_host = proxy_host
+ self.proxy_port = proxy_port
+ self.proxy_rdns = proxy_rdns
+ self.proxy_user = proxy_user
+ self.proxy_pass = proxy_pass
+
+ def astuple(self):
+ return (self.proxy_type, self.proxy_host, self.proxy_port,
+ self.proxy_rdns, self.proxy_user, self.proxy_pass)
+
+ def isgood(self):
+ return (self.proxy_host != None) and (self.proxy_port != None)
+
+ def applies_to(self, hostname):
+ return not self.bypass_host(hostname)
+
+ def bypass_host(self, hostname):
+ """Has this host been excluded from the proxy config"""
+ if self.bypass_hosts is AllHosts:
+ return True
+
+ bypass = False
+ for domain in self.bypass_hosts:
+ if hostname.endswith(domain):
+ bypass = True
+
+ return bypass
+
+
+def proxy_info_from_environment(method='http'):
+ """
+ Read proxy info from the environment variables.
+ """
+ if method not in ['http', 'https']:
+ return
+
+ env_var = method + '_proxy'
+ url = os.environ.get(env_var, os.environ.get(env_var.upper()))
+ if not url:
+ return
+ pi = proxy_info_from_url(url, method)
+
+ no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', ''))
+ bypass_hosts = []
+ if no_proxy:
+ bypass_hosts = no_proxy.split(',')
+ # special case, no_proxy=* means all hosts bypassed
+ if no_proxy == '*':
+ bypass_hosts = AllHosts
+
+ pi.bypass_hosts = bypass_hosts
+ return pi
+
+def proxy_info_from_url(url, method='http'):
+ """
+ Construct a ProxyInfo from a URL (such as http_proxy env var)
+ """
+ url = urlparse.urlparse(url)
+ username = None
+ password = None
+ port = None
+ if '@' in url[1]:
+ ident, host_port = url[1].split('@', 1)
+ if ':' in ident:
+ username, password = ident.split(':', 1)
+ else:
+ password = ident
+ else:
+ host_port = url[1]
+ if ':' in host_port:
+ host, port = host_port.split(':', 1)
+ else:
+ host = host_port
+
+ if port:
+ port = int(port)
+ else:
+ port = dict(https=443, http=80)[method]
+
+ proxy_type = 3 # socks.PROXY_TYPE_HTTP
+ return ProxyInfo(
+ proxy_type = proxy_type,
+ proxy_host = host,
+ proxy_port = port,
+ proxy_user = username or None,
+ proxy_pass = password or None,
+ )
+
+
+class HTTPConnectionWithTimeout(httplib.HTTPConnection):
+ """
+ HTTPConnection subclass that supports timeouts
+
+ All timeouts are in seconds. If None is passed for timeout then
+ Python's default timeout for sockets will be used. See for example
+ the docs of socket.setdefaulttimeout():
+ http://docs.python.org/library/socket.html#socket.setdefaulttimeout
+ """
+
+ def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None):
+ httplib.HTTPConnection.__init__(self, host, port, strict)
+ self.timeout = timeout
+ self.proxy_info = proxy_info
+
+ def connect(self):
+ """Connect to the host and port specified in __init__."""
+ # Mostly verbatim from httplib.py.
+ if self.proxy_info and socks is None:
+ raise ProxiesUnavailableError(
+ 'Proxy support missing but proxy use was requested!')
+ msg = "getaddrinfo returns an empty list"
+ if self.proxy_info and self.proxy_info.isgood():
+ use_proxy = True
+ proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple()
+ else:
+ use_proxy = False
+ if use_proxy and proxy_rdns:
+ host = proxy_host
+ port = proxy_port
+ else:
+ host = self.host
+ port = self.port
+
+ for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
+ af, socktype, proto, canonname, sa = res
+ try:
+ if use_proxy:
+ self.sock = socks.socksocket(af, socktype, proto)
+ self.sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)
+ else:
+ self.sock = socket.socket(af, socktype, proto)
+ self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ # Different from httplib: support timeouts.
+ if has_timeout(self.timeout):
+ self.sock.settimeout(self.timeout)
+ # End of difference from httplib.
+ if self.debuglevel > 0:
+ print "connect: (%s, %s) ************" % (self.host, self.port)
+ if use_proxy:
+ print "proxy: %s ************" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass))
+
+ self.sock.connect((self.host, self.port) + sa[2:])
+ except socket.error, msg:
+ if self.debuglevel > 0:
+ print "connect fail: (%s, %s)" % (self.host, self.port)
+ if use_proxy:
+ print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass))
+ if self.sock:
+ self.sock.close()
+ self.sock = None
+ continue
+ break
+ if not self.sock:
+ raise socket.error, msg
+
+class HTTPSConnectionWithTimeout(httplib.HTTPSConnection):
+ """
+ This class allows communication via SSL.
+
+ All timeouts are in seconds. If None is passed for timeout then
+ Python's default timeout for sockets will be used. See for example
+ the docs of socket.setdefaulttimeout():
+ http://docs.python.org/library/socket.html#socket.setdefaulttimeout
+ """
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=None, proxy_info=None,
+ ca_certs=None, disable_ssl_certificate_validation=False):
+ httplib.HTTPSConnection.__init__(self, host, port=port,
+ key_file=key_file,
+ cert_file=cert_file, strict=strict)
+ self.timeout = timeout
+ self.proxy_info = proxy_info
+ if ca_certs is None:
+ ca_certs = CA_CERTS
+ self.ca_certs = ca_certs
+ self.disable_ssl_certificate_validation = \
+ disable_ssl_certificate_validation
+
+ # The following two methods were adapted from https_wrapper.py, released
+ # with the Google Appengine SDK at
+ # http://googleappengine.googlecode.com/svn-history/r136/trunk/python/google/appengine/tools/https_wrapper.py
+ # under the following license:
+ #
+ # Copyright 2007 Google Inc.
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #
+
+ def _GetValidHostsForCert(self, cert):
+ """Returns a list of valid host globs for an SSL certificate.
+
+ Args:
+ cert: A dictionary representing an SSL certificate.
+ Returns:
+ list: A list of valid host globs.
+ """
+ if 'subjectAltName' in cert:
+ return [x[1] for x in cert['subjectAltName']
+ if x[0].lower() == 'dns']
+ else:
+ return [x[0][1] for x in cert['subject']
+ if x[0][0].lower() == 'commonname']
+
+ def _ValidateCertificateHostname(self, cert, hostname):
+ """Validates that a given hostname is valid for an SSL certificate.
+
+ Args:
+ cert: A dictionary representing an SSL certificate.
+ hostname: The hostname to test.
+ Returns:
+ bool: Whether or not the hostname is valid for this certificate.
+ """
+ hosts = self._GetValidHostsForCert(cert)
+ for host in hosts:
+ host_re = host.replace('.', '\.').replace('*', '[^.]*')
+ if re.search('^%s$' % (host_re,), hostname, re.I):
+ return True
+ return False
+
+ def connect(self):
+ "Connect to a host on a given (SSL) port."
+
+ msg = "getaddrinfo returns an empty list"
+ if self.proxy_info and self.proxy_info.isgood():
+ use_proxy = True
+ proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple()
+ else:
+ use_proxy = False
+ if use_proxy and proxy_rdns:
+ host = proxy_host
+ port = proxy_port
+ else:
+ host = self.host
+ port = self.port
+
+ address_info = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
+ for family, socktype, proto, canonname, sockaddr in address_info:
+ try:
+ if use_proxy:
+ sock = socks.socksocket(family, socktype, proto)
+
+ sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)
+ else:
+ sock = socket.socket(family, socktype, proto)
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+ if has_timeout(self.timeout):
+ sock.settimeout(self.timeout)
+ sock.connect((self.host, self.port))
+ self.sock =_ssl_wrap_socket(
+ sock, self.key_file, self.cert_file,
+ self.disable_ssl_certificate_validation, self.ca_certs)
+ if self.debuglevel > 0:
+ print "connect: (%s, %s)" % (self.host, self.port)
+ if use_proxy:
+ print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass))
+ if not self.disable_ssl_certificate_validation:
+ cert = self.sock.getpeercert()
+ hostname = self.host.split(':', 0)[0]
+ if not self._ValidateCertificateHostname(cert, hostname):
+ raise CertificateHostnameMismatch(
+ 'Server presented certificate that does not match '
+ 'host %s: %s' % (hostname, cert), hostname, cert)
+ except ssl_SSLError, e:
+ if sock:
+ sock.close()
+ if self.sock:
+ self.sock.close()
+ self.sock = None
+ # Unfortunately the ssl module doesn't seem to provide any way
+ # to get at more detailed error information, in particular
+ # whether the error is due to certificate validation or
+ # something else (such as SSL protocol mismatch).
+ if e.errno == ssl.SSL_ERROR_SSL:
+ raise SSLHandshakeError(e)
+ else:
+ raise
+ except (socket.timeout, socket.gaierror):
+ raise
+ except socket.error, msg:
+ if self.debuglevel > 0:
+ print "connect fail: (%s, %s)" % (self.host, self.port)
+ if use_proxy:
+ print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass))
+ if self.sock:
+ self.sock.close()
+ self.sock = None
+ continue
+ break
+ if not self.sock:
+ raise socket.error, msg
+
+SCHEME_TO_CONNECTION = {
+ 'http': HTTPConnectionWithTimeout,
+ 'https': HTTPSConnectionWithTimeout
+}
+
+# Use a different connection object for Google App Engine
+try:
+ try:
+ from google.appengine.api import apiproxy_stub_map
+ if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None:
+ raise ImportError # Bail out; we're not actually running on App Engine.
+ from google.appengine.api.urlfetch import fetch
+ from google.appengine.api.urlfetch import InvalidURLError
+ except (ImportError, AttributeError):
+ from google3.apphosting.api import apiproxy_stub_map
+ if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None:
+ raise ImportError # Bail out; we're not actually running on App Engine.
+ from google3.apphosting.api.urlfetch import fetch
+ from google3.apphosting.api.urlfetch import InvalidURLError
+
+ def _new_fixed_fetch(validate_certificate):
+ def fixed_fetch(url, payload=None, method="GET", headers={},
+ allow_truncated=False, follow_redirects=True,
+ deadline=5):
+ return fetch(url, payload=payload, method=method, headers=headers,
+ allow_truncated=allow_truncated,
+ follow_redirects=follow_redirects, deadline=deadline,
+ validate_certificate=validate_certificate)
+ return fixed_fetch
+
+ class AppEngineHttpConnection(httplib.HTTPConnection):
+ """Use httplib on App Engine, but compensate for its weirdness.
+
+ The parameters key_file, cert_file, proxy_info, ca_certs, and
+ disable_ssl_certificate_validation are all dropped on the ground.
+ """
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=None, proxy_info=None, ca_certs=None,
+ disable_ssl_certificate_validation=False):
+ httplib.HTTPConnection.__init__(self, host, port=port,
+ strict=strict, timeout=timeout)
+
+ class AppEngineHttpsConnection(httplib.HTTPSConnection):
+ """Same as AppEngineHttpConnection, but for HTTPS URIs."""
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=None, proxy_info=None, ca_certs=None,
+ disable_ssl_certificate_validation=False):
+ httplib.HTTPSConnection.__init__(self, host, port=port,
+ key_file=key_file,
+ cert_file=cert_file, strict=strict,
+ timeout=timeout)
+ self._fetch = _new_fixed_fetch(
+ not disable_ssl_certificate_validation)
+
+ # Update the connection classes to use the Googel App Engine specific ones.
+ SCHEME_TO_CONNECTION = {
+ 'http': AppEngineHttpConnection,
+ 'https': AppEngineHttpsConnection
+ }
+except (ImportError, AttributeError):
+ pass
+
+
+class Http(object):
+ """An HTTP client that handles:
+
+ - all methods
+ - caching
+ - ETags
+ - compression,
+ - HTTPS
+ - Basic
+ - Digest
+ - WSSE
+
+ and more.
+ """
+ def __init__(self, cache=None, timeout=None,
+ proxy_info=proxy_info_from_environment,
+ ca_certs=None, disable_ssl_certificate_validation=False):
+ """If 'cache' is a string then it is used as a directory name for
+ a disk cache. Otherwise it must be an object that supports the
+ same interface as FileCache.
+
+ All timeouts are in seconds. If None is passed for timeout
+ then Python's default timeout for sockets will be used. See
+ for example the docs of socket.setdefaulttimeout():
+ http://docs.python.org/library/socket.html#socket.setdefaulttimeout
+
+ `proxy_info` may be:
+ - a callable that takes the http scheme ('http' or 'https') and
+ returns a ProxyInfo instance per request. By default, uses
+ proxy_nfo_from_environment.
+ - a ProxyInfo instance (static proxy config).
+ - None (proxy disabled).
+
+ ca_certs is the path of a file containing root CA certificates for SSL
+ server certificate validation. By default, a CA cert file bundled with
+ httplib2 is used.
+
+ If disable_ssl_certificate_validation is true, SSL cert validation will
+ not be performed.
+ """
+ self.proxy_info = proxy_info
+ self.ca_certs = ca_certs
+ self.disable_ssl_certificate_validation = \
+ disable_ssl_certificate_validation
+
+ # Map domain name to an httplib connection
+ self.connections = {}
+ # The location of the cache, for now a directory
+ # where cached responses are held.
+ if cache and isinstance(cache, basestring):
+ self.cache = FileCache(cache)
+ else:
+ self.cache = cache
+
+ # Name/password
+ self.credentials = Credentials()
+
+ # Key/cert
+ self.certificates = KeyCerts()
+
+ # authorization objects
+ self.authorizations = []
+
+ # If set to False then no redirects are followed, even safe ones.
+ self.follow_redirects = True
+
+ # Which HTTP methods do we apply optimistic concurrency to, i.e.
+ # which methods get an "if-match:" etag header added to them.
+ self.optimistic_concurrency_methods = ["PUT", "PATCH"]
+
+ # If 'follow_redirects' is True, and this is set to True then
+ # all redirecs are followed, including unsafe ones.
+ self.follow_all_redirects = False
+
+ self.ignore_etag = False
+
+ self.force_exception_to_status_code = False
+
+ self.timeout = timeout
+
+ # Keep Authorization: headers on a redirect.
+ self.forward_authorization_headers = False
+
+ def __getstate__(self):
+ state_dict = copy.copy(self.__dict__)
+ # In case request is augmented by some foreign object such as
+ # credentials which handle auth
+ if 'request' in state_dict:
+ del state_dict['request']
+ if 'connections' in state_dict:
+ del state_dict['connections']
+ return state_dict
+
+ def __setstate__(self, state):
+ self.__dict__.update(state)
+ self.connections = {}
+
+ def _auth_from_challenge(self, host, request_uri, headers, response, content):
+ """A generator that creates Authorization objects
+ that can be applied to requests.
+ """
+ challenges = _parse_www_authenticate(response, 'www-authenticate')
+ for cred in self.credentials.iter(host):
+ for scheme in AUTH_SCHEME_ORDER:
+ if challenges.has_key(scheme):
+ yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self)
+
+ def add_credentials(self, name, password, domain=""):
+ """Add a name and password that will be used
+ any time a request requires authentication."""
+ self.credentials.add(name, password, domain)
+
+ def add_certificate(self, key, cert, domain):
+ """Add a key and cert that will be used
+ any time a request requires authentication."""
+ self.certificates.add(key, cert, domain)
+
+ def clear_credentials(self):
+ """Remove all the names and passwords
+ that are used for authentication"""
+ self.credentials.clear()
+ self.authorizations = []
+
+ def _conn_request(self, conn, request_uri, method, body, headers):
+ for i in range(RETRIES):
+ try:
+ if hasattr(conn, 'sock') and conn.sock is None:
+ conn.connect()
+ conn.request(method, request_uri, body, headers)
+ except socket.timeout:
+ raise
+ except socket.gaierror:
+ conn.close()
+ raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
+ except ssl_SSLError:
+ conn.close()
+ raise
+ except socket.error, e:
+ err = 0
+ if hasattr(e, 'args'):
+ err = getattr(e, 'args')[0]
+ else:
+ err = e.errno
+ if err == errno.ECONNREFUSED: # Connection refused
+ raise
+ except httplib.HTTPException:
+ # Just because the server closed the connection doesn't apparently mean
+ # that the server didn't send a response.
+ if hasattr(conn, 'sock') and conn.sock is None:
+ if i < RETRIES-1:
+ conn.close()
+ conn.connect()
+ continue
+ else:
+ conn.close()
+ raise
+ if i < RETRIES-1:
+ conn.close()
+ conn.connect()
+ continue
+ try:
+ response = conn.getresponse()
+ except (socket.error, httplib.HTTPException):
+ if i < RETRIES-1:
+ conn.close()
+ conn.connect()
+ continue
+ else:
+ conn.close()
+ raise
+ else:
+ content = ""
+ if method == "HEAD":
+ conn.close()
+ else:
+ content = response.read()
+ response = Response(response)
+ if method != "HEAD":
+ content = _decompressContent(response, content)
+ break
+ return (response, content)
+
+
+ def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey):
+ """Do the actual request using the connection object
+ and also follow one level of redirects if necessary"""
+
+ auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)]
+ auth = auths and sorted(auths)[0][1] or None
+ if auth:
+ auth.request(method, request_uri, headers, body)
+
+ (response, content) = self._conn_request(conn, request_uri, method, body, headers)
+
+ if auth:
+ if auth.response(response, body):
+ auth.request(method, request_uri, headers, body)
+ (response, content) = self._conn_request(conn, request_uri, method, body, headers )
+ response._stale_digest = 1
+
+ if response.status == 401:
+ for authorization in self._auth_from_challenge(host, request_uri, headers, response, content):
+ authorization.request(method, request_uri, headers, body)
+ (response, content) = self._conn_request(conn, request_uri, method, body, headers, )
+ if response.status != 401:
+ self.authorizations.append(authorization)
+ authorization.response(response, body)
+ break
+
+ if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303):
+ if self.follow_redirects and response.status in [300, 301, 302, 303, 307]:
+ # Pick out the location header and basically start from the beginning
+ # remembering first to strip the ETag header and decrement our 'depth'
+ if redirections:
+ if not response.has_key('location') and response.status != 300:
+ raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content)
+ # Fix-up relative redirects (which violate an RFC 2616 MUST)
+ if response.has_key('location'):
+ location = response['location']
+ (scheme, authority, path, query, fragment) = parse_uri(location)
+ if authority == None:
+ response['location'] = urlparse.urljoin(absolute_uri, location)
+ if response.status == 301 and method in ["GET", "HEAD"]:
+ response['-x-permanent-redirect-url'] = response['location']
+ if not response.has_key('content-location'):
+ response['content-location'] = absolute_uri
+ _updateCache(headers, response, content, self.cache, cachekey)
+ if headers.has_key('if-none-match'):
+ del headers['if-none-match']
+ if headers.has_key('if-modified-since'):
+ del headers['if-modified-since']
+ if 'authorization' in headers and not self.forward_authorization_headers:
+ del headers['authorization']
+ if response.has_key('location'):
+ location = response['location']
+ old_response = copy.deepcopy(response)
+ if not old_response.has_key('content-location'):
+ old_response['content-location'] = absolute_uri
+ redirect_method = method
+ if response.status in [302, 303]:
+ redirect_method = "GET"
+ body = None
+ (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1)
+ response.previous = old_response
+ else:
+ raise RedirectLimit("Redirected more times than rediection_limit allows.", response, content)
+ elif response.status in [200, 203] and method in ["GET", "HEAD"]:
+ # Don't cache 206's since we aren't going to handle byte range requests
+ if not response.has_key('content-location'):
+ response['content-location'] = absolute_uri
+ _updateCache(headers, response, content, self.cache, cachekey)
+
+ return (response, content)
+
+ def _normalize_headers(self, headers):
+ return _normalize_headers(headers)
+
+# Need to catch and rebrand some exceptions
+# Then need to optionally turn all exceptions into status codes
+# including all socket.* and httplib.* exceptions.
+
+
+ def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None):
+ """ Performs a single HTTP request.
+
+ The 'uri' is the URI of the HTTP resource and can begin with either
+ 'http' or 'https'. The value of 'uri' must be an absolute URI.
+
+ The 'method' is the HTTP method to perform, such as GET, POST, DELETE,
+ etc. There is no restriction on the methods allowed.
+
+ The 'body' is the entity body to be sent with the request. It is a
+ string object.
+
+ Any extra headers that are to be sent with the request should be
+ provided in the 'headers' dictionary.
+
+ The maximum number of redirect to follow before raising an
+ exception is 'redirections. The default is 5.
+
+ The return value is a tuple of (response, content), the first
+ being and instance of the 'Response' class, the second being
+ a string that contains the response entity body.
+ """
+ try:
+ if headers is None:
+ headers = {}
+ else:
+ headers = self._normalize_headers(headers)
+
+ if not headers.has_key('user-agent'):
+ headers['user-agent'] = "Python-httplib2/%s (gzip)" % __version__
+
+ uri = iri2uri(uri)
+
+ (scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
+ domain_port = authority.split(":")[0:2]
+ if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http':
+ scheme = 'https'
+ authority = domain_port[0]
+
+ proxy_info = self._get_proxy_info(scheme, authority)
+
+ conn_key = scheme+":"+authority
+ if conn_key in self.connections:
+ conn = self.connections[conn_key]
+ else:
+ if not connection_type:
+ connection_type = SCHEME_TO_CONNECTION[scheme]
+ certs = list(self.certificates.iter(authority))
+ if scheme == 'https':
+ if certs:
+ conn = self.connections[conn_key] = connection_type(
+ authority, key_file=certs[0][0],
+ cert_file=certs[0][1], timeout=self.timeout,
+ proxy_info=proxy_info,
+ ca_certs=self.ca_certs,
+ disable_ssl_certificate_validation=
+ self.disable_ssl_certificate_validation)
+ else:
+ conn = self.connections[conn_key] = connection_type(
+ authority, timeout=self.timeout,
+ proxy_info=proxy_info,
+ ca_certs=self.ca_certs,
+ disable_ssl_certificate_validation=
+ self.disable_ssl_certificate_validation)
+ else:
+ conn = self.connections[conn_key] = connection_type(
+ authority, timeout=self.timeout,
+ proxy_info=proxy_info)
+ conn.set_debuglevel(debuglevel)
+
+ if 'range' not in headers and 'accept-encoding' not in headers:
+ headers['accept-encoding'] = 'gzip, deflate'
+
+ info = email.Message.Message()
+ cached_value = None
+ if self.cache:
+ cachekey = defrag_uri
+ cached_value = self.cache.get(cachekey)
+ if cached_value:
+ # info = email.message_from_string(cached_value)
+ #
+ # Need to replace the line above with the kludge below
+ # to fix the non-existent bug not fixed in this
+ # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html
+ try:
+ info, content = cached_value.split('\r\n\r\n', 1)
+ feedparser = email.FeedParser.FeedParser()
+ feedparser.feed(info)
+ info = feedparser.close()
+ feedparser._parse = None
+ except (IndexError, ValueError):
+ self.cache.delete(cachekey)
+ cachekey = None
+ cached_value = None
+ else:
+ cachekey = None
+
+ if method in self.optimistic_concurrency_methods and self.cache and info.has_key('etag') and not self.ignore_etag and 'if-match' not in headers:
+ # http://www.w3.org/1999/04/Editing/
+ headers['if-match'] = info['etag']
+
+ if method not in ["GET", "HEAD"] and self.cache and cachekey:
+ # RFC 2616 Section 13.10
+ self.cache.delete(cachekey)
+
+ # Check the vary header in the cache to see if this request
+ # matches what varies in the cache.
+ if method in ['GET', 'HEAD'] and 'vary' in info:
+ vary = info['vary']
+ vary_headers = vary.lower().replace(' ', '').split(',')
+ for header in vary_headers:
+ key = '-varied-%s' % header
+ value = info[key]
+ if headers.get(header, None) != value:
+ cached_value = None
+ break
+
+ if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers:
+ if info.has_key('-x-permanent-redirect-url'):
+ # Should cached permanent redirects be counted in our redirection count? For now, yes.
+ if redirections <= 0:
+ raise RedirectLimit("Redirected more times than rediection_limit allows.", {}, "")
+ (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1)
+ response.previous = Response(info)
+ response.previous.fromcache = True
+ else:
+ # Determine our course of action:
+ # Is the cached entry fresh or stale?
+ # Has the client requested a non-cached response?
+ #
+ # There seems to be three possible answers:
+ # 1. [FRESH] Return the cache entry w/o doing a GET
+ # 2. [STALE] Do the GET (but add in cache validators if available)
+ # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request
+ entry_disposition = _entry_disposition(info, headers)
+
+ if entry_disposition == "FRESH":
+ if not cached_value:
+ info['status'] = '504'
+ content = ""
+ response = Response(info)
+ if cached_value:
+ response.fromcache = True
+ return (response, content)
+
+ if entry_disposition == "STALE":
+ if info.has_key('etag') and not self.ignore_etag and not 'if-none-match' in headers:
+ headers['if-none-match'] = info['etag']
+ if info.has_key('last-modified') and not 'last-modified' in headers:
+ headers['if-modified-since'] = info['last-modified']
+ elif entry_disposition == "TRANSPARENT":
+ pass
+
+ (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
+
+ if response.status == 304 and method == "GET":
+ # Rewrite the cache entry with the new end-to-end headers
+ # Take all headers that are in response
+ # and overwrite their values in info.
+ # unless they are hop-by-hop, or are listed in the connection header.
+
+ for key in _get_end2end_headers(response):
+ info[key] = response[key]
+ merged_response = Response(info)
+ if hasattr(response, "_stale_digest"):
+ merged_response._stale_digest = response._stale_digest
+ _updateCache(headers, merged_response, content, self.cache, cachekey)
+ response = merged_response
+ response.status = 200
+ response.fromcache = True
+
+ elif response.status == 200:
+ content = new_content
+ else:
+ self.cache.delete(cachekey)
+ content = new_content
+ else:
+ cc = _parse_cache_control(headers)
+ if cc.has_key('only-if-cached'):
+ info['status'] = '504'
+ response = Response(info)
+ content = ""
+ else:
+ (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
+ except Exception, e:
+ if self.force_exception_to_status_code:
+ if isinstance(e, HttpLib2ErrorWithResponse):
+ response = e.response
+ content = e.content
+ response.status = 500
+ response.reason = str(e)
+ elif isinstance(e, socket.timeout):
+ content = "Request Timeout"
+ response = Response({
+ "content-type": "text/plain",
+ "status": "408",
+ "content-length": len(content)
+ })
+ response.reason = "Request Timeout"
+ else:
+ content = str(e)
+ response = Response({
+ "content-type": "text/plain",
+ "status": "400",
+ "content-length": len(content)
+ })
+ response.reason = "Bad Request"
+ else:
+ raise
+
+
+ return (response, content)
+
+ def _get_proxy_info(self, scheme, authority):
+ """Return a ProxyInfo instance (or None) based on the scheme
+ and authority.
+ """
+ hostname, port = urllib.splitport(authority)
+ proxy_info = self.proxy_info
+ if callable(proxy_info):
+ proxy_info = proxy_info(scheme)
+
+ if (hasattr(proxy_info, 'applies_to')
+ and not proxy_info.applies_to(hostname)):
+ proxy_info = None
+ return proxy_info
+
+
+class Response(dict):
+ """An object more like email.Message than httplib.HTTPResponse."""
+
+ """Is this response from our local cache"""
+ fromcache = False
+
+ """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """
+ version = 11
+
+ "Status code returned by server. "
+ status = 200
+
+ """Reason phrase returned by server."""
+ reason = "Ok"
+
+ previous = None
+
+ def __init__(self, info):
+ # info is either an email.Message or
+ # an httplib.HTTPResponse object.
+ if isinstance(info, httplib.HTTPResponse):
+ for key, value in info.getheaders():
+ self[key.lower()] = value
+ self.status = info.status
+ self['status'] = str(self.status)
+ self.reason = info.reason
+ self.version = info.version
+ elif isinstance(info, email.Message.Message):
+ for key, value in info.items():
+ self[key.lower()] = value
+ self.status = int(self['status'])
+ else:
+ for key, value in info.iteritems():
+ self[key.lower()] = value
+ self.status = int(self.get('status', self.status))
+ self.reason = self.get('reason', self.reason)
+
+
+ def __getattr__(self, name):
+ if name == 'dict':
+ return self
+ else:
+ raise AttributeError, name
diff --git a/py/minijack/httplib2/cacerts.txt b/py/minijack/httplib2/cacerts.txt
new file mode 100644
index 0000000..d8a0027
--- /dev/null
+++ b/py/minijack/httplib2/cacerts.txt
@@ -0,0 +1,739 @@
+# Certifcate Authority certificates for validating SSL connections.
+#
+# This file contains PEM format certificates generated from
+# http://mxr.mozilla.org/seamonkey/source/security/nss/lib/ckfw/builtins/certdata.txt
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is the Netscape security libraries.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1994-2000
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+Verisign/RSA Secure Server CA
+=============================
+
+-----BEGIN CERTIFICATE-----
+MIICNDCCAaECEAKtZn5ORf5eV288mBle3cAwDQYJKoZIhvcNAQECBQAwXzELMAkG
+A1UEBhMCVVMxIDAeBgNVBAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYD
+VQQLEyVTZWN1cmUgU2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk0
+MTEwOTAwMDAwMFoXDTEwMDEwNzIzNTk1OVowXzELMAkGA1UEBhMCVVMxIDAeBgNV
+BAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYDVQQLEyVTZWN1cmUgU2Vy
+dmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGbMA0GCSqGSIb3DQEBAQUAA4GJ
+ADCBhQJ+AJLOesGugz5aqomDV6wlAXYMra6OLDfO6zV4ZFQD5YRAUcm/jwjiioII
+0haGN1XpsSECrXZogZoFokvJSyVmIlZsiAeP94FZbYQHZXATcXY+m3dM41CJVphI
+uR2nKRoTLkoRWZweFdVJVCxzOmmCsZc5nG1wZ0jl3S3WyB57AgMBAAEwDQYJKoZI
+hvcNAQECBQADfgBl3X7hsuyw4jrg7HFGmhkRuNPHoLQDQCYCPgmc4RKz0Vr2N6W3
+YQO2WxZpO8ZECAyIUwxrl0nHPjXcbLm7qt9cuzovk2C2qUtN8iD3zV9/ZHuO3ABc
+1/p3yjkWWW8O6tO1g39NTUJWdrTJXwT4OPjr0l91X817/OWOgHz8UA==
+-----END CERTIFICATE-----
+
+Thawte Personal Basic CA
+========================
+
+-----BEGIN CERTIFICATE-----
+MIIDITCCAoqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD
+VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT
+ZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFBlcnNvbmFsIEJhc2lj
+IENBMSgwJgYJKoZIhvcNAQkBFhlwZXJzb25hbC1iYXNpY0B0aGF3dGUuY29tMB4X
+DTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgcsxCzAJBgNVBAYTAlpBMRUw
+EwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEaMBgGA1UE
+ChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRpb24gU2Vy
+dmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQZXJzb25hbCBCYXNpYyBD
+QTEoMCYGCSqGSIb3DQEJARYZcGVyc29uYWwtYmFzaWNAdGhhd3RlLmNvbTCBnzAN
+BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvLyTU23AUE+CFeZIlDWmWr5vQvoPR+53
+dXLdjUmbllegeNTKP1GzaQuRdhciB5dqxFGTS+CN7zeVoQxN2jSQHReJl+A1OFdK
+wPQIcOk8RHtQfmGakOMj04gRRif1CwcOu93RfyAKiLlWCy4cgNrx454p7xS9CkT7
+G1sY0b8jkyECAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQQF
+AAOBgQAt4plrsD16iddZopQBHyvdEktTwq1/qqcAXJFAVyVKOKqEcLnZgA+le1z7
+c8a914phXAPjLSeoF+CEhULcXpvGt7Jtu3Sv5D/Lp7ew4F2+eIMllNLbgQ95B21P
+9DkVWlIBe94y1k049hJcBlDfBVu9FEuh3ym6O0GN92NWod8isQ==
+-----END CERTIFICATE-----
+
+Thawte Personal Premium CA
+==========================
+
+-----BEGIN CERTIFICATE-----
+MIIDKTCCApKgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBzzELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD
+VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT
+ZXJ2aWNlcyBEaXZpc2lvbjEjMCEGA1UEAxMaVGhhd3RlIFBlcnNvbmFsIFByZW1p
+dW0gQ0ExKjAoBgkqhkiG9w0BCQEWG3BlcnNvbmFsLXByZW1pdW1AdGhhd3RlLmNv
+bTAeFw05NjAxMDEwMDAwMDBaFw0yMDEyMzEyMzU5NTlaMIHPMQswCQYDVQQGEwJa
+QTEVMBMGA1UECBMMV2VzdGVybiBDYXBlMRIwEAYDVQQHEwlDYXBlIFRvd24xGjAY
+BgNVBAoTEVRoYXd0ZSBDb25zdWx0aW5nMSgwJgYDVQQLEx9DZXJ0aWZpY2F0aW9u
+IFNlcnZpY2VzIERpdmlzaW9uMSMwIQYDVQQDExpUaGF3dGUgUGVyc29uYWwgUHJl
+bWl1bSBDQTEqMCgGCSqGSIb3DQEJARYbcGVyc29uYWwtcHJlbWl1bUB0aGF3dGUu
+Y29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDJZtn4B0TPuYwu8KHvE0Vs
+Bd/eJxZRNkERbGw77f4QfRKe5ZtCmv5gMcNmt3M6SK5O0DI3lIi1DbbZ8/JE2dWI
+Et12TfIa/G8jHnrx2JhFTgcQ7xZC0EN1bUre4qrJMf8fAHB8Zs8QJQi6+u4A6UYD
+ZicRFTuqW/KY3TZCstqIdQIDAQABoxMwETAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
+SIb3DQEBBAUAA4GBAGk2ifc0KjNyL2071CKyuG+axTZmDhs8obF1Wub9NdP4qPIH
+b4Vnjt4rueIXsDqg8A6iAJrf8xQVbrvIhVqYgPn/vnQdPfP+MCXRNzRn+qVxeTBh
+KXLA4CxM+1bkOqhv5TJZUtt1KFBZDPgLGeSs2a+WjS9Q2wfD6h+rM+D1KzGJ
+-----END CERTIFICATE-----
+
+Thawte Personal Freemail CA
+===========================
+
+-----BEGIN CERTIFICATE-----
+MIIDLTCCApagAwIBAgIBADANBgkqhkiG9w0BAQQFADCB0TELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD
+VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT
+ZXJ2aWNlcyBEaXZpc2lvbjEkMCIGA1UEAxMbVGhhd3RlIFBlcnNvbmFsIEZyZWVt
+YWlsIENBMSswKQYJKoZIhvcNAQkBFhxwZXJzb25hbC1mcmVlbWFpbEB0aGF3dGUu
+Y29tMB4XDTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgdExCzAJBgNVBAYT
+AlpBMRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEa
+MBgGA1UEChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRp
+b24gU2VydmljZXMgRGl2aXNpb24xJDAiBgNVBAMTG1RoYXd0ZSBQZXJzb25hbCBG
+cmVlbWFpbCBDQTErMCkGCSqGSIb3DQEJARYccGVyc29uYWwtZnJlZW1haWxAdGhh
+d3RlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA1GnX1LCUZFtx6UfY
+DFG26nKRsIRefS0Nj3sS34UldSh0OkIsYyeflXtL734Zhx2G6qPduc6WZBrCFG5E
+rHzmj+hND3EfQDimAKOHePb5lIZererAXnbr2RSjXW56fAylS1V/Bhkpf56aJtVq
+uzgkCGqYx7Hao5iR/Xnb5VrEHLkCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zAN
+BgkqhkiG9w0BAQQFAAOBgQDH7JJ+Tvj1lqVnYiqk8E0RYNBvjWBYYawmu1I1XAjP
+MPuoSpaKH2JCI4wXD/S6ZJwXrEcp352YXtJsYHFcoqzceePnbgBHH7UNKOgCneSa
+/RP0ptl8sfjcXyMmCZGAc9AUG95DqYMl8uacLxXK/qarigd1iwzdUYRr5PjRznei
+gQ==
+-----END CERTIFICATE-----
+
+Thawte Server CA
+================
+
+-----BEGIN CERTIFICATE-----
+MIIDEzCCAnygAwIBAgIBATANBgkqhkiG9w0BAQQFADCBxDELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD
+VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv
+biBTZXJ2aWNlcyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEm
+MCQGCSqGSIb3DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wHhcNOTYwODAx
+MDAwMDAwWhcNMjAxMjMxMjM1OTU5WjCBxDELMAkGA1UEBhMCWkExFTATBgNVBAgT
+DFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYDVQQKExRUaGF3
+dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNl
+cyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEmMCQGCSqGSIb3
+DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQAD
+gY0AMIGJAoGBANOkUG7I/1Zr5s9dtuoMaHVHoqrC2oQl/Kj0R1HahbUgdJSGHg91
+yekIYfUGbTBuFRkC6VLAYttNmZ7iagxEOM3+vuNkCXDF/rFrKbYvScg71CcEJRCX
+L+eQbcAoQpnXTEPew/UhbVSfXcNY4cDk2VuwuNy0e982OsK1ZiIS1ocNAgMBAAGj
+EzARMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEEBQADgYEAB/pMaVz7lcxG
+7oWDTSEwjsrZqG9JGubaUeNgcGyEYRGhGshIPllDfU+VPaGLtwtimHp1it2ITk6e
+QNuozDJ0uW8NxuOzRAvZim+aKZuZGCg70eNAKJpaPNW15yAbi8qkq43pUdniTCxZ
+qdq5snUb9kLy78fyGPmJvKP/iiMucEc=
+-----END CERTIFICATE-----
+
+Thawte Premium Server CA
+========================
+
+-----BEGIN CERTIFICATE-----
+MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD
+VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv
+biBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFByZW1pdW0gU2Vy
+dmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNlcnZlckB0aGF3dGUuY29t
+MB4XDTk2MDgwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgc4xCzAJBgNVBAYTAlpB
+MRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEdMBsG
+A1UEChMUVGhhd3RlIENvbnN1bHRpbmcgY2MxKDAmBgNVBAsTH0NlcnRpZmljYXRp
+b24gU2VydmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQcmVtaXVtIFNl
+cnZlciBDQTEoMCYGCSqGSIb3DQEJARYZcHJlbWl1bS1zZXJ2ZXJAdGhhd3RlLmNv
+bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA0jY2aovXwlue2oFBYo847kkE
+VdbQ7xwblRZH7xhINTpS9CtqBo87L+pW46+GjZ4X9560ZXUCTe/LCaIhUdib0GfQ
+ug2SBhRz1JPLlyoAnFxODLz6FVL88kRu2hFKbgifLy3j+ao6hnO2RlNYyIkFvYMR
+uHM/qgeN9EJN50CdHDcCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG
+9w0BAQQFAAOBgQAmSCwWwlj66BZ0DKqqX1Q/8tfJeGBeXm43YyJ3Nn6yF8Q0ufUI
+hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM
+pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg==
+-----END CERTIFICATE-----
+
+Equifax Secure CA
+=================
+
+-----BEGIN CERTIFICATE-----
+MIIDIDCCAomgAwIBAgIENd70zzANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV
+UzEQMA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2Vy
+dGlmaWNhdGUgQXV0aG9yaXR5MB4XDTk4MDgyMjE2NDE1MVoXDTE4MDgyMjE2NDE1
+MVowTjELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0VxdWlmYXgxLTArBgNVBAsTJEVx
+dWlmYXggU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eTCBnzANBgkqhkiG9w0B
+AQEFAAOBjQAwgYkCgYEAwV2xWGcIYu6gmi0fCG2RFGiYCh7+2gRvE4RiIcPRfM6f
+BeC4AfBONOziipUEZKzxa1NfBbPLZ4C/QgKO/t0BCezhABRP/PvwDN1Dulsr4R+A
+cJkVV5MW8Q+XarfCaCMczE1ZMKxRHjuvK9buY0V7xdlfUNLjUA86iOe/FP3gx7kC
+AwEAAaOCAQkwggEFMHAGA1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEQ
+MA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2VydGlm
+aWNhdGUgQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMBoGA1UdEAQTMBGBDzIwMTgw
+ODIyMTY0MTUxWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUSOZo+SvSspXXR9gj
+IBBPM5iQn9QwHQYDVR0OBBYEFEjmaPkr0rKV10fYIyAQTzOYkJ/UMAwGA1UdEwQF
+MAMBAf8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUA
+A4GBAFjOKer89961zgK5F7WF0bnj4JXMJTENAKaSbn+2kmOeUJXRmm/kEd5jhW6Y
+7qj/WsjTVbJmcVfewCHrPSqnI0kBBIZCe/zuf6IWUrVnZ9NA2zsmWLIodz2uFHdh
+1voqZiegDfqnc1zqcPGUIWVEX/r87yloqaKHee9570+sB3c4
+-----END CERTIFICATE-----
+
+Verisign Class 1 Public Primary Certification Authority
+=======================================================
+
+-----BEGIN CERTIFICATE-----
+MIICPTCCAaYCEQDNun9W8N/kvFT+IqyzcqpVMA0GCSqGSIb3DQEBAgUAMF8xCzAJ
+BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xh
+c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05
+NjAxMjkwMDAwMDBaFw0yODA4MDEyMzU5NTlaMF8xCzAJBgNVBAYTAlVTMRcwFQYD
+VQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xhc3MgMSBQdWJsaWMgUHJp
+bWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCBnzANBgkqhkiG9w0BAQEFAAOB
+jQAwgYkCgYEA5Rm/baNWYS2ZSHH2Z965jeu3noaACpEO+jglr0aIguVzqKCbJF0N
+H8xlbgyw0FaEGIeaBpsQoXPftFg5a27B9hXVqKg/qhIGjTGsf7A01480Z4gJzRQR
+4k5FVmkfeAKA2txHkSm7NsljXMXg1y2He6G3MrB7MLoqLzGq7qNn2tsCAwEAATAN
+BgkqhkiG9w0BAQIFAAOBgQBMP7iLxmjf7kMzDl3ppssHhE16M/+SG/Q2rdiVIjZo
+EWx8QszznC7EBz8UsA9P/5CSdvnivErpj82ggAr3xSnxgiJduLHdgSOjeyUVRjB5
+FvjqBUuUfx3CHMjjt/QQQDwTw18fU+hI5Ia0e6E1sHslurjTjqs/OJ0ANACY89Fx
+lA==
+-----END CERTIFICATE-----
+
+Verisign Class 2 Public Primary Certification Authority
+=======================================================
+
+-----BEGIN CERTIFICATE-----
+MIICPDCCAaUCEC0b/EoXjaOR6+f/9YtFvgswDQYJKoZIhvcNAQECBQAwXzELMAkG
+A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz
+cyAyIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2
+MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV
+BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAyIFB1YmxpYyBQcmlt
+YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN
+ADCBiQKBgQC2WoujDWojg4BrzzmH9CETMwZMJaLtVRKXxaeAufqDwSCg+i8VDXyh
+YGt+eSz6Bg86rvYbb7HS/y8oUl+DfUvEerf4Zh+AVPy3wo5ZShRXRtGak75BkQO7
+FYCTXOvnzAhsPz6zSvz/S2wj1VCCJkQZjiPDceoZJEcEnnW/yKYAHwIDAQABMA0G
+CSqGSIb3DQEBAgUAA4GBAIobK/o5wXTXXtgZZKJYSi034DNHD6zt96rbHuSLBlxg
+J8pFUs4W7z8GZOeUaHxgMxURaa+dYo2jA1Rrpr7l7gUYYAS/QoD90KioHgE796Nc
+r6Pc5iaAIzy4RHT3Cq5Ji2F4zCS/iIqnDupzGUH9TQPwiNHleI2lKk/2lw0Xd8rY
+-----END CERTIFICATE-----
+
+Verisign Class 3 Public Primary Certification Authority
+=======================================================
+
+-----BEGIN CERTIFICATE-----
+MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzELMAkG
+A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz
+cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2
+MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV
+BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAzIFB1YmxpYyBQcmlt
+YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN
+ADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69qRUCPhAwL0TPZ2RHP7gJYHyX3KqhE
+BarsAx94f56TuZoAqiN91qyFomNFx3InzPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/is
+I19wKTakyYbnsZogy1Olhec9vn2a/iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0G
+CSqGSIb3DQEBAgUAA4GBALtMEivPLCYATxQT3ab7/AoRhIzzKBxnki98tsX63/Do
+lbwdj2wsqFHMc9ikwFPwTtYmwHYBV4GSXiHx0bH/59AhWM1pF+NEHJwZRDmJXNyc
+AA9WjQKZ7aKQRUzkuxCkPfAyAw7xzvjoyVGM5mKf5p/AfbdynMk2OmufTqj/ZA1k
+-----END CERTIFICATE-----
+
+Verisign Class 1 Public Primary Certification Authority - G2
+============================================================
+
+-----BEGIN CERTIFICATE-----
+MIIDAjCCAmsCEEzH6qqYPnHTkxD4PTqJkZIwDQYJKoZIhvcNAQEFBQAwgcExCzAJ
+BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh
+c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy
+MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp
+emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X
+DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw
+FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMSBQdWJsaWMg
+UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo
+YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5
+MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB
+AQUAA4GNADCBiQKBgQCq0Lq+Fi24g9TK0g+8djHKlNgdk4xWArzZbxpvUjZudVYK
+VdPfQ4chEWWKfo+9Id5rMj8bhDSVBZ1BNeuS65bdqlk/AVNtmU/t5eIqWpDBucSm
+Fc/IReumXY6cPvBkJHalzasab7bYe1FhbqZ/h8jit+U03EGI6glAvnOSPWvndQID
+AQABMA0GCSqGSIb3DQEBBQUAA4GBAKlPww3HZ74sy9mozS11534Vnjty637rXC0J
+h9ZrbWB85a7FkCMMXErQr7Fd88e2CtvgFZMN3QO8x3aKtd1Pw5sTdbgBwObJW2ul
+uIncrKTdcu1OofdPvAbT6shkdHvClUGcZXNY8ZCaPGqxmMnEh7zPRW1F4m4iP/68
+DzFc6PLZ
+-----END CERTIFICATE-----
+
+Verisign Class 2 Public Primary Certification Authority - G2
+============================================================
+
+-----BEGIN CERTIFICATE-----
+MIIDAzCCAmwCEQC5L2DMiJ+hekYJuFtwbIqvMA0GCSqGSIb3DQEBBQUAMIHBMQsw
+CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0Ns
+YXNzIDIgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBH
+MjE6MDgGA1UECxMxKGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9y
+aXplZCB1c2Ugb25seTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazAe
+Fw05ODA1MTgwMDAwMDBaFw0yODA4MDEyMzU5NTlaMIHBMQswCQYDVQQGEwJVUzEX
+MBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0NsYXNzIDIgUHVibGlj
+IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBHMjE6MDgGA1UECxMx
+KGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25s
+eTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazCBnzANBgkqhkiG9w0B
+AQEFAAOBjQAwgYkCgYEAp4gBIXQs5xoD8JjhlzwPIQjxnNuX6Zr8wgQGE75fUsjM
+HiwSViy4AWkszJkfrbCWrnkE8hM5wXuYuggs6MKEEyyqaekJ9MepAqRCwiNPStjw
+DqL7MWzJ5m+ZJwf15vRMeJ5t60aG+rmGyVTyssSv1EYcWskVMP8NbPUtDm3Of3cC
+AwEAATANBgkqhkiG9w0BAQUFAAOBgQByLvl/0fFx+8Se9sVeUYpAmLho+Jscg9ji
+nb3/7aHmZuovCfTK1+qlK5X2JGCGTUQug6XELaDTrnhpb3LabK4I8GOSN+a7xDAX
+rXfMSTWqz9iP0b63GJZHc2pUIjRkLbYWm1lbtFFZOrMLFPQS32eg9K0yZF6xRnIn
+jBJ7xUS0rg==
+-----END CERTIFICATE-----
+
+Verisign Class 3 Public Primary Certification Authority - G2
+============================================================
+
+-----BEGIN CERTIFICATE-----
+MIIDAjCCAmsCEH3Z/gfPqB63EHln+6eJNMYwDQYJKoZIhvcNAQEFBQAwgcExCzAJ
+BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh
+c3MgMyBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy
+MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp
+emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X
+DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw
+FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMyBQdWJsaWMg
+UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo
+YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5
+MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB
+AQUAA4GNADCBiQKBgQDMXtERXVxp0KvTuWpMmR9ZmDCOFoUgRm1HP9SFIIThbbP4
+pO0M8RcPO/mn+SXXwc+EY/J8Y8+iR/LGWzOOZEAEaMGAuWQcRXfH2G71lSk8UOg0
+13gfqLptQ5GVj0VXXn7F+8qkBOvqlzdUMG+7AUcyM83cV5tkaWH4mx0ciU9cZwID
+AQABMA0GCSqGSIb3DQEBBQUAA4GBAFFNzb5cy5gZnBWyATl4Lk0PZ3BwmcYQWpSk
+U01UbSuvDV1Ai2TT1+7eVmGSX6bEHRBhNtMsJzzoKQm5EWR0zLVznxxIqbxhAe7i
+F6YM40AIOw7n60RzKprxaZLvcRTDOaxxp5EJb+RxBrO6WVcmeQD2+A2iMzAo1KpY
+oJ2daZH9
+-----END CERTIFICATE-----
+
+Verisign Class 4 Public Primary Certification Authority - G2
+============================================================
+
+-----BEGIN CERTIFICATE-----
+MIIDAjCCAmsCEDKIjprS9esTR/h/xCA3JfgwDQYJKoZIhvcNAQEFBQAwgcExCzAJ
+BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh
+c3MgNCBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy
+MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp
+emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X
+DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw
+FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgNCBQdWJsaWMg
+UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo
+YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5
+MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB
+AQUAA4GNADCBiQKBgQC68OTP+cSuhVS5B1f5j8V/aBH4xBewRNzjMHPVKmIquNDM
+HO0oW369atyzkSTKQWI8/AIBvxwWMZQFl3Zuoq29YRdsTjCG8FE3KlDHqGKB3FtK
+qsGgtG7rL+VXxbErQHDbWk2hjh+9Ax/YA9SPTJlxvOKCzFjomDqG04Y48wApHwID
+AQABMA0GCSqGSIb3DQEBBQUAA4GBAIWMEsGnuVAVess+rLhDityq3RS6iYF+ATwj
+cSGIL4LcY/oCRaxFWdcqWERbt5+BO5JoPeI3JPV7bI92NZYJqFmduc4jq3TWg/0y
+cyfYaT5DdPauxYma51N86Xv2S/PBZYPejYqcPIiNOVn8qj8ijaHBZlCBckztImRP
+T8qAkbYp
+-----END CERTIFICATE-----
+
+Verisign Class 1 Public Primary Certification Authority - G3
+============================================================
+
+-----BEGIN CERTIFICATE-----
+MIIEGjCCAwICEQCLW3VWhFSFCwDPrzhIzrGkMA0GCSqGSIb3DQEBBQUAMIHKMQsw
+CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl
+cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu
+LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT
+aWduIENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp
+dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD
+VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT
+aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ
+bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu
+IENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg
+LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN2E1Lm0+afY8wR4
+nN493GwTFtl63SRRZsDHJlkNrAYIwpTRMx/wgzUfbhvI3qpuFU5UJ+/EbRrsC+MO
+8ESlV8dAWB6jRx9x7GD2bZTIGDnt/kIYVt/kTEkQeE4BdjVjEjbdZrwBBDajVWjV
+ojYJrKshJlQGrT/KFOCsyq0GHZXi+J3x4GD/wn91K0zM2v6HmSHquv4+VNfSWXjb
+PG7PoBMAGrgnoeS+Z5bKoMWznN3JdZ7rMJpfo83ZrngZPyPpXNspva1VyBtUjGP2
+6KbqxzcSXKMpHgLZ2x87tNcPVkeBFQRKr4Mn0cVYiMHd9qqnoxjaaKptEVHhv2Vr
+n5Z20T0CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAq2aN17O6x5q25lXQBfGfMY1a
+qtmqRiYPce2lrVNWYgFHKkTp/j90CxObufRNG7LRX7K20ohcs5/Ny9Sn2WCVhDr4
+wTcdYcrnsMXlkdpUpqwxga6X3s0IrLjAl4B/bnKk52kTlWUfxJM8/XmPBNQ+T+r3
+ns7NZ3xPZQL/kYVUc8f/NveGLezQXk//EZ9yBta4GvFMDSZl4kSAHsef493oCtrs
+pSCAaWihT37ha88HQfqDjrw43bAuEbFrskLMmrz5SCJ5ShkPshw+IHTZasO+8ih4
+E1Z5T21Q6huwtVexN2ZYI/PcD98Kh8TvhgXVOBRgmaNL3gaWcSzy27YfpO8/7g==
+-----END CERTIFICATE-----
+
+Verisign Class 2 Public Primary Certification Authority - G3
+============================================================
+
+-----BEGIN CERTIFICATE-----
+MIIEGTCCAwECEGFwy0mMX5hFKeewptlQW3owDQYJKoZIhvcNAQEFBQAwgcoxCzAJ
+BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVy
+aVNpZ24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24s
+IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNp
+Z24gQ2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0
+eSAtIEczMB4XDTk5MTAwMTAwMDAwMFoXDTM2MDcxNjIzNTk1OVowgcoxCzAJBgNV
+BAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVyaVNp
+Z24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24sIElu
+Yy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNpZ24g
+Q2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAt
+IEczMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArwoNwtUs22e5LeWU
+J92lvuCwTY+zYVY81nzD9M0+hsuiiOLh2KRpxbXiv8GmR1BeRjmL1Za6tW8UvxDO
+JxOeBUebMXoT2B/Z0wI3i60sR/COgQanDTAM6/c8DyAd3HJG7qUCyFvDyVZpTMUY
+wZF7C9UTAJu878NIPkZgIIUq1ZC2zYugzDLdt/1AVbJQHFauzI13TccgTacxdu9o
+koqQHgiBVrKtaaNS0MscxCM9H5n+TOgWY47GCI72MfbS+uV23bUckqNJzc0BzWjN
+qWm6o+sdDZykIKbBoMXRRkwXbdKsZj+WjOCE1Db/IlnF+RFgqF8EffIa9iVCYQ/E
+Srg+iQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQA0JhU8wI1NQ0kdvekhktdmnLfe
+xbjQ5F1fdiLAJvmEOjr5jLX77GDx6M4EsMjdpwOPMPOY36TmpDHf0xwLRtxyID+u
+7gU8pDM/CzmscHhzS5kr3zDCVLCoO1Wh/hYozUK9dG6A2ydEp85EXdQbkJgNHkKU
+sQAsBNB0owIFImNjzYO1+8FtYmtpdf1dcEG59b98377BMnMiIYtYgXsVkXq642RI
+sH/7NiXaldDxJBQX3RiAa0YjOVT1jmIJBB2UkKab5iXiQkWquJCtvgiPqQtCGJTP
+cjnhsUPgKM+351psE2tJs//jGHyJizNdrDPXp/naOlXJWBD5qu9ats9LS98q
+-----END CERTIFICATE-----
+
+Verisign Class 3 Public Primary Certification Authority - G3
+============================================================
+
+-----BEGIN CERTIFICATE-----
+MIIEGjCCAwICEQCbfgZJoz5iudXukEhxKe9XMA0GCSqGSIb3DQEBBQUAMIHKMQsw
+CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl
+cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu
+LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT
+aWduIENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp
+dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD
+VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT
+aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ
+bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu
+IENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg
+LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMu6nFL8eB8aHm8b
+N3O9+MlrlBIwT/A2R/XQkQr1F8ilYcEWQE37imGQ5XYgwREGfassbqb1EUGO+i2t
+KmFZpGcmTNDovFJbcCAEWNF6yaRpvIMXZK0Fi7zQWM6NjPXr8EJJC52XJ2cybuGu
+kxUccLwgTS8Y3pKI6GyFVxEa6X7jJhFUokWWVYPKMIno3Nij7SqAP395ZVc+FSBm
+CC+Vk7+qRy+oRpfwEuL+wgorUeZ25rdGt+INpsyow0xZVYnm6FNcHOqd8GIWC6fJ
+Xwzw3sJ2zq/3avL6QaaiMxTJ5Xpj055iN9WFZZ4O5lMkdBteHRJTW8cs54NJOxWu
+imi5V5cCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAERSWwauSCPc/L8my/uRan2Te
+2yFPhpk0djZX3dAVL8WtfxUfN2JzPtTnX84XA9s1+ivbrmAJXx5fj267Cz3qWhMe
+DGBvtcC1IyIuBwvLqXTLR7sdwdela8wv0kL9Sd2nic9TutoAWii/gt/4uhMdUIaC
+/Y4wjylGsB49Ndo4YhYYSq3mtlFs3q9i6wHQHiT+eo8SGhJouPtmmRQURVyu565p
+F4ErWjfJXir0xuKhXFSbplQAz/DxwceYMBo7Nhbbo27q/a2ywtrvAkcTisDxszGt
+TxzhT5yvDwyd93gN2PQ1VoDat20Xj50egWTh/sVFuq1ruQp6Tk9LhO5L8X3dEQ==
+-----END CERTIFICATE-----
+
+Verisign Class 4 Public Primary Certification Authority - G3
+============================================================
+
+-----BEGIN CERTIFICATE-----
+MIIEGjCCAwICEQDsoKeLbnVqAc/EfMwvlF7XMA0GCSqGSIb3DQEBBQUAMIHKMQsw
+CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl
+cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu
+LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT
+aWduIENsYXNzIDQgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp
+dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD
+VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT
+aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ
+bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu
+IENsYXNzIDQgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg
+LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK3LpRFpxlmr8Y+1
+GQ9Wzsy1HyDkniYlS+BzZYlZ3tCD5PUPtbut8XzoIfzk6AzufEUiGXaStBO3IFsJ
++mGuqPKljYXCKtbeZjbSmwL0qJJgfJxptI8kHtCGUvYynEFYHiK9zUVilQhu0Gbd
+U6LM8BDcVHOLBKFGMzNcF0C5nk3T875Vg+ixiY5afJqWIpA7iCXy0lOIAgwLePLm
+NxdLMEYH5IBtptiWLugs+BGzOA1mppvqySNb247i8xOOGlktqgLw7KSHZtzBP/XY
+ufTsgsbSPZUd5cBPhMnZo0QoBmrXRazwa2rvTl/4EYIeOGM0ZlDUPpNz+jDDZq3/
+ky2X7wMCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAj/ola09b5KROJ1WrIhVZPMq1
+CtRK26vdoV9TxaBXOcLORyu+OshWv8LZJxA6sQU8wHcxuzrTBXttmhwwjIDLk5Mq
+g6sFUYICABFna/OIYUdfA5PVWw3g8dShMjWFsjrbsIKr0csKvE+MW8VLADsfKoKm
+fjaF3H48ZwC15DtS4KjrXRX5xm3wrR0OhbepmnMUWluPQSjA1egtTaRezarZ7c7c
+2NU8Qh0XwRJdRTjDOPP8hS6DRkiy1yBfkjaP53kPmF6Z6PDQpLv1U70qzlmwr25/
+bLvSHgCwIe34QWKCudiyxLtGUPMxxY8BqHTr9Xgn2uf3ZkPznoM+IKrDNWCRzg==
+-----END CERTIFICATE-----
+
+Equifax Secure Global eBusiness CA
+==================================
+
+-----BEGIN CERTIFICATE-----
+MIICkDCCAfmgAwIBAgIBATANBgkqhkiG9w0BAQQFADBaMQswCQYDVQQGEwJVUzEc
+MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEtMCsGA1UEAxMkRXF1aWZheCBT
+ZWN1cmUgR2xvYmFsIGVCdXNpbmVzcyBDQS0xMB4XDTk5MDYyMTA0MDAwMFoXDTIw
+MDYyMTA0MDAwMFowWjELMAkGA1UEBhMCVVMxHDAaBgNVBAoTE0VxdWlmYXggU2Vj
+dXJlIEluYy4xLTArBgNVBAMTJEVxdWlmYXggU2VjdXJlIEdsb2JhbCBlQnVzaW5l
+c3MgQ0EtMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAuucXkAJlsTRVPEnC
+UdXfp9E3j9HngXNBUmCbnaEXJnitx7HoJpQytd4zjTov2/KaelpzmKNc6fuKcxtc
+58O/gGzNqfTWK8D3+ZmqY6KxRwIP1ORROhI8bIpaVIRw28HFkM9yRcuoWcDNM50/
+o5brhTMhHD4ePmBudpxnhcXIw2ECAwEAAaNmMGQwEQYJYIZIAYb4QgEBBAQDAgAH
+MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUvqigdHJQa0S3ySPY+6j/s1dr
+aGwwHQYDVR0OBBYEFL6ooHRyUGtEt8kj2Puo/7NXa2hsMA0GCSqGSIb3DQEBBAUA
+A4GBADDiAVGqx+pf2rnQZQ8w1j7aDRRJbpGTJxQx78T3LUX47Me/okENI7SS+RkA
+Z70Br83gcfxaz2TE4JaY0KNA4gGK7ycH8WUBikQtBmV1UsCGECAhX2xrD2yuCRyv
+8qIYNMR1pHMc8Y3c7635s3a0kr/clRAevsvIO1qEYBlWlKlV
+-----END CERTIFICATE-----
+
+Equifax Secure eBusiness CA 1
+=============================
+
+-----BEGIN CERTIFICATE-----
+MIICgjCCAeugAwIBAgIBBDANBgkqhkiG9w0BAQQFADBTMQswCQYDVQQGEwJVUzEc
+MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEmMCQGA1UEAxMdRXF1aWZheCBT
+ZWN1cmUgZUJ1c2luZXNzIENBLTEwHhcNOTkwNjIxMDQwMDAwWhcNMjAwNjIxMDQw
+MDAwWjBTMQswCQYDVQQGEwJVUzEcMBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5j
+LjEmMCQGA1UEAxMdRXF1aWZheCBTZWN1cmUgZUJ1c2luZXNzIENBLTEwgZ8wDQYJ
+KoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vGbwXt3fek6lfWg0XTzQaDJj0ItlZ1MRo
+RvC0NcWFAyDGr0WlIVFFQesWWDYyb+JQYmT5/VGcqiTZ9J2DKocKIdMSODRsjQBu
+WqDZQu4aIZX5UkxVWsUPOE9G+m34LjXWHXzr4vCwdYDIqROsvojvOm6rXyo4YgKw
+Env+j6YDAgMBAAGjZjBkMBEGCWCGSAGG+EIBAQQEAwIABzAPBgNVHRMBAf8EBTAD
+AQH/MB8GA1UdIwQYMBaAFEp4MlIR21kWNl7fwRQ2QGpHfEyhMB0GA1UdDgQWBBRK
+eDJSEdtZFjZe38EUNkBqR3xMoTANBgkqhkiG9w0BAQQFAAOBgQB1W6ibAxHm6VZM
+zfmpTMANmvPMZWnmJXbMWbfWVMMdzZmsGd20hdXgPfxiIKeES1hl8eL5lSE/9dR+
+WB5Hh1Q+WKG1tfgq73HnvMP2sUlG4tega+VWeponmHxGYhTnyfxuAxJ5gDgdSIKN
+/Bf+KpYrtWKmpj29f5JZzVoqgrI3eQ==
+-----END CERTIFICATE-----
+
+Equifax Secure eBusiness CA 2
+=============================
+
+-----BEGIN CERTIFICATE-----
+MIIDIDCCAomgAwIBAgIEN3DPtTANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV
+UzEXMBUGA1UEChMORXF1aWZheCBTZWN1cmUxJjAkBgNVBAsTHUVxdWlmYXggU2Vj
+dXJlIGVCdXNpbmVzcyBDQS0yMB4XDTk5MDYyMzEyMTQ0NVoXDTE5MDYyMzEyMTQ0
+NVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkVxdWlmYXggU2VjdXJlMSYwJAYD
+VQQLEx1FcXVpZmF4IFNlY3VyZSBlQnVzaW5lc3MgQ0EtMjCBnzANBgkqhkiG9w0B
+AQEFAAOBjQAwgYkCgYEA5Dk5kx5SBhsoNviyoynF7Y6yEb3+6+e0dMKP/wXn2Z0G
+vxLIPw7y1tEkshHe0XMJitSxLJgJDR5QRrKDpkWNYmi7hRsgcDKqQM2mll/EcTc/
+BPO3QSQ5BxoeLmFYoBIL5aXfxavqN3HMHMg3OrmXUqesxWoklE6ce8/AatbfIb0C
+AwEAAaOCAQkwggEFMHAGA1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEX
+MBUGA1UEChMORXF1aWZheCBTZWN1cmUxJjAkBgNVBAsTHUVxdWlmYXggU2VjdXJl
+IGVCdXNpbmVzcyBDQS0yMQ0wCwYDVQQDEwRDUkwxMBoGA1UdEAQTMBGBDzIwMTkw
+NjIzMTIxNDQ1WjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUUJ4L6q9euSBIplBq
+y/3YIHqngnYwHQYDVR0OBBYEFFCeC+qvXrkgSKZQasv92CB6p4J2MAwGA1UdEwQF
+MAMBAf8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUA
+A4GBAAyGgq3oThr1jokn4jVYPSm0B482UJW/bsGe68SQsoWou7dC4A8HOd/7npCy
+0cE+U58DRLB+S/Rv5Hwf5+Kx5Lia78O9zt4LMjTZ3ijtM2vE1Nc9ElirfQkty3D1
+E4qUoSek1nDFbZS1yX2doNLGCEnZZpum0/QL3MUmV+GRMOrN
+-----END CERTIFICATE-----
+
+Thawte Time Stamping CA
+=======================
+
+-----BEGIN CERTIFICATE-----
+MIICoTCCAgqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBizELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzAN
+BgNVBAoTBlRoYXd0ZTEdMBsGA1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAd
+BgNVBAMTFlRoYXd0ZSBUaW1lc3RhbXBpbmcgQ0EwHhcNOTcwMTAxMDAwMDAwWhcN
+MjAxMjMxMjM1OTU5WjCBizELMAkGA1UEBhMCWkExFTATBgNVBAgTDFdlc3Rlcm4g
+Q2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzANBgNVBAoTBlRoYXd0ZTEdMBsG
+A1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAdBgNVBAMTFlRoYXd0ZSBUaW1l
+c3RhbXBpbmcgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYrWHhhRYZT
+6jR7UZztsOYuGA7+4F+oJ9O0yeB8WU4WDnNUYMF/9p8u6TqFJBU820cEY8OexJQa
+Wt9MevPZQx08EHp5JduQ/vBR5zDWQQD9nyjfeb6Uu522FOMjhdepQeBMpHmwKxqL
+8vg7ij5FrHGSALSQQZj7X+36ty6K+Ig3AgMBAAGjEzARMA8GA1UdEwEB/wQFMAMB
+Af8wDQYJKoZIhvcNAQEEBQADgYEAZ9viwuaHPUCDhjc1fR/OmsMMZiCouqoEiYbC
+9RAIDb/LogWK0E02PvTX72nGXuSwlG9KuefeW4i2e9vjJ+V2w/A1wcu1J5szedyQ
+pgCed/r8zSeUQhac0xxo7L9c3eWpexAKMnRUEzGLhQOEkbdYATAUOK8oyvyxUBkZ
+CayJSdM=
+-----END CERTIFICATE-----
+
+thawte Primary Root CA
+======================
+
+-----BEGIN CERTIFICATE-----
+MIIEIDCCAwigAwIBAgIQNE7VVyDV7exJ9C/ON9srbTANBgkqhkiG9w0BAQUFADCB
+qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf
+Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw
+MDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxHzAdBgNV
+BAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwHhcNMDYxMTE3MDAwMDAwWhcNMzYw
+NzE2MjM1OTU5WjCBqTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5j
+LjEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYG
+A1UECxMvKGMpIDIwMDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNl
+IG9ubHkxHzAdBgNVBAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCsoPD7gFnUnMekz52hWXMJEEUMDSxuaPFs
+W0hoSVk3/AszGcJ3f8wQLZU0HObrTQmnHNK4yZc2AreJ1CRfBsDMRJSUjQJib+ta
+3RGNKJpchJAQeg29dGYvajig4tVUROsdB58Hum/u6f1OCyn1PoSgAfGcq/gcfomk
+6KHYcWUNo1F77rzSImANuVud37r8UVsLr5iy6S7pBOhih94ryNdOwUxkHt3Ph1i6
+Sk/KaAcdHJ1KxtUvkcx8cXIcxcBn6zL9yZJclNqFwJu/U30rCfSMnZEfl2pSy94J
+NqR32HuHUETVPm4pafs5SSYeCaWAe0At6+gnhcn+Yf1+5nyXHdWdAgMBAAGjQjBA
+MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBR7W0XP
+r87Lev0xkhpqtvNG61dIUDANBgkqhkiG9w0BAQUFAAOCAQEAeRHAS7ORtvzw6WfU
+DW5FvlXok9LOAz/t2iWwHVfLHjp2oEzsUHboZHIMpKnxuIvW1oeEuzLlQRHAd9mz
+YJ3rG9XRbkREqaYB7FViHXe4XI5ISXycO1cRrK1zN44veFyQaEfZYGDm/Ac9IiAX
+xPcW6cTYcvnIc3zfFi8VqT79aie2oetaupgf1eNNZAqdE8hhuvU5HIe6uL17In/2
+/qxAeeWsEG89jxt5dovEN7MhGITlNgDrYyCZuen+MwS7QcjBAvlEYyCegc5C09Y/
+LHbTY5xZ3Y+m4Q6gLkH3LpVHz7z9M/P2C2F+fpErgUfCJzDupxBdN49cOSvkBPB7
+jVaMaA==
+-----END CERTIFICATE-----
+
+VeriSign Class 3 Public Primary Certification Authority - G5
+============================================================
+
+-----BEGIN CERTIFICATE-----
+MIIE0zCCA7ugAwIBAgIQGNrRniZ96LtKIVjNzGs7SjANBgkqhkiG9w0BAQUFADCB
+yjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL
+ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJp
+U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxW
+ZXJpU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0
+aG9yaXR5IC0gRzUwHhcNMDYxMTA4MDAwMDAwWhcNMzYwNzE2MjM1OTU5WjCByjEL
+MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW
+ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJpU2ln
+biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp
+U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y
+aXR5IC0gRzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvJAgIKXo1
+nmAMqudLO07cfLw8RRy7K+D+KQL5VwijZIUVJ/XxrcgxiV0i6CqqpkKzj/i5Vbex
+t0uz/o9+B1fs70PbZmIVYc9gDaTY3vjgw2IIPVQT60nKWVSFJuUrjxuf6/WhkcIz
+SdhDY2pSS9KP6HBRTdGJaXvHcPaz3BJ023tdS1bTlr8Vd6Gw9KIl8q8ckmcY5fQG
+BO+QueQA5N06tRn/Arr0PO7gi+s3i+z016zy9vA9r911kTMZHRxAy3QkGSGT2RT+
+rCpSx4/VBEnkjWNHiDxpg8v+R70rfk/Fla4OndTRQ8Bnc+MUCH7lP59zuDMKz10/
+NIeWiu5T6CUVAgMBAAGjgbIwga8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E
+BAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJaW1hZ2UvZ2lmMCEwHzAH
+BgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYjaHR0cDovL2xvZ28udmVy
+aXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFH/TZafC3ey78DAJ80M5+gKv
+MzEzMA0GCSqGSIb3DQEBBQUAA4IBAQCTJEowX2LP2BqYLz3q3JktvXf2pXkiOOzE
+p6B4Eq1iDkVwZMXnl2YtmAl+X6/WzChl8gGqCBpH3vn5fJJaCGkgDdk+bW48DW7Y
+5gaRQBi5+MHt39tBquCWIMnNZBU4gcmU7qKEKQsTb47bDN0lAtukixlE0kF6BWlK
+WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ
+4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N
+hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq
+-----END CERTIFICATE-----
+
+Entrust.net Secure Server Certification Authority
+=================================================
+
+-----BEGIN CERTIFICATE-----
+MIIE2DCCBEGgAwIBAgIEN0rSQzANBgkqhkiG9w0BAQUFADCBwzELMAkGA1UEBhMC
+VVMxFDASBgNVBAoTC0VudHJ1c3QubmV0MTswOQYDVQQLEzJ3d3cuZW50cnVzdC5u
+ZXQvQ1BTIGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxpYWIuKTElMCMGA1UECxMc
+KGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDE6MDgGA1UEAxMxRW50cnVzdC5u
+ZXQgU2VjdXJlIFNlcnZlciBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05OTA1
+MjUxNjA5NDBaFw0xOTA1MjUxNjM5NDBaMIHDMQswCQYDVQQGEwJVUzEUMBIGA1UE
+ChMLRW50cnVzdC5uZXQxOzA5BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5j
+b3JwLiBieSByZWYuIChsaW1pdHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBF
+bnRydXN0Lm5ldCBMaW1pdGVkMTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUg
+U2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGdMA0GCSqGSIb3DQEBAQUA
+A4GLADCBhwKBgQDNKIM0VBuJ8w+vN5Ex/68xYMmo6LIQaO2f55M28Qpku0f1BBc/
+I0dNxScZgSYMVHINiC3ZH5oSn7yzcdOAGT9HZnuMNSjSuQrfJNqc1lB5gXpa0zf3
+wkrYKZImZNHkmGw6AIr1NJtl+O3jEP/9uElY3KDegjlrgbEWGWG5VLbmQwIBA6OC
+AdcwggHTMBEGCWCGSAGG+EIBAQQEAwIABzCCARkGA1UdHwSCARAwggEMMIHeoIHb
+oIHYpIHVMIHSMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLRW50cnVzdC5uZXQxOzA5
+BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5jb3JwLiBieSByZWYuIChsaW1p
+dHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBFbnRydXN0Lm5ldCBMaW1pdGVk
+MTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUgU2VydmVyIENlcnRpZmljYXRp
+b24gQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMCmgJ6AlhiNodHRwOi8vd3d3LmVu
+dHJ1c3QubmV0L0NSTC9uZXQxLmNybDArBgNVHRAEJDAigA8xOTk5MDUyNTE2MDk0
+MFqBDzIwMTkwNTI1MTYwOTQwWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAU8Bdi
+E1U9s/8KAGv7UISX8+1i0BowHQYDVR0OBBYEFPAXYhNVPbP/CgBr+1CEl/PtYtAa
+MAwGA1UdEwQFMAMBAf8wGQYJKoZIhvZ9B0EABAwwChsEVjQuMAMCBJAwDQYJKoZI
+hvcNAQEFBQADgYEAkNwwAvpkdMKnCqV8IY00F6j7Rw7/JXyNEwr75Ji174z4xRAN
+95K+8cPV1ZVqBLssziY2ZcgxxufuP+NXdYR6Ee9GTxj005i7qIcyunL2POI9n9cd
+2cNgQ4xYDiKWL2KjLB+6rQXvqzJ4h6BUcxm1XAX5Uj5tLUUL9wqT6u0G+bI=
+-----END CERTIFICATE-----
+
+Go Daddy Certification Authority Root Certificate Bundle
+========================================================
+
+-----BEGIN CERTIFICATE-----
+MIIE3jCCA8agAwIBAgICAwEwDQYJKoZIhvcNAQEFBQAwYzELMAkGA1UEBhMCVVMx
+ITAfBgNVBAoTGFRoZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28g
+RGFkZHkgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjExMTYw
+MTU0MzdaFw0yNjExMTYwMTU0MzdaMIHKMQswCQYDVQQGEwJVUzEQMA4GA1UECBMH
+QXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEaMBgGA1UEChMRR29EYWRkeS5j
+b20sIEluYy4xMzAxBgNVBAsTKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5j
+b20vcmVwb3NpdG9yeTEwMC4GA1UEAxMnR28gRGFkZHkgU2VjdXJlIENlcnRpZmlj
+YXRpb24gQXV0aG9yaXR5MREwDwYDVQQFEwgwNzk2OTI4NzCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBAMQt1RWMnCZM7DI161+4WQFapmGBWTtwY6vj3D3H
+KrjJM9N55DrtPDAjhI6zMBS2sofDPZVUBJ7fmd0LJR4h3mUpfjWoqVTr9vcyOdQm
+VZWt7/v+WIbXnvQAjYwqDL1CBM6nPwT27oDyqu9SoWlm2r4arV3aLGbqGmu75RpR
+SgAvSMeYddi5Kcju+GZtCpyz8/x4fKL4o/K1w/O5epHBp+YlLpyo7RJlbmr2EkRT
+cDCVw5wrWCs9CHRK8r5RsL+H0EwnWGu1NcWdrxcx+AuP7q2BNgWJCJjPOq8lh8BJ
+6qf9Z/dFjpfMFDniNoW1fho3/Rb2cRGadDAW/hOUoz+EDU8CAwEAAaOCATIwggEu
+MB0GA1UdDgQWBBT9rGEyk2xF1uLuhV+auud2mWjM5zAfBgNVHSMEGDAWgBTSxLDS
+kdRMEXGzYcs9of7dqGrU4zASBgNVHRMBAf8ECDAGAQH/AgEAMDMGCCsGAQUFBwEB
+BCcwJTAjBggrBgEFBQcwAYYXaHR0cDovL29jc3AuZ29kYWRkeS5jb20wRgYDVR0f
+BD8wPTA7oDmgN4Y1aHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNvbS9yZXBv
+c2l0b3J5L2dkcm9vdC5jcmwwSwYDVR0gBEQwQjBABgRVHSAAMDgwNgYIKwYBBQUH
+AgEWKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5jb20vcmVwb3NpdG9yeTAO
+BgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEFBQADggEBANKGwOy9+aG2Z+5mC6IG
+OgRQjhVyrEp0lVPLN8tESe8HkGsz2ZbwlFalEzAFPIUyIXvJxwqoJKSQ3kbTJSMU
+A2fCENZvD117esyfxVgqwcSeIaha86ykRvOe5GPLL5CkKSkB2XIsKd83ASe8T+5o
+0yGPwLPk9Qnt0hCqU7S+8MxZC9Y7lhyVJEnfzuz9p0iRFEUOOjZv2kWzRaJBydTX
+RE4+uXR21aITVSzGh6O1mawGhId/dQb8vxRMDsxuxN89txJx9OjxUUAiKEngHUuH
+qDTMBqLdElrRhjZkAzVvb3du6/KFUJheqwNTrZEjYx8WnM25sgVjOuH0aBsXBTWV
+U+4=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIE+zCCBGSgAwIBAgICAQ0wDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1Zh
+bGlDZXJ0IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIElu
+Yy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24g
+QXV0aG9yaXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAe
+BgkqhkiG9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTA0MDYyOTE3MDYyMFoX
+DTI0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRoZSBHbyBE
+YWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3MgMiBDZXJ0
+aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgC
+ggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCAPVYYYwhv
+2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6wwdhFJ2+q
+N1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXiEqITLdiO
+r18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMYavx4A6lN
+f4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+YihfukEH
+U1jPEX44dMX4/7VpkI+EdOqXG68CAQOjggHhMIIB3TAdBgNVHQ4EFgQU0sSw0pHU
+TBFxs2HLPaH+3ahq1OMwgdIGA1UdIwSByjCBx6GBwaSBvjCBuzEkMCIGA1UEBxMb
+VmFsaUNlcnQgVmFsaWRhdGlvbiBOZXR3b3JrMRcwFQYDVQQKEw5WYWxpQ2VydCwg
+SW5jLjE1MDMGA1UECxMsVmFsaUNlcnQgQ2xhc3MgMiBQb2xpY3kgVmFsaWRhdGlv
+biBBdXRob3JpdHkxITAfBgNVBAMTGGh0dHA6Ly93d3cudmFsaWNlcnQuY29tLzEg
+MB4GCSqGSIb3DQEJARYRaW5mb0B2YWxpY2VydC5jb22CAQEwDwYDVR0TAQH/BAUw
+AwEB/zAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly9vY3NwLmdv
+ZGFkZHkuY29tMEQGA1UdHwQ9MDswOaA3oDWGM2h0dHA6Ly9jZXJ0aWZpY2F0ZXMu
+Z29kYWRkeS5jb20vcmVwb3NpdG9yeS9yb290LmNybDBLBgNVHSAERDBCMEAGBFUd
+IAAwODA2BggrBgEFBQcCARYqaHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNv
+bS9yZXBvc2l0b3J5MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOBgQC1
+QPmnHfbq/qQaQlpE9xXUhUaJwL6e4+PrxeNYiY+Sn1eocSxI0YGyeR+sBjUZsE4O
+WBsUs5iB0QQeyAfJg594RAoYC5jcdnplDQ1tgMQLARzLrUc+cb53S8wGd9D0Vmsf
+SxOaFIqII6hR8INMqzW/Rn453HWkrugp++85j09VZw==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0
+IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz
+BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y
+aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG
+9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNjAwMTk1NFoXDTE5MDYy
+NjAwMTk1NFowgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y
+azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs
+YXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw
+Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl
+cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDOOnHK5avIWZJV16vY
+dA757tn2VUdZZUcOBVXc65g2PFxTXdMwzzjsvUGJ7SVCCSRrCl6zfN1SLUzm1NZ9
+WlmpZdRJEy0kTRxQb7XBhVQ7/nHk01xC+YDgkRoKWzk2Z/M/VXwbP7RfZHM047QS
+v4dk+NoS/zcnwbNDu+97bi5p9wIDAQABMA0GCSqGSIb3DQEBBQUAA4GBADt/UG9v
+UJSZSWI4OB9L+KXIPqeCgfYrx+jFzug6EILLGACOTb2oWH+heQC1u+mNr0HZDzTu
+IYEZoDJJKPTEjlbVUjP9UNV+mWwD5MlM/Mtsq2azSiGM5bUMMj4QssxsodyamEwC
+W/POuZ6lcg5Ktz885hZo+L7tdEy8W9ViH0Pd
+-----END CERTIFICATE-----
+
+GeoTrust Global CA
+==================
+
+-----BEGIN CERTIFICATE-----
+MIIDfTCCAuagAwIBAgIDErvmMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVT
+MRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0
+aWZpY2F0ZSBBdXRob3JpdHkwHhcNMDIwNTIxMDQwMDAwWhcNMTgwODIxMDQwMDAw
+WjBCMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UE
+AxMSR2VvVHJ1c3QgR2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEA2swYYzD99BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9m
+OSm9BXiLnTjoBbdqfnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIu
+T8rxh0PBFpVXLVDviS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6c
+JmTM386DGXHKTubU1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmR
+Cw7+OC7RHQWa9k0+bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5asz
+PeE4uwc2hGKceeoWMPRfwCvocWvk+QIDAQABo4HwMIHtMB8GA1UdIwQYMBaAFEjm
+aPkr0rKV10fYIyAQTzOYkJ/UMB0GA1UdDgQWBBTAephojYn7qwVkDBF9qn1luMrM
+TjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjA6BgNVHR8EMzAxMC+g
+LaArhilodHRwOi8vY3JsLmdlb3RydXN0LmNvbS9jcmxzL3NlY3VyZWNhLmNybDBO
+BgNVHSAERzBFMEMGBFUdIAAwOzA5BggrBgEFBQcCARYtaHR0cHM6Ly93d3cuZ2Vv
+dHJ1c3QuY29tL3Jlc291cmNlcy9yZXBvc2l0b3J5MA0GCSqGSIb3DQEBBQUAA4GB
+AHbhEm5OSxYShjAGsoEIz/AIx8dxfmbuwu3UOx//8PDITtZDOLC5MH0Y0FWDomrL
+NhGc6Ehmo21/uBPUR/6LWlxz/K7ZGzIZOKuXNBSqltLroxwUCEm2u+WR74M26x1W
+b8ravHNjkOR/ez4iyz0H7V84dJzjA1BOoa+Y7mHyhD8S
+-----END CERTIFICATE-----
+
diff --git a/py/minijack/httplib2/iri2uri.py b/py/minijack/httplib2/iri2uri.py
new file mode 100644
index 0000000..d88c91f
--- /dev/null
+++ b/py/minijack/httplib2/iri2uri.py
@@ -0,0 +1,110 @@
+"""
+iri2uri
+
+Converts an IRI to a URI.
+
+"""
+__author__ = "Joe Gregorio (joe@bitworking.org)"
+__copyright__ = "Copyright 2006, Joe Gregorio"
+__contributors__ = []
+__version__ = "1.0.0"
+__license__ = "MIT"
+__history__ = """
+"""
+
+import urlparse
+
+
+# Convert an IRI to a URI following the rules in RFC 3987
+#
+# The characters we need to enocde and escape are defined in the spec:
+#
+# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD
+# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF
+# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD
+# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD
+# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD
+# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD
+# / %xD0000-DFFFD / %xE1000-EFFFD
+
+escape_range = [
+ (0xA0, 0xD7FF),
+ (0xE000, 0xF8FF),
+ (0xF900, 0xFDCF),
+ (0xFDF0, 0xFFEF),
+ (0x10000, 0x1FFFD),
+ (0x20000, 0x2FFFD),
+ (0x30000, 0x3FFFD),
+ (0x40000, 0x4FFFD),
+ (0x50000, 0x5FFFD),
+ (0x60000, 0x6FFFD),
+ (0x70000, 0x7FFFD),
+ (0x80000, 0x8FFFD),
+ (0x90000, 0x9FFFD),
+ (0xA0000, 0xAFFFD),
+ (0xB0000, 0xBFFFD),
+ (0xC0000, 0xCFFFD),
+ (0xD0000, 0xDFFFD),
+ (0xE1000, 0xEFFFD),
+ (0xF0000, 0xFFFFD),
+ (0x100000, 0x10FFFD),
+]
+
+def encode(c):
+ retval = c
+ i = ord(c)
+ for low, high in escape_range:
+ if i < low:
+ break
+ if i >= low and i <= high:
+ retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')])
+ break
+ return retval
+
+
+def iri2uri(uri):
+ """Convert an IRI to a URI. Note that IRIs must be
+ passed in a unicode strings. That is, do not utf-8 encode
+ the IRI before passing it into the function."""
+ if isinstance(uri ,unicode):
+ (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri)
+ authority = authority.encode('idna')
+ # For each character in 'ucschar' or 'iprivate'
+ # 1. encode as utf-8
+ # 2. then %-encode each octet of that utf-8
+ uri = urlparse.urlunsplit((scheme, authority, path, query, fragment))
+ uri = "".join([encode(c) for c in uri])
+ return uri
+
+if __name__ == "__main__":
+ import unittest
+
+ class Test(unittest.TestCase):
+
+ def test_uris(self):
+ """Test that URIs are invariant under the transformation."""
+ invariant = [
+ u"ftp://ftp.is.co.za/rfc/rfc1808.txt",
+ u"http://www.ietf.org/rfc/rfc2396.txt",
+ u"ldap://[2001:db8::7]/c=GB?objectClass?one",
+ u"mailto:John.Doe@example.com",
+ u"news:comp.infosystems.www.servers.unix",
+ u"tel:+1-816-555-1212",
+ u"telnet://192.0.2.16:80/",
+ u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ]
+ for uri in invariant:
+ self.assertEqual(uri, iri2uri(uri))
+
+ def test_iri(self):
+ """ Test that the right type of escaping is done for each part of the URI."""
+ self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}"))
+ self.assertEqual("http://bitworking.org/?fred=%E2%98%84", iri2uri(u"http://bitworking.org/?fred=\N{COMET}"))
+ self.assertEqual("http://bitworking.org/#%E2%98%84", iri2uri(u"http://bitworking.org/#\N{COMET}"))
+ self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}"))
+ self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"))
+ self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")))
+ self.assertNotEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode('utf-8')))
+
+ unittest.main()
+
+
diff --git a/py/minijack/httplib2/socks.py b/py/minijack/httplib2/socks.py
new file mode 100644
index 0000000..0991f4c
--- /dev/null
+++ b/py/minijack/httplib2/socks.py
@@ -0,0 +1,438 @@
+"""SocksiPy - Python SOCKS module.
+Version 1.00
+
+Copyright 2006 Dan-Haim. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+3. Neither the name of Dan Haim nor the names of his contributors may be used
+ to endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE.
+
+
+This module provides a standard socket-like interface for Python
+for tunneling connections through SOCKS proxies.
+
+"""
+
+"""
+
+Minor modifications made by Christopher Gilbert (http://motomastyle.com/)
+for use in PyLoris (http://pyloris.sourceforge.net/)
+
+Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/)
+mainly to merge bug fixes found in Sourceforge
+
+"""
+
+import base64
+import socket
+import struct
+import sys
+
+if getattr(socket, 'socket', None) is None:
+ raise ImportError('socket.socket missing, proxy support unusable')
+
+PROXY_TYPE_SOCKS4 = 1
+PROXY_TYPE_SOCKS5 = 2
+PROXY_TYPE_HTTP = 3
+PROXY_TYPE_HTTP_NO_TUNNEL = 4
+
+_defaultproxy = None
+_orgsocket = socket.socket
+
+class ProxyError(Exception): pass
+class GeneralProxyError(ProxyError): pass
+class Socks5AuthError(ProxyError): pass
+class Socks5Error(ProxyError): pass
+class Socks4Error(ProxyError): pass
+class HTTPError(ProxyError): pass
+
+_generalerrors = ("success",
+ "invalid data",
+ "not connected",
+ "not available",
+ "bad proxy type",
+ "bad input")
+
+_socks5errors = ("succeeded",
+ "general SOCKS server failure",
+ "connection not allowed by ruleset",
+ "Network unreachable",
+ "Host unreachable",
+ "Connection refused",
+ "TTL expired",
+ "Command not supported",
+ "Address type not supported",
+ "Unknown error")
+
+_socks5autherrors = ("succeeded",
+ "authentication is required",
+ "all offered authentication methods were rejected",
+ "unknown username or invalid password",
+ "unknown error")
+
+_socks4errors = ("request granted",
+ "request rejected or failed",
+ "request rejected because SOCKS server cannot connect to identd on the client",
+ "request rejected because the client program and identd report different user-ids",
+ "unknown error")
+
+def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
+ """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
+ Sets a default proxy which all further socksocket objects will use,
+ unless explicitly changed.
+ """
+ global _defaultproxy
+ _defaultproxy = (proxytype, addr, port, rdns, username, password)
+
+def wrapmodule(module):
+ """wrapmodule(module)
+ Attempts to replace a module's socket library with a SOCKS socket. Must set
+ a default proxy using setdefaultproxy(...) first.
+ This will only work on modules that import socket directly into the namespace;
+ most of the Python Standard Library falls into this category.
+ """
+ if _defaultproxy != None:
+ module.socket.socket = socksocket
+ else:
+ raise GeneralProxyError((4, "no proxy specified"))
+
+class socksocket(socket.socket):
+ """socksocket([family[, type[, proto]]]) -> socket object
+ Open a SOCKS enabled socket. The parameters are the same as
+ those of the standard socket init. In order for SOCKS to work,
+ you must specify family=AF_INET, type=SOCK_STREAM and proto=0.
+ """
+
+ def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None):
+ _orgsocket.__init__(self, family, type, proto, _sock)
+ if _defaultproxy != None:
+ self.__proxy = _defaultproxy
+ else:
+ self.__proxy = (None, None, None, None, None, None)
+ self.__proxysockname = None
+ self.__proxypeername = None
+ self.__httptunnel = True
+
+ def __recvall(self, count):
+ """__recvall(count) -> data
+ Receive EXACTLY the number of bytes requested from the socket.
+ Blocks until the required number of bytes have been received.
+ """
+ data = self.recv(count)
+ while len(data) < count:
+ d = self.recv(count-len(data))
+ if not d: raise GeneralProxyError((0, "connection closed unexpectedly"))
+ data = data + d
+ return data
+
+ def sendall(self, content, *args):
+ """ override socket.socket.sendall method to rewrite the header
+ for non-tunneling proxies if needed
+ """
+ if not self.__httptunnel:
+ content = self.__rewriteproxy(content)
+ return super(socksocket, self).sendall(content, *args)
+
+ def __rewriteproxy(self, header):
+ """ rewrite HTTP request headers to support non-tunneling proxies
+ (i.e. those which do not support the CONNECT method).
+ This only works for HTTP (not HTTPS) since HTTPS requires tunneling.
+ """
+ host, endpt = None, None
+ hdrs = header.split("\r\n")
+ for hdr in hdrs:
+ if hdr.lower().startswith("host:"):
+ host = hdr
+ elif hdr.lower().startswith("get") or hdr.lower().startswith("post"):
+ endpt = hdr
+ if host and endpt:
+ hdrs.remove(host)
+ hdrs.remove(endpt)
+ host = host.split(" ")[1]
+ endpt = endpt.split(" ")
+ if (self.__proxy[4] != None and self.__proxy[5] != None):
+ hdrs.insert(0, self.__getauthheader())
+ hdrs.insert(0, "Host: %s" % host)
+ hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2]))
+ return "\r\n".join(hdrs)
+
+ def __getauthheader(self):
+ auth = self.__proxy[4] + ":" + self.__proxy[5]
+ return "Proxy-Authorization: Basic " + base64.b64encode(auth)
+
+ def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
+ """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
+ Sets the proxy to be used.
+ proxytype - The type of the proxy to be used. Three types
+ are supported: PROXY_TYPE_SOCKS4 (including socks4a),
+ PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP
+ addr - The address of the server (IP or DNS).
+ port - The port of the server. Defaults to 1080 for SOCKS
+ servers and 8080 for HTTP proxy servers.
+ rdns - Should DNS queries be preformed on the remote side
+ (rather than the local side). The default is True.
+ Note: This has no effect with SOCKS4 servers.
+ username - Username to authenticate with to the server.
+ The default is no authentication.
+ password - Password to authenticate with to the server.
+ Only relevant when username is also provided.
+ """
+ self.__proxy = (proxytype, addr, port, rdns, username, password)
+
+ def __negotiatesocks5(self, destaddr, destport):
+ """__negotiatesocks5(self,destaddr,destport)
+ Negotiates a connection through a SOCKS5 server.
+ """
+ # First we'll send the authentication packages we support.
+ if (self.__proxy[4]!=None) and (self.__proxy[5]!=None):
+ # The username/password details were supplied to the
+ # setproxy method so we support the USERNAME/PASSWORD
+ # authentication (in addition to the standard none).
+ self.sendall(struct.pack('BBBB', 0x05, 0x02, 0x00, 0x02))
+ else:
+ # No username/password were entered, therefore we
+ # only support connections with no authentication.
+ self.sendall(struct.pack('BBB', 0x05, 0x01, 0x00))
+ # We'll receive the server's response to determine which
+ # method was selected
+ chosenauth = self.__recvall(2)
+ if chosenauth[0:1] != chr(0x05).encode():
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ # Check the chosen authentication method
+ if chosenauth[1:2] == chr(0x00).encode():
+ # No authentication is required
+ pass
+ elif chosenauth[1:2] == chr(0x02).encode():
+ # Okay, we need to perform a basic username/password
+ # authentication.
+ self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5])
+ authstat = self.__recvall(2)
+ if authstat[0:1] != chr(0x01).encode():
+ # Bad response
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ if authstat[1:2] != chr(0x00).encode():
+ # Authentication failed
+ self.close()
+ raise Socks5AuthError((3, _socks5autherrors[3]))
+ # Authentication succeeded
+ else:
+ # Reaching here is always bad
+ self.close()
+ if chosenauth[1] == chr(0xFF).encode():
+ raise Socks5AuthError((2, _socks5autherrors[2]))
+ else:
+ raise GeneralProxyError((1, _generalerrors[1]))
+ # Now we can request the actual connection
+ req = struct.pack('BBB', 0x05, 0x01, 0x00)
+ # If the given destination address is an IP address, we'll
+ # use the IPv4 address request even if remote resolving was specified.
+ try:
+ ipaddr = socket.inet_aton(destaddr)
+ req = req + chr(0x01).encode() + ipaddr
+ except socket.error:
+ # Well it's not an IP number, so it's probably a DNS name.
+ if self.__proxy[3]:
+ # Resolve remotely
+ ipaddr = None
+ req = req + chr(0x03).encode() + chr(len(destaddr)).encode() + destaddr
+ else:
+ # Resolve locally
+ ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
+ req = req + chr(0x01).encode() + ipaddr
+ req = req + struct.pack(">H", destport)
+ self.sendall(req)
+ # Get the response
+ resp = self.__recvall(4)
+ if resp[0:1] != chr(0x05).encode():
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ elif resp[1:2] != chr(0x00).encode():
+ # Connection failed
+ self.close()
+ if ord(resp[1:2])<=8:
+ raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])]))
+ else:
+ raise Socks5Error((9, _socks5errors[9]))
+ # Get the bound address/port
+ elif resp[3:4] == chr(0x01).encode():
+ boundaddr = self.__recvall(4)
+ elif resp[3:4] == chr(0x03).encode():
+ resp = resp + self.recv(1)
+ boundaddr = self.__recvall(ord(resp[4:5]))
+ else:
+ self.close()
+ raise GeneralProxyError((1,_generalerrors[1]))
+ boundport = struct.unpack(">H", self.__recvall(2))[0]
+ self.__proxysockname = (boundaddr, boundport)
+ if ipaddr != None:
+ self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
+ else:
+ self.__proxypeername = (destaddr, destport)
+
+ def getproxysockname(self):
+ """getsockname() -> address info
+ Returns the bound IP address and port number at the proxy.
+ """
+ return self.__proxysockname
+
+ def getproxypeername(self):
+ """getproxypeername() -> address info
+ Returns the IP and port number of the proxy.
+ """
+ return _orgsocket.getpeername(self)
+
+ def getpeername(self):
+ """getpeername() -> address info
+ Returns the IP address and port number of the destination
+ machine (note: getproxypeername returns the proxy)
+ """
+ return self.__proxypeername
+
+ def __negotiatesocks4(self,destaddr,destport):
+ """__negotiatesocks4(self,destaddr,destport)
+ Negotiates a connection through a SOCKS4 server.
+ """
+ # Check if the destination address provided is an IP address
+ rmtrslv = False
+ try:
+ ipaddr = socket.inet_aton(destaddr)
+ except socket.error:
+ # It's a DNS name. Check where it should be resolved.
+ if self.__proxy[3]:
+ ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01)
+ rmtrslv = True
+ else:
+ ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
+ # Construct the request packet
+ req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr
+ # The username parameter is considered userid for SOCKS4
+ if self.__proxy[4] != None:
+ req = req + self.__proxy[4]
+ req = req + chr(0x00).encode()
+ # DNS name if remote resolving is required
+ # NOTE: This is actually an extension to the SOCKS4 protocol
+ # called SOCKS4A and may not be supported in all cases.
+ if rmtrslv:
+ req = req + destaddr + chr(0x00).encode()
+ self.sendall(req)
+ # Get the response from the server
+ resp = self.__recvall(8)
+ if resp[0:1] != chr(0x00).encode():
+ # Bad data
+ self.close()
+ raise GeneralProxyError((1,_generalerrors[1]))
+ if resp[1:2] != chr(0x5A).encode():
+ # Server returned an error
+ self.close()
+ if ord(resp[1:2]) in (91, 92, 93):
+ self.close()
+ raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90]))
+ else:
+ raise Socks4Error((94, _socks4errors[4]))
+ # Get the bound address/port
+ self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0])
+ if rmtrslv != None:
+ self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
+ else:
+ self.__proxypeername = (destaddr, destport)
+
+ def __negotiatehttp(self, destaddr, destport):
+ """__negotiatehttp(self,destaddr,destport)
+ Negotiates a connection through an HTTP server.
+ """
+ # If we need to resolve locally, we do this now
+ if not self.__proxy[3]:
+ addr = socket.gethostbyname(destaddr)
+ else:
+ addr = destaddr
+ headers = ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"]
+ headers += ["Host: ", destaddr, "\r\n"]
+ if (self.__proxy[4] != None and self.__proxy[5] != None):
+ headers += [self.__getauthheader(), "\r\n"]
+ headers.append("\r\n")
+ self.sendall("".join(headers).encode())
+ # We read the response until we get the string "\r\n\r\n"
+ resp = self.recv(1)
+ while resp.find("\r\n\r\n".encode()) == -1:
+ resp = resp + self.recv(1)
+ # We just need the first line to check if the connection
+ # was successful
+ statusline = resp.splitlines()[0].split(" ".encode(), 2)
+ if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()):
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ try:
+ statuscode = int(statusline[1])
+ except ValueError:
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ if statuscode != 200:
+ self.close()
+ raise HTTPError((statuscode, statusline[2]))
+ self.__proxysockname = ("0.0.0.0", 0)
+ self.__proxypeername = (addr, destport)
+
+ def connect(self, destpair):
+ """connect(self, despair)
+ Connects to the specified destination through a proxy.
+ destpar - A tuple of the IP/DNS address and the port number.
+ (identical to socket's connect).
+ To select the proxy server use setproxy().
+ """
+ # Do a minimal input check first
+ if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (not isinstance(destpair[0], basestring)) or (type(destpair[1]) != int):
+ raise GeneralProxyError((5, _generalerrors[5]))
+ if self.__proxy[0] == PROXY_TYPE_SOCKS5:
+ if self.__proxy[2] != None:
+ portnum = self.__proxy[2]
+ else:
+ portnum = 1080
+ _orgsocket.connect(self, (self.__proxy[1], portnum))
+ self.__negotiatesocks5(destpair[0], destpair[1])
+ elif self.__proxy[0] == PROXY_TYPE_SOCKS4:
+ if self.__proxy[2] != None:
+ portnum = self.__proxy[2]
+ else:
+ portnum = 1080
+ _orgsocket.connect(self,(self.__proxy[1], portnum))
+ self.__negotiatesocks4(destpair[0], destpair[1])
+ elif self.__proxy[0] == PROXY_TYPE_HTTP:
+ if self.__proxy[2] != None:
+ portnum = self.__proxy[2]
+ else:
+ portnum = 8080
+ _orgsocket.connect(self,(self.__proxy[1], portnum))
+ self.__negotiatehttp(destpair[0], destpair[1])
+ elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL:
+ if self.__proxy[2] != None:
+ portnum = self.__proxy[2]
+ else:
+ portnum = 8080
+ _orgsocket.connect(self,(self.__proxy[1],portnum))
+ if destpair[1] == 443:
+ self.__negotiatehttp(destpair[0],destpair[1])
+ else:
+ self.__httptunnel = False
+ elif self.__proxy[0] == None:
+ _orgsocket.connect(self, (destpair[0], destpair[1]))
+ else:
+ raise GeneralProxyError((4, _generalerrors[4]))
diff --git a/py/minijack/models.py b/py/minijack/models.py
index 43de956..0e451a6 100644
--- a/py/minijack/models.py
+++ b/py/minijack/models.py
@@ -7,72 +7,76 @@
class Event(models.Model):
- event_id = models.TextField(primary_key=True, db_index=True)
- device_id = models.TextField(db_index=True)
- time = models.TextField()
- event = models.TextField(db_index=True)
- seq = models.IntegerField()
- log_id = models.TextField()
- prefix = models.TextField()
- boot_id = models.TextField()
- boot_sequence = models.IntegerField()
+ event_id = models.TextField(primary_key=True, db_index=True)
+ device_id = models.TextField(db_index=True)
+ time = models.TextField()
+ event = models.TextField(db_index=True)
+ seq = models.IntegerField()
+ log_id = models.TextField()
+ prefix = models.TextField()
+ boot_id = models.TextField()
+ boot_sequence = models.IntegerField()
factory_md5sum = models.TextField()
- reimage_id = models.TextField()
+ reimage_id = models.TextField()
class Attr(models.Model):
# No primary_key for the Attr table for speed-up. Duplication check is
# done using the Event table.
event_id = models.TextField(db_index=True)
- attr = models.TextField()
- value = models.TextField()
+ attr = models.TextField()
+ value = models.TextField()
+ nested_parent = Event
+ nested_name = 'attrs'
class Test(models.Model):
- invocation = models.TextField(primary_key=True)
- event_id = models.TextField()
- event_seq = models.IntegerField()
- device_id = models.TextField(db_index=True)
+ invocation = models.TextField(primary_key=True)
+ event_id = models.TextField()
+ event_seq = models.IntegerField()
+ device_id = models.TextField(db_index=True)
factory_md5sum = models.TextField()
- reimage_id = models.TextField()
- path = models.TextField(db_index=True)
- pytest_name = models.TextField()
- status = models.TextField()
- start_time = models.TextField()
- end_time = models.TextField()
- duration = models.FloatField()
- error_msg = models.TextField()
+ reimage_id = models.TextField()
+ path = models.TextField(db_index=True)
+ pytest_name = models.TextField()
+ status = models.TextField()
+ start_time = models.TextField()
+ end_time = models.TextField()
+ duration = models.FloatField()
+ error_msg = models.TextField()
class Device(models.Model):
- device_id = models.TextField(primary_key=True, db_index=True)
- goofy_init_time = models.TextField()
- serial = models.TextField()
- mlb_serial = models.TextField()
- hwid = models.TextField()
- ips = models.TextField()
- ips_time = models.TextField()
- latest_test = models.TextField()
- latest_test_time = models.TextField()
- latest_ended_test = models.TextField()
+ device_id = models.TextField(primary_key=True, db_index=True)
+ goofy_init_time = models.TextField()
+ serial = models.TextField()
+ mlb_serial = models.TextField()
+ hwid = models.TextField()
+ ips = models.TextField()
+ ips_time = models.TextField()
+ latest_test = models.TextField()
+ latest_test_time = models.TextField()
+ latest_ended_test = models.TextField()
latest_ended_status = models.TextField()
- count_passed = models.IntegerField()
- count_failed = models.IntegerField()
- minijack_status = models.TextField()
- latest_note_level = models.TextField()
- latest_note_name = models.TextField()
- latest_note_text = models.TextField()
- latest_note_time = models.TextField()
+ count_passed = models.IntegerField()
+ count_failed = models.IntegerField()
+ minijack_status = models.TextField()
+ latest_note_level = models.TextField()
+ latest_note_name = models.TextField()
+ latest_note_text = models.TextField()
+ latest_note_time = models.TextField()
class Component(models.Model):
- device_id = models.TextField(primary_key=True, db_index=True)
+ device_id = models.TextField(primary_key=True, db_index=True)
component_class = models.TextField(primary_key=True, db_index=True)
- component_name = models.TextField()
+ component_name = models.TextField()
class ComponentDetail(models.Model):
- device_id = models.TextField(primary_key=True, db_index=True)
+ device_id = models.TextField(primary_key=True, db_index=True)
component_class = models.TextField(primary_key=True, db_index=True)
- field_name = models.TextField(primary_key=True, db_index=True)
- field_value = models.TextField()
+ field_name = models.TextField(primary_key=True, db_index=True)
+ field_value = models.TextField()
+ nested_parent = Component
+ nested_name = 'details'
diff --git a/py/minijack/oauth2client/__init__.py b/py/minijack/oauth2client/__init__.py
new file mode 100644
index 0000000..7e4e122
--- /dev/null
+++ b/py/minijack/oauth2client/__init__.py
@@ -0,0 +1,5 @@
+__version__ = "1.1"
+
+GOOGLE_AUTH_URI = 'https://accounts.google.com/o/oauth2/auth'
+GOOGLE_REVOKE_URI = 'https://accounts.google.com/o/oauth2/revoke'
+GOOGLE_TOKEN_URI = 'https://accounts.google.com/o/oauth2/token'
diff --git a/py/minijack/oauth2client/anyjson.py b/py/minijack/oauth2client/anyjson.py
new file mode 100644
index 0000000..ae21c33
--- /dev/null
+++ b/py/minijack/oauth2client/anyjson.py
@@ -0,0 +1,32 @@
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utility module to import a JSON module
+
+Hides all the messy details of exactly where
+we get a simplejson module from.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+
+try: # pragma: no cover
+ # Should work for Python2.6 and higher.
+ import json as simplejson
+except ImportError: # pragma: no cover
+ try:
+ import simplejson
+ except ImportError:
+ # Try to import from django, should work on App Engine
+ from django.utils import simplejson
diff --git a/py/minijack/oauth2client/appengine.py b/py/minijack/oauth2client/appengine.py
new file mode 100644
index 0000000..a6d88df
--- /dev/null
+++ b/py/minijack/oauth2client/appengine.py
@@ -0,0 +1,896 @@
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for Google App Engine
+
+Utilities for making it easier to use OAuth 2.0 on Google App Engine.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+import base64
+import cgi
+import httplib2
+import logging
+import os
+import pickle
+import time
+
+from google.appengine.api import app_identity
+from google.appengine.api import memcache
+from google.appengine.api import users
+from google.appengine.ext import db
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp.util import login_required
+from google.appengine.ext.webapp.util import run_wsgi_app
+from oauth2client import GOOGLE_AUTH_URI
+from oauth2client import GOOGLE_REVOKE_URI
+from oauth2client import GOOGLE_TOKEN_URI
+from oauth2client import clientsecrets
+from oauth2client import util
+from oauth2client import xsrfutil
+from oauth2client.anyjson import simplejson
+from oauth2client.client import AccessTokenRefreshError
+from oauth2client.client import AssertionCredentials
+from oauth2client.client import Credentials
+from oauth2client.client import Flow
+from oauth2client.client import OAuth2WebServerFlow
+from oauth2client.client import Storage
+
+# TODO(dhermes): Resolve import issue.
+# This is a temporary fix for a Google internal issue.
+try:
+ from google.appengine.ext import ndb
+except ImportError:
+ ndb = None
+
+
+logger = logging.getLogger(__name__)
+
+OAUTH2CLIENT_NAMESPACE = 'oauth2client#ns'
+
+XSRF_MEMCACHE_ID = 'xsrf_secret_key'
+
+
+def _safe_html(s):
+ """Escape text to make it safe to display.
+
+ Args:
+ s: string, The text to escape.
+
+ Returns:
+ The escaped text as a string.
+ """
+ return cgi.escape(s, quote=1).replace("'", ''')
+
+
+class InvalidClientSecretsError(Exception):
+ """The client_secrets.json file is malformed or missing required fields."""
+
+
+class InvalidXsrfTokenError(Exception):
+ """The XSRF token is invalid or expired."""
+
+
+class SiteXsrfSecretKey(db.Model):
+ """Storage for the sites XSRF secret key.
+
+ There will only be one instance stored of this model, the one used for the
+ site.
+ """
+ secret = db.StringProperty()
+
+if ndb is not None:
+ class SiteXsrfSecretKeyNDB(ndb.Model):
+ """NDB Model for storage for the sites XSRF secret key.
+
+ Since this model uses the same kind as SiteXsrfSecretKey, it can be used
+ interchangeably. This simply provides an NDB model for interacting with the
+ same data the DB model interacts with.
+
+ There should only be one instance stored of this model, the one used for the
+ site.
+ """
+ secret = ndb.StringProperty()
+
+ @classmethod
+ def _get_kind(cls):
+ """Return the kind name for this class."""
+ return 'SiteXsrfSecretKey'
+
+
+def _generate_new_xsrf_secret_key():
+ """Returns a random XSRF secret key.
+ """
+ return os.urandom(16).encode("hex")
+
+
+def xsrf_secret_key():
+ """Return the secret key for use for XSRF protection.
+
+ If the Site entity does not have a secret key, this method will also create
+ one and persist it.
+
+ Returns:
+ The secret key.
+ """
+ secret = memcache.get(XSRF_MEMCACHE_ID, namespace=OAUTH2CLIENT_NAMESPACE)
+ if not secret:
+ # Load the one and only instance of SiteXsrfSecretKey.
+ model = SiteXsrfSecretKey.get_or_insert(key_name='site')
+ if not model.secret:
+ model.secret = _generate_new_xsrf_secret_key()
+ model.put()
+ secret = model.secret
+ memcache.add(XSRF_MEMCACHE_ID, secret, namespace=OAUTH2CLIENT_NAMESPACE)
+
+ return str(secret)
+
+
+class AppAssertionCredentials(AssertionCredentials):
+ """Credentials object for App Engine Assertion Grants
+
+ This object will allow an App Engine application to identify itself to Google
+ and other OAuth 2.0 servers that can verify assertions. It can be used for the
+ purpose of accessing data stored under an account assigned to the App Engine
+ application itself.
+
+ This credential does not require a flow to instantiate because it represents
+ a two legged flow, and therefore has all of the required information to
+ generate and refresh its own access tokens.
+ """
+
+ @util.positional(2)
+ def __init__(self, scope, **kwargs):
+ """Constructor for AppAssertionCredentials
+
+ Args:
+ scope: string or iterable of strings, scope(s) of the credentials being
+ requested.
+ """
+ self.scope = util.scopes_to_string(scope)
+
+ # Assertion type is no longer used, but still in the parent class signature.
+ super(AppAssertionCredentials, self).__init__(None)
+
+ @classmethod
+ def from_json(cls, json):
+ data = simplejson.loads(json)
+ return AppAssertionCredentials(data['scope'])
+
+ def _refresh(self, http_request):
+ """Refreshes the access_token.
+
+ Since the underlying App Engine app_identity implementation does its own
+ caching we can skip all the storage hoops and just to a refresh using the
+ API.
+
+ Args:
+ http_request: callable, a callable that matches the method signature of
+ httplib2.Http.request, used to make the refresh request.
+
+ Raises:
+ AccessTokenRefreshError: When the refresh fails.
+ """
+ try:
+ scopes = self.scope.split()
+ (token, _) = app_identity.get_access_token(scopes)
+ except app_identity.Error, e:
+ raise AccessTokenRefreshError(str(e))
+ self.access_token = token
+
+
+class FlowProperty(db.Property):
+ """App Engine datastore Property for Flow.
+
+ Utility property that allows easy storage and retrieval of an
+ oauth2client.Flow"""
+
+ # Tell what the user type is.
+ data_type = Flow
+
+ # For writing to datastore.
+ def get_value_for_datastore(self, model_instance):
+ flow = super(FlowProperty,
+ self).get_value_for_datastore(model_instance)
+ return db.Blob(pickle.dumps(flow))
+
+ # For reading from datastore.
+ def make_value_from_datastore(self, value):
+ if value is None:
+ return None
+ return pickle.loads(value)
+
+ def validate(self, value):
+ if value is not None and not isinstance(value, Flow):
+ raise db.BadValueError('Property %s must be convertible '
+ 'to a FlowThreeLegged instance (%s)' %
+ (self.name, value))
+ return super(FlowProperty, self).validate(value)
+
+ def empty(self, value):
+ return not value
+
+
+if ndb is not None:
+ class FlowNDBProperty(ndb.PickleProperty):
+ """App Engine NDB datastore Property for Flow.
+
+ Serves the same purpose as the DB FlowProperty, but for NDB models. Since
+ PickleProperty inherits from BlobProperty, the underlying representation of
+ the data in the datastore will be the same as in the DB case.
+
+ Utility property that allows easy storage and retrieval of an
+ oauth2client.Flow
+ """
+
+ def _validate(self, value):
+ """Validates a value as a proper Flow object.
+
+ Args:
+ value: A value to be set on the property.
+
+ Raises:
+ TypeError if the value is not an instance of Flow.
+ """
+ logger.info('validate: Got type %s', type(value))
+ if value is not None and not isinstance(value, Flow):
+ raise TypeError('Property %s must be convertible to a flow '
+ 'instance; received: %s.' % (self._name, value))
+
+
+class CredentialsProperty(db.Property):
+ """App Engine datastore Property for Credentials.
+
+ Utility property that allows easy storage and retrieval of
+ oath2client.Credentials
+ """
+
+ # Tell what the user type is.
+ data_type = Credentials
+
+ # For writing to datastore.
+ def get_value_for_datastore(self, model_instance):
+ logger.info("get: Got type " + str(type(model_instance)))
+ cred = super(CredentialsProperty,
+ self).get_value_for_datastore(model_instance)
+ if cred is None:
+ cred = ''
+ else:
+ cred = cred.to_json()
+ return db.Blob(cred)
+
+ # For reading from datastore.
+ def make_value_from_datastore(self, value):
+ logger.info("make: Got type " + str(type(value)))
+ if value is None:
+ return None
+ if len(value) == 0:
+ return None
+ try:
+ credentials = Credentials.new_from_json(value)
+ except ValueError:
+ credentials = None
+ return credentials
+
+ def validate(self, value):
+ value = super(CredentialsProperty, self).validate(value)
+ logger.info("validate: Got type " + str(type(value)))
+ if value is not None and not isinstance(value, Credentials):
+ raise db.BadValueError('Property %s must be convertible '
+ 'to a Credentials instance (%s)' %
+ (self.name, value))
+ #if value is not None and not isinstance(value, Credentials):
+ # return None
+ return value
+
+
+if ndb is not None:
+ # TODO(dhermes): Turn this into a JsonProperty and overhaul the Credentials
+ # and subclass mechanics to use new_from_dict, to_dict,
+ # from_dict, etc.
+ class CredentialsNDBProperty(ndb.BlobProperty):
+ """App Engine NDB datastore Property for Credentials.
+
+ Serves the same purpose as the DB CredentialsProperty, but for NDB models.
+ Since CredentialsProperty stores data as a blob and this inherits from
+ BlobProperty, the data in the datastore will be the same as in the DB case.
+
+ Utility property that allows easy storage and retrieval of Credentials and
+ subclasses.
+ """
+ def _validate(self, value):
+ """Validates a value as a proper credentials object.
+
+ Args:
+ value: A value to be set on the property.
+
+ Raises:
+ TypeError if the value is not an instance of Credentials.
+ """
+ logger.info('validate: Got type %s', type(value))
+ if value is not None and not isinstance(value, Credentials):
+ raise TypeError('Property %s must be convertible to a credentials '
+ 'instance; received: %s.' % (self._name, value))
+
+ def _to_base_type(self, value):
+ """Converts our validated value to a JSON serialized string.
+
+ Args:
+ value: A value to be set in the datastore.
+
+ Returns:
+ A JSON serialized version of the credential, else '' if value is None.
+ """
+ if value is None:
+ return ''
+ else:
+ return value.to_json()
+
+ def _from_base_type(self, value):
+ """Converts our stored JSON string back to the desired type.
+
+ Args:
+ value: A value from the datastore to be converted to the desired type.
+
+ Returns:
+ A deserialized Credentials (or subclass) object, else None if the
+ value can't be parsed.
+ """
+ if not value:
+ return None
+ try:
+ # Uses the from_json method of the implied class of value
+ credentials = Credentials.new_from_json(value)
+ except ValueError:
+ credentials = None
+ return credentials
+
+
+class StorageByKeyName(Storage):
+ """Store and retrieve a credential to and from the App Engine datastore.
+
+ This Storage helper presumes the Credentials have been stored as a
+ CredentialsProperty or CredentialsNDBProperty on a datastore model class, and
+ that entities are stored by key_name.
+ """
+
+ @util.positional(4)
+ def __init__(self, model, key_name, property_name, cache=None):
+ """Constructor for Storage.
+
+ Args:
+ model: db.Model or ndb.Model, model class
+ key_name: string, key name for the entity that has the credentials
+ property_name: string, name of the property that is a CredentialsProperty
+ or CredentialsNDBProperty.
+ cache: memcache, a write-through cache to put in front of the datastore.
+ If the model you are using is an NDB model, using a cache will be
+ redundant since the model uses an instance cache and memcache for you.
+ """
+ self._model = model
+ self._key_name = key_name
+ self._property_name = property_name
+ self._cache = cache
+
+ def _is_ndb(self):
+ """Determine whether the model of the instance is an NDB model.
+
+ Returns:
+ Boolean indicating whether or not the model is an NDB or DB model.
+ """
+ # issubclass will fail if one of the arguments is not a class, only need
+ # worry about new-style classes since ndb and db models are new-style
+ if isinstance(self._model, type):
+ if ndb is not None and issubclass(self._model, ndb.Model):
+ return True
+ elif issubclass(self._model, db.Model):
+ return False
+
+ raise TypeError('Model class not an NDB or DB model: %s.' % (self._model,))
+
+ def _get_entity(self):
+ """Retrieve entity from datastore.
+
+ Uses a different model method for db or ndb models.
+
+ Returns:
+ Instance of the model corresponding to the current storage object
+ and stored using the key name of the storage object.
+ """
+ if self._is_ndb():
+ return self._model.get_by_id(self._key_name)
+ else:
+ return self._model.get_by_key_name(self._key_name)
+
+ def _delete_entity(self):
+ """Delete entity from datastore.
+
+ Attempts to delete using the key_name stored on the object, whether or not
+ the given key is in the datastore.
+ """
+ if self._is_ndb():
+ ndb.Key(self._model, self._key_name).delete()
+ else:
+ entity_key = db.Key.from_path(self._model.kind(), self._key_name)
+ db.delete(entity_key)
+
+ def locked_get(self):
+ """Retrieve Credential from datastore.
+
+ Returns:
+ oauth2client.Credentials
+ """
+ if self._cache:
+ json = self._cache.get(self._key_name)
+ if json:
+ return Credentials.new_from_json(json)
+
+ credentials = None
+ entity = self._get_entity()
+ if entity is not None:
+ credentials = getattr(entity, self._property_name)
+ if credentials and hasattr(credentials, 'set_store'):
+ credentials.set_store(self)
+ if self._cache:
+ self._cache.set(self._key_name, credentials.to_json())
+
+ return credentials
+
+ def locked_put(self, credentials):
+ """Write a Credentials to the datastore.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+ """
+ entity = self._model.get_or_insert(self._key_name)
+ setattr(entity, self._property_name, credentials)
+ entity.put()
+ if self._cache:
+ self._cache.set(self._key_name, credentials.to_json())
+
+ def locked_delete(self):
+ """Delete Credential from datastore."""
+
+ if self._cache:
+ self._cache.delete(self._key_name)
+
+ self._delete_entity()
+
+
+class CredentialsModel(db.Model):
+ """Storage for OAuth 2.0 Credentials
+
+ Storage of the model is keyed by the user.user_id().
+ """
+ credentials = CredentialsProperty()
+
+
+if ndb is not None:
+ class CredentialsNDBModel(ndb.Model):
+ """NDB Model for storage of OAuth 2.0 Credentials
+
+ Since this model uses the same kind as CredentialsModel and has a property
+ which can serialize and deserialize Credentials correctly, it can be used
+ interchangeably with a CredentialsModel to access, insert and delete the
+ same entities. This simply provides an NDB model for interacting with the
+ same data the DB model interacts with.
+
+ Storage of the model is keyed by the user.user_id().
+ """
+ credentials = CredentialsNDBProperty()
+
+ @classmethod
+ def _get_kind(cls):
+ """Return the kind name for this class."""
+ return 'CredentialsModel'
+
+
+def _build_state_value(request_handler, user):
+ """Composes the value for the 'state' parameter.
+
+ Packs the current request URI and an XSRF token into an opaque string that
+ can be passed to the authentication server via the 'state' parameter.
+
+ Args:
+ request_handler: webapp.RequestHandler, The request.
+ user: google.appengine.api.users.User, The current user.
+
+ Returns:
+ The state value as a string.
+ """
+ uri = request_handler.request.url
+ token = xsrfutil.generate_token(xsrf_secret_key(), user.user_id(),
+ action_id=str(uri))
+ return uri + ':' + token
+
+
+def _parse_state_value(state, user):
+ """Parse the value of the 'state' parameter.
+
+ Parses the value and validates the XSRF token in the state parameter.
+
+ Args:
+ state: string, The value of the state parameter.
+ user: google.appengine.api.users.User, The current user.
+
+ Raises:
+ InvalidXsrfTokenError: if the XSRF token is invalid.
+
+ Returns:
+ The redirect URI.
+ """
+ uri, token = state.rsplit(':', 1)
+ if not xsrfutil.validate_token(xsrf_secret_key(), token, user.user_id(),
+ action_id=uri):
+ raise InvalidXsrfTokenError()
+
+ return uri
+
+
+class OAuth2Decorator(object):
+ """Utility for making OAuth 2.0 easier.
+
+ Instantiate and then use with oauth_required or oauth_aware
+ as decorators on webapp.RequestHandler methods.
+
+ Example:
+
+ decorator = OAuth2Decorator(
+ client_id='837...ent.com',
+ client_secret='Qh...wwI',
+ scope='https://www.googleapis.com/auth/plus')
+
+
+ class MainHandler(webapp.RequestHandler):
+
+ @decorator.oauth_required
+ def get(self):
+ http = decorator.http()
+ # http is authorized with the user's Credentials and can be used
+ # in API calls
+
+ """
+
+ @util.positional(4)
+ def __init__(self, client_id, client_secret, scope,
+ auth_uri=GOOGLE_AUTH_URI,
+ token_uri=GOOGLE_TOKEN_URI,
+ revoke_uri=GOOGLE_REVOKE_URI,
+ user_agent=None,
+ message=None,
+ callback_path='/oauth2callback',
+ token_response_param=None,
+ **kwargs):
+
+ """Constructor for OAuth2Decorator
+
+ Args:
+ client_id: string, client identifier.
+ client_secret: string client secret.
+ scope: string or iterable of strings, scope(s) of the credentials being
+ requested.
+ auth_uri: string, URI for authorization endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ token_uri: string, URI for token endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ revoke_uri: string, URI for revoke endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ user_agent: string, User agent of your application, default to None.
+ message: Message to display if there are problems with the OAuth 2.0
+ configuration. The message may contain HTML and will be presented on the
+ web interface for any method that uses the decorator.
+ callback_path: string, The absolute path to use as the callback URI. Note
+ that this must match up with the URI given when registering the
+ application in the APIs Console.
+ token_response_param: string. If provided, the full JSON response
+ to the access token request will be encoded and included in this query
+ parameter in the callback URI. This is useful with providers (e.g.
+ wordpress.com) that include extra fields that the client may want.
+ **kwargs: dict, Keyword arguments are be passed along as kwargs to the
+ OAuth2WebServerFlow constructor.
+ """
+ self.flow = None
+ self.credentials = None
+ self._client_id = client_id
+ self._client_secret = client_secret
+ self._scope = util.scopes_to_string(scope)
+ self._auth_uri = auth_uri
+ self._token_uri = token_uri
+ self._revoke_uri = revoke_uri
+ self._user_agent = user_agent
+ self._kwargs = kwargs
+ self._message = message
+ self._in_error = False
+ self._callback_path = callback_path
+ self._token_response_param = token_response_param
+
+ def _display_error_message(self, request_handler):
+ request_handler.response.out.write('<html><body>')
+ request_handler.response.out.write(_safe_html(self._message))
+ request_handler.response.out.write('</body></html>')
+
+ def oauth_required(self, method):
+ """Decorator that starts the OAuth 2.0 dance.
+
+ Starts the OAuth dance for the logged in user if they haven't already
+ granted access for this application.
+
+ Args:
+ method: callable, to be decorated method of a webapp.RequestHandler
+ instance.
+ """
+
+ def check_oauth(request_handler, *args, **kwargs):
+ if self._in_error:
+ self._display_error_message(request_handler)
+ return
+
+ user = users.get_current_user()
+ # Don't use @login_decorator as this could be used in a POST request.
+ if not user:
+ request_handler.redirect(users.create_login_url(
+ request_handler.request.uri))
+ return
+
+ self._create_flow(request_handler)
+
+ # Store the request URI in 'state' so we can use it later
+ self.flow.params['state'] = _build_state_value(request_handler, user)
+ self.credentials = StorageByKeyName(
+ CredentialsModel, user.user_id(), 'credentials').get()
+
+ if not self.has_credentials():
+ return request_handler.redirect(self.authorize_url())
+ try:
+ return method(request_handler, *args, **kwargs)
+ except AccessTokenRefreshError:
+ return request_handler.redirect(self.authorize_url())
+
+ return check_oauth
+
+ def _create_flow(self, request_handler):
+ """Create the Flow object.
+
+ The Flow is calculated lazily since we don't know where this app is
+ running until it receives a request, at which point redirect_uri can be
+ calculated and then the Flow object can be constructed.
+
+ Args:
+ request_handler: webapp.RequestHandler, the request handler.
+ """
+ if self.flow is None:
+ redirect_uri = request_handler.request.relative_url(
+ self._callback_path) # Usually /oauth2callback
+ self.flow = OAuth2WebServerFlow(self._client_id, self._client_secret,
+ self._scope, redirect_uri=redirect_uri,
+ user_agent=self._user_agent,
+ auth_uri=self._auth_uri,
+ token_uri=self._token_uri,
+ revoke_uri=self._revoke_uri,
+ **self._kwargs)
+
+ def oauth_aware(self, method):
+ """Decorator that sets up for OAuth 2.0 dance, but doesn't do it.
+
+ Does all the setup for the OAuth dance, but doesn't initiate it.
+ This decorator is useful if you want to create a page that knows
+ whether or not the user has granted access to this application.
+ From within a method decorated with @oauth_aware the has_credentials()
+ and authorize_url() methods can be called.
+
+ Args:
+ method: callable, to be decorated method of a webapp.RequestHandler
+ instance.
+ """
+
+ def setup_oauth(request_handler, *args, **kwargs):
+ if self._in_error:
+ self._display_error_message(request_handler)
+ return
+
+ user = users.get_current_user()
+ # Don't use @login_decorator as this could be used in a POST request.
+ if not user:
+ request_handler.redirect(users.create_login_url(
+ request_handler.request.uri))
+ return
+
+ self._create_flow(request_handler)
+
+ self.flow.params['state'] = _build_state_value(request_handler, user)
+ self.credentials = StorageByKeyName(
+ CredentialsModel, user.user_id(), 'credentials').get()
+ return method(request_handler, *args, **kwargs)
+ return setup_oauth
+
+ def has_credentials(self):
+ """True if for the logged in user there are valid access Credentials.
+
+ Must only be called from with a webapp.RequestHandler subclassed method
+ that had been decorated with either @oauth_required or @oauth_aware.
+ """
+ return self.credentials is not None and not self.credentials.invalid
+
+ def authorize_url(self):
+ """Returns the URL to start the OAuth dance.
+
+ Must only be called from with a webapp.RequestHandler subclassed method
+ that had been decorated with either @oauth_required or @oauth_aware.
+ """
+ url = self.flow.step1_get_authorize_url()
+ return str(url)
+
+ def http(self):
+ """Returns an authorized http instance.
+
+ Must only be called from within an @oauth_required decorated method, or
+ from within an @oauth_aware decorated method where has_credentials()
+ returns True.
+ """
+ return self.credentials.authorize(httplib2.Http())
+
+ @property
+ def callback_path(self):
+ """The absolute path where the callback will occur.
+
+ Note this is the absolute path, not the absolute URI, that will be
+ calculated by the decorator at runtime. See callback_handler() for how this
+ should be used.
+
+ Returns:
+ The callback path as a string.
+ """
+ return self._callback_path
+
+
+ def callback_handler(self):
+ """RequestHandler for the OAuth 2.0 redirect callback.
+
+ Usage:
+ app = webapp.WSGIApplication([
+ ('/index', MyIndexHandler),
+ ...,
+ (decorator.callback_path, decorator.callback_handler())
+ ])
+
+ Returns:
+ A webapp.RequestHandler that handles the redirect back from the
+ server during the OAuth 2.0 dance.
+ """
+ decorator = self
+
+ class OAuth2Handler(webapp.RequestHandler):
+ """Handler for the redirect_uri of the OAuth 2.0 dance."""
+
+ @login_required
+ def get(self):
+ error = self.request.get('error')
+ if error:
+ errormsg = self.request.get('error_description', error)
+ self.response.out.write(
+ 'The authorization request failed: %s' % _safe_html(errormsg))
+ else:
+ user = users.get_current_user()
+ decorator._create_flow(self)
+ credentials = decorator.flow.step2_exchange(self.request.params)
+ StorageByKeyName(
+ CredentialsModel, user.user_id(), 'credentials').put(credentials)
+ redirect_uri = _parse_state_value(str(self.request.get('state')),
+ user)
+
+ if decorator._token_response_param and credentials.token_response:
+ resp_json = simplejson.dumps(credentials.token_response)
+ redirect_uri = util._add_query_parameter(
+ redirect_uri, decorator._token_response_param, resp_json)
+
+ self.redirect(redirect_uri)
+
+ return OAuth2Handler
+
+ def callback_application(self):
+ """WSGI application for handling the OAuth 2.0 redirect callback.
+
+ If you need finer grained control use `callback_handler` which returns just
+ the webapp.RequestHandler.
+
+ Returns:
+ A webapp.WSGIApplication that handles the redirect back from the
+ server during the OAuth 2.0 dance.
+ """
+ return webapp.WSGIApplication([
+ (self.callback_path, self.callback_handler())
+ ])
+
+
+class OAuth2DecoratorFromClientSecrets(OAuth2Decorator):
+ """An OAuth2Decorator that builds from a clientsecrets file.
+
+ Uses a clientsecrets file as the source for all the information when
+ constructing an OAuth2Decorator.
+
+ Example:
+
+ decorator = OAuth2DecoratorFromClientSecrets(
+ os.path.join(os.path.dirname(__file__), 'client_secrets.json')
+ scope='https://www.googleapis.com/auth/plus')
+
+
+ class MainHandler(webapp.RequestHandler):
+
+ @decorator.oauth_required
+ def get(self):
+ http = decorator.http()
+ # http is authorized with the user's Credentials and can be used
+ # in API calls
+ """
+
+ @util.positional(3)
+ def __init__(self, filename, scope, message=None, cache=None):
+ """Constructor
+
+ Args:
+ filename: string, File name of client secrets.
+ scope: string or iterable of strings, scope(s) of the credentials being
+ requested.
+ message: string, A friendly string to display to the user if the
+ clientsecrets file is missing or invalid. The message may contain HTML
+ and will be presented on the web interface for any method that uses the
+ decorator.
+ cache: An optional cache service client that implements get() and set()
+ methods. See clientsecrets.loadfile() for details.
+ """
+ client_type, client_info = clientsecrets.loadfile(filename, cache=cache)
+ if client_type not in [
+ clientsecrets.TYPE_WEB, clientsecrets.TYPE_INSTALLED]:
+ raise InvalidClientSecretsError(
+ 'OAuth2Decorator doesn\'t support this OAuth 2.0 flow.')
+ constructor_kwargs = {
+ 'auth_uri': client_info['auth_uri'],
+ 'token_uri': client_info['token_uri'],
+ 'message': message,
+ }
+ revoke_uri = client_info.get('revoke_uri')
+ if revoke_uri is not None:
+ constructor_kwargs['revoke_uri'] = revoke_uri
+ super(OAuth2DecoratorFromClientSecrets, self).__init__(
+ client_info['client_id'], client_info['client_secret'],
+ scope, **constructor_kwargs)
+ if message is not None:
+ self._message = message
+ else:
+ self._message = 'Please configure your application for OAuth 2.0.'
+
+
+@util.positional(2)
+def oauth2decorator_from_clientsecrets(filename, scope,
+ message=None, cache=None):
+ """Creates an OAuth2Decorator populated from a clientsecrets file.
+
+ Args:
+ filename: string, File name of client secrets.
+ scope: string or list of strings, scope(s) of the credentials being
+ requested.
+ message: string, A friendly string to display to the user if the
+ clientsecrets file is missing or invalid. The message may contain HTML and
+ will be presented on the web interface for any method that uses the
+ decorator.
+ cache: An optional cache service client that implements get() and set()
+ methods. See clientsecrets.loadfile() for details.
+
+ Returns: An OAuth2Decorator
+
+ """
+ return OAuth2DecoratorFromClientSecrets(filename, scope,
+ message=message, cache=cache)
diff --git a/py/minijack/oauth2client/client.py b/py/minijack/oauth2client/client.py
new file mode 100644
index 0000000..6b580a0
--- /dev/null
+++ b/py/minijack/oauth2client/client.py
@@ -0,0 +1,1364 @@
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An OAuth 2.0 client.
+
+Tools for interacting with OAuth 2.0 protected resources.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+import base64
+import clientsecrets
+import copy
+import datetime
+import httplib2
+import logging
+import os
+import sys
+import time
+import urllib
+import urlparse
+
+from oauth2client import GOOGLE_AUTH_URI
+from oauth2client import GOOGLE_REVOKE_URI
+from oauth2client import GOOGLE_TOKEN_URI
+from oauth2client import util
+from oauth2client.anyjson import simplejson
+
+HAS_OPENSSL = False
+HAS_CRYPTO = False
+try:
+ from oauth2client import crypt
+ HAS_CRYPTO = True
+ if crypt.OpenSSLVerifier is not None:
+ HAS_OPENSSL = True
+except ImportError:
+ pass
+
+try:
+ from urlparse import parse_qsl
+except ImportError:
+ from cgi import parse_qsl
+
+logger = logging.getLogger(__name__)
+
+# Expiry is stored in RFC3339 UTC format
+EXPIRY_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
+# Which certs to use to validate id_tokens received.
+ID_TOKEN_VERIFICATON_CERTS = 'https://www.googleapis.com/oauth2/v1/certs'
+
+# Constant to use for the out of band OAuth 2.0 flow.
+OOB_CALLBACK_URN = 'urn:ietf:wg:oauth:2.0:oob'
+
+# Google Data client libraries may need to set this to [401, 403].
+REFRESH_STATUS_CODES = [401]
+
+
+class Error(Exception):
+ """Base error for this module."""
+
+
+class FlowExchangeError(Error):
+ """Error trying to exchange an authorization grant for an access token."""
+
+
+class AccessTokenRefreshError(Error):
+ """Error trying to refresh an expired access token."""
+
+
+class TokenRevokeError(Error):
+ """Error trying to revoke a token."""
+
+
+class UnknownClientSecretsFlowError(Error):
+ """The client secrets file called for an unknown type of OAuth 2.0 flow. """
+
+
+class AccessTokenCredentialsError(Error):
+ """Having only the access_token means no refresh is possible."""
+
+
+class VerifyJwtTokenError(Error):
+ """Could on retrieve certificates for validation."""
+
+
+class NonAsciiHeaderError(Error):
+ """Header names and values must be ASCII strings."""
+
+
+def _abstract():
+ raise NotImplementedError('You need to override this function')
+
+
+class MemoryCache(object):
+ """httplib2 Cache implementation which only caches locally."""
+
+ def __init__(self):
+ self.cache = {}
+
+ def get(self, key):
+ return self.cache.get(key)
+
+ def set(self, key, value):
+ self.cache[key] = value
+
+ def delete(self, key):
+ self.cache.pop(key, None)
+
+
+class Credentials(object):
+ """Base class for all Credentials objects.
+
+ Subclasses must define an authorize() method that applies the credentials to
+ an HTTP transport.
+
+ Subclasses must also specify a classmethod named 'from_json' that takes a JSON
+ string as input and returns an instaniated Credentials object.
+ """
+
+ NON_SERIALIZED_MEMBERS = ['store']
+
+ def authorize(self, http):
+ """Take an httplib2.Http instance (or equivalent) and authorizes it.
+
+ Authorizes it for the set of credentials, usually by replacing
+ http.request() with a method that adds in the appropriate headers and then
+ delegates to the original Http.request() method.
+
+ Args:
+ http: httplib2.Http, an http object to be used to make the refresh
+ request.
+ """
+ _abstract()
+
+ def refresh(self, http):
+ """Forces a refresh of the access_token.
+
+ Args:
+ http: httplib2.Http, an http object to be used to make the refresh
+ request.
+ """
+ _abstract()
+
+ def revoke(self, http):
+ """Revokes a refresh_token and makes the credentials void.
+
+ Args:
+ http: httplib2.Http, an http object to be used to make the revoke
+ request.
+ """
+ _abstract()
+
+ def apply(self, headers):
+ """Add the authorization to the headers.
+
+ Args:
+ headers: dict, the headers to add the Authorization header to.
+ """
+ _abstract()
+
+ def _to_json(self, strip):
+ """Utility function that creates JSON repr. of a Credentials object.
+
+ Args:
+ strip: array, An array of names of members to not include in the JSON.
+
+ Returns:
+ string, a JSON representation of this instance, suitable to pass to
+ from_json().
+ """
+ t = type(self)
+ d = copy.copy(self.__dict__)
+ for member in strip:
+ if member in d:
+ del d[member]
+ if 'token_expiry' in d and isinstance(d['token_expiry'], datetime.datetime):
+ d['token_expiry'] = d['token_expiry'].strftime(EXPIRY_FORMAT)
+ # Add in information we will need later to reconsistitue this instance.
+ d['_class'] = t.__name__
+ d['_module'] = t.__module__
+ return simplejson.dumps(d)
+
+ def to_json(self):
+ """Creating a JSON representation of an instance of Credentials.
+
+ Returns:
+ string, a JSON representation of this instance, suitable to pass to
+ from_json().
+ """
+ return self._to_json(Credentials.NON_SERIALIZED_MEMBERS)
+
+ @classmethod
+ def new_from_json(cls, s):
+ """Utility class method to instantiate a Credentials subclass from a JSON
+ representation produced by to_json().
+
+ Args:
+ s: string, JSON from to_json().
+
+ Returns:
+ An instance of the subclass of Credentials that was serialized with
+ to_json().
+ """
+ data = simplejson.loads(s)
+ # Find and call the right classmethod from_json() to restore the object.
+ module = data['_module']
+ try:
+ m = __import__(module)
+ except ImportError:
+ # In case there's an object from the old package structure, update it
+ module = module.replace('.apiclient', '')
+ m = __import__(module)
+
+ m = __import__(module, fromlist=module.split('.')[:-1])
+ kls = getattr(m, data['_class'])
+ from_json = getattr(kls, 'from_json')
+ return from_json(s)
+
+ @classmethod
+ def from_json(cls, s):
+ """Instantiate a Credentials object from a JSON description of it.
+
+ The JSON should have been produced by calling .to_json() on the object.
+
+ Args:
+ data: dict, A deserialized JSON object.
+
+ Returns:
+ An instance of a Credentials subclass.
+ """
+ return Credentials()
+
+
+class Flow(object):
+ """Base class for all Flow objects."""
+ pass
+
+
+class Storage(object):
+ """Base class for all Storage objects.
+
+ Store and retrieve a single credential. This class supports locking
+ such that multiple processes and threads can operate on a single
+ store.
+ """
+
+ def acquire_lock(self):
+ """Acquires any lock necessary to access this Storage.
+
+ This lock is not reentrant.
+ """
+ pass
+
+ def release_lock(self):
+ """Release the Storage lock.
+
+ Trying to release a lock that isn't held will result in a
+ RuntimeError.
+ """
+ pass
+
+ def locked_get(self):
+ """Retrieve credential.
+
+ The Storage lock must be held when this is called.
+
+ Returns:
+ oauth2client.client.Credentials
+ """
+ _abstract()
+
+ def locked_put(self, credentials):
+ """Write a credential.
+
+ The Storage lock must be held when this is called.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+ """
+ _abstract()
+
+ def locked_delete(self):
+ """Delete a credential.
+
+ The Storage lock must be held when this is called.
+ """
+ _abstract()
+
+ def get(self):
+ """Retrieve credential.
+
+ The Storage lock must *not* be held when this is called.
+
+ Returns:
+ oauth2client.client.Credentials
+ """
+ self.acquire_lock()
+ try:
+ return self.locked_get()
+ finally:
+ self.release_lock()
+
+ def put(self, credentials):
+ """Write a credential.
+
+ The Storage lock must be held when this is called.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+ """
+ self.acquire_lock()
+ try:
+ self.locked_put(credentials)
+ finally:
+ self.release_lock()
+
+ def delete(self):
+ """Delete credential.
+
+ Frees any resources associated with storing the credential.
+ The Storage lock must *not* be held when this is called.
+
+ Returns:
+ None
+ """
+ self.acquire_lock()
+ try:
+ return self.locked_delete()
+ finally:
+ self.release_lock()
+
+
+def clean_headers(headers):
+ """Forces header keys and values to be strings, i.e not unicode.
+
+ The httplib module just concats the header keys and values in a way that may
+ make the message header a unicode string, which, if it then tries to
+ contatenate to a binary request body may result in a unicode decode error.
+
+ Args:
+ headers: dict, A dictionary of headers.
+
+ Returns:
+ The same dictionary but with all the keys converted to strings.
+ """
+ clean = {}
+ try:
+ for k, v in headers.iteritems():
+ clean[str(k)] = str(v)
+ except UnicodeEncodeError:
+ raise NonAsciiHeaderError(k + ': ' + v)
+ return clean
+
+
+def _update_query_params(uri, params):
+ """Updates a URI with new query parameters.
+
+ Args:
+ uri: string, A valid URI, with potential existing query parameters.
+ params: dict, A dictionary of query parameters.
+
+ Returns:
+ The same URI but with the new query parameters added.
+ """
+ parts = list(urlparse.urlparse(uri))
+ query_params = dict(parse_qsl(parts[4])) # 4 is the index of the query part
+ query_params.update(params)
+ parts[4] = urllib.urlencode(query_params)
+ return urlparse.urlunparse(parts)
+
+
+class OAuth2Credentials(Credentials):
+ """Credentials object for OAuth 2.0.
+
+ Credentials can be applied to an httplib2.Http object using the authorize()
+ method, which then adds the OAuth 2.0 access token to each request.
+
+ OAuth2Credentials objects may be safely pickled and unpickled.
+ """
+
+ @util.positional(8)
+ def __init__(self, access_token, client_id, client_secret, refresh_token,
+ token_expiry, token_uri, user_agent, revoke_uri=None,
+ id_token=None, token_response=None):
+ """Create an instance of OAuth2Credentials.
+
+ This constructor is not usually called by the user, instead
+ OAuth2Credentials objects are instantiated by the OAuth2WebServerFlow.
+
+ Args:
+ access_token: string, access token.
+ client_id: string, client identifier.
+ client_secret: string, client secret.
+ refresh_token: string, refresh token.
+ token_expiry: datetime, when the access_token expires.
+ token_uri: string, URI of token endpoint.
+ user_agent: string, The HTTP User-Agent to provide for this application.
+ revoke_uri: string, URI for revoke endpoint. Defaults to None; a token
+ can't be revoked if this is None.
+ id_token: object, The identity of the resource owner.
+ token_response: dict, the decoded response to the token request. None
+ if a token hasn't been requested yet. Stored because some providers
+ (e.g. wordpress.com) include extra fields that clients may want.
+
+ Notes:
+ store: callable, A callable that when passed a Credential
+ will store the credential back to where it came from.
+ This is needed to store the latest access_token if it
+ has expired and been refreshed.
+ """
+ self.access_token = access_token
+ self.client_id = client_id
+ self.client_secret = client_secret
+ self.refresh_token = refresh_token
+ self.store = None
+ self.token_expiry = token_expiry
+ self.token_uri = token_uri
+ self.user_agent = user_agent
+ self.revoke_uri = revoke_uri
+ self.id_token = id_token
+ self.token_response = token_response
+
+ # True if the credentials have been revoked or expired and can't be
+ # refreshed.
+ self.invalid = False
+
+ def authorize(self, http):
+ """Authorize an httplib2.Http instance with these credentials.
+
+ The modified http.request method will add authentication headers to each
+ request and will refresh access_tokens when a 401 is received on a
+ request. In addition the http.request method has a credentials property,
+ http.request.credentials, which is the Credentials object that authorized
+ it.
+
+ Args:
+ http: An instance of httplib2.Http
+ or something that acts like it.
+
+ Returns:
+ A modified instance of http that was passed in.
+
+ Example:
+
+ h = httplib2.Http()
+ h = credentials.authorize(h)
+
+ You can't create a new OAuth subclass of httplib2.Authenication
+ because it never gets passed the absolute URI, which is needed for
+ signing. So instead we have to overload 'request' with a closure
+ that adds in the Authorization header and then calls the original
+ version of 'request()'.
+ """
+ request_orig = http.request
+
+ # The closure that will replace 'httplib2.Http.request'.
+ @util.positional(1)
+ def new_request(uri, method='GET', body=None, headers=None,
+ redirections=httplib2.DEFAULT_MAX_REDIRECTS,
+ connection_type=None):
+ if not self.access_token:
+ logger.info('Attempting refresh to obtain initial access_token')
+ self._refresh(request_orig)
+
+ # Modify the request headers to add the appropriate
+ # Authorization header.
+ if headers is None:
+ headers = {}
+ self.apply(headers)
+
+ if self.user_agent is not None:
+ if 'user-agent' in headers:
+ headers['user-agent'] = self.user_agent + ' ' + headers['user-agent']
+ else:
+ headers['user-agent'] = self.user_agent
+
+ resp, content = request_orig(uri, method, body, clean_headers(headers),
+ redirections, connection_type)
+
+ if resp.status in REFRESH_STATUS_CODES:
+ logger.info('Refreshing due to a %s' % str(resp.status))
+ self._refresh(request_orig)
+ self.apply(headers)
+ return request_orig(uri, method, body, clean_headers(headers),
+ redirections, connection_type)
+ else:
+ return (resp, content)
+
+ # Replace the request method with our own closure.
+ http.request = new_request
+
+ # Set credentials as a property of the request method.
+ setattr(http.request, 'credentials', self)
+
+ return http
+
+ def refresh(self, http):
+ """Forces a refresh of the access_token.
+
+ Args:
+ http: httplib2.Http, an http object to be used to make the refresh
+ request.
+ """
+ self._refresh(http.request)
+
+ def revoke(self, http):
+ """Revokes a refresh_token and makes the credentials void.
+
+ Args:
+ http: httplib2.Http, an http object to be used to make the revoke
+ request.
+ """
+ self._revoke(http.request)
+
+ def apply(self, headers):
+ """Add the authorization to the headers.
+
+ Args:
+ headers: dict, the headers to add the Authorization header to.
+ """
+ headers['Authorization'] = 'Bearer ' + self.access_token
+
+ def to_json(self):
+ return self._to_json(Credentials.NON_SERIALIZED_MEMBERS)
+
+ @classmethod
+ def from_json(cls, s):
+ """Instantiate a Credentials object from a JSON description of it. The JSON
+ should have been produced by calling .to_json() on the object.
+
+ Args:
+ data: dict, A deserialized JSON object.
+
+ Returns:
+ An instance of a Credentials subclass.
+ """
+ data = simplejson.loads(s)
+ if 'token_expiry' in data and not isinstance(data['token_expiry'],
+ datetime.datetime):
+ try:
+ data['token_expiry'] = datetime.datetime.strptime(
+ data['token_expiry'], EXPIRY_FORMAT)
+ except:
+ data['token_expiry'] = None
+ retval = cls(
+ data['access_token'],
+ data['client_id'],
+ data['client_secret'],
+ data['refresh_token'],
+ data['token_expiry'],
+ data['token_uri'],
+ data['user_agent'],
+ revoke_uri=data.get('revoke_uri', None),
+ id_token=data.get('id_token', None),
+ token_response=data.get('token_response', None))
+ retval.invalid = data['invalid']
+ return retval
+
+ @property
+ def access_token_expired(self):
+ """True if the credential is expired or invalid.
+
+ If the token_expiry isn't set, we assume the token doesn't expire.
+ """
+ if self.invalid:
+ return True
+
+ if not self.token_expiry:
+ return False
+
+ now = datetime.datetime.utcnow()
+ if now >= self.token_expiry:
+ logger.info('access_token is expired. Now: %s, token_expiry: %s',
+ now, self.token_expiry)
+ return True
+ return False
+
+ def set_store(self, store):
+ """Set the Storage for the credential.
+
+ Args:
+ store: Storage, an implementation of Stroage object.
+ This is needed to store the latest access_token if it
+ has expired and been refreshed. This implementation uses
+ locking to check for updates before updating the
+ access_token.
+ """
+ self.store = store
+
+ def _updateFromCredential(self, other):
+ """Update this Credential from another instance."""
+ self.__dict__.update(other.__getstate__())
+
+ def __getstate__(self):
+ """Trim the state down to something that can be pickled."""
+ d = copy.copy(self.__dict__)
+ del d['store']
+ return d
+
+ def __setstate__(self, state):
+ """Reconstitute the state of the object from being pickled."""
+ self.__dict__.update(state)
+ self.store = None
+
+ def _generate_refresh_request_body(self):
+ """Generate the body that will be used in the refresh request."""
+ body = urllib.urlencode({
+ 'grant_type': 'refresh_token',
+ 'client_id': self.client_id,
+ 'client_secret': self.client_secret,
+ 'refresh_token': self.refresh_token,
+ })
+ return body
+
+ def _generate_refresh_request_headers(self):
+ """Generate the headers that will be used in the refresh request."""
+ headers = {
+ 'content-type': 'application/x-www-form-urlencoded',
+ }
+
+ if self.user_agent is not None:
+ headers['user-agent'] = self.user_agent
+
+ return headers
+
+ def _refresh(self, http_request):
+ """Refreshes the access_token.
+
+ This method first checks by reading the Storage object if available.
+ If a refresh is still needed, it holds the Storage lock until the
+ refresh is completed.
+
+ Args:
+ http_request: callable, a callable that matches the method signature of
+ httplib2.Http.request, used to make the refresh request.
+
+ Raises:
+ AccessTokenRefreshError: When the refresh fails.
+ """
+ if not self.store:
+ self._do_refresh_request(http_request)
+ else:
+ self.store.acquire_lock()
+ try:
+ new_cred = self.store.locked_get()
+ if (new_cred and not new_cred.invalid and
+ new_cred.access_token != self.access_token):
+ logger.info('Updated access_token read from Storage')
+ self._updateFromCredential(new_cred)
+ else:
+ self._do_refresh_request(http_request)
+ finally:
+ self.store.release_lock()
+
+ def _do_refresh_request(self, http_request):
+ """Refresh the access_token using the refresh_token.
+
+ Args:
+ http_request: callable, a callable that matches the method signature of
+ httplib2.Http.request, used to make the refresh request.
+
+ Raises:
+ AccessTokenRefreshError: When the refresh fails.
+ """
+ body = self._generate_refresh_request_body()
+ headers = self._generate_refresh_request_headers()
+
+ logger.info('Refreshing access_token')
+ resp, content = http_request(
+ self.token_uri, method='POST', body=body, headers=headers)
+ if resp.status == 200:
+ # TODO(jcgregorio) Raise an error if loads fails?
+ d = simplejson.loads(content)
+ self.token_response = d
+ self.access_token = d['access_token']
+ self.refresh_token = d.get('refresh_token', self.refresh_token)
+ if 'expires_in' in d:
+ self.token_expiry = datetime.timedelta(
+ seconds=int(d['expires_in'])) + datetime.datetime.utcnow()
+ else:
+ self.token_expiry = None
+ if self.store:
+ self.store.locked_put(self)
+ else:
+ # An {'error':...} response body means the token is expired or revoked,
+ # so we flag the credentials as such.
+ logger.info('Failed to retrieve access token: %s' % content)
+ error_msg = 'Invalid response %s.' % resp['status']
+ try:
+ d = simplejson.loads(content)
+ if 'error' in d:
+ error_msg = d['error']
+ self.invalid = True
+ if self.store:
+ self.store.locked_put(self)
+ except StandardError:
+ pass
+ raise AccessTokenRefreshError(error_msg)
+
+ def _revoke(self, http_request):
+ """Revokes the refresh_token and deletes the store if available.
+
+ Args:
+ http_request: callable, a callable that matches the method signature of
+ httplib2.Http.request, used to make the revoke request.
+ """
+ self._do_revoke(http_request, self.refresh_token)
+
+ def _do_revoke(self, http_request, token):
+ """Revokes the credentials and deletes the store if available.
+
+ Args:
+ http_request: callable, a callable that matches the method signature of
+ httplib2.Http.request, used to make the refresh request.
+ token: A string used as the token to be revoked. Can be either an
+ access_token or refresh_token.
+
+ Raises:
+ TokenRevokeError: If the revoke request does not return with a 200 OK.
+ """
+ logger.info('Revoking token')
+ query_params = {'token': token}
+ token_revoke_uri = _update_query_params(self.revoke_uri, query_params)
+ resp, content = http_request(token_revoke_uri)
+ if resp.status == 200:
+ self.invalid = True
+ else:
+ error_msg = 'Invalid response %s.' % resp.status
+ try:
+ d = simplejson.loads(content)
+ if 'error' in d:
+ error_msg = d['error']
+ except StandardError:
+ pass
+ raise TokenRevokeError(error_msg)
+
+ if self.store:
+ self.store.delete()
+
+
+class AccessTokenCredentials(OAuth2Credentials):
+ """Credentials object for OAuth 2.0.
+
+ Credentials can be applied to an httplib2.Http object using the
+ authorize() method, which then signs each request from that object
+ with the OAuth 2.0 access token. This set of credentials is for the
+ use case where you have acquired an OAuth 2.0 access_token from
+ another place such as a JavaScript client or another web
+ application, and wish to use it from Python. Because only the
+ access_token is present it can not be refreshed and will in time
+ expire.
+
+ AccessTokenCredentials objects may be safely pickled and unpickled.
+
+ Usage:
+ credentials = AccessTokenCredentials('<an access token>',
+ 'my-user-agent/1.0')
+ http = httplib2.Http()
+ http = credentials.authorize(http)
+
+ Exceptions:
+ AccessTokenCredentialsExpired: raised when the access_token expires or is
+ revoked.
+ """
+
+ def __init__(self, access_token, user_agent, revoke_uri=None):
+ """Create an instance of OAuth2Credentials
+
+ This is one of the few types if Credentials that you should contrust,
+ Credentials objects are usually instantiated by a Flow.
+
+ Args:
+ access_token: string, access token.
+ user_agent: string, The HTTP User-Agent to provide for this application.
+ revoke_uri: string, URI for revoke endpoint. Defaults to None; a token
+ can't be revoked if this is None.
+ """
+ super(AccessTokenCredentials, self).__init__(
+ access_token,
+ None,
+ None,
+ None,
+ None,
+ None,
+ user_agent,
+ revoke_uri=revoke_uri)
+
+
+ @classmethod
+ def from_json(cls, s):
+ data = simplejson.loads(s)
+ retval = AccessTokenCredentials(
+ data['access_token'],
+ data['user_agent'])
+ return retval
+
+ def _refresh(self, http_request):
+ raise AccessTokenCredentialsError(
+ 'The access_token is expired or invalid and can\'t be refreshed.')
+
+ def _revoke(self, http_request):
+ """Revokes the access_token and deletes the store if available.
+
+ Args:
+ http_request: callable, a callable that matches the method signature of
+ httplib2.Http.request, used to make the revoke request.
+ """
+ self._do_revoke(http_request, self.access_token)
+
+
+class AssertionCredentials(OAuth2Credentials):
+ """Abstract Credentials object used for OAuth 2.0 assertion grants.
+
+ This credential does not require a flow to instantiate because it
+ represents a two legged flow, and therefore has all of the required
+ information to generate and refresh its own access tokens. It must
+ be subclassed to generate the appropriate assertion string.
+
+ AssertionCredentials objects may be safely pickled and unpickled.
+ """
+
+ @util.positional(2)
+ def __init__(self, assertion_type, user_agent=None,
+ token_uri=GOOGLE_TOKEN_URI,
+ revoke_uri=GOOGLE_REVOKE_URI,
+ **unused_kwargs):
+ """Constructor for AssertionFlowCredentials.
+
+ Args:
+ assertion_type: string, assertion type that will be declared to the auth
+ server
+ user_agent: string, The HTTP User-Agent to provide for this application.
+ token_uri: string, URI for token endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ revoke_uri: string, URI for revoke endpoint.
+ """
+ super(AssertionCredentials, self).__init__(
+ None,
+ None,
+ None,
+ None,
+ None,
+ token_uri,
+ user_agent,
+ revoke_uri=revoke_uri)
+ self.assertion_type = assertion_type
+
+ def _generate_refresh_request_body(self):
+ assertion = self._generate_assertion()
+
+ body = urllib.urlencode({
+ 'assertion': assertion,
+ 'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+ })
+
+ return body
+
+ def _generate_assertion(self):
+ """Generate the assertion string that will be used in the access token
+ request.
+ """
+ _abstract()
+
+ def _revoke(self, http_request):
+ """Revokes the access_token and deletes the store if available.
+
+ Args:
+ http_request: callable, a callable that matches the method signature of
+ httplib2.Http.request, used to make the revoke request.
+ """
+ self._do_revoke(http_request, self.access_token)
+
+
+if HAS_CRYPTO:
+ # PyOpenSSL and PyCrypto are not prerequisites for oauth2client, so if it is
+ # missing then don't create the SignedJwtAssertionCredentials or the
+ # verify_id_token() method.
+
+ class SignedJwtAssertionCredentials(AssertionCredentials):
+ """Credentials object used for OAuth 2.0 Signed JWT assertion grants.
+
+ This credential does not require a flow to instantiate because it represents
+ a two legged flow, and therefore has all of the required information to
+ generate and refresh its own access tokens.
+
+ SignedJwtAssertionCredentials requires either PyOpenSSL, or PyCrypto 2.6 or
+ later. For App Engine you may also consider using AppAssertionCredentials.
+ """
+
+ MAX_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds
+
+ @util.positional(4)
+ def __init__(self,
+ service_account_name,
+ private_key,
+ scope,
+ private_key_password='notasecret',
+ user_agent=None,
+ token_uri=GOOGLE_TOKEN_URI,
+ revoke_uri=GOOGLE_REVOKE_URI,
+ **kwargs):
+ """Constructor for SignedJwtAssertionCredentials.
+
+ Args:
+ service_account_name: string, id for account, usually an email address.
+ private_key: string, private key in PKCS12 or PEM format.
+ scope: string or iterable of strings, scope(s) of the credentials being
+ requested.
+ private_key_password: string, password for private_key, unused if
+ private_key is in PEM format.
+ user_agent: string, HTTP User-Agent to provide for this application.
+ token_uri: string, URI for token endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ revoke_uri: string, URI for revoke endpoint.
+ kwargs: kwargs, Additional parameters to add to the JWT token, for
+ example prn=joe@xample.org."""
+
+ super(SignedJwtAssertionCredentials, self).__init__(
+ None,
+ user_agent=user_agent,
+ token_uri=token_uri,
+ revoke_uri=revoke_uri,
+ )
+
+ self.scope = util.scopes_to_string(scope)
+
+ # Keep base64 encoded so it can be stored in JSON.
+ self.private_key = base64.b64encode(private_key)
+
+ self.private_key_password = private_key_password
+ self.service_account_name = service_account_name
+ self.kwargs = kwargs
+
+ @classmethod
+ def from_json(cls, s):
+ data = simplejson.loads(s)
+ retval = SignedJwtAssertionCredentials(
+ data['service_account_name'],
+ base64.b64decode(data['private_key']),
+ data['scope'],
+ private_key_password=data['private_key_password'],
+ user_agent=data['user_agent'],
+ token_uri=data['token_uri'],
+ **data['kwargs']
+ )
+ retval.invalid = data['invalid']
+ retval.access_token = data['access_token']
+ return retval
+
+ def _generate_assertion(self):
+ """Generate the assertion that will be used in the request."""
+ now = long(time.time())
+ payload = {
+ 'aud': self.token_uri,
+ 'scope': self.scope,
+ 'iat': now,
+ 'exp': now + SignedJwtAssertionCredentials.MAX_TOKEN_LIFETIME_SECS,
+ 'iss': self.service_account_name
+ }
+ payload.update(self.kwargs)
+ logger.debug(str(payload))
+
+ private_key = base64.b64decode(self.private_key)
+ return crypt.make_signed_jwt(crypt.Signer.from_string(
+ private_key, self.private_key_password), payload)
+
+ # Only used in verify_id_token(), which is always calling to the same URI
+ # for the certs.
+ _cached_http = httplib2.Http(MemoryCache())
+
+ @util.positional(2)
+ def verify_id_token(id_token, audience, http=None,
+ cert_uri=ID_TOKEN_VERIFICATON_CERTS):
+ """Verifies a signed JWT id_token.
+
+ This function requires PyOpenSSL and because of that it does not work on
+ App Engine.
+
+ Args:
+ id_token: string, A Signed JWT.
+ audience: string, The audience 'aud' that the token should be for.
+ http: httplib2.Http, instance to use to make the HTTP request. Callers
+ should supply an instance that has caching enabled.
+ cert_uri: string, URI of the certificates in JSON format to
+ verify the JWT against.
+
+ Returns:
+ The deserialized JSON in the JWT.
+
+ Raises:
+ oauth2client.crypt.AppIdentityError if the JWT fails to verify.
+ """
+ if http is None:
+ http = _cached_http
+
+ resp, content = http.request(cert_uri)
+
+ if resp.status == 200:
+ certs = simplejson.loads(content)
+ return crypt.verify_signed_jwt_with_certs(id_token, certs, audience)
+ else:
+ raise VerifyJwtTokenError('Status code: %d' % resp.status)
+
+
+def _urlsafe_b64decode(b64string):
+ # Guard against unicode strings, which base64 can't handle.
+ b64string = b64string.encode('ascii')
+ padded = b64string + '=' * (4 - len(b64string) % 4)
+ return base64.urlsafe_b64decode(padded)
+
+
+def _extract_id_token(id_token):
+ """Extract the JSON payload from a JWT.
+
+ Does the extraction w/o checking the signature.
+
+ Args:
+ id_token: string, OAuth 2.0 id_token.
+
+ Returns:
+ object, The deserialized JSON payload.
+ """
+ segments = id_token.split('.')
+
+ if (len(segments) != 3):
+ raise VerifyJwtTokenError(
+ 'Wrong number of segments in token: %s' % id_token)
+
+ return simplejson.loads(_urlsafe_b64decode(segments[1]))
+
+
+def _parse_exchange_token_response(content):
+ """Parses response of an exchange token request.
+
+ Most providers return JSON but some (e.g. Facebook) return a
+ url-encoded string.
+
+ Args:
+ content: The body of a response
+
+ Returns:
+ Content as a dictionary object. Note that the dict could be empty,
+ i.e. {}. That basically indicates a failure.
+ """
+ resp = {}
+ try:
+ resp = simplejson.loads(content)
+ except StandardError:
+ # different JSON libs raise different exceptions,
+ # so we just do a catch-all here
+ resp = dict(parse_qsl(content))
+
+ # some providers respond with 'expires', others with 'expires_in'
+ if resp and 'expires' in resp:
+ resp['expires_in'] = resp.pop('expires')
+
+ return resp
+
+
+@util.positional(4)
+def credentials_from_code(client_id, client_secret, scope, code,
+ redirect_uri='postmessage', http=None,
+ user_agent=None, token_uri=GOOGLE_TOKEN_URI,
+ auth_uri=GOOGLE_AUTH_URI,
+ revoke_uri=GOOGLE_REVOKE_URI):
+ """Exchanges an authorization code for an OAuth2Credentials object.
+
+ Args:
+ client_id: string, client identifier.
+ client_secret: string, client secret.
+ scope: string or iterable of strings, scope(s) to request.
+ code: string, An authroization code, most likely passed down from
+ the client
+ redirect_uri: string, this is generally set to 'postmessage' to match the
+ redirect_uri that the client specified
+ http: httplib2.Http, optional http instance to use to do the fetch
+ token_uri: string, URI for token endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ auth_uri: string, URI for authorization endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ revoke_uri: string, URI for revoke endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+
+ Returns:
+ An OAuth2Credentials object.
+
+ Raises:
+ FlowExchangeError if the authorization code cannot be exchanged for an
+ access token
+ """
+ flow = OAuth2WebServerFlow(client_id, client_secret, scope,
+ redirect_uri=redirect_uri, user_agent=user_agent,
+ auth_uri=auth_uri, token_uri=token_uri,
+ revoke_uri=revoke_uri)
+
+ credentials = flow.step2_exchange(code, http=http)
+ return credentials
+
+
+@util.positional(3)
+def credentials_from_clientsecrets_and_code(filename, scope, code,
+ message = None,
+ redirect_uri='postmessage',
+ http=None,
+ cache=None):
+ """Returns OAuth2Credentials from a clientsecrets file and an auth code.
+
+ Will create the right kind of Flow based on the contents of the clientsecrets
+ file or will raise InvalidClientSecretsError for unknown types of Flows.
+
+ Args:
+ filename: string, File name of clientsecrets.
+ scope: string or iterable of strings, scope(s) to request.
+ code: string, An authorization code, most likely passed down from
+ the client
+ message: string, A friendly string to display to the user if the
+ clientsecrets file is missing or invalid. If message is provided then
+ sys.exit will be called in the case of an error. If message in not
+ provided then clientsecrets.InvalidClientSecretsError will be raised.
+ redirect_uri: string, this is generally set to 'postmessage' to match the
+ redirect_uri that the client specified
+ http: httplib2.Http, optional http instance to use to do the fetch
+ cache: An optional cache service client that implements get() and set()
+ methods. See clientsecrets.loadfile() for details.
+
+ Returns:
+ An OAuth2Credentials object.
+
+ Raises:
+ FlowExchangeError if the authorization code cannot be exchanged for an
+ access token
+ UnknownClientSecretsFlowError if the file describes an unknown kind of Flow.
+ clientsecrets.InvalidClientSecretsError if the clientsecrets file is
+ invalid.
+ """
+ flow = flow_from_clientsecrets(filename, scope, message=message, cache=cache,
+ redirect_uri=redirect_uri)
+ credentials = flow.step2_exchange(code, http=http)
+ return credentials
+
+
+class OAuth2WebServerFlow(Flow):
+ """Does the Web Server Flow for OAuth 2.0.
+
+ OAuth2WebServerFlow objects may be safely pickled and unpickled.
+ """
+
+ @util.positional(4)
+ def __init__(self, client_id, client_secret, scope,
+ redirect_uri=None,
+ user_agent=None,
+ auth_uri=GOOGLE_AUTH_URI,
+ token_uri=GOOGLE_TOKEN_URI,
+ revoke_uri=GOOGLE_REVOKE_URI,
+ **kwargs):
+ """Constructor for OAuth2WebServerFlow.
+
+ The kwargs argument is used to set extra query parameters on the
+ auth_uri. For example, the access_type and approval_prompt
+ query parameters can be set via kwargs.
+
+ Args:
+ client_id: string, client identifier.
+ client_secret: string client secret.
+ scope: string or iterable of strings, scope(s) of the credentials being
+ requested.
+ redirect_uri: string, Either the string 'urn:ietf:wg:oauth:2.0:oob' for
+ a non-web-based application, or a URI that handles the callback from
+ the authorization server.
+ user_agent: string, HTTP User-Agent to provide for this application.
+ auth_uri: string, URI for authorization endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ token_uri: string, URI for token endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ revoke_uri: string, URI for revoke endpoint. For convenience
+ defaults to Google's endpoints but any OAuth 2.0 provider can be used.
+ **kwargs: dict, The keyword arguments are all optional and required
+ parameters for the OAuth calls.
+ """
+ self.client_id = client_id
+ self.client_secret = client_secret
+ self.scope = util.scopes_to_string(scope)
+ self.redirect_uri = redirect_uri
+ self.user_agent = user_agent
+ self.auth_uri = auth_uri
+ self.token_uri = token_uri
+ self.revoke_uri = revoke_uri
+ self.params = {
+ 'access_type': 'offline',
+ 'response_type': 'code',
+ }
+ self.params.update(kwargs)
+
+ @util.positional(1)
+ def step1_get_authorize_url(self, redirect_uri=None):
+ """Returns a URI to redirect to the provider.
+
+ Args:
+ redirect_uri: string, Either the string 'urn:ietf:wg:oauth:2.0:oob' for
+ a non-web-based application, or a URI that handles the callback from
+ the authorization server. This parameter is deprecated, please move to
+ passing the redirect_uri in via the constructor.
+
+ Returns:
+ A URI as a string to redirect the user to begin the authorization flow.
+ """
+ if redirect_uri is not None:
+ logger.warning(('The redirect_uri parameter for'
+ 'OAuth2WebServerFlow.step1_get_authorize_url is deprecated. Please'
+ 'move to passing the redirect_uri in via the constructor.'))
+ self.redirect_uri = redirect_uri
+
+ if self.redirect_uri is None:
+ raise ValueError('The value of redirect_uri must not be None.')
+
+ query_params = {
+ 'client_id': self.client_id,
+ 'redirect_uri': self.redirect_uri,
+ 'scope': self.scope,
+ }
+ query_params.update(self.params)
+ return _update_query_params(self.auth_uri, query_params)
+
+ @util.positional(2)
+ def step2_exchange(self, code, http=None):
+ """Exhanges a code for OAuth2Credentials.
+
+ Args:
+ code: string or dict, either the code as a string, or a dictionary
+ of the query parameters to the redirect_uri, which contains
+ the code.
+ http: httplib2.Http, optional http instance to use to do the fetch
+
+ Returns:
+ An OAuth2Credentials object that can be used to authorize requests.
+
+ Raises:
+ FlowExchangeError if a problem occured exchanging the code for a
+ refresh_token.
+ """
+
+ if not (isinstance(code, str) or isinstance(code, unicode)):
+ if 'code' not in code:
+ if 'error' in code:
+ error_msg = code['error']
+ else:
+ error_msg = 'No code was supplied in the query parameters.'
+ raise FlowExchangeError(error_msg)
+ else:
+ code = code['code']
+
+ body = urllib.urlencode({
+ 'grant_type': 'authorization_code',
+ 'client_id': self.client_id,
+ 'client_secret': self.client_secret,
+ 'code': code,
+ 'redirect_uri': self.redirect_uri,
+ 'scope': self.scope,
+ })
+ headers = {
+ 'content-type': 'application/x-www-form-urlencoded',
+ }
+
+ if self.user_agent is not None:
+ headers['user-agent'] = self.user_agent
+
+ if http is None:
+ http = httplib2.Http()
+
+ resp, content = http.request(self.token_uri, method='POST', body=body,
+ headers=headers)
+ d = _parse_exchange_token_response(content)
+ if resp.status == 200 and 'access_token' in d:
+ access_token = d['access_token']
+ refresh_token = d.get('refresh_token', None)
+ token_expiry = None
+ if 'expires_in' in d:
+ token_expiry = datetime.datetime.utcnow() + datetime.timedelta(
+ seconds=int(d['expires_in']))
+
+ if 'id_token' in d:
+ d['id_token'] = _extract_id_token(d['id_token'])
+
+ logger.info('Successfully retrieved access token')
+ return OAuth2Credentials(access_token, self.client_id,
+ self.client_secret, refresh_token, token_expiry,
+ self.token_uri, self.user_agent,
+ revoke_uri=self.revoke_uri,
+ id_token=d.get('id_token', None),
+ token_response=d)
+ else:
+ logger.info('Failed to retrieve access token: %s' % content)
+ if 'error' in d:
+ # you never know what those providers got to say
+ error_msg = unicode(d['error'])
+ else:
+ error_msg = 'Invalid response: %s.' % str(resp.status)
+ raise FlowExchangeError(error_msg)
+
+
+@util.positional(2)
+def flow_from_clientsecrets(filename, scope, redirect_uri=None,
+ message=None, cache=None):
+ """Create a Flow from a clientsecrets file.
+
+ Will create the right kind of Flow based on the contents of the clientsecrets
+ file or will raise InvalidClientSecretsError for unknown types of Flows.
+
+ Args:
+ filename: string, File name of client secrets.
+ scope: string or iterable of strings, scope(s) to request.
+ redirect_uri: string, Either the string 'urn:ietf:wg:oauth:2.0:oob' for
+ a non-web-based application, or a URI that handles the callback from
+ the authorization server.
+ message: string, A friendly string to display to the user if the
+ clientsecrets file is missing or invalid. If message is provided then
+ sys.exit will be called in the case of an error. If message in not
+ provided then clientsecrets.InvalidClientSecretsError will be raised.
+ cache: An optional cache service client that implements get() and set()
+ methods. See clientsecrets.loadfile() for details.
+
+ Returns:
+ A Flow object.
+
+ Raises:
+ UnknownClientSecretsFlowError if the file describes an unknown kind of Flow.
+ clientsecrets.InvalidClientSecretsError if the clientsecrets file is
+ invalid.
+ """
+ try:
+ client_type, client_info = clientsecrets.loadfile(filename, cache=cache)
+ if client_type in (clientsecrets.TYPE_WEB, clientsecrets.TYPE_INSTALLED):
+ constructor_kwargs = {
+ 'redirect_uri': redirect_uri,
+ 'auth_uri': client_info['auth_uri'],
+ 'token_uri': client_info['token_uri'],
+ }
+ revoke_uri = client_info.get('revoke_uri')
+ if revoke_uri is not None:
+ constructor_kwargs['revoke_uri'] = revoke_uri
+ return OAuth2WebServerFlow(
+ client_info['client_id'], client_info['client_secret'],
+ scope, **constructor_kwargs)
+
+ except clientsecrets.InvalidClientSecretsError:
+ if message:
+ sys.exit(message)
+ else:
+ raise
+ else:
+ raise UnknownClientSecretsFlowError(
+ 'This OAuth 2.0 flow is unsupported: %r' % client_type)
diff --git a/py/minijack/oauth2client/clientsecrets.py b/py/minijack/oauth2client/clientsecrets.py
new file mode 100644
index 0000000..ac99aae
--- /dev/null
+++ b/py/minijack/oauth2client/clientsecrets.py
@@ -0,0 +1,153 @@
+# Copyright (C) 2011 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for reading OAuth 2.0 client secret files.
+
+A client_secrets.json file contains all the information needed to interact with
+an OAuth 2.0 protected service.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+
+from anyjson import simplejson
+
+# Properties that make a client_secrets.json file valid.
+TYPE_WEB = 'web'
+TYPE_INSTALLED = 'installed'
+
+VALID_CLIENT = {
+ TYPE_WEB: {
+ 'required': [
+ 'client_id',
+ 'client_secret',
+ 'redirect_uris',
+ 'auth_uri',
+ 'token_uri',
+ ],
+ 'string': [
+ 'client_id',
+ 'client_secret',
+ ],
+ },
+ TYPE_INSTALLED: {
+ 'required': [
+ 'client_id',
+ 'client_secret',
+ 'redirect_uris',
+ 'auth_uri',
+ 'token_uri',
+ ],
+ 'string': [
+ 'client_id',
+ 'client_secret',
+ ],
+ },
+}
+
+
+class Error(Exception):
+ """Base error for this module."""
+ pass
+
+
+class InvalidClientSecretsError(Error):
+ """Format of ClientSecrets file is invalid."""
+ pass
+
+
+def _validate_clientsecrets(obj):
+ if obj is None or len(obj) != 1:
+ raise InvalidClientSecretsError('Invalid file format.')
+ client_type = obj.keys()[0]
+ if client_type not in VALID_CLIENT.keys():
+ raise InvalidClientSecretsError('Unknown client type: %s.' % client_type)
+ client_info = obj[client_type]
+ for prop_name in VALID_CLIENT[client_type]['required']:
+ if prop_name not in client_info:
+ raise InvalidClientSecretsError(
+ 'Missing property "%s" in a client type of "%s".' % (prop_name,
+ client_type))
+ for prop_name in VALID_CLIENT[client_type]['string']:
+ if client_info[prop_name].startswith('[['):
+ raise InvalidClientSecretsError(
+ 'Property "%s" is not configured.' % prop_name)
+ return client_type, client_info
+
+
+def load(fp):
+ obj = simplejson.load(fp)
+ return _validate_clientsecrets(obj)
+
+
+def loads(s):
+ obj = simplejson.loads(s)
+ return _validate_clientsecrets(obj)
+
+
+def _loadfile(filename):
+ try:
+ fp = file(filename, 'r')
+ try:
+ obj = simplejson.load(fp)
+ finally:
+ fp.close()
+ except IOError:
+ raise InvalidClientSecretsError('File not found: "%s"' % filename)
+ return _validate_clientsecrets(obj)
+
+
+def loadfile(filename, cache=None):
+ """Loading of client_secrets JSON file, optionally backed by a cache.
+
+ Typical cache storage would be App Engine memcache service,
+ but you can pass in any other cache client that implements
+ these methods:
+ - get(key, namespace=ns)
+ - set(key, value, namespace=ns)
+
+ Usage:
+ # without caching
+ client_type, client_info = loadfile('secrets.json')
+ # using App Engine memcache service
+ from google.appengine.api import memcache
+ client_type, client_info = loadfile('secrets.json', cache=memcache)
+
+ Args:
+ filename: string, Path to a client_secrets.json file on a filesystem.
+ cache: An optional cache service client that implements get() and set()
+ methods. If not specified, the file is always being loaded from
+ a filesystem.
+
+ Raises:
+ InvalidClientSecretsError: In case of a validation error or some
+ I/O failure. Can happen only on cache miss.
+
+ Returns:
+ (client_type, client_info) tuple, as _loadfile() normally would.
+ JSON contents is validated only during first load. Cache hits are not
+ validated.
+ """
+ _SECRET_NAMESPACE = 'oauth2client:secrets#ns'
+
+ if not cache:
+ return _loadfile(filename)
+
+ obj = cache.get(filename, namespace=_SECRET_NAMESPACE)
+ if obj is None:
+ client_type, client_info = _loadfile(filename)
+ obj = {client_type: client_info}
+ cache.set(filename, obj, namespace=_SECRET_NAMESPACE)
+
+ return obj.iteritems().next()
diff --git a/py/minijack/oauth2client/crypt.py b/py/minijack/oauth2client/crypt.py
new file mode 100644
index 0000000..2d31815
--- /dev/null
+++ b/py/minijack/oauth2client/crypt.py
@@ -0,0 +1,377 @@
+#!/usr/bin/python2.4
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import hashlib
+import logging
+import time
+
+from anyjson import simplejson
+
+
+CLOCK_SKEW_SECS = 300 # 5 minutes in seconds
+AUTH_TOKEN_LIFETIME_SECS = 300 # 5 minutes in seconds
+MAX_TOKEN_LIFETIME_SECS = 86400 # 1 day in seconds
+
+
+logger = logging.getLogger(__name__)
+
+
+class AppIdentityError(Exception):
+ pass
+
+
+try:
+ from OpenSSL import crypto
+
+
+ class OpenSSLVerifier(object):
+ """Verifies the signature on a message."""
+
+ def __init__(self, pubkey):
+ """Constructor.
+
+ Args:
+ pubkey, OpenSSL.crypto.PKey, The public key to verify with.
+ """
+ self._pubkey = pubkey
+
+ def verify(self, message, signature):
+ """Verifies a message against a signature.
+
+ Args:
+ message: string, The message to verify.
+ signature: string, The signature on the message.
+
+ Returns:
+ True if message was signed by the private key associated with the public
+ key that this object was constructed with.
+ """
+ try:
+ crypto.verify(self._pubkey, signature, message, 'sha256')
+ return True
+ except:
+ return False
+
+ @staticmethod
+ def from_string(key_pem, is_x509_cert):
+ """Construct a Verified instance from a string.
+
+ Args:
+ key_pem: string, public key in PEM format.
+ is_x509_cert: bool, True if key_pem is an X509 cert, otherwise it is
+ expected to be an RSA key in PEM format.
+
+ Returns:
+ Verifier instance.
+
+ Raises:
+ OpenSSL.crypto.Error if the key_pem can't be parsed.
+ """
+ if is_x509_cert:
+ pubkey = crypto.load_certificate(crypto.FILETYPE_PEM, key_pem)
+ else:
+ pubkey = crypto.load_privatekey(crypto.FILETYPE_PEM, key_pem)
+ return OpenSSLVerifier(pubkey)
+
+
+ class OpenSSLSigner(object):
+ """Signs messages with a private key."""
+
+ def __init__(self, pkey):
+ """Constructor.
+
+ Args:
+ pkey, OpenSSL.crypto.PKey (or equiv), The private key to sign with.
+ """
+ self._key = pkey
+
+ def sign(self, message):
+ """Signs a message.
+
+ Args:
+ message: string, Message to be signed.
+
+ Returns:
+ string, The signature of the message for the given key.
+ """
+ return crypto.sign(self._key, message, 'sha256')
+
+ @staticmethod
+ def from_string(key, password='notasecret'):
+ """Construct a Signer instance from a string.
+
+ Args:
+ key: string, private key in PKCS12 or PEM format.
+ password: string, password for the private key file.
+
+ Returns:
+ Signer instance.
+
+ Raises:
+ OpenSSL.crypto.Error if the key can't be parsed.
+ """
+ if key.startswith('-----BEGIN '):
+ pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
+ else:
+ pkey = crypto.load_pkcs12(key, password).get_privatekey()
+ return OpenSSLSigner(pkey)
+
+except ImportError:
+ OpenSSLVerifier = None
+ OpenSSLSigner = None
+
+
+try:
+ from Crypto.PublicKey import RSA
+ from Crypto.Hash import SHA256
+ from Crypto.Signature import PKCS1_v1_5
+
+
+ class PyCryptoVerifier(object):
+ """Verifies the signature on a message."""
+
+ def __init__(self, pubkey):
+ """Constructor.
+
+ Args:
+ pubkey, OpenSSL.crypto.PKey (or equiv), The public key to verify with.
+ """
+ self._pubkey = pubkey
+
+ def verify(self, message, signature):
+ """Verifies a message against a signature.
+
+ Args:
+ message: string, The message to verify.
+ signature: string, The signature on the message.
+
+ Returns:
+ True if message was signed by the private key associated with the public
+ key that this object was constructed with.
+ """
+ try:
+ return PKCS1_v1_5.new(self._pubkey).verify(
+ SHA256.new(message), signature)
+ except:
+ return False
+
+ @staticmethod
+ def from_string(key_pem, is_x509_cert):
+ """Construct a Verified instance from a string.
+
+ Args:
+ key_pem: string, public key in PEM format.
+ is_x509_cert: bool, True if key_pem is an X509 cert, otherwise it is
+ expected to be an RSA key in PEM format.
+
+ Returns:
+ Verifier instance.
+
+ Raises:
+ NotImplementedError if is_x509_cert is true.
+ """
+ if is_x509_cert:
+ raise NotImplementedError(
+ 'X509 certs are not supported by the PyCrypto library. '
+ 'Try using PyOpenSSL if native code is an option.')
+ else:
+ pubkey = RSA.importKey(key_pem)
+ return PyCryptoVerifier(pubkey)
+
+
+ class PyCryptoSigner(object):
+ """Signs messages with a private key."""
+
+ def __init__(self, pkey):
+ """Constructor.
+
+ Args:
+ pkey, OpenSSL.crypto.PKey (or equiv), The private key to sign with.
+ """
+ self._key = pkey
+
+ def sign(self, message):
+ """Signs a message.
+
+ Args:
+ message: string, Message to be signed.
+
+ Returns:
+ string, The signature of the message for the given key.
+ """
+ return PKCS1_v1_5.new(self._key).sign(SHA256.new(message))
+
+ @staticmethod
+ def from_string(key, password='notasecret'):
+ """Construct a Signer instance from a string.
+
+ Args:
+ key: string, private key in PEM format.
+ password: string, password for private key file. Unused for PEM files.
+
+ Returns:
+ Signer instance.
+
+ Raises:
+ NotImplementedError if they key isn't in PEM format.
+ """
+ if key.startswith('-----BEGIN '):
+ pkey = RSA.importKey(key)
+ else:
+ raise NotImplementedError(
+ 'PKCS12 format is not supported by the PyCrpto library. '
+ 'Try converting to a "PEM" '
+ '(openssl pkcs12 -in xxxxx.p12 -nodes -nocerts > privatekey.pem) '
+ 'or using PyOpenSSL if native code is an option.')
+ return PyCryptoSigner(pkey)
+
+except ImportError:
+ PyCryptoVerifier = None
+ PyCryptoSigner = None
+
+
+if OpenSSLSigner:
+ Signer = OpenSSLSigner
+ Verifier = OpenSSLVerifier
+elif PyCryptoSigner:
+ Signer = PyCryptoSigner
+ Verifier = PyCryptoVerifier
+else:
+ raise ImportError('No encryption library found. Please install either '
+ 'PyOpenSSL, or PyCrypto 2.6 or later')
+
+
+def _urlsafe_b64encode(raw_bytes):
+ return base64.urlsafe_b64encode(raw_bytes).rstrip('=')
+
+
+def _urlsafe_b64decode(b64string):
+ # Guard against unicode strings, which base64 can't handle.
+ b64string = b64string.encode('ascii')
+ padded = b64string + '=' * (4 - len(b64string) % 4)
+ return base64.urlsafe_b64decode(padded)
+
+
+def _json_encode(data):
+ return simplejson.dumps(data, separators = (',', ':'))
+
+
+def make_signed_jwt(signer, payload):
+ """Make a signed JWT.
+
+ See http://self-issued.info/docs/draft-jones-json-web-token.html.
+
+ Args:
+ signer: crypt.Signer, Cryptographic signer.
+ payload: dict, Dictionary of data to convert to JSON and then sign.
+
+ Returns:
+ string, The JWT for the payload.
+ """
+ header = {'typ': 'JWT', 'alg': 'RS256'}
+
+ segments = [
+ _urlsafe_b64encode(_json_encode(header)),
+ _urlsafe_b64encode(_json_encode(payload)),
+ ]
+ signing_input = '.'.join(segments)
+
+ signature = signer.sign(signing_input)
+ segments.append(_urlsafe_b64encode(signature))
+
+ logger.debug(str(segments))
+
+ return '.'.join(segments)
+
+
+def verify_signed_jwt_with_certs(jwt, certs, audience):
+ """Verify a JWT against public certs.
+
+ See http://self-issued.info/docs/draft-jones-json-web-token.html.
+
+ Args:
+ jwt: string, A JWT.
+ certs: dict, Dictionary where values of public keys in PEM format.
+ audience: string, The audience, 'aud', that this JWT should contain. If
+ None then the JWT's 'aud' parameter is not verified.
+
+ Returns:
+ dict, The deserialized JSON payload in the JWT.
+
+ Raises:
+ AppIdentityError if any checks are failed.
+ """
+ segments = jwt.split('.')
+
+ if (len(segments) != 3):
+ raise AppIdentityError(
+ 'Wrong number of segments in token: %s' % jwt)
+ signed = '%s.%s' % (segments[0], segments[1])
+
+ signature = _urlsafe_b64decode(segments[2])
+
+ # Parse token.
+ json_body = _urlsafe_b64decode(segments[1])
+ try:
+ parsed = simplejson.loads(json_body)
+ except:
+ raise AppIdentityError('Can\'t parse token: %s' % json_body)
+
+ # Check signature.
+ verified = False
+ for (keyname, pem) in certs.items():
+ verifier = Verifier.from_string(pem, True)
+ if (verifier.verify(signed, signature)):
+ verified = True
+ break
+ if not verified:
+ raise AppIdentityError('Invalid token signature: %s' % jwt)
+
+ # Check creation timestamp.
+ iat = parsed.get('iat')
+ if iat is None:
+ raise AppIdentityError('No iat field in token: %s' % json_body)
+ earliest = iat - CLOCK_SKEW_SECS
+
+ # Check expiration timestamp.
+ now = long(time.time())
+ exp = parsed.get('exp')
+ if exp is None:
+ raise AppIdentityError('No exp field in token: %s' % json_body)
+ if exp >= now + MAX_TOKEN_LIFETIME_SECS:
+ raise AppIdentityError(
+ 'exp field too far in future: %s' % json_body)
+ latest = exp + CLOCK_SKEW_SECS
+
+ if now < earliest:
+ raise AppIdentityError('Token used too early, %d < %d: %s' %
+ (now, earliest, json_body))
+ if now > latest:
+ raise AppIdentityError('Token used too late, %d > %d: %s' %
+ (now, latest, json_body))
+
+ # Check audience.
+ if audience is not None:
+ aud = parsed.get('aud')
+ if aud is None:
+ raise AppIdentityError('No aud field in token: %s' % json_body)
+ if aud != audience:
+ raise AppIdentityError('Wrong recipient, %s != %s: %s' %
+ (aud, audience, json_body))
+
+ return parsed
diff --git a/py/minijack/oauth2client/django_orm.py b/py/minijack/oauth2client/django_orm.py
new file mode 100644
index 0000000..d54d20c
--- /dev/null
+++ b/py/minijack/oauth2client/django_orm.py
@@ -0,0 +1,134 @@
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""OAuth 2.0 utilities for Django.
+
+Utilities for using OAuth 2.0 in conjunction with
+the Django datastore.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+import oauth2client
+import base64
+import pickle
+
+from django.db import models
+from oauth2client.client import Storage as BaseStorage
+
+class CredentialsField(models.Field):
+
+ __metaclass__ = models.SubfieldBase
+
+ def __init__(self, *args, **kwargs):
+ if 'null' not in kwargs:
+ kwargs['null'] = True
+ super(CredentialsField, self).__init__(*args, **kwargs)
+
+ def get_internal_type(self):
+ return "TextField"
+
+ def to_python(self, value):
+ if value is None:
+ return None
+ if isinstance(value, oauth2client.client.Credentials):
+ return value
+ return pickle.loads(base64.b64decode(value))
+
+ def get_db_prep_value(self, value, connection, prepared=False):
+ if value is None:
+ return None
+ return base64.b64encode(pickle.dumps(value))
+
+
+class FlowField(models.Field):
+
+ __metaclass__ = models.SubfieldBase
+
+ def __init__(self, *args, **kwargs):
+ if 'null' not in kwargs:
+ kwargs['null'] = True
+ super(FlowField, self).__init__(*args, **kwargs)
+
+ def get_internal_type(self):
+ return "TextField"
+
+ def to_python(self, value):
+ if value is None:
+ return None
+ if isinstance(value, oauth2client.client.Flow):
+ return value
+ return pickle.loads(base64.b64decode(value))
+
+ def get_db_prep_value(self, value, connection, prepared=False):
+ if value is None:
+ return None
+ return base64.b64encode(pickle.dumps(value))
+
+
+class Storage(BaseStorage):
+ """Store and retrieve a single credential to and from
+ the datastore.
+
+ This Storage helper presumes the Credentials
+ have been stored as a CredenialsField
+ on a db model class.
+ """
+
+ def __init__(self, model_class, key_name, key_value, property_name):
+ """Constructor for Storage.
+
+ Args:
+ model: db.Model, model class
+ key_name: string, key name for the entity that has the credentials
+ key_value: string, key value for the entity that has the credentials
+ property_name: string, name of the property that is an CredentialsProperty
+ """
+ self.model_class = model_class
+ self.key_name = key_name
+ self.key_value = key_value
+ self.property_name = property_name
+
+ def locked_get(self):
+ """Retrieve Credential from datastore.
+
+ Returns:
+ oauth2client.Credentials
+ """
+ credential = None
+
+ query = {self.key_name: self.key_value}
+ entities = self.model_class.objects.filter(**query)
+ if len(entities) > 0:
+ credential = getattr(entities[0], self.property_name)
+ if credential and hasattr(credential, 'set_store'):
+ credential.set_store(self)
+ return credential
+
+ def locked_put(self, credentials):
+ """Write a Credentials to the datastore.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+ """
+ args = {self.key_name: self.key_value}
+ entity = self.model_class(**args)
+ setattr(entity, self.property_name, credentials)
+ entity.save()
+
+ def locked_delete(self):
+ """Delete Credentials from the datastore."""
+
+ query = {self.key_name: self.key_value}
+ entities = self.model_class.objects.filter(**query).delete()
diff --git a/py/minijack/oauth2client/file.py b/py/minijack/oauth2client/file.py
new file mode 100644
index 0000000..1895f94
--- /dev/null
+++ b/py/minijack/oauth2client/file.py
@@ -0,0 +1,124 @@
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for OAuth.
+
+Utilities for making it easier to work with OAuth 2.0
+credentials.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+import os
+import stat
+import threading
+
+from anyjson import simplejson
+from client import Storage as BaseStorage
+from client import Credentials
+
+
+class CredentialsFileSymbolicLinkError(Exception):
+ """Credentials files must not be symbolic links."""
+
+
+class Storage(BaseStorage):
+ """Store and retrieve a single credential to and from a file."""
+
+ def __init__(self, filename):
+ self._filename = filename
+ self._lock = threading.Lock()
+
+ def _validate_file(self):
+ if os.path.islink(self._filename):
+ raise CredentialsFileSymbolicLinkError(
+ 'File: %s is a symbolic link.' % self._filename)
+
+ def acquire_lock(self):
+ """Acquires any lock necessary to access this Storage.
+
+ This lock is not reentrant."""
+ self._lock.acquire()
+
+ def release_lock(self):
+ """Release the Storage lock.
+
+ Trying to release a lock that isn't held will result in a
+ RuntimeError.
+ """
+ self._lock.release()
+
+ def locked_get(self):
+ """Retrieve Credential from file.
+
+ Returns:
+ oauth2client.client.Credentials
+
+ Raises:
+ CredentialsFileSymbolicLinkError if the file is a symbolic link.
+ """
+ credentials = None
+ self._validate_file()
+ try:
+ f = open(self._filename, 'rb')
+ content = f.read()
+ f.close()
+ except IOError:
+ return credentials
+
+ try:
+ credentials = Credentials.new_from_json(content)
+ credentials.set_store(self)
+ except ValueError:
+ pass
+
+ return credentials
+
+ def _create_file_if_needed(self):
+ """Create an empty file if necessary.
+
+ This method will not initialize the file. Instead it implements a
+ simple version of "touch" to ensure the file has been created.
+ """
+ if not os.path.exists(self._filename):
+ old_umask = os.umask(0177)
+ try:
+ open(self._filename, 'a+b').close()
+ finally:
+ os.umask(old_umask)
+
+ def locked_put(self, credentials):
+ """Write Credentials to file.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+
+ Raises:
+ CredentialsFileSymbolicLinkError if the file is a symbolic link.
+ """
+
+ self._create_file_if_needed()
+ self._validate_file()
+ f = open(self._filename, 'wb')
+ f.write(credentials.to_json())
+ f.close()
+
+ def locked_delete(self):
+ """Delete Credentials file.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+ """
+
+ os.unlink(self._filename)
diff --git a/py/minijack/oauth2client/gce.py b/py/minijack/oauth2client/gce.py
new file mode 100644
index 0000000..c7fd7c1
--- /dev/null
+++ b/py/minijack/oauth2client/gce.py
@@ -0,0 +1,90 @@
+# Copyright (C) 2012 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for Google Compute Engine
+
+Utilities for making it easier to use OAuth 2.0 on Google Compute Engine.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+import httplib2
+import logging
+import uritemplate
+
+from oauth2client import util
+from oauth2client.anyjson import simplejson
+from oauth2client.client import AccessTokenRefreshError
+from oauth2client.client import AssertionCredentials
+
+logger = logging.getLogger(__name__)
+
+# URI Template for the endpoint that returns access_tokens.
+META = ('http://metadata.google.internal/0.1/meta-data/service-accounts/'
+ 'default/acquire{?scope}')
+
+
+class AppAssertionCredentials(AssertionCredentials):
+ """Credentials object for Compute Engine Assertion Grants
+
+ This object will allow a Compute Engine instance to identify itself to
+ Google and other OAuth 2.0 servers that can verify assertions. It can be used
+ for the purpose of accessing data stored under an account assigned to the
+ Compute Engine instance itself.
+
+ This credential does not require a flow to instantiate because it represents
+ a two legged flow, and therefore has all of the required information to
+ generate and refresh its own access tokens.
+ """
+
+ @util.positional(2)
+ def __init__(self, scope, **kwargs):
+ """Constructor for AppAssertionCredentials
+
+ Args:
+ scope: string or iterable of strings, scope(s) of the credentials being
+ requested.
+ """
+ self.scope = util.scopes_to_string(scope)
+
+ # Assertion type is no longer used, but still in the parent class signature.
+ super(AppAssertionCredentials, self).__init__(None)
+
+ @classmethod
+ def from_json(cls, json):
+ data = simplejson.loads(json)
+ return AppAssertionCredentials(data['scope'])
+
+ def _refresh(self, http_request):
+ """Refreshes the access_token.
+
+ Skip all the storage hoops and just refresh using the API.
+
+ Args:
+ http_request: callable, a callable that matches the method signature of
+ httplib2.Http.request, used to make the refresh request.
+
+ Raises:
+ AccessTokenRefreshError: When the refresh fails.
+ """
+ uri = uritemplate.expand(META, {'scope': self.scope})
+ response, content = http_request(uri)
+ if response.status == 200:
+ try:
+ d = simplejson.loads(content)
+ except StandardError, e:
+ raise AccessTokenRefreshError(str(e))
+ self.access_token = d['accessToken']
+ else:
+ raise AccessTokenRefreshError(content)
diff --git a/py/minijack/oauth2client/keyring_storage.py b/py/minijack/oauth2client/keyring_storage.py
new file mode 100644
index 0000000..efe2949
--- /dev/null
+++ b/py/minijack/oauth2client/keyring_storage.py
@@ -0,0 +1,109 @@
+# Copyright (C) 2012 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A keyring based Storage.
+
+A Storage for Credentials that uses the keyring module.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+
+import keyring
+import threading
+
+from client import Storage as BaseStorage
+from client import Credentials
+
+
+class Storage(BaseStorage):
+ """Store and retrieve a single credential to and from the keyring.
+
+ To use this module you must have the keyring module installed. See
+ <http://pypi.python.org/pypi/keyring/>. This is an optional module and is not
+ installed with oauth2client by default because it does not work on all the
+ platforms that oauth2client supports, such as Google App Engine.
+
+ The keyring module <http://pypi.python.org/pypi/keyring/> is a cross-platform
+ library for access the keyring capabilities of the local system. The user will
+ be prompted for their keyring password when this module is used, and the
+ manner in which the user is prompted will vary per platform.
+
+ Usage:
+ from oauth2client.keyring_storage import Storage
+
+ s = Storage('name_of_application', 'user1')
+ credentials = s.get()
+
+ """
+
+ def __init__(self, service_name, user_name):
+ """Constructor.
+
+ Args:
+ service_name: string, The name of the service under which the credentials
+ are stored.
+ user_name: string, The name of the user to store credentials for.
+ """
+ self._service_name = service_name
+ self._user_name = user_name
+ self._lock = threading.Lock()
+
+ def acquire_lock(self):
+ """Acquires any lock necessary to access this Storage.
+
+ This lock is not reentrant."""
+ self._lock.acquire()
+
+ def release_lock(self):
+ """Release the Storage lock.
+
+ Trying to release a lock that isn't held will result in a
+ RuntimeError.
+ """
+ self._lock.release()
+
+ def locked_get(self):
+ """Retrieve Credential from file.
+
+ Returns:
+ oauth2client.client.Credentials
+ """
+ credentials = None
+ content = keyring.get_password(self._service_name, self._user_name)
+
+ if content is not None:
+ try:
+ credentials = Credentials.new_from_json(content)
+ credentials.set_store(self)
+ except ValueError:
+ pass
+
+ return credentials
+
+ def locked_put(self, credentials):
+ """Write Credentials to file.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+ """
+ keyring.set_password(self._service_name, self._user_name,
+ credentials.to_json())
+
+ def locked_delete(self):
+ """Delete Credentials file.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+ """
+ keyring.set_password(self._service_name, self._user_name, '')
diff --git a/py/minijack/oauth2client/locked_file.py b/py/minijack/oauth2client/locked_file.py
new file mode 100644
index 0000000..26f783e
--- /dev/null
+++ b/py/minijack/oauth2client/locked_file.py
@@ -0,0 +1,361 @@
+# Copyright 2011 Google Inc. All Rights Reserved.
+
+"""Locked file interface that should work on Unix and Windows pythons.
+
+This module first tries to use fcntl locking to ensure serialized access
+to a file, then falls back on a lock file if that is unavialable.
+
+Usage:
+ f = LockedFile('filename', 'r+b', 'rb')
+ f.open_and_lock()
+ if f.is_locked():
+ print 'Acquired filename with r+b mode'
+ f.file_handle().write('locked data')
+ else:
+ print 'Aquired filename with rb mode'
+ f.unlock_and_close()
+"""
+
+__author__ = 'cache@google.com (David T McWherter)'
+
+import errno
+import logging
+import os
+import time
+
+from oauth2client import util
+
+logger = logging.getLogger(__name__)
+
+
+class CredentialsFileSymbolicLinkError(Exception):
+ """Credentials files must not be symbolic links."""
+
+
+class AlreadyLockedException(Exception):
+ """Trying to lock a file that has already been locked by the LockedFile."""
+ pass
+
+
+def validate_file(filename):
+ if os.path.islink(filename):
+ raise CredentialsFileSymbolicLinkError(
+ 'File: %s is a symbolic link.' % filename)
+
+class _Opener(object):
+ """Base class for different locking primitives."""
+
+ def __init__(self, filename, mode, fallback_mode):
+ """Create an Opener.
+
+ Args:
+ filename: string, The pathname of the file.
+ mode: string, The preferred mode to access the file with.
+ fallback_mode: string, The mode to use if locking fails.
+ """
+ self._locked = False
+ self._filename = filename
+ self._mode = mode
+ self._fallback_mode = fallback_mode
+ self._fh = None
+
+ def is_locked(self):
+ """Was the file locked."""
+ return self._locked
+
+ def file_handle(self):
+ """The file handle to the file. Valid only after opened."""
+ return self._fh
+
+ def filename(self):
+ """The filename that is being locked."""
+ return self._filename
+
+ def open_and_lock(self, timeout, delay):
+ """Open the file and lock it.
+
+ Args:
+ timeout: float, How long to try to lock for.
+ delay: float, How long to wait between retries.
+ """
+ pass
+
+ def unlock_and_close(self):
+ """Unlock and close the file."""
+ pass
+
+
+class _PosixOpener(_Opener):
+ """Lock files using Posix advisory lock files."""
+
+ def open_and_lock(self, timeout, delay):
+ """Open the file and lock it.
+
+ Tries to create a .lock file next to the file we're trying to open.
+
+ Args:
+ timeout: float, How long to try to lock for.
+ delay: float, How long to wait between retries.
+
+ Raises:
+ AlreadyLockedException: if the lock is already acquired.
+ IOError: if the open fails.
+ CredentialsFileSymbolicLinkError if the file is a symbolic link.
+ """
+ if self._locked:
+ raise AlreadyLockedException('File %s is already locked' %
+ self._filename)
+ self._locked = False
+
+ validate_file(self._filename)
+ try:
+ self._fh = open(self._filename, self._mode)
+ except IOError, e:
+ # If we can't access with _mode, try _fallback_mode and don't lock.
+ if e.errno == errno.EACCES:
+ self._fh = open(self._filename, self._fallback_mode)
+ return
+
+ lock_filename = self._posix_lockfile(self._filename)
+ start_time = time.time()
+ while True:
+ try:
+ self._lock_fd = os.open(lock_filename,
+ os.O_CREAT|os.O_EXCL|os.O_RDWR)
+ self._locked = True
+ break
+
+ except OSError, e:
+ if e.errno != errno.EEXIST:
+ raise
+ if (time.time() - start_time) >= timeout:
+ logger.warn('Could not acquire lock %s in %s seconds' % (
+ lock_filename, timeout))
+ # Close the file and open in fallback_mode.
+ if self._fh:
+ self._fh.close()
+ self._fh = open(self._filename, self._fallback_mode)
+ return
+ time.sleep(delay)
+
+ def unlock_and_close(self):
+ """Unlock a file by removing the .lock file, and close the handle."""
+ if self._locked:
+ lock_filename = self._posix_lockfile(self._filename)
+ os.close(self._lock_fd)
+ os.unlink(lock_filename)
+ self._locked = False
+ self._lock_fd = None
+ if self._fh:
+ self._fh.close()
+
+ def _posix_lockfile(self, filename):
+ """The name of the lock file to use for posix locking."""
+ return '%s.lock' % filename
+
+
+try:
+ import fcntl
+
+ class _FcntlOpener(_Opener):
+ """Open, lock, and unlock a file using fcntl.lockf."""
+
+ def open_and_lock(self, timeout, delay):
+ """Open the file and lock it.
+
+ Args:
+ timeout: float, How long to try to lock for.
+ delay: float, How long to wait between retries
+
+ Raises:
+ AlreadyLockedException: if the lock is already acquired.
+ IOError: if the open fails.
+ CredentialsFileSymbolicLinkError if the file is a symbolic link.
+ """
+ if self._locked:
+ raise AlreadyLockedException('File %s is already locked' %
+ self._filename)
+ start_time = time.time()
+
+ validate_file(self._filename)
+ try:
+ self._fh = open(self._filename, self._mode)
+ except IOError, e:
+ # If we can't access with _mode, try _fallback_mode and don't lock.
+ if e.errno == errno.EACCES:
+ self._fh = open(self._filename, self._fallback_mode)
+ return
+
+ # We opened in _mode, try to lock the file.
+ while True:
+ try:
+ fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX)
+ self._locked = True
+ return
+ except IOError, e:
+ # If not retrying, then just pass on the error.
+ if timeout == 0:
+ raise e
+ if e.errno != errno.EACCES:
+ raise e
+ # We could not acquire the lock. Try again.
+ if (time.time() - start_time) >= timeout:
+ logger.warn('Could not lock %s in %s seconds' % (
+ self._filename, timeout))
+ if self._fh:
+ self._fh.close()
+ self._fh = open(self._filename, self._fallback_mode)
+ return
+ time.sleep(delay)
+
+ def unlock_and_close(self):
+ """Close and unlock the file using the fcntl.lockf primitive."""
+ if self._locked:
+ fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN)
+ self._locked = False
+ if self._fh:
+ self._fh.close()
+except ImportError:
+ _FcntlOpener = None
+
+
+try:
+ import pywintypes
+ import win32con
+ import win32file
+
+ class _Win32Opener(_Opener):
+ """Open, lock, and unlock a file using windows primitives."""
+
+ # Error #33:
+ # 'The process cannot access the file because another process'
+ FILE_IN_USE_ERROR = 33
+
+ # Error #158:
+ # 'The segment is already unlocked.'
+ FILE_ALREADY_UNLOCKED_ERROR = 158
+
+ def open_and_lock(self, timeout, delay):
+ """Open the file and lock it.
+
+ Args:
+ timeout: float, How long to try to lock for.
+ delay: float, How long to wait between retries
+
+ Raises:
+ AlreadyLockedException: if the lock is already acquired.
+ IOError: if the open fails.
+ CredentialsFileSymbolicLinkError if the file is a symbolic link.
+ """
+ if self._locked:
+ raise AlreadyLockedException('File %s is already locked' %
+ self._filename)
+ start_time = time.time()
+
+ validate_file(self._filename)
+ try:
+ self._fh = open(self._filename, self._mode)
+ except IOError, e:
+ # If we can't access with _mode, try _fallback_mode and don't lock.
+ if e.errno == errno.EACCES:
+ self._fh = open(self._filename, self._fallback_mode)
+ return
+
+ # We opened in _mode, try to lock the file.
+ while True:
+ try:
+ hfile = win32file._get_osfhandle(self._fh.fileno())
+ win32file.LockFileEx(
+ hfile,
+ (win32con.LOCKFILE_FAIL_IMMEDIATELY|
+ win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000,
+ pywintypes.OVERLAPPED())
+ self._locked = True
+ return
+ except pywintypes.error, e:
+ if timeout == 0:
+ raise e
+
+ # If the error is not that the file is already in use, raise.
+ if e[0] != _Win32Opener.FILE_IN_USE_ERROR:
+ raise
+
+ # We could not acquire the lock. Try again.
+ if (time.time() - start_time) >= timeout:
+ logger.warn('Could not lock %s in %s seconds' % (
+ self._filename, timeout))
+ if self._fh:
+ self._fh.close()
+ self._fh = open(self._filename, self._fallback_mode)
+ return
+ time.sleep(delay)
+
+ def unlock_and_close(self):
+ """Close and unlock the file using the win32 primitive."""
+ if self._locked:
+ try:
+ hfile = win32file._get_osfhandle(self._fh.fileno())
+ win32file.UnlockFileEx(hfile, 0, -0x10000, pywintypes.OVERLAPPED())
+ except pywintypes.error, e:
+ if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR:
+ raise
+ self._locked = False
+ if self._fh:
+ self._fh.close()
+except ImportError:
+ _Win32Opener = None
+
+
+class LockedFile(object):
+ """Represent a file that has exclusive access."""
+
+ @util.positional(4)
+ def __init__(self, filename, mode, fallback_mode, use_native_locking=True):
+ """Construct a LockedFile.
+
+ Args:
+ filename: string, The path of the file to open.
+ mode: string, The mode to try to open the file with.
+ fallback_mode: string, The mode to use if locking fails.
+ use_native_locking: bool, Whether or not fcntl/win32 locking is used.
+ """
+ opener = None
+ if not opener and use_native_locking:
+ if _Win32Opener:
+ opener = _Win32Opener(filename, mode, fallback_mode)
+ if _FcntlOpener:
+ opener = _FcntlOpener(filename, mode, fallback_mode)
+
+ if not opener:
+ opener = _PosixOpener(filename, mode, fallback_mode)
+
+ self._opener = opener
+
+ def filename(self):
+ """Return the filename we were constructed with."""
+ return self._opener._filename
+
+ def file_handle(self):
+ """Return the file_handle to the opened file."""
+ return self._opener.file_handle()
+
+ def is_locked(self):
+ """Return whether we successfully locked the file."""
+ return self._opener.is_locked()
+
+ def open_and_lock(self, timeout=0, delay=0.05):
+ """Open the file, trying to lock it.
+
+ Args:
+ timeout: float, The number of seconds to try to acquire the lock.
+ delay: float, The number of seconds to wait between retry attempts.
+
+ Raises:
+ AlreadyLockedException: if the lock is already acquired.
+ IOError: if the open fails.
+ """
+ self._opener.open_and_lock(timeout, delay)
+
+ def unlock_and_close(self):
+ """Unlock and close a file."""
+ self._opener.unlock_and_close()
diff --git a/py/minijack/oauth2client/multistore_file.py b/py/minijack/oauth2client/multistore_file.py
new file mode 100644
index 0000000..e1b39f7
--- /dev/null
+++ b/py/minijack/oauth2client/multistore_file.py
@@ -0,0 +1,409 @@
+# Copyright 2011 Google Inc. All Rights Reserved.
+
+"""Multi-credential file store with lock support.
+
+This module implements a JSON credential store where multiple
+credentials can be stored in one file. That file supports locking
+both in a single process and across processes.
+
+The credential themselves are keyed off of:
+* client_id
+* user_agent
+* scope
+
+The format of the stored data is like so:
+{
+ 'file_version': 1,
+ 'data': [
+ {
+ 'key': {
+ 'clientId': '<client id>',
+ 'userAgent': '<user agent>',
+ 'scope': '<scope>'
+ },
+ 'credential': {
+ # JSON serialized Credentials.
+ }
+ }
+ ]
+}
+"""
+
+__author__ = 'jbeda@google.com (Joe Beda)'
+
+import base64
+import errno
+import logging
+import os
+import threading
+
+from anyjson import simplejson
+from oauth2client.client import Storage as BaseStorage
+from oauth2client.client import Credentials
+from oauth2client import util
+from locked_file import LockedFile
+
+logger = logging.getLogger(__name__)
+
+# A dict from 'filename'->_MultiStore instances
+_multistores = {}
+_multistores_lock = threading.Lock()
+
+
+class Error(Exception):
+ """Base error for this module."""
+ pass
+
+
+class NewerCredentialStoreError(Error):
+ """The credential store is a newer version that supported."""
+ pass
+
+
+@util.positional(4)
+def get_credential_storage(filename, client_id, user_agent, scope,
+ warn_on_readonly=True):
+ """Get a Storage instance for a credential.
+
+ Args:
+ filename: The JSON file storing a set of credentials
+ client_id: The client_id for the credential
+ user_agent: The user agent for the credential
+ scope: string or iterable of strings, Scope(s) being requested
+ warn_on_readonly: if True, log a warning if the store is readonly
+
+ Returns:
+ An object derived from client.Storage for getting/setting the
+ credential.
+ """
+ # Recreate the legacy key with these specific parameters
+ key = {'clientId': client_id, 'userAgent': user_agent,
+ 'scope': util.scopes_to_string(scope)}
+ return get_credential_storage_custom_key(
+ filename, key, warn_on_readonly=warn_on_readonly)
+
+
+@util.positional(2)
+def get_credential_storage_custom_string_key(
+ filename, key_string, warn_on_readonly=True):
+ """Get a Storage instance for a credential using a single string as a key.
+
+ Allows you to provide a string as a custom key that will be used for
+ credential storage and retrieval.
+
+ Args:
+ filename: The JSON file storing a set of credentials
+ key_string: A string to use as the key for storing this credential.
+ warn_on_readonly: if True, log a warning if the store is readonly
+
+ Returns:
+ An object derived from client.Storage for getting/setting the
+ credential.
+ """
+ # Create a key dictionary that can be used
+ key_dict = {'key': key_string}
+ return get_credential_storage_custom_key(
+ filename, key_dict, warn_on_readonly=warn_on_readonly)
+
+
+@util.positional(2)
+def get_credential_storage_custom_key(
+ filename, key_dict, warn_on_readonly=True):
+ """Get a Storage instance for a credential using a dictionary as a key.
+
+ Allows you to provide a dictionary as a custom key that will be used for
+ credential storage and retrieval.
+
+ Args:
+ filename: The JSON file storing a set of credentials
+ key_dict: A dictionary to use as the key for storing this credential. There
+ is no ordering of the keys in the dictionary. Logically equivalent
+ dictionaries will produce equivalent storage keys.
+ warn_on_readonly: if True, log a warning if the store is readonly
+
+ Returns:
+ An object derived from client.Storage for getting/setting the
+ credential.
+ """
+ filename = os.path.expanduser(filename)
+ _multistores_lock.acquire()
+ try:
+ multistore = _multistores.setdefault(
+ filename, _MultiStore(filename, warn_on_readonly=warn_on_readonly))
+ finally:
+ _multistores_lock.release()
+ key = util.dict_to_tuple_key(key_dict)
+ return multistore._get_storage(key)
+
+
+class _MultiStore(object):
+ """A file backed store for multiple credentials."""
+
+ @util.positional(2)
+ def __init__(self, filename, warn_on_readonly=True):
+ """Initialize the class.
+
+ This will create the file if necessary.
+ """
+ self._file = LockedFile(filename, 'r+b', 'rb')
+ self._thread_lock = threading.Lock()
+ self._read_only = False
+ self._warn_on_readonly = warn_on_readonly
+
+ self._create_file_if_needed()
+
+ # Cache of deserialized store. This is only valid after the
+ # _MultiStore is locked or _refresh_data_cache is called. This is
+ # of the form of:
+ #
+ # ((key, value), (key, value)...) -> OAuth2Credential
+ #
+ # If this is None, then the store hasn't been read yet.
+ self._data = None
+
+ class _Storage(BaseStorage):
+ """A Storage object that knows how to read/write a single credential."""
+
+ def __init__(self, multistore, key):
+ self._multistore = multistore
+ self._key = key
+
+ def acquire_lock(self):
+ """Acquires any lock necessary to access this Storage.
+
+ This lock is not reentrant.
+ """
+ self._multistore._lock()
+
+ def release_lock(self):
+ """Release the Storage lock.
+
+ Trying to release a lock that isn't held will result in a
+ RuntimeError.
+ """
+ self._multistore._unlock()
+
+ def locked_get(self):
+ """Retrieve credential.
+
+ The Storage lock must be held when this is called.
+
+ Returns:
+ oauth2client.client.Credentials
+ """
+ credential = self._multistore._get_credential(self._key)
+ if credential:
+ credential.set_store(self)
+ return credential
+
+ def locked_put(self, credentials):
+ """Write a credential.
+
+ The Storage lock must be held when this is called.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+ """
+ self._multistore._update_credential(self._key, credentials)
+
+ def locked_delete(self):
+ """Delete a credential.
+
+ The Storage lock must be held when this is called.
+
+ Args:
+ credentials: Credentials, the credentials to store.
+ """
+ self._multistore._delete_credential(self._key)
+
+ def _create_file_if_needed(self):
+ """Create an empty file if necessary.
+
+ This method will not initialize the file. Instead it implements a
+ simple version of "touch" to ensure the file has been created.
+ """
+ if not os.path.exists(self._file.filename()):
+ old_umask = os.umask(0177)
+ try:
+ open(self._file.filename(), 'a+b').close()
+ finally:
+ os.umask(old_umask)
+
+ def _lock(self):
+ """Lock the entire multistore."""
+ self._thread_lock.acquire()
+ self._file.open_and_lock()
+ if not self._file.is_locked():
+ self._read_only = True
+ if self._warn_on_readonly:
+ logger.warn('The credentials file (%s) is not writable. Opening in '
+ 'read-only mode. Any refreshed credentials will only be '
+ 'valid for this run.' % self._file.filename())
+ if os.path.getsize(self._file.filename()) == 0:
+ logger.debug('Initializing empty multistore file')
+ # The multistore is empty so write out an empty file.
+ self._data = {}
+ self._write()
+ elif not self._read_only or self._data is None:
+ # Only refresh the data if we are read/write or we haven't
+ # cached the data yet. If we are readonly, we assume is isn't
+ # changing out from under us and that we only have to read it
+ # once. This prevents us from whacking any new access keys that
+ # we have cached in memory but were unable to write out.
+ self._refresh_data_cache()
+
+ def _unlock(self):
+ """Release the lock on the multistore."""
+ self._file.unlock_and_close()
+ self._thread_lock.release()
+
+ def _locked_json_read(self):
+ """Get the raw content of the multistore file.
+
+ The multistore must be locked when this is called.
+
+ Returns:
+ The contents of the multistore decoded as JSON.
+ """
+ assert self._thread_lock.locked()
+ self._file.file_handle().seek(0)
+ return simplejson.load(self._file.file_handle())
+
+ def _locked_json_write(self, data):
+ """Write a JSON serializable data structure to the multistore.
+
+ The multistore must be locked when this is called.
+
+ Args:
+ data: The data to be serialized and written.
+ """
+ assert self._thread_lock.locked()
+ if self._read_only:
+ return
+ self._file.file_handle().seek(0)
+ simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2)
+ self._file.file_handle().truncate()
+
+ def _refresh_data_cache(self):
+ """Refresh the contents of the multistore.
+
+ The multistore must be locked when this is called.
+
+ Raises:
+ NewerCredentialStoreError: Raised when a newer client has written the
+ store.
+ """
+ self._data = {}
+ try:
+ raw_data = self._locked_json_read()
+ except Exception:
+ logger.warn('Credential data store could not be loaded. '
+ 'Will ignore and overwrite.')
+ return
+
+ version = 0
+ try:
+ version = raw_data['file_version']
+ except Exception:
+ logger.warn('Missing version for credential data store. It may be '
+ 'corrupt or an old version. Overwriting.')
+ if version > 1:
+ raise NewerCredentialStoreError(
+ 'Credential file has file_version of %d. '
+ 'Only file_version of 1 is supported.' % version)
+
+ credentials = []
+ try:
+ credentials = raw_data['data']
+ except (TypeError, KeyError):
+ pass
+
+ for cred_entry in credentials:
+ try:
+ (key, credential) = self._decode_credential_from_json(cred_entry)
+ self._data[key] = credential
+ except:
+ # If something goes wrong loading a credential, just ignore it
+ logger.info('Error decoding credential, skipping', exc_info=True)
+
+ def _decode_credential_from_json(self, cred_entry):
+ """Load a credential from our JSON serialization.
+
+ Args:
+ cred_entry: A dict entry from the data member of our format
+
+ Returns:
+ (key, cred) where the key is the key tuple and the cred is the
+ OAuth2Credential object.
+ """
+ raw_key = cred_entry['key']
+ key = util.dict_to_tuple_key(raw_key)
+ credential = None
+ credential = Credentials.new_from_json(simplejson.dumps(cred_entry['credential']))
+ return (key, credential)
+
+ def _write(self):
+ """Write the cached data back out.
+
+ The multistore must be locked.
+ """
+ raw_data = {'file_version': 1}
+ raw_creds = []
+ raw_data['data'] = raw_creds
+ for (cred_key, cred) in self._data.items():
+ raw_key = dict(cred_key)
+ raw_cred = simplejson.loads(cred.to_json())
+ raw_creds.append({'key': raw_key, 'credential': raw_cred})
+ self._locked_json_write(raw_data)
+
+ def _get_credential(self, key):
+ """Get a credential from the multistore.
+
+ The multistore must be locked.
+
+ Args:
+ key: The key used to retrieve the credential
+
+ Returns:
+ The credential specified or None if not present
+ """
+ return self._data.get(key, None)
+
+ def _update_credential(self, key, cred):
+ """Update a credential and write the multistore.
+
+ This must be called when the multistore is locked.
+
+ Args:
+ key: The key used to retrieve the credential
+ cred: The OAuth2Credential to update/set
+ """
+ self._data[key] = cred
+ self._write()
+
+ def _delete_credential(self, key):
+ """Delete a credential and write the multistore.
+
+ This must be called when the multistore is locked.
+
+ Args:
+ key: The key used to retrieve the credential
+ """
+ try:
+ del self._data[key]
+ except KeyError:
+ pass
+ self._write()
+
+ def _get_storage(self, key):
+ """Get a Storage object to get/set a credential.
+
+ This Storage is a 'view' into the multistore.
+
+ Args:
+ key: The key used to retrieve the credential
+
+ Returns:
+ A Storage object that can be used to get/set this cred
+ """
+ return self._Storage(self, key)
diff --git a/py/minijack/oauth2client/tools.py b/py/minijack/oauth2client/tools.py
new file mode 100644
index 0000000..93b0171
--- /dev/null
+++ b/py/minijack/oauth2client/tools.py
@@ -0,0 +1,205 @@
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Command-line tools for authenticating via OAuth 2.0
+
+Do the OAuth 2.0 Web Server dance for a command line application. Stores the
+generated credentials in a common file that is used by other example apps in
+the same directory.
+"""
+
+__author__ = 'jcgregorio@google.com (Joe Gregorio)'
+__all__ = ['run']
+
+
+import BaseHTTPServer
+import gflags
+import socket
+import sys
+import webbrowser
+
+from oauth2client.client import FlowExchangeError
+from oauth2client.client import OOB_CALLBACK_URN
+from oauth2client import util
+
+try:
+ from urlparse import parse_qsl
+except ImportError:
+ from cgi import parse_qsl
+
+
+FLAGS = gflags.FLAGS
+
+gflags.DEFINE_boolean('auth_local_webserver', True,
+ ('Run a local web server to handle redirects during '
+ 'OAuth authorization.'))
+
+gflags.DEFINE_string('auth_host_name', 'localhost',
+ ('Host name to use when running a local web server to '
+ 'handle redirects during OAuth authorization.'))
+
+gflags.DEFINE_multi_int('auth_host_port', [8080, 8090],
+ ('Port to use when running a local web server to '
+ 'handle redirects during OAuth authorization.'))
+
+
+class ClientRedirectServer(BaseHTTPServer.HTTPServer):
+ """A server to handle OAuth 2.0 redirects back to localhost.
+
+ Waits for a single request and parses the query parameters
+ into query_params and then stops serving.
+ """
+ query_params = {}
+
+
+class ClientRedirectHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+ """A handler for OAuth 2.0 redirects back to localhost.
+
+ Waits for a single request and parses the query parameters
+ into the servers query_params and then stops serving.
+ """
+
+ def do_GET(s):
+ """Handle a GET request.
+
+ Parses the query parameters and prints a message
+ if the flow has completed. Note that we can't detect
+ if an error occurred.
+ """
+ s.send_response(200)
+ s.send_header("Content-type", "text/html")
+ s.end_headers()
+ query = s.path.split('?', 1)[-1]
+ query = dict(parse_qsl(query))
+ s.server.query_params = query
+ s.wfile.write("<html><head><title>Authentication Status</title></head>")
+ s.wfile.write("<body><p>The authentication flow has completed.</p>")
+ s.wfile.write("</body></html>")
+
+ def log_message(self, format, *args):
+ """Do not log messages to stdout while running as command line program."""
+ pass
+
+
+@util.positional(2)
+def run(flow, storage, http=None):
+ """Core code for a command-line application.
+
+ The run() function is called from your application and runs through all the
+ steps to obtain credentials. It takes a Flow argument and attempts to open an
+ authorization server page in the user's default web browser. The server asks
+ the user to grant your application access to the user's data. If the user
+ grants access, the run() function returns new credentials. The new credentials
+ are also stored in the Storage argument, which updates the file associated
+ with the Storage object.
+
+ It presumes it is run from a command-line application and supports the
+ following flags:
+
+ --auth_host_name: Host name to use when running a local web server
+ to handle redirects during OAuth authorization.
+ (default: 'localhost')
+
+ --auth_host_port: Port to use when running a local web server to handle
+ redirects during OAuth authorization.;
+ repeat this option to specify a list of values
+ (default: '[8080, 8090]')
+ (an integer)
+
+ --[no]auth_local_webserver: Run a local web server to handle redirects
+ during OAuth authorization.
+ (default: 'true')
+
+ Since it uses flags make sure to initialize the gflags module before calling
+ run().
+
+ Args:
+ flow: Flow, an OAuth 2.0 Flow to step through.
+ storage: Storage, a Storage to store the credential in.
+ http: An instance of httplib2.Http.request
+ or something that acts like it.
+
+ Returns:
+ Credentials, the obtained credential.
+ """
+ if FLAGS.auth_local_webserver:
+ success = False
+ port_number = 0
+ for port in FLAGS.auth_host_port:
+ port_number = port
+ try:
+ httpd = ClientRedirectServer((FLAGS.auth_host_name, port),
+ ClientRedirectHandler)
+ except socket.error, e:
+ pass
+ else:
+ success = True
+ break
+ FLAGS.auth_local_webserver = success
+ if not success:
+ print 'Failed to start a local webserver listening on either port 8080'
+ print 'or port 9090. Please check your firewall settings and locally'
+ print 'running programs that may be blocking or using those ports.'
+ print
+ print 'Falling back to --noauth_local_webserver and continuing with',
+ print 'authorization.'
+ print
+
+ if FLAGS.auth_local_webserver:
+ oauth_callback = 'http://%s:%s/' % (FLAGS.auth_host_name, port_number)
+ else:
+ oauth_callback = OOB_CALLBACK_URN
+ flow.redirect_uri = oauth_callback
+ authorize_url = flow.step1_get_authorize_url()
+
+ if FLAGS.auth_local_webserver:
+ webbrowser.open(authorize_url, new=1, autoraise=True)
+ print 'Your browser has been opened to visit:'
+ print
+ print ' ' + authorize_url
+ print
+ print 'If your browser is on a different machine then exit and re-run this'
+ print 'application with the command-line parameter '
+ print
+ print ' --noauth_local_webserver'
+ print
+ else:
+ print 'Go to the following link in your browser:'
+ print
+ print ' ' + authorize_url
+ print
+
+ code = None
+ if FLAGS.auth_local_webserver:
+ httpd.handle_request()
+ if 'error' in httpd.query_params:
+ sys.exit('Authentication request was rejected.')
+ if 'code' in httpd.query_params:
+ code = httpd.query_params['code']
+ else:
+ print 'Failed to find "code" in the query parameters of the redirect.'
+ sys.exit('Try running with --noauth_local_webserver.')
+ else:
+ code = raw_input('Enter verification code: ').strip()
+
+ try:
+ credential = flow.step2_exchange(code, http=http)
+ except FlowExchangeError, e:
+ sys.exit('Authentication has failed: %s' % e)
+
+ storage.put(credential)
+ credential.set_store(storage)
+ print 'Authentication successful.'
+
+ return credential
diff --git a/py/minijack/oauth2client/util.py b/py/minijack/oauth2client/util.py
new file mode 100644
index 0000000..c94eae9
--- /dev/null
+++ b/py/minijack/oauth2client/util.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+#
+# Copyright 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Common utility library."""
+
+__author__ = ['rafek@google.com (Rafe Kaplan)',
+ 'guido@google.com (Guido van Rossum)',
+]
+__all__ = [
+ 'positional',
+]
+
+import gflags
+import inspect
+import logging
+import types
+import urllib
+import urlparse
+
+try:
+ from urlparse import parse_qsl
+except ImportError:
+ from cgi import parse_qsl
+
+logger = logging.getLogger(__name__)
+
+FLAGS = gflags.FLAGS
+
+gflags.DEFINE_enum('positional_parameters_enforcement', 'WARNING',
+ ['EXCEPTION', 'WARNING', 'IGNORE'],
+ 'The action when an oauth2client.util.positional declaration is violated.')
+
+
+def positional(max_positional_args):
+ """A decorator to declare that only the first N arguments my be positional.
+
+ This decorator makes it easy to support Python 3 style key-word only
+ parameters. For example, in Python 3 it is possible to write:
+
+ def fn(pos1, *, kwonly1=None, kwonly1=None):
+ ...
+
+ All named parameters after * must be a keyword:
+
+ fn(10, 'kw1', 'kw2') # Raises exception.
+ fn(10, kwonly1='kw1') # Ok.
+
+ Example:
+ To define a function like above, do:
+
+ @positional(1)
+ def fn(pos1, kwonly1=None, kwonly2=None):
+ ...
+
+ If no default value is provided to a keyword argument, it becomes a required
+ keyword argument:
+
+ @positional(0)
+ def fn(required_kw):
+ ...
+
+ This must be called with the keyword parameter:
+
+ fn() # Raises exception.
+ fn(10) # Raises exception.
+ fn(required_kw=10) # Ok.
+
+ When defining instance or class methods always remember to account for
+ 'self' and 'cls':
+
+ class MyClass(object):
+
+ @positional(2)
+ def my_method(self, pos1, kwonly1=None):
+ ...
+
+ @classmethod
+ @positional(2)
+ def my_method(cls, pos1, kwonly1=None):
+ ...
+
+ The positional decorator behavior is controlled by the
+ --positional_parameters_enforcement flag. The flag may be set to 'EXCEPTION',
+ 'WARNING' or 'IGNORE' to raise an exception, log a warning, or do nothing,
+ respectively, if a declaration is violated.
+
+ Args:
+ max_positional_arguments: Maximum number of positional arguments. All
+ parameters after the this index must be keyword only.
+
+ Returns:
+ A decorator that prevents using arguments after max_positional_args from
+ being used as positional parameters.
+
+ Raises:
+ TypeError if a key-word only argument is provided as a positional parameter,
+ but only if the --positional_parameters_enforcement flag is set to
+ 'EXCEPTION'.
+ """
+ def positional_decorator(wrapped):
+ def positional_wrapper(*args, **kwargs):
+ if len(args) > max_positional_args:
+ plural_s = ''
+ if max_positional_args != 1:
+ plural_s = 's'
+ message = '%s() takes at most %d positional argument%s (%d given)' % (
+ wrapped.__name__, max_positional_args, plural_s, len(args))
+ if FLAGS.positional_parameters_enforcement == 'EXCEPTION':
+ raise TypeError(message)
+ elif FLAGS.positional_parameters_enforcement == 'WARNING':
+ import traceback
+ traceback.print_stack()
+ logger.warning(message)
+ else: # IGNORE
+ pass
+ return wrapped(*args, **kwargs)
+ return positional_wrapper
+
+ if isinstance(max_positional_args, (int, long)):
+ return positional_decorator
+ else:
+ args, _, _, defaults = inspect.getargspec(max_positional_args)
+ return positional(len(args) - len(defaults))(max_positional_args)
+
+
+def scopes_to_string(scopes):
+ """Converts scope value to a string.
+
+ If scopes is a string then it is simply passed through. If scopes is an
+ iterable then a string is returned that is all the individual scopes
+ concatenated with spaces.
+
+ Args:
+ scopes: string or iterable of strings, the scopes.
+
+ Returns:
+ The scopes formatted as a single string.
+ """
+ if isinstance(scopes, types.StringTypes):
+ return scopes
+ else:
+ return ' '.join(scopes)
+
+
+def dict_to_tuple_key(dictionary):
+ """Converts a dictionary to a tuple that can be used as an immutable key.
+
+ The resulting key is always sorted so that logically equivalent dictionaries
+ always produce an identical tuple for a key.
+
+ Args:
+ dictionary: the dictionary to use as the key.
+
+ Returns:
+ A tuple representing the dictionary in it's naturally sorted ordering.
+ """
+ return tuple(sorted(dictionary.items()))
+
+
+def _add_query_parameter(url, name, value):
+ """Adds a query parameter to a url.
+
+ Replaces the current value if it already exists in the URL.
+
+ Args:
+ url: string, url to add the query parameter to.
+ name: string, query parameter name.
+ value: string, query parameter value.
+
+ Returns:
+ Updated query parameter. Does not update the url if value is None.
+ """
+ if value is None:
+ return url
+ else:
+ parsed = list(urlparse.urlparse(url))
+ q = dict(parse_qsl(parsed[4]))
+ q[name] = value
+ parsed[4] = urllib.urlencode(q)
+ return urlparse.urlunparse(parsed)
diff --git a/py/minijack/oauth2client/xsrfutil.py b/py/minijack/oauth2client/xsrfutil.py
new file mode 100644
index 0000000..7e1fe5c
--- /dev/null
+++ b/py/minijack/oauth2client/xsrfutil.py
@@ -0,0 +1,113 @@
+#!/usr/bin/python2.5
+#
+# Copyright 2010 the Melange authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper methods for creating & verifying XSRF tokens."""
+
+__authors__ = [
+ '"Doug Coker" <dcoker@google.com>',
+ '"Joe Gregorio" <jcgregorio@google.com>',
+]
+
+
+import base64
+import hmac
+import os # for urandom
+import time
+
+from oauth2client import util
+
+
+# Delimiter character
+DELIMITER = ':'
+
+# 1 hour in seconds
+DEFAULT_TIMEOUT_SECS = 1*60*60
+
+@util.positional(2)
+def generate_token(key, user_id, action_id="", when=None):
+ """Generates a URL-safe token for the given user, action, time tuple.
+
+ Args:
+ key: secret key to use.
+ user_id: the user ID of the authenticated user.
+ action_id: a string identifier of the action they requested
+ authorization for.
+ when: the time in seconds since the epoch at which the user was
+ authorized for this action. If not set the current time is used.
+
+ Returns:
+ A string XSRF protection token.
+ """
+ when = when or int(time.time())
+ digester = hmac.new(key)
+ digester.update(str(user_id))
+ digester.update(DELIMITER)
+ digester.update(action_id)
+ digester.update(DELIMITER)
+ digester.update(str(when))
+ digest = digester.digest()
+
+ token = base64.urlsafe_b64encode('%s%s%d' % (digest,
+ DELIMITER,
+ when))
+ return token
+
+
+@util.positional(3)
+def validate_token(key, token, user_id, action_id="", current_time=None):
+ """Validates that the given token authorizes the user for the action.
+
+ Tokens are invalid if the time of issue is too old or if the token
+ does not match what generateToken outputs (i.e. the token was forged).
+
+ Args:
+ key: secret key to use.
+ token: a string of the token generated by generateToken.
+ user_id: the user ID of the authenticated user.
+ action_id: a string identifier of the action they requested
+ authorization for.
+
+ Returns:
+ A boolean - True if the user is authorized for the action, False
+ otherwise.
+ """
+ if not token:
+ return False
+ try:
+ decoded = base64.urlsafe_b64decode(str(token))
+ token_time = long(decoded.split(DELIMITER)[-1])
+ except (TypeError, ValueError):
+ return False
+ if current_time is None:
+ current_time = time.time()
+ # If the token is too old it's not valid.
+ if current_time - token_time > DEFAULT_TIMEOUT_SECS:
+ return False
+
+ # The given token should match the generated one with the same time.
+ expected_token = generate_token(key, user_id, action_id=action_id,
+ when=token_time)
+ if len(token) != len(expected_token):
+ return False
+
+ # Perform constant time comparison to avoid timing attacks
+ different = 0
+ for x, y in zip(token, expected_token):
+ different |= ord(x) ^ ord(y)
+ if different:
+ return False
+
+ return True
diff --git a/py/minijack/schemas/component.json b/py/minijack/schemas/component.json
new file mode 100644
index 0000000..6d63102
--- /dev/null
+++ b/py/minijack/schemas/component.json
@@ -0,0 +1 @@
+[{"name": "device_id", "type": "STRING"}, {"name": "component_class", "type": "STRING"}, {"name": "component_name", "type": "STRING"}, {"fields": [{"name": "field_value", "type": "STRING"}, {"name": "field_name", "type": "STRING"}], "mode": "REPEATED", "name": "attrs", "type": "RECORD"}]
diff --git a/py/minijack/schemas/device.json b/py/minijack/schemas/device.json
new file mode 100644
index 0000000..ba57f49
--- /dev/null
+++ b/py/minijack/schemas/device.json
@@ -0,0 +1 @@
+[{"name": "latest_note_time", "type": "STRING"}, {"name": "minijack_status", "type": "STRING"}, {"name": "latest_ended_test", "type": "STRING"}, {"name": "goofy_init_time", "type": "STRING"}, {"name": "count_passed", "type": "INTEGER"}, {"name": "latest_note_text", "type": "STRING"}, {"name": "count_failed", "type": "INTEGER"}, {"name": "hwid", "type": "STRING"}, {"name": "ips_time", "type": "STRING"}, {"name": "ips", "type": "STRING"}, {"name": "latest_note_name", "type": "STRING"}, {"name": "latest_test", "type": "STRING"}, {"name": "latest_ended_status", "type": "STRING"}, {"name": "mlb_serial", "type": "STRING"}, {"name": "serial", "type": "STRING"}, {"name": "latest_test_time", "type": "STRING"}, {"name": "latest_note_level", "type": "STRING"}, {"name": "device_id", "type": "STRING"}]
diff --git a/py/minijack/schemas/event.json b/py/minijack/schemas/event.json
new file mode 100644
index 0000000..12d1da5
--- /dev/null
+++ b/py/minijack/schemas/event.json
@@ -0,0 +1 @@
+[{"name": "seq", "type": "INTEGER"}, {"name": "event_id", "type": "STRING"}, {"name": "boot_id", "type": "STRING"}, {"name": "reimage_id", "type": "STRING"}, {"name": "log_id", "type": "STRING"}, {"name": "prefix", "type": "STRING"}, {"name": "boot_sequence", "type": "INTEGER"}, {"name": "time", "type": "STRING"}, {"name": "event", "type": "STRING"}, {"name": "factory_md5sum", "type": "STRING"}, {"name": "device_id", "type": "STRING"}, {"fields": [{"name": "attr", "type": "STRING"}, {"name": "value", "type": "STRING"}], "mode": "REPEATED", "name": "attrs", "type": "RECORD"}]
diff --git a/py/minijack/schemas/test.json b/py/minijack/schemas/test.json
new file mode 100644
index 0000000..c24b69b
--- /dev/null
+++ b/py/minijack/schemas/test.json
@@ -0,0 +1 @@
+[{"name": "status", "type": "STRING"}, {"name": "event_id", "type": "STRING"}, {"name": "start_time", "type": "STRING"}, {"name": "event_seq", "type": "INTEGER"}, {"name": "error_msg", "type": "STRING"}, {"name": "reimage_id", "type": "STRING"}, {"name": "duration", "type": "FLOAT"}, {"name": "end_time", "type": "STRING"}, {"name": "invocation", "type": "STRING"}, {"name": "path", "type": "STRING"}, {"name": "pytest_name", "type": "STRING"}, {"name": "factory_md5sum", "type": "STRING"}, {"name": "device_id", "type": "STRING"}]
diff --git a/py/minijack/settings.py b/py/minijack/settings.py
index 8935dc7..5776847 100644
--- a/py/minijack/settings.py
+++ b/py/minijack/settings.py
@@ -40,15 +40,34 @@
STATIC_URL = '/static/'
-RELEASE_ROOT = '/var/db/factory'
-# Search the default path of Minijack DB.
-# TODO(waihong): Make it a command line option.
-for path in [os.path.join(PROJECT_ROOT, 'minijack_db'),
- os.path.join(PROJECT_ROOT, '..', 'minijack_db'),
- os.path.join(RELEASE_ROOT, 'minijack_db')]:
- if os.path.exists(path):
- MINIJACK_DB_PATH = path
- break
+IS_APPENGINE = ('APPENGINE_RUNTIME' in os.environ)
+
+if IS_APPENGINE:
+ # See README on how to obtain this file.
+ try:
+ with file('privatekey.pem', 'r') as f:
+ GOOGLE_API_PRIVATE_KEY = f.read()
+ except IOError:
+ logging.exception('private key for Google API Service Account not found.' +
+ ' (should be at privatekey.pem)')
+ sys.exit(os.EX_DATAERR)
+
+ # The google api id associated with the private key.
+ from settings_bigquery import GOOGLE_API_ID # pylint: disable=W0611
+ # The project id and dataset id for data in bigquery.
+ from settings_bigquery import PROJECT_ID # pylint: disable=W0611
+ from settings_bigquery import DATASET_ID # pylint: disable=W0611
+
else:
- logging.exception('Minijack database not found.')
- sys.exit(os.EX_DATAERR)
+ RELEASE_ROOT = '/var/db/factory'
+ # Search the default path of Minijack DB.
+ # TODO(waihong): Make it a command line option.
+ for path in [os.path.join(PROJECT_ROOT, 'minijack_db'),
+ os.path.join(PROJECT_ROOT, '..', 'minijack_db'),
+ os.path.join(RELEASE_ROOT, 'minijack_db')]:
+ if os.path.exists(path):
+ MINIJACK_DB_PATH = path
+ break
+ else:
+ logging.exception('Minijack database not found.')
+ sys.exit(os.EX_DATAERR)
diff --git a/py/minijack/settings_bigquery.py.TEMPLATE b/py/minijack/settings_bigquery.py.TEMPLATE
new file mode 100644
index 0000000..bd61104
--- /dev/null
+++ b/py/minijack/settings_bigquery.py.TEMPLATE
@@ -0,0 +1,14 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# The email address for the API Service Account
+# (...@developer.gserviceaccount.com)
+GOOGLE_API_ID = ''
+
+# Used by BigQuery API
+# The project id which the uploaded data is in.
+PROJECT_ID = ''
+
+# The dataset id which the uploaded data is in.
+DATASET_ID = ''
diff --git a/py/minijack/uritemplate/__init__.py b/py/minijack/uritemplate/__init__.py
new file mode 100644
index 0000000..f796f67
--- /dev/null
+++ b/py/minijack/uritemplate/__init__.py
@@ -0,0 +1,147 @@
+# Early, and incomplete implementation of -04.
+#
+import re
+import urllib
+
+RESERVED = ":/?#[]@!$&'()*+,;="
+OPERATOR = "+./;?|!@"
+EXPLODE = "*+"
+MODIFIER = ":^"
+TEMPLATE = re.compile(r"{(?P<operator>[\+\./;\?|!@])?(?P<varlist>[^}]+)}", re.UNICODE)
+VAR = re.compile(r"^(?P<varname>[^=\+\*:\^]+)((?P<explode>[\+\*])|(?P<partial>[:\^]-?[0-9]+))?(=(?P<default>.*))?$", re.UNICODE)
+
+def _tostring(varname, value, explode, operator, safe=""):
+ if type(value) == type([]):
+ if explode == "+":
+ return ",".join([varname + "." + urllib.quote(x, safe) for x in value])
+ else:
+ return ",".join([urllib.quote(x, safe) for x in value])
+ if type(value) == type({}):
+ keys = value.keys()
+ keys.sort()
+ if explode == "+":
+ return ",".join([varname + "." + urllib.quote(key, safe) + "," + urllib.quote(value[key], safe) for key in keys])
+ else:
+ return ",".join([urllib.quote(key, safe) + "," + urllib.quote(value[key], safe) for key in keys])
+ else:
+ return urllib.quote(value, safe)
+
+
+def _tostring_path(varname, value, explode, operator, safe=""):
+ joiner = operator
+ if type(value) == type([]):
+ if explode == "+":
+ return joiner.join([varname + "." + urllib.quote(x, safe) for x in value])
+ elif explode == "*":
+ return joiner.join([urllib.quote(x, safe) for x in value])
+ else:
+ return ",".join([urllib.quote(x, safe) for x in value])
+ elif type(value) == type({}):
+ keys = value.keys()
+ keys.sort()
+ if explode == "+":
+ return joiner.join([varname + "." + urllib.quote(key, safe) + joiner + urllib.quote(value[key], safe) for key in keys])
+ elif explode == "*":
+ return joiner.join([urllib.quote(key, safe) + joiner + urllib.quote(value[key], safe) for key in keys])
+ else:
+ return ",".join([urllib.quote(key, safe) + "," + urllib.quote(value[key], safe) for key in keys])
+ else:
+ if value:
+ return urllib.quote(value, safe)
+ else:
+ return ""
+
+def _tostring_query(varname, value, explode, operator, safe=""):
+ joiner = operator
+ varprefix = ""
+ if operator == "?":
+ joiner = "&"
+ varprefix = varname + "="
+ if type(value) == type([]):
+ if 0 == len(value):
+ return ""
+ if explode == "+":
+ return joiner.join([varname + "=" + urllib.quote(x, safe) for x in value])
+ elif explode == "*":
+ return joiner.join([urllib.quote(x, safe) for x in value])
+ else:
+ return varprefix + ",".join([urllib.quote(x, safe) for x in value])
+ elif type(value) == type({}):
+ if 0 == len(value):
+ return ""
+ keys = value.keys()
+ keys.sort()
+ if explode == "+":
+ return joiner.join([varname + "." + urllib.quote(key, safe) + "=" + urllib.quote(value[key], safe) for key in keys])
+ elif explode == "*":
+ return joiner.join([urllib.quote(key, safe) + "=" + urllib.quote(value[key], safe) for key in keys])
+ else:
+ return varprefix + ",".join([urllib.quote(key, safe) + "," + urllib.quote(value[key], safe) for key in keys])
+ else:
+ if value:
+ return varname + "=" + urllib.quote(value, safe)
+ else:
+ return varname
+
+TOSTRING = {
+ "" : _tostring,
+ "+": _tostring,
+ ";": _tostring_query,
+ "?": _tostring_query,
+ "/": _tostring_path,
+ ".": _tostring_path,
+ }
+
+
+def expand(template, vars):
+ def _sub(match):
+ groupdict = match.groupdict()
+ operator = groupdict.get('operator')
+ if operator is None:
+ operator = ''
+ varlist = groupdict.get('varlist')
+
+ safe = "@"
+ if operator == '+':
+ safe = RESERVED
+ varspecs = varlist.split(",")
+ varnames = []
+ defaults = {}
+ for varspec in varspecs:
+ m = VAR.search(varspec)
+ groupdict = m.groupdict()
+ varname = groupdict.get('varname')
+ explode = groupdict.get('explode')
+ partial = groupdict.get('partial')
+ default = groupdict.get('default')
+ if default:
+ defaults[varname] = default
+ varnames.append((varname, explode, partial))
+
+ retval = []
+ joiner = operator
+ prefix = operator
+ if operator == "+":
+ prefix = ""
+ joiner = ","
+ if operator == "?":
+ joiner = "&"
+ if operator == "":
+ joiner = ","
+ for varname, explode, partial in varnames:
+ if varname in vars:
+ value = vars[varname]
+ #if not value and (type(value) == type({}) or type(value) == type([])) and varname in defaults:
+ if not value and value != "" and varname in defaults:
+ value = defaults[varname]
+ elif varname in defaults:
+ value = defaults[varname]
+ else:
+ continue
+ retval.append(TOSTRING[operator](varname, value, explode, operator, safe=safe))
+ if "".join(retval):
+ return prefix + joiner.join(retval)
+ else:
+ return ""
+
+ return TEMPLATE.sub(_sub, template)
