Revision created by MOE tool init_codebases.

Revision created by MOE tool push_codebase.
MOE_MIGRATION=


git-svn-id: http://google-safe-browsing.googlecode.com/svn/trunk/testing@103 2195c2fd-d934-0410-ae3f-cd772a4098b8
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index 229ff03..0000000
--- a/LICENSE
+++ /dev/null
@@ -1,13 +0,0 @@
-Copyright 2009 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
diff --git a/external_test_pb2.py b/external_test_pb2.py
deleted file mode 100755
index 3b86228..0000000
--- a/external_test_pb2.py
+++ /dev/null
@@ -1,202 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-
-from google.protobuf import descriptor
-from google.protobuf import message
-from google.protobuf import reflection
-from google.protobuf import service
-from google.protobuf import service_reflection
-from google.protobuf import descriptor_pb2
-
-
-
-_CGIPARAM = descriptor.Descriptor(
-  name='CGIParam',
-  full_name='CGIParam',
-  filename='external_test.proto',
-  containing_type=None,
-  fields=[
-    descriptor.FieldDescriptor(
-      name='Name', full_name='CGIParam.Name', index=0,
-      number=1, type=12, cpp_type=9, label=2,
-      default_value="",
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='Value', full_name='CGIParam.Value', index=1,
-      number=2, type=12, cpp_type=9, label=2,
-      default_value="",
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],  # TODO(robinson): Implement.
-  enum_types=[
-  ],
-  options=None)
-
-
-_REQUESTDATA = descriptor.Descriptor(
-  name='RequestData',
-  full_name='RequestData',
-  filename='external_test.proto',
-  containing_type=None,
-  fields=[
-    descriptor.FieldDescriptor(
-      name='RequestPath', full_name='RequestData.RequestPath', index=0,
-      number=1, type=12, cpp_type=9, label=2,
-      default_value="",
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='Params', full_name='RequestData.Params', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='PostData', full_name='RequestData.PostData', index=2,
-      number=3, type=12, cpp_type=9, label=1,
-      default_value="",
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='ServerResponse', full_name='RequestData.ServerResponse', index=3,
-      number=4, type=12, cpp_type=9, label=2,
-      default_value="",
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],  # TODO(robinson): Implement.
-  enum_types=[
-  ],
-  options=None)
-
-
-_HASHREQUEST = descriptor.Descriptor(
-  name='HashRequest',
-  full_name='HashRequest',
-  filename='external_test.proto',
-  containing_type=None,
-  fields=[
-    descriptor.FieldDescriptor(
-      name='HashPrefix', full_name='HashRequest.HashPrefix', index=0,
-      number=1, type=12, cpp_type=9, label=2,
-      default_value="",
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='ServerResponse', full_name='HashRequest.ServerResponse', index=1,
-      number=2, type=12, cpp_type=9, label=2,
-      default_value="",
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='Expression', full_name='HashRequest.Expression', index=2,
-      number=3, type=12, cpp_type=9, label=1,
-      default_value="",
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],  # TODO(robinson): Implement.
-  enum_types=[
-  ],
-  options=None)
-
-
-_STEPDATA = descriptor.Descriptor(
-  name='StepData',
-  full_name='StepData',
-  filename='external_test.proto',
-  containing_type=None,
-  fields=[
-    descriptor.FieldDescriptor(
-      name='Requests', full_name='StepData.Requests', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='Hashes', full_name='StepData.Hashes', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],  # TODO(robinson): Implement.
-  enum_types=[
-  ],
-  options=None)
-
-
-_TESTDATA = descriptor.Descriptor(
-  name='TestData',
-  full_name='TestData',
-  filename='external_test.proto',
-  containing_type=None,
-  fields=[
-    descriptor.FieldDescriptor(
-      name='Steps', full_name='TestData.Steps', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='ClientKey', full_name='TestData.ClientKey', index=1,
-      number=2, type=12, cpp_type=9, label=2,
-      default_value="",
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],  # TODO(robinson): Implement.
-  enum_types=[
-  ],
-  options=None)
-
-
-_REQUESTDATA.fields_by_name['Params'].message_type = _CGIPARAM
-_STEPDATA.fields_by_name['Requests'].message_type = _REQUESTDATA
-_STEPDATA.fields_by_name['Hashes'].message_type = _HASHREQUEST
-_TESTDATA.fields_by_name['Steps'].message_type = _STEPDATA
-
-class CGIParam(message.Message):
-  __metaclass__ = reflection.GeneratedProtocolMessageType
-  DESCRIPTOR = _CGIPARAM
-
-class RequestData(message.Message):
-  __metaclass__ = reflection.GeneratedProtocolMessageType
-  DESCRIPTOR = _REQUESTDATA
-
-class HashRequest(message.Message):
-  __metaclass__ = reflection.GeneratedProtocolMessageType
-  DESCRIPTOR = _HASHREQUEST
-
-class StepData(message.Message):
-  __metaclass__ = reflection.GeneratedProtocolMessageType
-  DESCRIPTOR = _STEPDATA
-
-class TestData(message.Message):
-  __metaclass__ = reflection.GeneratedProtocolMessageType
-  DESCRIPTOR = _TESTDATA
-
diff --git a/safebrowsing_test_server.py b/safebrowsing_test_server.py
deleted file mode 100755
index d730f2a..0000000
--- a/safebrowsing_test_server.py
+++ /dev/null
@@ -1,392 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2009 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Test server for Safebrowsing protocol v2.
-
-To test an implementation of the safebrowsing protocol, this server should
-be run on the same machine as the client implementation.  The client should
-connect to this server at localhost:port where port is specified as a command
-line flag (--port) and perform updates normally, except that each request
-should have an additional CGI param "test_step" that specifies which update
-request this is for the client.  That is, it should be incremented after the
-complete parsing of a downloads request so a downloads request and its
-associated redirects should all have the same test_step. The client should
-also make a newkey request and then a getlists requests before making the
-first update request and should use test_step=1 for these requests (test_step
-is 1 indexed). When the client believes that it is done with testing (because
-it recieves a response from an update request with no new data), it should
-make a "/test_complete" request. This will return either "yes" or "no" if the
-test is complete or not.
-"""
-
-__author__ = 'gcasto@google.com (Garrett Casto)'
-
-import BaseHTTPServer
-import binascii
-import base64
-import cgi
-import hmac
-from optparse import OptionParser
-import re
-import sha
-import sys
-from threading import Timer
-import time
-import urlparse
-
-import external_test_pb2
-
-DEFAULT_PORT = 40101
-DEFAULT_DATAFILE_LOCATION = "testing_input.dat"
-POST_DATA_KEY = "post_data"
-GETHASH_PATH = "/safebrowsing/gethash"
-RESET_PATH="/reset"
-DOWNLOADS_PATH = "/safebrowsing/downloads"
-TEST_COMPLETE_PATH = "/test_complete"
-DATABASE_VALIDATION_PATH = "/safebrowsing/verify_database"
-
-# Dict of step -> List of (request_path, param key, response)
-response_data_by_step = {}
-# Dict of step -> Dict of hash_prefix ->
-# (full length hashes responses, num times requested)
-hash_data_by_step = {}
-client_key = ''
-enforce_caching = False
-validate_database = True
-server_port = -1
-datafile_location = ''
-
-def EndServer():
-  sys.exit(0)
-
-def CGIParamsToListOfTuples(cgi_params):
-  return [(param.Name, param.Value) for param in cgi_params]
-
-def SortedTupleFromParamsAndPostData(params,
-                                     post_data):
-  """ Make a sorted tuple from the request such that it can be inserted as
-  a key in a map. params is a list of (name, value) tuples and post_data is
-  a string (or None).
-  """
-  if post_data:
-    params.append((POST_DATA_KEY, tuple(sorted(post_data.split('\n')))))
-  return tuple(sorted(params))
-
-def LoadData(filename):
-  """ Load data from filename to be used by the testing server.
-  """
-  global response_data_by_step
-  global client_key
-  data_file = open(filename, 'rb')
-  str_data = data_file.read()
-  test_data = external_test_pb2.TestData()
-  test_data.ParseFromString(str_data)
-  print "Data Loaded"
-  client_key = test_data.ClientKey
-  step = 0
-  for step_data in test_data.Steps:
-    step += 1
-    step_list = []
-    for request_data in step_data.Requests:
-      params_tuple = SortedTupleFromParamsAndPostData(
-          CGIParamsToListOfTuples(request_data.Params),
-          request_data.PostData)
-      step_list.append((request_data.RequestPath,
-                        params_tuple,
-                        request_data.ServerResponse))
-    response_data_by_step[step] = step_list
-
-    hash_step_dict = {}
-    for hash_request in step_data.Hashes:
-      hash_step_dict[hash_request.HashPrefix] = (hash_request.ServerResponse,
-                                                 hash_request.Expression,
-                                                 0)
-    hash_data_by_step[step] = hash_step_dict
-  print "Data Parsed"
-
-def VerifyTestComplete():
-  """ Returns true if all the necessary requests have been made by the client.
-  """
-  global response_data_by_step
-  global hash_data_by_step
-  global enforce_caching
-
-  complete = True
-  for (step, step_list) in response_data_by_step.iteritems():
-    if len(step_list):
-      print ("Step %s has %d request(s) that were not made %s" %
-             (step, len(step_list), step_list))
-      complete = False
-
-  for (step, hash_step_dict) in hash_data_by_step.iteritems():
-    for (prefix,
-         (response, expression, num_requests)) in hash_step_dict.iteritems():
-      if ((enforce_caching and num_requests != 1) or
-          num_requests == 0):
-        print ("Hash prefix %s not requested the correct number of times"
-               "(%d requests). Requests originated because of expression"
-               " %s. Prefix is located in the following locations" %
-               (binascii.hexlify(prefix),
-                num_requests,
-                expression))
-        cur_index = 0
-        while cur_index < len(response):
-          end_header_index = response.find('\n', cur_index + 1)
-          header = response[cur_index:end_header_index]
-          (listname, chunk_num, hashdatalen) = header.split(":")
-          print "   List '%s' in add chunk num %d" % (listname, chunk_num)
-          cur_index = end_header_index + hashdatalen + 1
-
-        complete = False
-
-  # TODO(gcasto): Have a check here that verifies that the client doesn't
-  # make too many hash requests during the test run.
-
-  return complete
-
-class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
-  def ParamDictToListOfTuples(self, params):
-    # params is a list cgi params to list of specified values.  Since we never
-    # expect a parameter to be specified multiple times, we just take the first
-    # one.
-    return [(name, value[0]) for (name, value) in params.iteritems()]
-
-  def MakeParamKey(self, params, post_data=None):
-    """ Make a lookup key from the request.
-    """
-    return SortedTupleFromParamsAndPostData(
-        self.ParamDictToListOfTuples(params),
-        post_data)
-
-  def MACResponse(self, response, is_downloads_request):
-    """ Returns the response wrapped with a MAC. Formatting will change
-    if this is a downloads_request or hashserver_request.
-    """
-    unescaped_mac = hmac.new(client_key, response, sha).digest()
-    return "%s%s\n%s" % (is_downloads_request and "m:" or "",
-                       base64.urlsafe_b64encode(unescaped_mac),
-                       response)
-
-  def VerifyRequest(self, is_post_request):
-    """ Verify that the request matches one loaded from the datafile and
-    give the corresponding response. If there is no match, try and give a
-    descriptive error message in the response.
-    """
-    parsed_url = urlparse.urlparse(self.path)
-    path = parsed_url[2]
-    params = cgi.parse_qs(parsed_url[4])
-
-    step = params.get("test_step")
-    if step is None or len(step) != 1:
-      self.send_response(400)
-      self.end_headers()
-      print "No test step present."
-      return
-    step = int(step[0])
-
-    if path == TEST_COMPLETE_PATH:
-      self.send_response(200)
-      self.end_headers()
-      if VerifyTestComplete():
-        self.wfile.write('yes')
-      else:
-        self.wfile.write('no')
-    elif path == GETHASH_PATH:
-      self.SynthesizeGethashResponse(step)
-    elif path == RESET_PATH:
-      LoadData(datafile_location)
-      self.send_response(200)
-      self.end_headers()
-      self.wfile.write('done')
-    else:
-      self.GetCannedResponse(path, params, step, is_post_request)
-
-  def SynthesizeGethashResponse(self, step):
-    """ Create a gethash response. This will possibly combine an arbitrary
-    number of hash requests from the protocol buffer.
-    """
-    global hash_data_by_step
-
-    hashes_for_step = hash_data_by_step.get(step, {})
-    if not hashes_for_step:
-      self.send_response(400)
-      self.end_headers()
-      print "No response for step %d" % step
-      return
-
-    post_data = self.rfile.read(int(self.headers['Content-Length']))
-    match = re.match(
-        r'(?P<prefixsize>\d+):(?P<totalsize>\d+)\n(?P<prefixes>.+)',
-        post_data,
-        re.MULTILINE | re.IGNORECASE | re.DOTALL)
-    if not match:
-      self.send_response(400)
-      self.end_headers()
-      print "Gethash request is malformed %s" % post_data
-      return
-
-    prefixsize = int(match.group('prefixsize'))
-    total_length = int(match.group('totalsize'))
-    if total_length % prefixsize != 0:
-      self.send_response(400)
-      self.end_headers()
-      print ("Gethash request is malformed, length should be a multiple of the "
-             " prefix size%s" % post_data)
-      return
-
-    response = ""
-    for n in range(total_length/prefixsize):
-      prefix = match.group('prefixes')[n*prefixsize:n*prefixsize + prefixsize]
-      hash_data = hashes_for_step.get(prefix)
-      if hash_data is not None:
-        # Reply with the correct response
-        response += hash_data[0]
-        # Remember that this hash has now been requested.
-        hashes_for_step[prefix] = (hash_data[0], hash_data[1], hash_data[2] + 1)
-
-    if not response:
-      self.send_response(204)
-      self.end_headers()
-      return
-
-    # Need to perform MACing before sending response out.
-    self.send_response(200)
-    self.end_headers()
-    self.wfile.write(self.MACResponse(response, False))
-
-  def GetCannedResponse(self, path, params, step, is_post_request):
-    """ Given the parameters of a request, see if a matching response is
-    found. If one is found, respond with with it, else respond with a 400.
-    """
-    responses_for_step = response_data_by_step.get(step)
-    if not responses_for_step:
-      self.send_response(400)
-      self.end_headers()
-      print "No responses for step %d" % step
-      return
-
-    # Delete unnecessary params
-    del params["test_step"]
-    if "client" in params:
-      del params["client"]
-    if "appver" in params:
-      del params["appver"]
-
-    param_key = self.MakeParamKey(
-        params,
-        is_post_request and
-        self.rfile.read(int(self.headers['Content-Length'])) or
-        None)
-
-    (expected_path, expected_params, server_response) = responses_for_step[0]
-    if expected_path != path or param_key != expected_params:
-      self.send_response(400)
-      self.end_headers()
-      print "Expected request with path %s and params %s." % (expected_path,
-                                                              expected_params)
-      print "Actual request path %s and params %s" % (path, param_key)
-      return
-
-    # Remove request that was just made
-    responses_for_step.pop(0)
-
-    # If the next request is not needed for this test run, remove it now.
-    # We do this after processing instead of before for cases where the
-    # data we are removing is the last requests in a step.
-    if responses_for_step:
-      (expected_path, _, _) = responses_for_step[0]
-      if expected_path == DATABASE_VALIDATION_PATH and not validate_database:
-        responses_for_step.pop(0)
-
-    if path == DOWNLOADS_PATH:
-      # Need to have the redirects point to the current port.
-      server_response = re.sub(r'localhost:\d+',
-                               'localhost:%d' % server_port,
-                               server_response)
-      # Remove the current MAC, because it's going to be wrong now.
-      server_response = server_response[server_response.find('\n')+1:]
-      # Add a new correct MAC.
-      server_response = self.MACResponse(server_response, True)
-
-    self.send_response(200)
-    self.end_headers()
-    self.wfile.write(server_response)
-
-  def do_GET(self):
-    self.VerifyRequest(False)
-
-  def do_POST(self):
-    self.VerifyRequest(True)
-
-
-def SetupServer(datafile_location,
-                port,
-                opt_enforce_caching,
-                opt_validate_database):
-  """Sets up the safebrowsing test server.
-
-  Arguments:
-    datafile_location: The file to load testing data from.
-    port: port that the server runs on.
-    opt_enforce_caching: Whether to require the client to implement caching.
-    opt_validate_database: Whether to require the client makes database
-         verification requests.
-
-  Returns:
-    An HTTPServer object which the caller should call serve_forever() on.
-  """
-  LoadData(datafile_location)
-  # TODO(gcasto):  Look into extending HTTPServer to remove global variables.
-  global enforce_caching
-  global validate_database
-  global server_port
-  enforce_caching = opt_enforce_caching
-  validate_database = opt_validate_database
-  server_port = port
-  return BaseHTTPServer.HTTPServer(('', port), RequestHandler)
-
-if __name__ == '__main__':
-  parser = OptionParser()
-  parser.add_option("--datafile", dest="datafile_location",
-                    default=DEFAULT_DATAFILE_LOCATION,
-                    help="Location to load testing data from.")
-  parser.add_option("--port", dest="port", type="int",
-                    default=DEFAULT_PORT, help="Port to run the server on.")
-  parser.add_option("--enforce_caching", dest="enforce_caching",
-                    action="store_true", default=False,
-                    help="Whether to require that the client"
-                    "has implemented caching or not.")
-  parser.add_option("--ignore_database_validation", dest="validate_database",
-                    action="store_false", default=True,
-                    help="Whether to requires that the client makes verify "
-                    "database requests or not.")
-  parser.add_option("--server_timeout_sec", dest="server_timeout_sec",
-                    type="int", default=600,
-                    help="How long to let the server run before shutting it "
-                    "down. If <=0, the server will never be down")
-  (options, _) = parser.parse_args()
-
-  datafile_location = options.datafile_location
-  server = SetupServer(options.datafile_location,
-                       options.port,
-                       options.enforce_caching,
-                       options.validate_database)
-
-  if (options.server_timeout_sec > 0):
-    tm = Timer(options.server_timeout_sec, EndServer)
-    tm.start()
-
-  server.serve_forever()
diff --git a/testing_input.dat b/testing_input.dat
deleted file mode 100644
index 9eb760f..0000000
--- a/testing_input.dat
+++ /dev/null
Binary files differ