Adding License and changing test_server to read using 'rd'

git-svn-id: http://google-safe-browsing.googlecode.com/svn/trunk/testing@100 2195c2fd-d934-0410-ae3f-cd772a4098b8
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..229ff03
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,13 @@
+Copyright 2009 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/external_test_pb2.py b/external_test_pb2.py
index 43c3daa..383342d 100755
--- a/external_test_pb2.py
+++ b/external_test_pb2.py
@@ -1,195 +1,254 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 
-from google.protobuf import descriptor
-from google.protobuf import message
-from google.protobuf import reflection
-from google.protobuf import service
-from google.protobuf import service_reflection
-from google.protobuf import descriptor_pb2
+from google3.net.proto2.python.public import descriptor
+from google3.net.proto2.python.public import message
+from google3.net.proto2.python.public import reflection
+from google3.net.proto2.proto import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+
+
+DESCRIPTOR = descriptor.FileDescriptor(
+  name='quality/trustrank/frontend/externaltest/external_test.proto',
+  package='',
+  serialized_pb='\n;quality/trustrank/frontend/externaltest/external_test.proto\"\'\n\x08\x43GIParam\x12\x0c\n\x04Name\x18\x01 \x02(\x0c\x12\r\n\x05Value\x18\x02 \x02(\x0c\"g\n\x0bRequestData\x12\x13\n\x0bRequestPath\x18\x01 \x02(\x0c\x12\x19\n\x06Params\x18\x02 \x03(\x0b\x32\t.CGIParam\x12\x10\n\x08PostData\x18\x03 \x01(\x0c\x12\x16\n\x0eServerResponse\x18\x04 \x02(\x0c\"M\n\x0bHashRequest\x12\x12\n\nHashPrefix\x18\x01 \x02(\x0c\x12\x16\n\x0eServerResponse\x18\x02 \x02(\x0c\x12\x12\n\nExpression\x18\x03 \x01(\x0c\"H\n\x08StepData\x12\x1e\n\x08Requests\x18\x01 \x03(\x0b\x32\x0c.RequestData\x12\x1c\n\x06Hashes\x18\x02 \x03(\x0b\x32\x0c.HashRequest\"7\n\x08TestData\x12\x18\n\x05Steps\x18\x01 \x03(\x0b\x32\t.StepData\x12\x11\n\tClientKey\x18\x02 \x02(\x0c')
+
 
 
 
 _CGIPARAM = descriptor.Descriptor(
   name='CGIParam',
   full_name='CGIParam',
-  filename='external_test.proto',
+  filename=None,
+  file=DESCRIPTOR,
   containing_type=None,
   fields=[
     descriptor.FieldDescriptor(
       name='Name', full_name='CGIParam.Name', index=0,
       number=1, type=12, cpp_type=9, label=2,
-      default_value="",
+      has_default_value=False, default_value="",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
     descriptor.FieldDescriptor(
       name='Value', full_name='CGIParam.Value', index=1,
       number=2, type=12, cpp_type=9, label=2,
-      default_value="",
+      has_default_value=False, default_value="",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
   ],
   extensions=[
   ],
-  nested_types=[],  # TODO(robinson): Implement.
+  nested_types=[],
   enum_types=[
   ],
-  options=None)
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=63,
+  serialized_end=102,
+)
 
 
 _REQUESTDATA = descriptor.Descriptor(
   name='RequestData',
   full_name='RequestData',
-  filename='external_test.proto',
+  filename=None,
+  file=DESCRIPTOR,
   containing_type=None,
   fields=[
     descriptor.FieldDescriptor(
       name='RequestPath', full_name='RequestData.RequestPath', index=0,
       number=1, type=12, cpp_type=9, label=2,
-      default_value="",
+      has_default_value=False, default_value="",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
     descriptor.FieldDescriptor(
       name='Params', full_name='RequestData.Params', index=1,
       number=2, type=11, cpp_type=10, label=3,
-      default_value=[],
+      has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
     descriptor.FieldDescriptor(
       name='PostData', full_name='RequestData.PostData', index=2,
       number=3, type=12, cpp_type=9, label=1,
-      default_value="",
+      has_default_value=False, default_value="",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
     descriptor.FieldDescriptor(
       name='ServerResponse', full_name='RequestData.ServerResponse', index=3,
       number=4, type=12, cpp_type=9, label=2,
-      default_value="",
+      has_default_value=False, default_value="",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
   ],
   extensions=[
   ],
-  nested_types=[],  # TODO(robinson): Implement.
+  nested_types=[],
   enum_types=[
   ],
-  options=None)
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=104,
+  serialized_end=207,
+)
 
 
 _HASHREQUEST = descriptor.Descriptor(
   name='HashRequest',
   full_name='HashRequest',
-  filename='external_test.proto',
+  filename=None,
+  file=DESCRIPTOR,
   containing_type=None,
   fields=[
     descriptor.FieldDescriptor(
       name='HashPrefix', full_name='HashRequest.HashPrefix', index=0,
       number=1, type=12, cpp_type=9, label=2,
-      default_value="",
+      has_default_value=False, default_value="",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
     descriptor.FieldDescriptor(
       name='ServerResponse', full_name='HashRequest.ServerResponse', index=1,
       number=2, type=12, cpp_type=9, label=2,
-      default_value="",
+      has_default_value=False, default_value="",
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    descriptor.FieldDescriptor(
+      name='Expression', full_name='HashRequest.Expression', index=2,
+      number=3, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value="",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
   ],
   extensions=[
   ],
-  nested_types=[],  # TODO(robinson): Implement.
+  nested_types=[],
   enum_types=[
   ],
-  options=None)
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=209,
+  serialized_end=286,
+)
 
 
 _STEPDATA = descriptor.Descriptor(
   name='StepData',
   full_name='StepData',
-  filename='external_test.proto',
+  filename=None,
+  file=DESCRIPTOR,
   containing_type=None,
   fields=[
     descriptor.FieldDescriptor(
       name='Requests', full_name='StepData.Requests', index=0,
       number=1, type=11, cpp_type=10, label=3,
-      default_value=[],
+      has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
     descriptor.FieldDescriptor(
       name='Hashes', full_name='StepData.Hashes', index=1,
       number=2, type=11, cpp_type=10, label=3,
-      default_value=[],
+      has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
   ],
   extensions=[
   ],
-  nested_types=[],  # TODO(robinson): Implement.
+  nested_types=[],
   enum_types=[
   ],
-  options=None)
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=288,
+  serialized_end=360,
+)
 
 
 _TESTDATA = descriptor.Descriptor(
   name='TestData',
   full_name='TestData',
-  filename='external_test.proto',
+  filename=None,
+  file=DESCRIPTOR,
   containing_type=None,
   fields=[
     descriptor.FieldDescriptor(
       name='Steps', full_name='TestData.Steps', index=0,
       number=1, type=11, cpp_type=10, label=3,
-      default_value=[],
+      has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
     descriptor.FieldDescriptor(
       name='ClientKey', full_name='TestData.ClientKey', index=1,
       number=2, type=12, cpp_type=9, label=2,
-      default_value="",
+      has_default_value=False, default_value="",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
   ],
   extensions=[
   ],
-  nested_types=[],  # TODO(robinson): Implement.
+  nested_types=[],
   enum_types=[
   ],
-  options=None)
-
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=362,
+  serialized_end=417,
+)
 
 _REQUESTDATA.fields_by_name['Params'].message_type = _CGIPARAM
 _STEPDATA.fields_by_name['Requests'].message_type = _REQUESTDATA
 _STEPDATA.fields_by_name['Hashes'].message_type = _HASHREQUEST
 _TESTDATA.fields_by_name['Steps'].message_type = _STEPDATA
+DESCRIPTOR.message_types_by_name['CGIParam'] = _CGIPARAM
+DESCRIPTOR.message_types_by_name['RequestData'] = _REQUESTDATA
+DESCRIPTOR.message_types_by_name['HashRequest'] = _HASHREQUEST
+DESCRIPTOR.message_types_by_name['StepData'] = _STEPDATA
+DESCRIPTOR.message_types_by_name['TestData'] = _TESTDATA
 
 class CGIParam(message.Message):
   __metaclass__ = reflection.GeneratedProtocolMessageType
   DESCRIPTOR = _CGIPARAM
+  
+  # @@protoc_insertion_point(class_scope:CGIParam)
 
 class RequestData(message.Message):
   __metaclass__ = reflection.GeneratedProtocolMessageType
   DESCRIPTOR = _REQUESTDATA
+  
+  # @@protoc_insertion_point(class_scope:RequestData)
 
 class HashRequest(message.Message):
   __metaclass__ = reflection.GeneratedProtocolMessageType
   DESCRIPTOR = _HASHREQUEST
+  
+  # @@protoc_insertion_point(class_scope:HashRequest)
 
 class StepData(message.Message):
   __metaclass__ = reflection.GeneratedProtocolMessageType
   DESCRIPTOR = _STEPDATA
+  
+  # @@protoc_insertion_point(class_scope:StepData)
 
 class TestData(message.Message):
   __metaclass__ = reflection.GeneratedProtocolMessageType
   DESCRIPTOR = _TESTDATA
+  
+  # @@protoc_insertion_point(class_scope:TestData)
 
+# @@protoc_insertion_point(module_scope)
diff --git a/safebrowsing_test_server.py b/safebrowsing_test_server.py
index 45ea701..cc6ef7a 100755
--- a/safebrowsing_test_server.py
+++ b/safebrowsing_test_server.py
@@ -35,6 +35,7 @@
 __author__ = 'gcasto@google.com (Garrett Casto)'
 
 import BaseHTTPServer
+import binascii
 import base64
 import cgi
 import hmac
@@ -82,7 +83,7 @@
   """
   global response_data_by_step
   global client_key
-  data_file = open(filename, 'r')
+  data_file = open(filename, 'rb')
   str_data = data_file.read()
   test_data = external_test_pb2.TestData()
   test_data.ParseFromString(str_data)
@@ -104,6 +105,7 @@
     hash_step_dict = {}
     for hash_request in step_data.Hashes:
       hash_step_dict[hash_request.HashPrefix] = (hash_request.ServerResponse,
+                                                 hash_request.Expression,
                                                  0)
     hash_data_by_step[step] = hash_step_dict
   print "Data Parsed"
@@ -122,12 +124,25 @@
              (step, len(step_list), step_list))
       complete = False
 
-  for (prefix, hash_step_dict) in hash_data_by_step.iteritems():
-    for (_, num_times_requested) in hash_step_dict.itervalues():
-      if ((enforce_caching and num_times_requested != 1) or
-          num_times_requested == 0):
+  for (step, hash_step_dict) in hash_data_by_step.iteritems():
+    for (prefix,
+         (response, expression, num_requests)) in hash_step_dict.iteritems():
+      if ((enforce_caching and num_requests != 1) or
+          num_requests == 0):
         print ("Hash prefix %s not requested the correct number of times"
-               "(%d requests)" % (prefix, num_times_requested))
+               "(%d requests). Requests originated because of expression"
+               " %s. Prefix is located in the following locations" %
+               (binascii.hexlify(prefix),
+                num_requests,
+                expression))
+        cur_index = 0
+        while cur_index < len(response):
+          end_header_index = response.find('\n', cur_index + 1)
+          header = response[cur_index:end_header_index]
+          (listname, chunk_num, hashdatalen) = header.split(":")
+          print "   List '%s' in add chunk num %d" % (listname, chunk_num)
+          cur_index = end_header_index + hashdatalen + 1
+
         complete = False
 
   # TODO(gcasto): Have a check here that verifies that the client doesn't
@@ -228,7 +243,7 @@
         # Reply with the correct response
         response += hash_data[0]
         # Remember that this hash has now been requested.
-        hashes_for_step[prefix] = (hash_data[0], hash_data[1] + 1)
+        hashes_for_step[prefix] = (hash_data[0], hash_data[1], hash_data[2] + 1)
 
     if not response:
       self.send_response(204)
diff --git a/testing_input.dat b/testing_input.dat
index 4259fe9..9eb760f 100644
--- a/testing_input.dat
+++ b/testing_input.dat
Binary files differ