Testlog: Add better defined schema for StationTestRun event
Previously, FIELDS in StationTestRun only checks for top-level
definition (Dict/List). Introducing the utils.schema to perform
a throughout check against Testlog specification.
Miscellaneous bugs are fixed as well:
1. CastFields misses super, but it should call that.
2. CastFields in StationTestRun can't work ideally since
__setitem__ is overrided.
3. status in StationTestRun['parameters'] use PASS or FAIL,
which is not consistent with Testlog playbook.
BUG=b:38484982
TEST=pass unittest
CQ-DEPEND=CL:504047
Change-Id: I13617bf0e5fdb829053f1f52d3194649aa3c589f
Reviewed-on: https://chromium-review.googlesource.com/509431
Commit-Ready: Chun-Tsen Kuo <chuntsen@chromium.org>
Tested-by: Chun-Tsen Kuo <chuntsen@chromium.org>
Reviewed-by: Chun-ta Lin <itspeter@chromium.org>
diff --git a/py/testlog/testlog.py b/py/testlog/testlog.py
index 99e6c32..ab63bc7 100644
--- a/py/testlog/testlog.py
+++ b/py/testlog/testlog.py
@@ -52,6 +52,7 @@
import testlog_utils
import testlog_validator
from utils import file_utils
+from utils import schema
from utils import sys_utils
from utils import time_utils
from utils import type_utils
@@ -234,7 +235,7 @@
metadata = last_test_run.pop(
Testlog.FIELDS._METADATA) # pylint: disable=protected-access
return {
- Testlog.FIELDS.LAST_TEST_RUN: Event.FromDict(last_test_run),
+ Testlog.FIELDS.LAST_TEST_RUN: Event.FromDict(last_test_run, False),
Testlog.FIELDS.LOG_ROOT: metadata[Testlog.FIELDS.LOG_ROOT],
Testlog.FIELDS.PRIMARY_JSON: metadata[Testlog.FIELDS.PRIMARY_JSON],
Testlog.FIELDS.SESSION_JSON: session_json_path,
@@ -666,16 +667,17 @@
def CheckMissingFields(self):
"""Returns a list of missing field."""
mro = self.__class__.__mro__
- missing_fileds = []
+ missing_fields = []
# Find the corresponding requirement.
for cls in mro:
if cls is object:
break
for field_name, metadata in cls.FIELDS.iteritems():
if metadata[0] and field_name not in self._data:
- missing_fileds.append(field_name)
+ missing_fields.append(field_name)
- return missing_fileds
+ if missing_fields != []:
+ raise testlog_utils.TestlogError('Missing fields: %s' % missing_fields)
@classmethod
def GetEventType(cls):
@@ -693,6 +695,7 @@
"""
for key in data.iterkeys():
data_type = type(data[key])
+ # TODO(chuntsen): raise exception when the empty list/dict has wrong key.
if data_type == list:
for value in data[key]:
self[key] = value
@@ -734,16 +737,18 @@
'Input event does not have a valid `type`.')
@classmethod
- def FromJSON(cls, json_string):
+ def FromJSON(cls, json_string, check_missing=True):
"""Converts JSON data into an Event instance."""
- return cls.FromDict(json.loads(json_string))
+ return cls.FromDict(json.loads(json_string), check_missing)
@classmethod
- def FromDict(cls, data):
+ def FromDict(cls, data, check_missing=True):
"""Converts Python dict data into an Event instance."""
event = cls.DetermineClass(data)()
event.Populate(data)
event.CastFields()
+ if check_missing:
+ event.CheckMissingFields()
return event
def ToJSON(self):
@@ -903,6 +908,76 @@
# TODO(itspeter): Document 'argument', 'operatorId', 'failures',
# 'serialNumbers', 'parameters' and 'series'.
+ @classmethod
+ def _NumericSchema(cls, label):
+ return schema.AnyOf([
+ schema.Scalar(label, int),
+ schema.Scalar(label, long),
+ schema.Scalar(label, float)])
+
+ def _ValidatorArgumentWrapper(*args, **kwargs): # pylint: disable=E0211
+ SCHEMA = schema.FixedDict(
+ 'arguments.value',
+ items={'value': schema.Scalar('value', object)},
+ optional_items={
+ 'description': schema.Scalar('description', basestring)})
+ kwargs['schema'] = SCHEMA
+ return testlog_validator.Validator.Dict(*args, **kwargs)
+
+ def _ValidatorFailureWrapper(*args, **kwargs): # pylint: disable=E0211
+ SCHEMA = schema.FixedDict(
+ 'failures.value',
+ items={'code': schema.Scalar('code', basestring),
+ 'details': schema.Scalar('details', basestring)})
+ kwargs['schema'] = SCHEMA
+ return testlog_validator.Validator.List(*args, **kwargs)
+
+ def _ValidatorSerialNumberWrapper(*args, **kwargs): # pylint: disable=E0211
+ SCHEMA = schema.Scalar('serialNumbers.value', basestring)
+ kwargs['schema'] = SCHEMA
+ return testlog_validator.Validator.Dict(*args, **kwargs)
+
+ def _ValidatorParameterWrapper(*args, **kwargs): # pylint: disable=E0211
+ SCHEMA = schema.FixedDict(
+ 'parameters.value',
+ items={},
+ optional_items={
+ 'description': schema.Scalar('description', basestring),
+ 'group': schema.Scalar('group', basestring),
+ 'status': schema.Scalar('status', basestring, ['PASS', 'FAIL']),
+ 'valueUnit': schema.Scalar('valueUnit', basestring),
+ 'numericValue': StationTestRun._NumericSchema('numericValue'),
+ 'expectedMinimum': StationTestRun._NumericSchema('expectedMinimum'),
+ 'expectedMaximum': StationTestRun._NumericSchema('expectedMaximum'),
+ 'textValue': schema.Scalar('textValue', basestring),
+ 'expectedRegex': schema.Scalar('expectedRegex', basestring)})
+ kwargs['schema'] = SCHEMA
+ return testlog_validator.Validator.Dict(*args, **kwargs)
+
+ def _ValidatorSeriesWrapper(*args, **kwargs): # pylint: disable=E0211
+ DATA_SCHEMA = schema.List('data', schema.FixedDict(
+ 'data',
+ items={
+ 'key': StationTestRun._NumericSchema('key')},
+ optional_items={
+ 'status': schema.Scalar('status', basestring, ['PASS', 'FAIL']),
+ 'numericValue': StationTestRun._NumericSchema('numericValue'),
+ 'expectedMinimum': StationTestRun._NumericSchema('expectedMinimum'),
+ 'expectedMaximum': StationTestRun._NumericSchema('expectedMaximum')
+ }))
+ SCHEMA = schema.FixedDict(
+ 'series.value',
+ items={},
+ optional_items={
+ 'description': schema.Scalar('description', basestring),
+ 'group': schema.Scalar('group', basestring),
+ 'status': schema.Scalar('status', basestring, ['PASS', 'FAIL']),
+ 'keyUnit': schema.Scalar('keyUnit', basestring),
+ 'valueUnit': schema.Scalar('valueUnit', basestring),
+ 'data': DATA_SCHEMA})
+ kwargs['schema'] = SCHEMA
+ return testlog_validator.Validator.Dict(*args, **kwargs)
+
def _ValidatorAttachmentWrapper(*args, **kwargs): # pylint: disable=E0211
# Because the FIELDS map must be assigned at the time of loading the
# module. However, Testlog singleton is not ready yet, we pass the
@@ -914,17 +989,17 @@
'testRunId': (True, testlog_validator.Validator.String),
'testName': (True, testlog_validator.Validator.String),
'testType': (True, testlog_validator.Validator.String),
- 'arguments': (False, testlog_validator.Validator.Dict),
+ 'arguments': (False, _ValidatorArgumentWrapper),
'status': (True, testlog_validator.Validator.Status),
'startTime': (True, testlog_validator.Validator.Time),
'endTime': (False, testlog_validator.Validator.Time),
- 'duration': (False, testlog_validator.Validator.Float),
+ 'duration': (False, testlog_validator.Validator.Number),
'operatorId': (False, testlog_validator.Validator.String),
'attachments': (False, _ValidatorAttachmentWrapper),
- 'failures': (False, testlog_validator.Validator.List),
- 'serialNumbers': (False, testlog_validator.Validator.Dict),
- 'parameters': (False, testlog_validator.Validator.Dict),
- 'series': (False, testlog_validator.Validator.Dict),
+ 'failures': (False, _ValidatorFailureWrapper),
+ 'serialNumbers': (False, _ValidatorSerialNumberWrapper),
+ 'parameters': (False, _ValidatorParameterWrapper),
+ 'series': (False, _ValidatorSeriesWrapper),
}
# Possible values for the `status` field.
@@ -938,10 +1013,11 @@
return 'station.test_run'
def CastFields(self):
+ super(StationTestRun, self).CastFields()
if 'series' in self:
s = Series(__METADATA__={})
s.update(self['series'])
- self['series'] = s
+ self._data.update(series=s)
@staticmethod
def _CheckParamArguments(value, description, value_unit,
diff --git a/py/testlog/testlog_unittest.py b/py/testlog/testlog_unittest.py
index f58e94d..ad2639b 100755
--- a/py/testlog/testlog_unittest.py
+++ b/py/testlog/testlog_unittest.py
@@ -55,7 +55,7 @@
def testEventSerializeUnserialize(self):
original = testlog.StationInit()
- output = testlog.Event.FromJSON(original.ToJSON())
+ output = testlog.Event.FromJSON(original.ToJSON(), False)
self.assertEquals(output, original)
def testNewEventTime(self):
@@ -79,8 +79,11 @@
def testCheckMissingFields(self):
event = testlog.StationInit()
event['failureMessage'] = 'Missed fields'
- self.assertItemsEqual(event.CheckMissingFields(),
- ['count', 'success', 'uuid', 'apiVersion', 'time'])
+ try:
+ event.CheckMissingFields()
+ except testlog_utils.TestlogError as e:
+ self.assertEqual(repr(e), 'TestlogError("Missing fields: [\'count\', '
+ '\'success\', \'uuid\', \'apiVersion\', \'time\']",)')
def testAddArgument(self):
event = testlog.StationTestRun()
diff --git a/py/testlog/testlog_validator.py b/py/testlog/testlog_validator.py
index 9c273f7..7335a76 100644
--- a/py/testlog/testlog_validator.py
+++ b/py/testlog/testlog_validator.py
@@ -32,7 +32,9 @@
Validator.Object(inst, key, value)
@staticmethod
- def Float(inst, key, value):
+ def Number(inst, key, value):
+ if isinstance(value, (int, long)):
+ value = float(value)
if not isinstance(value, float):
raise ValueError(
'key[%s] accepts type of float. Not %r '
@@ -56,7 +58,7 @@
Validator.Object(inst, key, value)
@staticmethod
- def Dict(inst, key, value):
+ def Dict(inst, key, value, schema=None):
"""Inserts an item into the inst._data[key].
Assuming inst._data[key] is a dictionary, the inserted element will be
@@ -68,6 +70,14 @@
'Validator.Dict accepts value in form of {%r:..., %r:...}, not %s' % (
'key', 'value', pprint.pformat(value)))
+ if not isinstance(value['key'], basestring):
+ raise ValueError(
+ 'The key of %s accepts type of basestring. Not %r '
+ 'Please convert before assign' % (key, type(value['key'])))
+
+ if schema:
+ schema.Validate(value['value'])
+
# pylint: disable=protected-access
updated_dict = inst._data[key] if key in inst._data else {}
sub_key = value['key']
@@ -80,8 +90,12 @@
inst._data[key] = updated_dict
@staticmethod
- def List(inst, key, value):
+ def List(inst, key, value, schema=None):
logging.debug('Validator.List called with (%s, %s)', key, value)
+
+ if schema:
+ schema.Validate(value)
+
# pylint: disable=protected-access
updated_list = inst._data[key] if key in inst._data else []
updated_list.append(value)
diff --git a/py/testlog/testlog_validator_unittest.py b/py/testlog/testlog_validator_unittest.py
index 5589bca..2bd9a63 100755
--- a/py/testlog/testlog_validator_unittest.py
+++ b/py/testlog/testlog_validator_unittest.py
@@ -17,6 +17,7 @@
import testlog
import testlog_utils
import testlog_validator
+from utils import schema
from utils import time_utils
@@ -25,7 +26,7 @@
"""A derived class from Event that can be initialized."""
FIELDS = {
'Long': (True, testlog_validator.Validator.Long),
- 'Float': (True, testlog_validator.Validator.Float),
+ 'Number': (True, testlog_validator.Validator.Number),
'String': (True, testlog_validator.Validator.String),
'Boolean': (True, testlog_validator.Validator.Boolean),
'Dict': (True, testlog_validator.Validator.Dict),
@@ -53,14 +54,18 @@
event['Long'] = 3333
self.assertEquals(event['Long'], 3333)
- def testFloatValidator(self):
+ def testNumberValidator(self):
event = self.TestEvent()
with self.assertRaises(ValueError):
- event['Float'] = '66'
+ event['Number'] = '66'
with self.assertRaises(ValueError):
- event['Float'] = 'aa'
- event['Float'] = 33.33
- self.assertAlmostEqual(event['Float'], 33.33)
+ event['Number'] = 'aa'
+ event['Number'] = 33.33
+ self.assertAlmostEqual(event['Number'], 33.33)
+ event['Number'] = -1
+ self.assertAlmostEqual(event['Number'], -1)
+ event['Number'] = 999999999999999
+ self.assertAlmostEqual(event['Number'], 999999999999999)
def testStringValidator(self):
event = self.TestEvent()
@@ -93,7 +98,7 @@
self.assertEquals(event['Dict'],
{'PARA1': 33, 'PARA2': 'aaa'})
# Converts to JSON and convert back.
- event2 = testlog.EventBase.FromJSON(event.ToJSON())
+ event2 = testlog.EventBase.FromJSON(event.ToJSON(), False)
self.assertEquals(event, event2)
def testListValidator(self):
@@ -112,8 +117,167 @@
self.assertEquals(event['Time'], copy.copy(valid_datetime))
-class StationTestRunValidatorTest(unittest.TestCase):
+class StationTestRunFieldsValidatorTest(unittest.TestCase):
+ """Checks the correctness of FIELDS' validator and SCHEMA's"""
+
+ def testArgumentValidator(self):
+ event = testlog.StationTestRun()
+ # Valid argument
+ event['arguments'] = {'key': 'KeyIsAString', 'value': {
+ 'value': 'This is a string.'}}
+ event['arguments'] = {'key': 'KeyIsANumber', 'value': {
+ 'value': 987.654321,
+ 'description': 'G'}}
+ event['arguments'] = {'key': 'KeyCanBeAnything', 'value': {
+ 'value': True}}
+ # Wrong top-level type
+ with self.assertRaises(schema.SchemaException):
+ event['arguments'] = {'key': 'A', 'value': 'Google'}
+ # Wrong key type
+ with self.assertRaises(ValueError):
+ event['arguments'] = {'key': 3, 'value': {'value': 0}}
+ # Additional item in Dict
+ with self.assertRaises(schema.SchemaException):
+ event['arguments'] = {'key': 'A', 'value': {'value': 0, 'A': 'B'}}
+ # Missing item
+ with self.assertRaises(schema.SchemaException):
+ event['arguments'] = {'key': 'A', 'value': {'description': 'yoyo'}}
+ # Empty dict
+ with self.assertRaises(schema.SchemaException):
+ event['arguments'] = {'key': 'A', 'value': {}}
+
+ def testFailuresValidator(self):
+ event = testlog.StationTestRun()
+ # Valid failure
+ event['failures'] = {
+ 'code': 'WiFiThroughputSpeedLimitFail',
+ 'details': '2.4G WiFi throughput test fail: ... bla bla bla'}
+ # Wrong top-level type
+ with self.assertRaises(schema.SchemaException):
+ event['failures'] = 'Google'
+ # Wrong value type
+ with self.assertRaises(schema.SchemaException):
+ event['failures'] = {'code': 'C', 'details': 3}
+ # Additional item in Dict
+ with self.assertRaises(schema.SchemaException):
+ event['failures'] = {'code': 'C', 'details': 'D', 'A': 'B'}
+ # Missing item
+ with self.assertRaises(schema.SchemaException):
+ event['failures'] = {'code': 'C'}
+ # Empty dict
+ with self.assertRaises(schema.SchemaException):
+ event['failures'] = {}
+
+ def testSerialNumbersValidator(self):
+ event = testlog.StationTestRun()
+ # Valid serial number
+ event['serialNumbers'] = {'key': 'mlb', 'value': 'A1234567890'}
+ # Wrong key type
+ with self.assertRaises(ValueError):
+ event['serialNumbers'] = {'key': 3, 'value': 'Google'}
+ # Wrong value type
+ with self.assertRaises(schema.SchemaException):
+ event['serialNumbers'] = {'key': 'A', 'value': {}}
+ with self.assertRaises(schema.SchemaException):
+ event['serialNumbers'] = {'key': 'A', 'value': 3}
+
+ def testParametersValidator(self):
+ event = testlog.StationTestRun()
+ # Valid parameter
+ event['parameters'] = {'key': 'a', 'value': {}}
+ event['parameters'] = {'key': 'b', 'value': {'status': 'PASS'}}
+ event['parameters'] = {'key': 'ec_firmware_version', 'value': {
+ 'description': 'Version of the EC firmware on the DUT.',
+ 'group': 'What is this?',
+ 'status': 'FAIL',
+ 'valueUnit': 'version',
+ 'numericValue': 0,
+ 'expectedMinimum': 1.2,
+ 'expectedMaximum': 3L,
+ 'textValue': '1.56a',
+ 'expectedRegex': '^1.5'
+ }}
+ # Wrong top-level type
+ with self.assertRaises(schema.SchemaException):
+ event['parameters'] = {'key': 'A', 'value': 'Google'}
+ # Wrong key type
+ with self.assertRaises(ValueError):
+ event['parameters'] = {'key': 3, 'value': {}}
+ # Wrong value type
+ with self.assertRaises(schema.SchemaException):
+ event['parameters'] = {'key': 'A', 'value': {'description': 0}}
+ with self.assertRaises(schema.SchemaException):
+ event['parameters'] = {'key': 'A', 'value': {'numericValue': 'Google'}}
+ # Value not in choices(PASS, FAIL)
+ with self.assertRaises(schema.SchemaException):
+ event['parameters'] = {'key': 'A', 'value': {'status': 'Google'}}
+ # Additional item in Dict
+ with self.assertRaises(schema.SchemaException):
+ event['parameters'] = {'key': 'A', 'value': {'NonExist': 'Hi'}}
+
+ def testSeriesValidator(self):
+ event = testlog.StationTestRun()
+ # Valid parameter
+ event['series'] = {'key': 'a', 'value': {}}
+ event['series'] = {'key': 'b', 'value': {'status': 'PASS', 'data': []}}
+ event['series'] = {'key': '5g_throughput', 'value': {
+ 'description': '5G throughput speeds at an interval of 1 per ... .',
+ 'group': 'Still do not know.',
+ 'status': 'FAIL',
+ 'keyUnit': 'second',
+ 'valueUnit': 'MBit/s',
+ 'data': [
+ {
+ 'key': 1,
+ 'status': 'FAIL',
+ 'numericValue': 19.8,
+ 'expectedMinimum': 20,
+ 'expectedMaximum': 999L
+ }, {
+ 'key': 2.0,
+ 'status': 'PASS'
+ }
+ ]
+ }}
+ # Wrong top-level type, value should be a Dict
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': 'Google'}
+ # Wrong second-level type, data should be a List
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {'data': 'Google'}}
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {'data': {}}}
+ # Wrong key type
+ with self.assertRaises(ValueError):
+ event['series'] = {'key': 3, 'value': {}}
+ # Wrong value type
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {'description': 0}}
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {'data': [{'key': 'Google'}]}}
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {
+ 'data': [{'key': 0, 'status': 0}]}}
+ # Value not in choices(PASS, FAIL)
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {'status': 'Google'}}
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {
+ 'data': [{'key': 0, 'status': 'Google'}]}}
+ # Additional item in Dict
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {'NonExist': 'Hi'}}
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {
+ 'data': [{'key': 0, 'A': 'B'}]}}
+ # Missing item
+ with self.assertRaises(schema.SchemaException):
+ event['series'] = {'key': 'A', 'value': {'data': [{}]}}
+
+
+class StationTestRunApiValidatorTest(unittest.TestCase):
"""Validators primarily serve for StationTestRun."""
+
def setUp(self):
self.state_dir = tempfile.mkdtemp()
self.tmp_dir = tempfile.mkdtemp()
@@ -268,6 +432,42 @@
# Make sure the file names are distinguished
self.assertEquals(len(paths), 2)
+ def testFromDict(self):
+ self._SimulateSubSession()
+ example_dict = {
+ 'uuid': '8b127476-2604-422a-b9b1-f05e4f14bf72',
+ 'type': 'station.test_run',
+ 'apiVersion': '0.1',
+ 'time': '2017-01-05T13:01:45.503Z',
+ 'seq': 8202191,
+ 'stationDeviceId': 'e7d3227e-f12d-42b3-9c64-0d9e8fa02f6d',
+ 'stationInstallationId': '92228272-056e-4329-a432-64d3ed6dfa0c',
+ 'testRunId': '8b127472-4593-4be8-9e94-79f228fc1adc',
+ 'testName': 'the_test',
+ 'testType': 'aaaa',
+ 'arguments': {},
+ 'status': 'PASSED',
+ 'startTime': '2017-01-05T13:01:45.489Z',
+ 'failures': [],
+ 'parameters': {},
+ 'series': {}
+ }
+ _unused_valid_event = testlog.EventBase.FromDict(example_dict)
+ example_dict['arguments']['A'] = {'value': 'yoyo'}
+ example_dict['arguments']['B'] = {'value': 9.53543, 'description': 'number'}
+ example_dict['arguments']['C'] = {'value': -9}
+ example_dict['failures'].append({'code': 'C', 'details': 'D'})
+ example_dict['serialNumbers'] = {}
+ example_dict['serialNumbers']['A'] = 'B'
+ example_dict['parameters']['A'] = {'description': 'D'}
+ example_dict['series']['A'] = {'description': 'D', 'data': [
+ {'key': 987, 'status': 'PASS'},
+ {'key': 7.8, 'status': 'FAIL'}]}
+ _unused_valid_event = testlog.EventBase.FromDict(example_dict)
+ with self.assertRaises(schema.SchemaException):
+ example_dict['arguments']['D'] = {}
+ _unused_invalid_event = testlog.EventBase.FromDict(example_dict)
+
if __name__ == '__main__':
logging.basicConfig(
