graphyte: Make "make test" python 2/3 compatible.
python2 is deprecated in 2020.
(1) Use list instead of map function to create lists.
(2) Decodes bytes input from socket to string.
(3) Encodes string output to socket to bytes.
(4) Adapts inspect library behavior changes.
(5) Replace iteritems() with items().
(6) Import packages from six.moves.
(7) Use python3 print function syntax.
BUG=chromium:1019481
TEST=make test PYTHON=python2; make test PYTHON=python3
Change-Id: I82dcf6efa00da4d7cb6e9f1ec63101c027f00047
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/graphyte/+/2058226
Commit-Queue: Cheng Yueh <cyueh@chromium.org>
Tested-by: Cheng Yueh <cyueh@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Yong Hong <yhong@chromium.org>
diff --git a/graphyte/controller_unittest.py b/graphyte/controller_unittest.py
index b0e4129..3b85df2 100755
--- a/graphyte/controller_unittest.py
+++ b/graphyte/controller_unittest.py
@@ -3,9 +3,16 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-import mock
+from __future__ import absolute_import
+
+import sys
import unittest
+if sys.version_info.major < 3:
+ import mock
+else:
+ import unittest.mock as mock
+
import graphyte_common # pylint: disable=unused-import
from graphyte import controller
from graphyte import testplan
diff --git a/graphyte/data_parser.py b/graphyte/data_parser.py
index 3b6f6cf..cf1a8e4 100644
--- a/graphyte/data_parser.py
+++ b/graphyte/data_parser.py
@@ -65,7 +65,8 @@
left, items, right = string[0], string[1:-1], string[-1]
if right != RIGHT_BRACKETS_DICT[left]:
raise ValueError('The right bracket is not found: %s', string)
- return BUILDER_DICT[left](map(_ParseLiteral, items.split(',')))
+ literals = list(map(_ParseLiteral, items.split(',')))
+ return BUILDER_DICT[left](literals)
result = _Parse(string)
if checker is not None and checker.CheckData(result) is False:
diff --git a/graphyte/data_parser_unittest.py b/graphyte/data_parser_unittest.py
index 5901b81..f2f2c05 100755
--- a/graphyte/data_parser_unittest.py
+++ b/graphyte/data_parser_unittest.py
@@ -60,7 +60,7 @@
def testCheckerStr(self):
self.assertEqual(
- "LiteralChecker([<type 'int'>, 'a', None])", str(self.checker))
+ "LiteralChecker([%s, 'a', None])" % int, str(self.checker))
def testValidData(self):
self.assertTrue(self.checker.CheckData(123))
@@ -80,7 +80,7 @@
def testCheckerStr(self):
self.assertEqual(
- "ListChecker([<type 'int'>, 'a', None])", str(self.checker))
+ "ListChecker([%s, 'a', None])" % int, str(self.checker))
def testValidData(self):
self.assertTrue(self.checker.CheckData([123, -55, 'a', None]))
@@ -98,7 +98,7 @@
def testCheckerStr(self):
self.assertEqual(
- "TupleChecker([<type 'int'>, None], ['hello', <type 'float'>])",
+ "TupleChecker([%s, None], ['hello', %s])" % (int, float),
str(self.checker))
def testValidData(self):
diff --git a/graphyte/link.py b/graphyte/link.py
index d57e9fb..5282810 100644
--- a/graphyte/link.py
+++ b/graphyte/link.py
@@ -7,26 +7,24 @@
import tempfile
import graphyte_common # pylint: disable=unused-import
-from graphyte.utils import file_utils
from graphyte.default_setting import logger
+from graphyte.utils import file_utils
CalledProcessError = subprocess.CalledProcessError
MODULE_PATH_MAPPING = {
- 'ADBLink': 'links.adb',
- 'SSHLink': 'links.ssh',
- 'SCPILink': 'links.scpi',
- 'GPIBLink': 'links.gpib'}
+ 'ADBLink': 'graphyte.links.adb',
+ 'SSHLink': 'graphyte.links.ssh',
+ 'SCPILink': 'graphyte.links.scpi',
+ 'GPIBLink': 'graphyte.links.gpib'}
class LinkNotReady(Exception):
"""Exception for link is not ready."""
- pass
class LinkOptionsError(Exception):
"""Exception for invalid link options."""
- pass
class DeviceLink(object):
@@ -149,7 +147,7 @@
exit_code = self.Call(command, stdin, stdout, stderr)
stdout.flush()
stdout.seek(0)
- output = stdout.read()
+ output = stdout.read().decode('utf-8')
if exit_code != 0:
raise CalledProcessError(
returncode=exit_code, cmd=command, output=output)
@@ -222,7 +220,7 @@
if spec.keywords is None:
# if the function accepts ** arguments, we can just pass everything into it
# so we only need to filter kargs if spec.keywords is None
- kargs = {k: v for (k, v) in kargs.iteritems() if k in spec.args}
+ kargs = {k: v for (k, v) in kargs.items() if k in spec.args}
return kargs
diff --git a/graphyte/link_unittest.py b/graphyte/link_unittest.py
index 86ec919..ab41b9c 100755
--- a/graphyte/link_unittest.py
+++ b/graphyte/link_unittest.py
@@ -49,7 +49,7 @@
return subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr)
def _WriteScript(self, script):
- os.write(self.temp_fd, script)
+ os.write(self.temp_fd, script.encode('utf-8'))
def testNormalCall(self):
self._WriteScript('exit 0')
@@ -89,7 +89,7 @@
with tempfile.NamedTemporaryFile() as stdin, \
tempfile.NamedTemporaryFile() as stdout, \
tempfile.NamedTemporaryFile() as stderr:
- stdin.write('hello')
+ stdin.write(b'hello')
stdin.flush()
stdin.seek(0)
ret = self.link_instance.Call(['sh', self.temp_file], stdin=stdin,
@@ -99,8 +99,8 @@
stderr.seek(0)
out_str = stdout.read()
err_str = stderr.read()
- self.assertEqual("output hello\n", out_str)
- self.assertEqual("error hello\n", err_str)
+ self.assertEqual(b"output hello\n", out_str)
+ self.assertEqual(b"error hello\n", err_str)
if __name__ == '__main__':
unittest.main()
diff --git a/graphyte/links/adb.py b/graphyte/links/adb.py
index 9e36c7f..af727e7 100644
--- a/graphyte/links/adb.py
+++ b/graphyte/links/adb.py
@@ -8,6 +8,8 @@
import subprocess
import uuid
+import six
+
import graphyte_common # pylint: disable=unused-import
from graphyte import link
from graphyte.default_setting import logger
@@ -89,7 +91,7 @@
# Convert list-style commands to single string because we need to run
# multiple commands in same session (and needs shell=True).
- if not isinstance(command, basestring):
+ if not isinstance(command, six.string_types):
command = ' '.join(pipes.quote(param) for param in command)
# ADB protocol currently mixes stderr and stdout in same channel (i.e., the
diff --git a/graphyte/links/local.py b/graphyte/links/local.py
index 32c2d4a..6b63f22 100644
--- a/graphyte/links/local.py
+++ b/graphyte/links/local.py
@@ -8,6 +8,8 @@
import shutil
import subprocess
+import six
+
import graphyte_common # pylint: disable=unused-import
from graphyte import link
@@ -45,7 +47,7 @@
# Android only has /system/bin/sh) then calling Popen may give you 'No such
# file or directory' error.
- if not isinstance(command, basestring):
+ if not isinstance(command, six.string_types):
command = ' '.join(pipes.quote(param) for param in command)
if self._shell_path:
diff --git a/graphyte/links/scpi.py b/graphyte/links/scpi.py
index 1e841f0..d765b36 100644
--- a/graphyte/links/scpi.py
+++ b/graphyte/links/scpi.py
@@ -93,7 +93,7 @@
if self._IsQueryCommand(command):
output = self._Query(command)
if stdout:
- stdout.write(output)
+ stdout.write(output.encode('utf-8'))
else:
self._Send(command)
return 0
@@ -159,7 +159,7 @@
with sync_utils.Timeout(self.timeout):
if not self.timeout:
logger.debug('Waiting for reading...')
- ret = self.rfile.readline().rstrip('\n')
+ ret = self.rfile.readline().decode('utf-8').rstrip('\n')
logger.debug('Read: %s', ret)
return ret
@@ -168,7 +168,7 @@
if '\n' in command:
raise SCPILinkError('Newline in command: %r' % command)
logger.debug('Write: %s', command)
- self.wfile.write(command + '\n')
+ self.wfile.write((command + '\n').encode('utf-8'))
def _IsQueryCommand(self, command):
return '?' in command
diff --git a/graphyte/links/scpi_mock.py b/graphyte/links/scpi_mock.py
index f1cd363..2bbb602 100644
--- a/graphyte/links/scpi_mock.py
+++ b/graphyte/links/scpi_mock.py
@@ -11,18 +11,18 @@
MockTestServer(('0.0.0.0', SERVER_PORT), MockServerHandler).serve_forever()
"""
-import logging
import inspect
+import logging
import re
-import types
-import SocketServer
+import six
+from six.moves import socketserver
-class MockTestServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+class MockTestServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
allow_reuse_address = True
-class MockServerHandler(SocketServer.StreamRequestHandler):
+class MockServerHandler(socketserver.StreamRequestHandler):
"""A mocking handler for socket.
This handler responses client based on its pre-defined lookup table.
@@ -49,8 +49,8 @@
"""
# Check if the response is one of the known types.
is_known_types = False
- if (isinstance(response, types.StringType) or
- isinstance(response, types.NoneType)):
+
+ if isinstance(response, six.string_types) or response is None:
is_known_types = True
elif inspect.isfunction(response) or inspect.ismethod(response):
is_known_types = True
@@ -60,16 +60,16 @@
def __init__(self, *args, **kwargs):
self.lookup = list(self.responses_lookup)
- SocketServer.StreamRequestHandler.__init__(self, *args, **kwargs)
+ socketserver.StreamRequestHandler.__init__(self, *args, **kwargs)
def handle(self):
regex_type = type(re.compile(''))
while True:
- line = self.rfile.readline().rstrip('\r\n')
+ line = self.rfile.readline().decode('utf-8').rstrip('\r\n')
if not line:
break
for rule, response in self.lookup:
- if isinstance(rule, str) and line == rule:
+ if isinstance(rule, six.string_types) and line == rule:
logging.info('Input %r matched.', line)
elif isinstance(rule, regex_type) and rule.match(line):
logging.info('Input %r matched with regexp %s', line, rule.pattern)
@@ -78,8 +78,8 @@
if inspect.isfunction(response) or inspect.ismethod(response):
response = response(line)
- if isinstance(response, types.StringType):
- self.wfile.write(response)
+ if isinstance(response, six.string_types):
+ self.wfile.write(response.encode('utf-8'))
break # Only the first match will be used.
else:
raise ValueError('Input %r is not matching any.' % line)
diff --git a/graphyte/links/ssh.py b/graphyte/links/ssh.py
index 7c69e12..7174405 100644
--- a/graphyte/links/ssh.py
+++ b/graphyte/links/ssh.py
@@ -5,7 +5,6 @@
"""Implementation of graphyte.link.DeviceLink using SSH."""
-import Queue
import os
import pipes
import platform
@@ -14,6 +13,9 @@
import threading
import time
+import six
+from six.moves import queue
+
import graphyte_common # pylint: disable=unused-import
from graphyte import link
from graphyte.default_setting import logger
@@ -61,7 +63,7 @@
if self.identity is not None:
try:
self.identity = graphyte_utils.SearchConfig(self.identity)
- os.chmod(self.identity, 0600)
+ os.chmod(self.identity, 0o600)
except Exception:
logger.error('SSHLink: Failed to locate identity file "%s"',
self.identity)
@@ -137,16 +139,15 @@
"""See DUTLink.Shell"""
remote_sig, options = self._Signature(False)
- if isinstance(command, basestring):
+ if isinstance(command, six.string_types):
# since the shell command is passed through SSH client,
# the entire command is argument for SSH command
# e.g. command = 'VAR=xxx cmd arg0 arg1...'
# will become: 'ssh' <SSH OPTIONS> 'VAR=xxx' 'cmd' 'arg0' 'arg1' ...
command = shlex.split(command)
- command = ['ssh'] + options + [remote_sig] + command
else:
- command = ['ssh'] + options + [remote_sig] + map(pipes.quote, command)
-
+ command = [pipes.quote(c) for c in command]
+ command = ['ssh'] + options + [remote_sig] + command
shell = False
logger.debug('SSHLink: Run [%r]', command)
@@ -175,7 +176,7 @@
self._link = link_instance
self._thread = threading.Thread(target=self.Run)
- self._proc_queue = Queue.Queue()
+ self._proc_queue = queue.Queue()
self._user = self._link.user
self._host = self._link.host
diff --git a/graphyte/plugin_shell.py b/graphyte/plugin_shell.py
index bf71d29..3db6828 100644
--- a/graphyte/plugin_shell.py
+++ b/graphyte/plugin_shell.py
@@ -28,10 +28,10 @@
from graphyte.device import DeviceType
from graphyte.utils import type_utils
-DEVICE_STATUS = type_utils.Enum(
- STOP='STOP', # Device is not initialized.
- IDLE='IDLE', # Device is initiailized but not running.
- RUNNING='RUNNING') # Device starts running the test case.
+DEVICE_STATUS = type_utils.Enum([
+ 'STOP', # Device is not initialized.
+ 'IDLE', # Device is initiailized but not running.
+ 'RUNNING']) # Device starts running the test case.
def CheckIn(name, item, arr):
diff --git a/graphyte/port_config.py b/graphyte/port_config.py
index 56dd77f..b8926ad 100644
--- a/graphyte/port_config.py
+++ b/graphyte/port_config.py
@@ -36,8 +36,11 @@
path2 is {2402: 21.0, 2404: 20.3}.
"""
+from __future__ import print_function
+
import csv
import json
+import math
import os
import sys
@@ -62,9 +65,9 @@
self.port_config = config['port_config']
# Load the pathloss table for each antenna
- for component_config in self.port_config.itervalues():
+ for component_config in self.port_config.values():
for antenna_config in component_config:
- for config_table in antenna_config.itervalues():
+ for config_table in antenna_config.values():
filename, title = config_table['pathloss']
config_table['pathloss'] = self.LoadPathloss(filename, title)
@@ -92,7 +95,7 @@
pathloss to float.
"""
ret = {}
- for key, value in pathloss.iteritems():
+ for key, value in pathloss.items():
if not value:
continue
ret[int(key)] = float(value)
@@ -132,7 +135,7 @@
return self.GetPortConfig(**kwarg)['pathloss']
def GetIterator(self):
- return self.port_config.iteritems()
+ return self.port_config.items()
class PathlossTable(object):
@@ -151,6 +154,8 @@
def __nonzero__(self):
return len(self.table) > 0
+ __bool__ = __nonzero__
+
def __getitem__(self, freq):
"""Returns the pathloss of the frequency.
@@ -170,8 +175,12 @@
if freq in self.table:
return self.table[freq]
- upper_dist = sys.maxint
- lower_dist = sys.maxint
+ try:
+ upper_dist = math.inf
+ lower_dist = math.inf
+ except AttributeError:
+ upper_dist = sys.maxint
+ lower_dist = sys.maxint
upper_freq = None
lower_freq = None
for k in self.table:
@@ -195,15 +204,15 @@
def main():
port_config = PortConfig('sample_port_config.json')
for component_name, antennas_config in port_config.GetIterator():
- print 'Component: %s' % component_name
+ print('Component: %s' % component_name)
for antenna_idx, antenna_config in enumerate(antennas_config):
- print ' Antenna %d:' % antenna_idx
+ print(' Antenna %d:' % antenna_idx)
for test_type in ['TX', 'RX']:
- print ' port Config for %s: ' % test_type
- print ' pathloss: %s' % antenna_config[test_type]['pathloss']
- print ' port_mapping: %s' % antenna_config[test_type]['port_mapping']
+ print(' port Config for %s: ' % test_type)
+ print(' pathloss: %s' % antenna_config[test_type]['pathloss'])
+ print(' port_mapping: %s' % antenna_config[test_type]['port_mapping'])
- print port_config.QueryPortConfig('WLAN_2G', 'TX')
+ print(port_config.QueryPortConfig('WLAN_2G', 'TX'))
if __name__ == '__main__':
diff --git a/graphyte/result_writer.py b/graphyte/result_writer.py
index d9ff100..e9fcab0 100644
--- a/graphyte/result_writer.py
+++ b/graphyte/result_writer.py
@@ -61,7 +61,7 @@
self._WriteResult(new_test_case, new_result)
def _WriteResult(self, test_case, result):
- for result_name, bound in test_case.result_limit.iteritems():
+ for result_name, bound in test_case.result_limit.items():
value = result.get(result_name, 'None')
data = {
'test_item': test_case.name,
diff --git a/graphyte/testplan.py b/graphyte/testplan.py
index 51d92be..1736e20 100644
--- a/graphyte/testplan.py
+++ b/graphyte/testplan.py
@@ -308,7 +308,7 @@
'rf_type': ListChecker([RF_TYPE]),
'component_name': ListChecker([str]),
'test_type': ListChecker(SUPPORTED_TEST_TYPES),
- 'center_freq': ListChecker([lambda freq: 2400 < freq < 6000]),
+ 'center_freq': ListChecker([lambda freq: 2400 < int(freq) < 6000]),
'power_level': ListChecker([int, float, 'auto']),
'standard': ListChecker(['A', 'AC', 'B', 'G', 'N']),
'bandwidth': ListChecker([20, 40, 80, 160]),
@@ -316,7 +316,7 @@
DATA_RATE_A_G +
DATA_RATE_N +
DATA_RATE_AC),
- 'chain_mask': ListChecker(range(1, 16)), # 4x4 MIMO supported
+ 'chain_mask': ListChecker(list(range(1, 16))), # 4x4 MIMO supported
'nss': ListChecker([None, 1, 2, 3, 4]),
'long_preamble': ListChecker([0, 1])}
@@ -411,7 +411,7 @@
TestCaseFactory.VerifyArguments(arg_checker, args)
# Check if the arguments are valid in the specified standard.
bandwidth = args['bandwidth']
- center_freq = args['center_freq']
+ center_freq = int(args['center_freq'])
data_rate = args['data_rate']
standard = args['standard']
TestCaseFactory.VerifyArguments(
diff --git a/graphyte/utils/file_utils.py b/graphyte/utils/file_utils.py
index 92b6a49..593a669 100644
--- a/graphyte/utils/file_utils.py
+++ b/graphyte/utils/file_utils.py
@@ -46,7 +46,7 @@
pass
-def MakeDirsUidGid(path, uid=-1, gid=-1, mode=0777):
+def MakeDirsUidGid(path, uid=-1, gid=-1, mode=0o777):
"""Recursive directory creation with specified uid, gid and mode.
Like os.makedirs, but it also chown() and chmod() to the directories it
@@ -56,7 +56,7 @@
path: Path to create recursively.
uid: User id. -1 means unchanged.
gid: Group id. -1 means unchanged.
- mode: Mode (numeric) of path. Default 0777.
+ mode: Mode (numeric) of path. Default 0o777.
"""
logging.debug('MakeDirsUidGid %r', path)
if not path:
@@ -69,7 +69,7 @@
os.mkdir(path)
os.chmod(path, mode)
os.chown(path, uid, gid)
- logging.debug('mkdir %r with mode 0%o uid %r gid %r', path, mode, uid, gid)
+ logging.debug('mkdir %r with mode 0o%o uid %r gid %r', path, mode, uid, gid)
class Glob(object):
@@ -90,7 +90,7 @@
elif isinstance(exclude, str):
self.exclude = [exclude]
else:
- raise TypeError, 'Unexpected exclude type %s' % type(exclude)
+ raise TypeError('Unexpected exclude type %s' % type(exclude))
def Match(self, root):
"""Returns files that match include but not exclude.
@@ -628,8 +628,8 @@
"""
matches = glob.glob(pattern)
if len(matches) != 1:
- raise ValueError, 'Expected one match for %s but got %s' % (
- pattern, matches)
+ raise ValueError('Expected one match for %s but got %s' % (
+ pattern, matches))
return matches[0]
diff --git a/graphyte/utils/graphyte_utils.py b/graphyte/utils/graphyte_utils.py
index fbcc7d0..42695ca 100644
--- a/graphyte/utils/graphyte_utils.py
+++ b/graphyte/utils/graphyte_utils.py
@@ -9,6 +9,7 @@
import json
import math
import os
+import sys
import graphyte_common # pylint: disable=unused-import
from graphyte.default_setting import CONFIG_DIR
@@ -31,7 +32,7 @@
Returns:
The new mapping object with values overridden.
"""
- for key, val in overrides.iteritems():
+ for key, val in overrides.items():
if isinstance(val, collections.Mapping):
base[key] = OverrideConfig(base.get(key, {}), val)
else:
@@ -118,7 +119,7 @@
lower, upper = bound
return lower or upper or 0
return dict([(key, _MakeInBoundValue(bound))
- for key, bound in result_limit.iteritems()])
+ for key, bound in result_limit.items()])
def CalculateAverage(values, average_type='Linear'):
@@ -132,7 +133,7 @@
the average value.
"""
length = len(values)
- values = map(float, values)
+ values = [float(x) for x in values]
if length == 0:
return float('nan')
if length == 1:
@@ -184,6 +185,12 @@
raise TypeError('The type should be list or a dict. %s' % results)
+if sys.version_info.major < 3:
+ IsMemberMethod = inspect.ismethod
+else:
+ IsMemberMethod = inspect.isfunction
+
+
def LogFunc(func, prefix=''):
"""The decorator for logging the function call."""
def Wrapper(*args, **kwargs):
@@ -192,7 +199,7 @@
real_args = args[1:]
else:
real_args = args[:]
- arg_str = ', '.join(map(str, real_args) +
+ arg_str = ', '.join([str(val) for val in real_args] +
['%s=%r' % (key, val) for key, val in kwargs.items()])
logger.debug('Calling %s(%s)', prefix + func.__name__, arg_str)
return func(*args, **kwargs)
@@ -202,7 +209,7 @@
def LogAllMethods(cls):
"""The class decorator that Logs all the public methods."""
prefix = cls.__name__ + '.'
- for func_name, func in inspect.getmembers(cls, inspect.ismethod):
+ for func_name, func in inspect.getmembers(cls, IsMemberMethod):
if not func_name.startswith('_'):
setattr(cls, func_name, LogFunc(func, prefix))
return cls
@@ -238,7 +245,7 @@
return Wrapper
def IsolateAllMethods(self, cls):
- for func_name, func in inspect.getmembers(cls, inspect.ismethod):
+ for func_name, func in inspect.getmembers(cls, IsMemberMethod):
setattr(cls, func_name, self.IsolateFunc(func))
return cls
diff --git a/graphyte/utils/graphyte_utils_unittest.py b/graphyte/utils/graphyte_utils_unittest.py
index e67e831..2794f04 100755
--- a/graphyte/utils/graphyte_utils_unittest.py
+++ b/graphyte/utils/graphyte_utils_unittest.py
@@ -98,14 +98,14 @@
def testLogAverage10(self):
db_10 = lambda x: 10 * math.log10(x)
answer = db_10(self.answer)
- values = map(db_10, self.values)
+ values = [db_10(x) for x in self.values]
self.assertEquals(
answer, graphyte_utils.CalculateAverage(values, '10Log10'))
def testLogAverage20(self):
db_20 = lambda x: 20 * math.log10(x)
answer = db_20(self.answer)
- values = map(db_20, self.values)
+ values = [db_20(x) for x in self.values]
self.assertEquals(
answer, graphyte_utils.CalculateAverage(values, '20Log10'))
diff --git a/graphyte/utils/platform_utils.py b/graphyte/utils/platform_utils.py
index 3a3c65e..bff20b3 100644
--- a/graphyte/utils/platform_utils.py
+++ b/graphyte/utils/platform_utils.py
@@ -10,6 +10,7 @@
import platform
import time
+import six
# Cache to speed up.
_CURRENT_PLATFORM_SYSTEM = platform.system()
@@ -44,7 +45,7 @@
systems: A list of supported platform systems.
"""
global _PROVIDER_MAP
- assert not isinstance(systems, basestring), "systems must be list."
+ assert not isinstance(systems, six.string_types), "systems must be list."
if api_name not in _PROVIDER_MAP:
_PROVIDER_MAP[api_name] = {}
def ProviderDecorator(func):
@@ -71,8 +72,8 @@
system = _CURRENT_PLATFORM_SYSTEM
func = systems.get(system, systems.get(_SYSTEM_DEFAULT, None))
if func is None:
- raise NotImplementedError, 'No implementation on %s for <%s>' % (
- system, api_name)
+ raise NotImplementedError('No implementation on %s for <%s>' % (
+ system, api_name))
return func
diff --git a/graphyte/utils/process_utils.py b/graphyte/utils/process_utils.py
index 4fb1adc..6a8cce6 100644
--- a/graphyte/utils/process_utils.py
+++ b/graphyte/utils/process_utils.py
@@ -4,8 +4,6 @@
from __future__ import print_function
-import Queue
-from StringIO import StringIO
import contextlib
import copy
import getpass
@@ -20,6 +18,9 @@
import time
import traceback
+from six.moves import queue
+from six.moves import StringIO
+
PIPE = subprocess.PIPE
@@ -367,7 +368,7 @@
def add_children(pid):
pids.append(pid)
- map(add_children, children.get(pid, []))
+ list(map(add_children, children.get(pid, [])))
add_children(root)
# Reverse the list to first kill children then parents.
# Note reversed(pids) will return an iterator instead of real list, so
@@ -386,7 +387,7 @@
os.kill(pid, sig)
except OSError:
pass
- pids = filter(IsProcessAlive, pids)
+ pids = list(filter(IsProcessAlive, pids))
if not pids:
return
time.sleep(0.2) # Sleep 200 ms and try again
@@ -429,7 +430,7 @@
raise ValueError('output_file must be specified')
stdout = kwargs.get('stdout', sys.stdout)
- message_queue = Queue.Queue()
+ message_queue = queue.Queue()
def Tee(out_fd):
"""Writes available data from message queue to stdout and output file."""
@@ -440,7 +441,7 @@
stdout.flush()
out_fd.write(line)
out_fd.flush()
- except Queue.Empty:
+ except queue.Empty:
return
def EnqueueOutput(in_fd):
diff --git a/graphyte/utils/sync_utils.py b/graphyte/utils/sync_utils.py
index 95b75ef..8a51f4e 100644
--- a/graphyte/utils/sync_utils.py
+++ b/graphyte/utils/sync_utils.py
@@ -117,7 +117,7 @@
if condition is None:
condition = lambda x: x
result = None
- for retry_time in xrange(max_retry_times):
+ for retry_time in range(max_retry_times):
try:
result = target(*args, **kwargs)
except Exception:
diff --git a/graphyte/utils/time_utils.py b/graphyte/utils/time_utils.py
index 119f099..7be9de0 100644
--- a/graphyte/utils/time_utils.py
+++ b/graphyte/utils/time_utils.py
@@ -6,14 +6,29 @@
import datetime
import time
-from uuid import uuid4
+import uuid
import graphyte_common # pylint: disable=unused-import
from graphyte.utils import platform_utils
-EPOCH_ZERO = datetime.datetime(1970, 1, 1)
+
MonotonicTime = platform_utils.GetProvider('MonotonicTime')
+
+# pylint: disable=unused-argument
+class TZUTC(datetime.tzinfo):
+ """A tzinfo about UTC."""
+
+ def utcoffset(self, dt):
+ return datetime.timedelta(0)
+
+ def dst(self, dt):
+ return datetime.timedelta(0)
+
+ def tzname(self, dt):
+ return 'UTC'
+
+
def FormatElapsedTime(elapsed_secs):
"""Formats an elapsed time.
@@ -51,8 +66,8 @@
milliseconds: Whether to include milliseconds.
"""
- if type(time_value) is datetime.datetime:
- t = (time_value - EPOCH_ZERO).total_seconds()
+ if isinstance(time_value, datetime.datetime):
+ t = DatetimeToUnixtime(time_value)
else:
t = time_value or time.time()
ret = time.strftime(
@@ -74,4 +89,21 @@
enough randomness to remain unique.
"""
return ('%08x' % (int(time.time() * 100) & 0xFFFFFFFF) +
- str(uuid4())[8:])
+ str(uuid.uuid4())[8:])
+
+
+EPOCH_ZERO = datetime.datetime(1970, 1, 1)
+EPOCH_ZERO_WITH_TZINFO = datetime.datetime(1970, 1, 1, tzinfo=TZUTC())
+
+
+def DatetimeToUnixtime(obj):
+ """Converts datetime.datetime to Unix time.
+
+ The function will use the time zone info if obj has; otherwise, it will treat
+ obj as in Coordinated Universal Time (UTC).
+ """
+ if not isinstance(obj, datetime.datetime):
+ raise ValueError('Expected datetime.datetime but found %s' % type(obj))
+ if obj.tzinfo is not None:
+ return (obj - EPOCH_ZERO_WITH_TZINFO).total_seconds()
+ return (obj - EPOCH_ZERO).total_seconds()
diff --git a/graphyte/utils/type_utils.py b/graphyte/utils/type_utils.py
index b1ffe02..ed6276a 100644
--- a/graphyte/utils/type_utils.py
+++ b/graphyte/utils/type_utils.py
@@ -5,11 +5,13 @@
"""Utilities for data types."""
import collections
+import functools
import inspect
-import itertools
-import Queue
import re
+import six
+from six import iteritems
+from six.moves import queue
# The regular expression used by Overrides.
_OVERRIDES_CLASS_RE = re.compile(r'^\s*class[^#]+\(\s*([^\s#]+)\s*\)\s*\:')
@@ -17,12 +19,19 @@
class Error(Exception):
"""Generic fatal error."""
- pass
+
+
+class TestFailure(Error):
+ """Failure of a test."""
+
+
+class TestListError(Error):
+ """TestList exception"""
class TimeoutError(Error):
"""Timeout error."""
- def __init__(self, message, output=None):
+ def __init__(self, message='Timed out', output=None):
Error.__init__(self)
self.message = message
self.output = output
@@ -40,25 +49,32 @@
def __repr__(self):
return repr(self.__dict__)
+ def __eq__(self, rhs):
+ return isinstance(rhs, Obj) and self.__dict__ == rhs.__dict__
-# TODO(akahuang): Update from latest factory repo.
-def Enum(*args, **kwargs):
- """Creates the immutable enumeration set.
+ def __ne__(self, rhs):
+ return not self.__eq__(rhs)
+
+
+class Enum(frozenset):
+ """An enumeration type.
Usage:
- 1. C-style enum. The value starts from 0 and increase orderly.
- A = Enum('foo', 'bar')
- then A.foo == 0, A.bar == 1
- 2. Key-value pair enum.
- B = Enum(foo='FOO', bar='BAR')
- then B.foo == 'FOO', B.bar == 'BAR'
+ To create a enum object:
+ dummy_enum = type_utils.Enum(['A', 'B', 'C'])
+
+ To access a enum object, use:
+ dummy_enum.A
+ dummy_enum.B
"""
- fields = dict(zip(args, itertools.count()), **kwargs)
- enum_type = collections.namedtuple('Enum', fields.keys())
- return enum_type(**fields)
+
+ def __getattr__(self, name):
+ if name in self:
+ return name
+ raise AttributeError
-def DrainQueue(queue):
+def DrainQueue(q):
"""Returns as many elements as can be obtained from a queue without blocking.
(This may be no elements at all.)
@@ -66,8 +82,8 @@
ret = []
while True:
try:
- ret.append(queue.get_nowait())
- except Queue.Empty:
+ ret.append(q.get_nowait())
+ except queue.Empty:
break
return ret
@@ -83,6 +99,17 @@
[])
+def FlattenTuple(tupl):
+ """Flattens a tuple, recursively including all items in contained tuples.
+
+ For example:
+
+ FlattenList((1,2,(3,4,()),5,6)) == (1,2,3,4,5,6)
+ """
+ return sum((FlattenTuple(x) if isinstance(x, tuple) else (x, ) for x in tupl),
+ ())
+
+
def MakeList(value):
"""Converts the given value to a list.
@@ -91,11 +118,33 @@
otherwise, a list contains only one element.
"""
if (isinstance(value, collections.Iterable) and
- not isinstance(value, basestring)):
+ not isinstance(value, six.string_types)):
return list(value)
return [value]
+def MakeTuple(value):
+ """Converts the given value to a tuple recursively.
+
+ This is helpful for using an iterable argument as dict keys especially
+ that arguments from JSON will always be list instead of tuple.
+
+ Returns:
+ A tuple of elements from "value" if it is iterable (except string)
+ recursively; otherwise, a tuple with only one element.
+ """
+ def ShouldExpand(v):
+ return (isinstance(v, collections.Iterable) and
+ not isinstance(v, six.string_types))
+
+ def Expand(v):
+ return tuple(Expand(e) if ShouldExpand(e) else e for e in v)
+
+ if ShouldExpand(value):
+ return Expand(value)
+ return (value,)
+
+
def MakeSet(value):
"""Converts the given value to a set.
@@ -104,7 +153,7 @@
otherwise, a set contains only one element.
"""
if (isinstance(value, collections.Iterable) and
- not isinstance(value, basestring)):
+ not isinstance(value, six.string_types)):
return set(value)
return set([value])
@@ -124,6 +173,26 @@
raise ValueError('Found extra keys: %s' % list(extra_keys))
+def GetDict(data, key_path, default_value=None):
+ """A simplified getter function to retrieve values inside dictionary.
+
+ This function is very similar to `dict.get`, except it accepts a key path
+ (can be a list or string delimited by dot, for example ['a', 'b'] or 'a.b')
+
+ Args:
+ data: A dictionary that may contain sub-dictionaries.
+ key_path: A list of keys, or one simple string delimited by dot.
+ default_value: The value to return if key_path does not exist.
+ """
+ if isinstance(key_path, six.string_types):
+ key_path = key_path.split('.')
+ for key in key_path:
+ if key not in data:
+ return default_value
+ data = data[key]
+ return data
+
+
class AttrDict(dict):
"""Attribute dictionary.
@@ -142,34 +211,18 @@
assertEqual(bar.z[0].m, 'value_z_0_m')
"""
- def _IsBuiltinDict(self, item):
- return (isinstance(item, dict) and
- item.__class__.__module__ == '__builtin__' and
- item.__class__.__name__ == 'dict')
-
- def _IsBuiltinList(self, item):
- return (isinstance(item, list) and
- item.__class__.__module__ == '__builtin__' and
- item.__class__.__name__ == 'list')
-
- def _ConvertList(self, itemlist):
- converted = []
- for item in itemlist:
- if self._IsBuiltinDict(item):
- converted.append(AttrDict(item))
- elif self._IsBuiltinList(item):
- converted.append(self._ConvertList(item))
- else:
- converted.append(item)
- return converted
+ @classmethod
+ def _Convert(cls, obj):
+ if isinstance(obj, list):
+ return [cls._Convert(val) for val in obj]
+ if isinstance(obj, dict):
+ return cls(obj)
+ return obj
def __init__(self, *args, **kwargs):
super(AttrDict, self).__init__(*args, **kwargs)
- for key, value in self.iteritems():
- if self._IsBuiltinDict(value):
- self[key] = AttrDict(value)
- elif self._IsBuiltinList(value):
- self[key] = self._ConvertList(value)
+ for key, val in iteritems(self):
+ self[key] = self._Convert(val)
self.__dict__ = self
@@ -256,6 +309,63 @@
return method
+class CachedGetter(object):
+ """A decorator for a cacheable getter function.
+
+ This is helpful for caching results for getter functions. For example::
+
+ @CacheGetter
+ def ReadDeviceID():
+ with open('/var/device_id') as f:
+ return f.read()
+
+ The real file I/O will occur only on first invocation of ``ReadDeviceID()``,
+ until ``ReadDeviceID.InvalidateCache()`` is called.
+
+ In current implementation, the getter may accept arguments, but the arguments
+ are ignored if there is already cache available. In other words::
+
+ @CacheGetter
+ def m(v):
+ return v + 1
+
+ m(0) # First call: returns 1
+ m(1) # Second call: return previous cached answer, 1.
+ """
+
+ def __init__(self, getter):
+ functools.update_wrapper(self, getter)
+ self._getter = getter
+ self._has_cached = False
+ self._cached_value = None
+
+ def InvalidateCache(self):
+ self._has_cached = False
+ self._cached_value = None
+
+ def Override(self, value):
+ self._has_cached = True
+ self._cached_value = value
+
+ def HasCached(self):
+ return self._has_cached
+
+ def __call__(self, *args, **kargs):
+ # TODO(hungte) Cache args/kargs as well, to return different values when the
+ # arguments are different.
+ if not self.HasCached():
+ self.Override(self._getter(*args, **kargs))
+ return self._cached_value
+
+
+def OverrideCacheableGetter(getter, value):
+ """Overrides a function decorated by CacheableGetter with some value."""
+ assert hasattr(getter, 'has_cached'), 'Need a CacheableGetter target.'
+ assert hasattr(getter, 'cached_value'), 'Need a CacheableGetter target.'
+ getter.has_cached = True
+ getter.cached_value = value
+
+
class LazyProperty(object):
"""A decorator for lazy loading properties.
@@ -275,6 +385,7 @@
def __init__(self, prop):
self._init_func = prop
self._prop_name = self.PROP_NAME_PREFIX + prop.__name__
+ functools.update_wrapper(self, prop)
def __get__(self, obj, ignored_obj_type):
if obj is None:
@@ -299,6 +410,23 @@
setattr(obj, cls.PROP_NAME_PREFIX + prop_name, value)
+class LazyObject(object):
+ """A proxy object for creating an object on demand.."""
+
+ def __init__(self, constructor, *args, **kargs):
+ self._proxy_constructor = lambda: constructor(*args, **kargs)
+ self._proxy_object = None
+
+ def __getattr__(self, name):
+ if self._proxy_constructor is not None:
+ self._proxy_object = self._proxy_constructor()
+ self._proxy_constructor = None
+ attr = getattr(self._proxy_object, name)
+ # We can't do 'setattr' here to speed up processing because the members in
+ # the proxy object may be volatile.
+ return attr
+
+
class UniqueStack(object):
""" A data structure very similar to a stack, but objects inside are unique.
@@ -344,7 +472,7 @@
This function should run in amortized O(1)
"""
with self._lock:
- while len(self._list) > 0:
+ while self._list:
if self._list[-1] in self._set:
return self._list[-1]
else:
@@ -353,15 +481,15 @@
def UnicodeToString(obj):
- '''Converts any Unicode strings in obj to UTF-8 strings.
+ """Converts any Unicode strings in obj to UTF-8 strings.
Recurses into lists, dicts, and tuples in obj.
- '''
+ """
if isinstance(obj, list):
return [UnicodeToString(x) for x in obj]
elif isinstance(obj, dict):
return dict((UnicodeToString(k), UnicodeToString(v))
- for k, v in obj.iteritems())
+ for k, v in iteritems(obj))
elif isinstance(obj, unicode):
return obj.encode('utf-8')
elif isinstance(obj, tuple):
@@ -373,17 +501,18 @@
def UnicodeToStringArgs(function):
- '''A function decorator that converts function's arguments from
+ """A function decorator that converts function's arguments from
Unicode to strings using UnicodeToString.
- '''
- return (lambda *args, **kwargs:
- function(*UnicodeToString(args),
- **UnicodeToString(kwargs)))
+ """
+ @functools.wraps(function)
+ def _Wrapper(*args, **kwargs):
+ return function(*UnicodeToString(args), **UnicodeToString(kwargs))
+ return _Wrapper
def UnicodeToStringClass(cls):
- '''A class decorator that converts all arguments of all
- methods in class from Unicode to strings using UnicodeToStringArgs.'''
+ """A class decorator that converts all arguments of all
+ methods in class from Unicode to strings using UnicodeToStringArgs."""
for k, v in cls.__dict__.items():
if callable(v):
setattr(cls, k, UnicodeToStringArgs(v))
@@ -410,3 +539,14 @@
if k[0] != '_' and k not in excluded_keys and (
not true_only or getattr(obj, k))])
+ ')')
+
+
+def BindFunction(func, *args, **kwargs):
+ """Bind arguments to a function.
+
+ The returned function have same __name__ and __doc__ with func.
+ """
+ @functools.wraps(func)
+ def _Wrapper():
+ return func(*args, **kwargs)
+ return _Wrapper
diff --git a/graphyte_common.py b/graphyte_common.py
new file mode 120000
index 0000000..e0fa44f
--- /dev/null
+++ b/graphyte_common.py
@@ -0,0 +1 @@
+graphyte/graphyte_common.py
\ No newline at end of file
diff --git a/unittest_runner.py b/unittest_runner.py
index ba98c83..48eef53 100755
--- a/unittest_runner.py
+++ b/unittest_runner.py
@@ -4,6 +4,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+from __future__ import print_function
+
import logging
import os
import sys
@@ -15,11 +17,11 @@
def PassMessage(message):
- print '\033[22;32m%s\033[22;0m' % message
+ print('\033[22;32m%s\033[22;0m' % message)
def FailMessage(message):
- print '\033[22;31m%s\033[22;0m' % message
+ print('\033[22;31m%s\033[22;0m' % message)
def LogFilePath(dir_name, filename):
@@ -45,7 +47,8 @@
suite = unittest.defaultTestLoader.loadTestsFromName(module_name)
start_time = time.time()
log_file = LogFilePath(temp_dir, filename)
- result = unittest.TextTestRunner(stream=open(log_file, 'w')).run(suite)
+ with open(log_file, 'w') as stream:
+ result = unittest.TextTestRunner(stream=stream).run(suite)
duration = time.time() - start_time
if result.wasSuccessful():
PassMessage('*** PASS [%.2f s] %s' % (duration, filename))
@@ -55,14 +58,14 @@
failed_tests.append(filename)
total_tests += 1
- print '%d/%d tests passed' % (passed_tests, total_tests)
+ print('%d/%d tests passed' % (passed_tests, total_tests))
if failed_tests:
- print ''
- print 'Logs of %d failed tests:' % len(failed_tests)
+ print('')
+ print('Logs of %d failed tests:' % len(failed_tests))
for failed_test in failed_tests:
FailMessage(LogFilePath(temp_dir, failed_test))
- print 'To re-test failed unittests, run:'
- print 'make test UNITTEST_WHITELIST="%s"' % ' '.join(failed_tests)
+ print('To re-test failed unittests, run:')
+ print('make test UNITTEST_WHITELIST="%s"' % ' '.join(failed_tests))
if __name__ == '__main__':
