|  | # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 
|  | # Use of this source code is governed by a BSD-style license that can be | 
|  | # found in the LICENSE file. | 
|  |  | 
|  | """Defines the PersistentMixIn utility class to easily convert classes to and | 
|  | from dict for serialization. | 
|  |  | 
|  | This class is aimed at json-compatible serialization, so it supports the limited | 
|  | set of structures supported by json; strings, numbers as int or float, list and | 
|  | dictionaries. | 
|  |  | 
|  | PersistentMixIn._persistent_members() returns a dict of each member with the | 
|  | tuple of expected types. Each member can be decoded in multiple types, for | 
|  | example, a subversion revision number could have (None, int, str), meaning that | 
|  | the revision could be None, when not known, an int or the int as a string | 
|  | representation. The tuple is listed in the prefered order of conversions. | 
|  |  | 
|  | Composites types that cannot be represented exactly in json like tuple, set and | 
|  | frozenset are converted from and back to list automatically. Any class instance | 
|  | that has been serialized can be unserialized in the same class instance or into | 
|  | a bare dict. | 
|  |  | 
|  | See tests/model_tests.py for examples. | 
|  | """ | 
|  |  | 
|  | import json | 
|  | import logging | 
|  | import os | 
|  |  | 
|  | # Set in the output dict to be able to know which class was serialized to help | 
|  | # deserialization. | 
|  | TYPE_FLAG = '__persistent_type__' | 
|  |  | 
|  | # Marker to tell the deserializer that we don't know the expected type, used in | 
|  | # composite types. | 
|  | _UNKNOWN = object() | 
|  |  | 
|  |  | 
|  | def as_dict(value): | 
|  | """Recursively converts an object into a dictionary. | 
|  |  | 
|  | Converts tuple,set,frozenset into list and recursively process each items. | 
|  | """ | 
|  | if hasattr(value, 'as_dict') and callable(value.as_dict): | 
|  | return value.as_dict() | 
|  | elif isinstance(value, (list, tuple, set, frozenset)): | 
|  | return [as_dict(v) for v in value] | 
|  | elif isinstance(value, dict): | 
|  | return dict((as_dict(k), as_dict(v)) | 
|  | for k, v in value.iteritems()) | 
|  | elif isinstance(value, (bool, float, int, basestring)) or value is None: | 
|  | return value | 
|  | else: | 
|  | raise AttributeError('Can\'t type %s into a dictionary' % type(value)) | 
|  |  | 
|  |  | 
|  | def _inner_from_dict(name, value, member_types): | 
|  | """Recursively regenerates an object. | 
|  |  | 
|  | For each of the allowable types, try to convert it. If None is an allowable | 
|  | type, any data that can't be parsed will be parsed as None and will be | 
|  | silently discarded. Otherwise, an exception will be raise. | 
|  | """ | 
|  | logging.debug('_inner_from_dict(%s, %r, %s)', name, value, member_types) | 
|  | result = None | 
|  | if member_types is _UNKNOWN: | 
|  | # Use guesswork a bit more and accept anything. | 
|  | if isinstance(value, dict): | 
|  | if TYPE_FLAG in value: | 
|  | result = PersistentMixIn.from_dict(value, _UNKNOWN) | 
|  | else: | 
|  | # Unserialize it as a raw dict. | 
|  | result =  dict( | 
|  | (_inner_from_dict(None, k, _UNKNOWN), | 
|  | _inner_from_dict(None, v, _UNKNOWN)) | 
|  | for k, v in value.iteritems()) | 
|  | elif isinstance(value, list): | 
|  | # All of these are serialized to list. | 
|  | result = [_inner_from_dict(None, v, _UNKNOWN) for v in value] | 
|  | elif isinstance(value, (bool, float, int, unicode)): | 
|  | result = value | 
|  | else: | 
|  | raise TypeError('No idea how to convert %r' % value) | 
|  | else: | 
|  | for member_type in member_types: | 
|  | # Explicitly leave None out of this loop. | 
|  | if issubclass(member_type, PersistentMixIn): | 
|  | if isinstance(value, dict) and TYPE_FLAG in value: | 
|  | result = PersistentMixIn.from_dict(value, member_type) | 
|  | break | 
|  | elif member_type is dict: | 
|  | if isinstance(value, dict): | 
|  | result =  dict( | 
|  | (_inner_from_dict(None, k, _UNKNOWN), | 
|  | _inner_from_dict(None, v, _UNKNOWN)) | 
|  | for k, v in value.iteritems()) | 
|  | break | 
|  | elif member_type in (list, tuple, set, frozenset): | 
|  | # All of these are serialized to list. | 
|  | if isinstance(value, list): | 
|  | result = member_type( | 
|  | _inner_from_dict(None, v, _UNKNOWN) for v in value) | 
|  | break | 
|  | elif member_type in (bool, float, int, str, unicode): | 
|  | if isinstance(value, member_type): | 
|  | result = member_type(value) | 
|  | break | 
|  | elif member_type is None.__class__ and value is None: | 
|  | result = None | 
|  | break | 
|  | else: | 
|  | logging.info( | 
|  | 'Ignored %s: didn\'t fit types %s; %s', | 
|  | name, | 
|  | ', '.join(i.__name__ for i in member_types), | 
|  | repr(value)[:200]) | 
|  | _check_type_value(name, result, member_types) | 
|  | return result | 
|  |  | 
|  |  | 
|  | def to_yaml(obj): | 
|  | """Converts a PersistentMixIn into a yaml-inspired format. | 
|  |  | 
|  | Warning: Not unit tested, use at your own risk! | 
|  | """ | 
|  | def align(x): | 
|  | y = x.splitlines(True) | 
|  | if len(y) > 1: | 
|  | return ''.join(y[0:1] + ['  ' + z for z in y[1:]]) | 
|  | return x | 
|  | def align_value(x): | 
|  | if '\n' in x: | 
|  | return '\n  ' + align(x) | 
|  | return x | 
|  |  | 
|  | if hasattr(obj, 'as_dict') and callable(obj.as_dict): | 
|  | out = (to_yaml(obj.as_dict()),) | 
|  | elif isinstance(obj, (bool, float, int, unicode)) or obj is None: | 
|  | out = (align(str(obj)),) | 
|  | elif isinstance(obj, dict): | 
|  | if TYPE_FLAG in obj: | 
|  | out = ['%s:' % obj[TYPE_FLAG]] | 
|  | else: | 
|  | out = [] | 
|  | for k, v in obj.iteritems(): | 
|  | # Skips many members resolving to bool() == False | 
|  | if k.startswith('__') or v in (None, '', False, 0): | 
|  | continue | 
|  | r = align_value(to_yaml(v)) | 
|  | if not r: | 
|  | continue | 
|  | out.append('- %s: %s' % (k, r)) | 
|  | elif hasattr(obj, '__iter__') and callable(obj.__iter__): | 
|  | out = ['- %s' % align(to_yaml(x)) for x in obj] | 
|  | else: | 
|  | out = ('%s' % obj.__class__.__name__,) | 
|  | return '\n'.join(out) | 
|  |  | 
|  |  | 
|  | def _default_value(member_types): | 
|  | """Returns an instance of the first allowed type. Special case None.""" | 
|  | if member_types[0] is None.__class__: | 
|  | return None | 
|  | else: | 
|  | return member_types[0]() | 
|  |  | 
|  |  | 
|  | def _check_type_value(name, value, member_types): | 
|  | """Raises a TypeError exception if value is not one of the allowed types in | 
|  | member_types. | 
|  | """ | 
|  | if not isinstance(value, member_types): | 
|  | prefix = '%s e' % name if name else 'E' | 
|  | raise TypeError( | 
|  | '%sxpected type(s) %s; got %r' % | 
|  | (prefix, ', '.join(i.__name__ for i in member_types), value)) | 
|  |  | 
|  |  | 
|  |  | 
|  | class PersistentMixIn(object): | 
|  | """Class to be used as a base class to persistent data in a simplistic way. | 
|  |  | 
|  | Persistent class member needs to be set to a tuple containing the instance | 
|  | member variable that needs to be saved or loaded. The first item will be | 
|  | default value, e.g.: | 
|  | foo = (None, str, dict) | 
|  | Will default initialize self.foo to None. | 
|  | """ | 
|  | # Cache of all the subclasses of PersistentMixIn. | 
|  | __persistent_classes_cache = None | 
|  |  | 
|  | _read_only = False | 
|  |  | 
|  | def __init__(self, **kwargs): | 
|  | """Initializes with the default members.""" | 
|  | super(PersistentMixIn, self).__init__() | 
|  | persistent_members = self._persistent_members() | 
|  | for member, member_types in persistent_members.iteritems(): | 
|  | if member in kwargs: | 
|  | value = kwargs.pop(member) | 
|  | if isinstance(value, str): | 
|  | # Assume UTF-8 all the time. Note: This is explicitly when the object | 
|  | # is constructed in the code. This code path is never used when | 
|  | # deserializing the object. | 
|  | value = value.decode('utf-8') | 
|  | else: | 
|  | value = _default_value(member_types) | 
|  | _check_type_value(member, value, member_types) | 
|  | setattr(self, member, value) | 
|  | if kwargs: | 
|  | raise AttributeError('Received unexpected initializers: %s' % kwargs) | 
|  |  | 
|  | def __setattr__(self, name, value): | 
|  | """Enforces immutability if cls._read_only is True.""" | 
|  | if self._read_only: | 
|  | raise TypeError() | 
|  | return super(PersistentMixIn, self).__setattr__(name, value) | 
|  |  | 
|  | @classmethod | 
|  | def _persistent_members(cls): | 
|  | """Returns the persistent items as a dict. | 
|  |  | 
|  | Each entry value can be a tuple when the member can be assigned different | 
|  | types. | 
|  | """ | 
|  | # Note that here, cls is the subclass, not PersistentMixIn. | 
|  | # TODO(maruel): Cache the results. It's tricky because setting | 
|  | # cls.__persistent_members_cache on a class will implicitly set it on its | 
|  | # subclass. So in a class hierarchy with A -> B -> PersistentMixIn, calling | 
|  | # B()._persistent_members() will incorrectly set the cache for A. | 
|  | persistent_members_cache = {} | 
|  | # Enumerate on the subclass, not on an instance. | 
|  | for item in dir(cls): | 
|  | if item.startswith('_'): | 
|  | continue | 
|  | item_value = getattr(cls, item) | 
|  | if isinstance(item_value, type): | 
|  | item_value = (item_value,) | 
|  | if not isinstance(item_value, tuple): | 
|  | continue | 
|  | if not all(i is None or i.__class__ == type for i in item_value): | 
|  | continue | 
|  | if any(i is str for i in item_value): | 
|  | raise TypeError( | 
|  | '%s is type \'str\' which is currently not supported' % item) | 
|  | item_value = tuple( | 
|  | f if f is not None else None.__class__ for f in item_value) | 
|  | persistent_members_cache[item] = item_value | 
|  | return persistent_members_cache | 
|  |  | 
|  | @staticmethod | 
|  | def _get_subclass(typename): | 
|  | """Returns the PersistentMixIn subclass with the name |typename|.""" | 
|  | subclass = None | 
|  | if PersistentMixIn.__persistent_classes_cache is not None: | 
|  | subclass = PersistentMixIn.__persistent_classes_cache.get(typename) | 
|  | if not subclass: | 
|  | # Get the subclasses recursively. | 
|  | PersistentMixIn.__persistent_classes_cache = {} | 
|  | def recurse(c): | 
|  | for s in c.__subclasses__(): | 
|  | assert s.__name__ not in PersistentMixIn.__persistent_classes_cache | 
|  | PersistentMixIn.__persistent_classes_cache[s.__name__] = s | 
|  | recurse(s) | 
|  | recurse(PersistentMixIn) | 
|  |  | 
|  | subclass = PersistentMixIn.__persistent_classes_cache.get(typename) | 
|  | if not subclass: | 
|  | raise KeyError('Couldn\'t find type %s' % typename) | 
|  | return subclass | 
|  |  | 
|  | def as_dict(self): | 
|  | """Create a dictionary out of this object, i.e. Serialize the object.""" | 
|  | out = {} | 
|  | for member, member_types in self._persistent_members().iteritems(): | 
|  | value = getattr(self, member) | 
|  | _check_type_value(member, value, member_types) | 
|  | out[member] = as_dict(value) | 
|  | out[TYPE_FLAG] = self.__class__.__name__ | 
|  | return out | 
|  |  | 
|  | @staticmethod | 
|  | def from_dict(data, subclass=_UNKNOWN): | 
|  | """Returns an instance of a class inheriting from PersistentMixIn, | 
|  | initialized with 'data' dict, i.e. Deserialize the object. | 
|  | """ | 
|  | logging.debug('from_dict(%r, %s)', data, subclass) | 
|  | if subclass is _UNKNOWN: | 
|  | subclass = PersistentMixIn._get_subclass(data[TYPE_FLAG]) | 
|  | # This initializes the instance with the default values. | 
|  |  | 
|  | # pylint: disable=W0212 | 
|  | kwargs = {} | 
|  | for member, member_types in subclass._persistent_members().iteritems(): | 
|  | if member in data: | 
|  | try: | 
|  | value = _inner_from_dict(member, data[member], member_types) | 
|  | except TypeError: | 
|  | # pylint: disable=E1103 | 
|  | logging.error( | 
|  | 'Failed to instantiate %s because of member %s', | 
|  | subclass.__name__, member) | 
|  | raise | 
|  | else: | 
|  | value = _default_value(member_types) | 
|  | _check_type_value(member, value, member_types) | 
|  | kwargs[member] = value | 
|  | try: | 
|  | obj = subclass(**kwargs) | 
|  | except TypeError: | 
|  | # pylint: disable=E1103 | 
|  | logging.error('Failed to instantiate %s: %r', subclass.__name__, kwargs) | 
|  | raise | 
|  | assert isinstance(obj, PersistentMixIn) and obj.__class__ != PersistentMixIn | 
|  | return obj | 
|  |  | 
|  | def __str__(self): | 
|  | return to_yaml(self) | 
|  |  | 
|  | def __eq__(self, _): | 
|  | raise TypeError() | 
|  |  | 
|  | # pylint: disable=R0201 | 
|  | def __ne__(self, _): | 
|  | raise TypeError() | 
|  |  | 
|  |  | 
|  | def is_equivalent(lhs, rhs): | 
|  | """Implements the equivalent of __eq__. | 
|  |  | 
|  | The reason for not implementing __eq__ is to not encourage bad behavior by | 
|  | implicitly and recursively using __eq__() in a list().remove() call. | 
|  | """ | 
|  | # pylint: disable=W0212 | 
|  | if lhs._persistent_members() != rhs._persistent_members(): | 
|  | return False | 
|  | for i in lhs._persistent_members(): | 
|  | if getattr(lhs, i) != getattr(rhs, i): | 
|  | return False | 
|  | return True | 
|  |  | 
|  |  | 
|  | def immutable(func): | 
|  | """Member function decorators that convert 'self' to an immutable object. | 
|  |  | 
|  | Member functions of the object can't be called unless they are immutable too. | 
|  | Properties can be looked up, this function assumes properties do not mutate | 
|  | the object. | 
|  |  | 
|  | Note: a user can still call classmethod and do mutation on the class, or they | 
|  | can lookup a member object and mutate this one. Don't be silly. | 
|  | """ | 
|  | class Immutable(object): | 
|  | def __init__(self, obj): | 
|  | object.__setattr__(self, '__ref', obj) | 
|  |  | 
|  | def __getattribute__(self, name): | 
|  | ref = object.__getattribute__(self, '__ref') | 
|  | value = getattr(ref, name) | 
|  | if not callable(value): | 
|  | return value | 
|  | if getattr(value, 'is_immutable', None): | 
|  | # It is immutable too. | 
|  | return value | 
|  | if getattr(value, 'im_self', None) == None: | 
|  | # It is static. | 
|  | return value | 
|  | raise TypeError( | 
|  | 'Can\'t call mutable member function \'%s\' on an immutable ' | 
|  | 'instance of %s' % (name, ref.__class__.__name__)) | 
|  |  | 
|  | def __setattr__(self, name, _value): | 
|  | ref = object.__getattribute__(self, '__ref') | 
|  | raise TypeError( | 
|  | 'Can\'t change attribute \'%s\' on an immutable instance of \'%s\'' % | 
|  | (name, ref.__class__.__name__)) | 
|  |  | 
|  | def __delattr__(self, name): | 
|  | ref = object.__getattribute__(self, '__ref') | 
|  | raise TypeError( | 
|  | 'Can\'t delete attribute \'%s\' on an immutable instance of \'%s\'' % | 
|  | (name, ref.__class__.__name__)) | 
|  |  | 
|  | def hook(self, *args, **kwargs): | 
|  | return func(Immutable(self), *args, **kwargs) | 
|  |  | 
|  | hook.is_immutable = True | 
|  | return hook | 
|  |  | 
|  |  | 
|  | def load_from_json_file(filename): | 
|  | """Loads one object from a JSON file.""" | 
|  | with open(filename, 'r') as f: | 
|  | return PersistentMixIn.from_dict(json.load(f)) | 
|  |  | 
|  |  | 
|  | def save_to_json_file(filename, obj): | 
|  | """Save one object in a JSON file.""" | 
|  | try: | 
|  | old = filename + '.old' | 
|  | if os.path.exists(filename): | 
|  | os.rename(filename, old) | 
|  | finally: | 
|  | with open(filename, 'wb') as f: | 
|  | json.dump(obj.as_dict(), f, sort_keys=True, indent=2) | 
|  | f.write('\n') |