blob: 467f7e53dedf7dde4040790f748b00c59e035718 [file] [log] [blame]
# Copyright 2013 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.
import contextlib
import datetime
import functools
import logging
import os
import sys
import time
import traceback
import urllib
from cStringIO import StringIO
def sentinel(name):
"""Create a sentinel object.
The sentinel's type is a class with the given name that has no behavior except
that it's string representation is also the given name. The sentinel is
intended for use where some special behavior is required where there is no
acceptable special value in the type of an argument. An identity check (x is
SENTINEL) can be used to check for the sentinel.
return type(name, (), {
'__repr__': lambda _: name,
'__copy__': lambda self: self,
'__deepcopy__': lambda self, _: self,
class RecipeAbort(Exception):
class ModuleInjectionError(AttributeError):
class ModuleInjectionSite(object):
def __init__(self, owner_module=None):
self.owner_module = owner_module
def __getattr__(self, key):
if self.owner_module is None:
raise ModuleInjectionError(
"RecipeApi has no dependency %r. (Add it to DEPS?)" % (key,))
raise ModuleInjectionError(
"Recipe Module %r has no dependency %r. (Add it to"
% (, key))
class Placeholder(object):
"""Base class for command line argument placeholders. Do not use directly."""
def __init__(self, name=None):
if name is not None:
assert isinstance(name, basestring), (
'Expect a string name for a placeholder, but got %r' % name) = name
self.namespaces = None
def backing_file(self): # pragma: no cover
"""Return path to a temp file that holds or receives the data.
Valid only after 'render' has been called.
raise NotImplementedError
def render(self, test): # pragma: no cover
"""Return [cmd items]*"""
raise NotImplementedError
def label(self):
if is None:
return "%s.%s" % self.namespaces
return "%s.%s[%s]" % (self.namespaces[0], self.namespaces[1],
class InputPlaceholder(Placeholder):
"""Base class for json/raw_io input placeholders. Do not use directly."""
def cleanup(self, test_enabled):
"""Called after step completion.
test_enabled (bool) - indicate whether running in simulation mode.
class OutputPlaceholder(Placeholder):
"""Base class for json/raw_io output placeholders. Do not use directly."""
def result(self, presentation, test):
"""Called after step completion.
presentation (StepPresentation) - for the current step.
test (PlaceholderTestData) - test data for this placeholder.
May optionally modify presentation as a side-effect.
Returned value will be added to the step result.
def static_wraps(func):
wrapped_fn = func
if isinstance(func, staticmethod):
# __get__(obj) is the way to get the function contained in the staticmethod.
# python 2.7+ has a __func__ member, but previous to this the attribute
# doesn't exist. It doesn't matter what the obj is, as long as it's not
# None.
wrapped_fn = func.__get__(object)
return functools.wraps(wrapped_fn)
def static_call(obj, func, *args, **kwargs):
if isinstance(func, staticmethod):
return func.__get__(obj)(*args, **kwargs)
return func(obj, *args, **kwargs)
def static_name(obj, func):
if isinstance(func, staticmethod):
return func.__get__(obj).__name__
return func.__name__
def returns_placeholder(func):
def inner(self, *args, **kwargs):
ret = static_call(self, func, *args, **kwargs)
assert isinstance(ret, Placeholder)
ret.namespaces = (, static_name(self, func))
return ret
# prevent this placeholder-returning function from becoming a composite_step.
inner._non_step = True # pylint: disable=protected-access
return inner
class StringListIO(object):
def __init__(self):
self.lines = [StringIO()]
def write(self, s):
while s:
i = s.find('\n')
if i == -1:
self.lines[-1] = self.lines[-1].getvalue()
s = s[i+1:]
def close(self):
if not isinstance(self.lines[-1], basestring):
self.lines[-1] = self.lines[-1].getvalue()
class exponential_retry(object):
"""Decorator which retries the function if an exception is encountered."""
def __init__(self, retries=None, delay=None, condition=None):
"""Creates a new exponential retry decorator.
retries (int): Maximum number of retries before giving up.
delay (datetime.timedelta): Amount of time to wait before retrying. This
will double every retry attempt (exponential).
condition (func): If not None, a function that will be passed the
exception as its one argument. Retries will only happen if this
function returns True. If None, retries will always happen.
self.retries = retries or 5
self.delay = delay or datetime.timedelta(seconds=1)
self.condition = condition or (lambda e: True)
def __call__(self, f):
def wrapper(*args, **kwargs):
retry_delay = self.delay
for i in xrange(self.retries):
return f(*args, **kwargs)
except Exception as e:
if (i+1) >= self.retries or not self.condition(e):
logging.exception('Exception encountered, retrying in %s',
retry_delay *= 2
return wrapper
class MultiException(Exception):
"""An exception that aggregates multiple exceptions and summarizes them."""
class Builder(object):
"""Iteratively constructs a MultiException."""
def __init__(self):
self._exceptions = []
def append(self, exc):
if exc is not None:
def get(self):
"""Returns (MultiException or None): The constructed MultiException.
If no exceptions have been appended, None will be returned.
return MultiException(*self._exceptions) if self._exceptions else (None)
def raise_if_any(self):
mexc = self.get()
if mexc is not None:
raise mexc
def catch(self, *exc_types):
"""ContextManager that catches any exception raised during its execution
and adds them to the MultiException.
exc_types (list): A list of exception classes to catch. If empty,
Exception will be used.
exc_types = exc_types or (Exception,)
except exc_types as exc:
def __init__(self, *base):
super(MultiException, self).__init__()
# Determine base Exception text.
if len(base) == 0:
self.message = 'No exceptions'
elif len(base) == 1:
self.message = str(base[0])
self.message = str(base[0]) + ', and %d more...' % (len(base)-1)
self._inner = base
def __nonzero__(self):
return bool(self._inner)
def __len__(self):
return len(self._inner)
def __getitem__(self, key):
return self._inner[key]
def __iter__(self):
return iter(self._inner)
def __str__(self):
return '%s(%s)' % (type(self).__name__, self.message)
def map_defer_exceptions(fn, it, *exc_types):
"""Executes "fn" for each element in "it". Any exceptions thrown by "fn" will
be deferred until the end of "it", then raised as a single MultiException.
fn (callable): A function to call for each element in "it".
it (iterable): An iterable to traverse.
exc_types (list): An optional list of specific exception types to defer.
If empty, Exception will be used. Any Exceptions not referenced by this
list will skip deferring and be immediately raised.
mexc_builder = MultiException.Builder()
for e in it:
with mexc_builder.catch(*exc_types):
def strip_unicode(obj):
"""Recursively re-encodes strings as utf-8 inside |obj|. Returns the result.
if isinstance(obj, unicode):
return obj.encode('utf-8', 'replace')
if isinstance(obj, list):
return map(strip_unicode, obj)
if isinstance(obj, dict):
new_obj = type(obj)(
(strip_unicode(k), strip_unicode(v)) for k, v in obj.iteritems() )
return new_obj
return obj