blob: 05b44ce86e33f9dcb03caeb889f43df0ab16bae6 [file] [log] [blame]
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
Module to allow a Twisted 'defer.inlineCallbacks'-style generator to
be used both asynchronously (through Twisted) and synchronously.
This module works by having code implement logic through framework-independent
'yield'-based coroutines similar to how Twisted uses 'inlineCallbacks'.
Hybrid methods are then invoked against a 'HybridBase' implementation, at which
point they pick up implementation-specific details and behaviors.
def readStrippedData(read_data_func):
data = yield read_data_func()
Functions decorated with 'hybrid.inlineCallbacks' can then be invoked in one of
the following ways:
1) Directly through a hybrid implementation's `call' method
2) Yielded in a hybrid coroutine.
3) Called directly as a member function of a class that uses a Hybrid MixIn.
Current implementations include:
- 'Twisted': Hybrid coroutines return and can yield Deferred objects in a
manner identical to 'defer.inlineCallbacks'.
- 'Synchronous': Coroutines are executed synchronously and return their
evaluated results.
For example, when executing the 'readStrippedData' hybrid coroutine through
the Twisted framework, one would define a Deferred-based data source and
execute the function through the Twisted `call' method:
# Example using `hybrid.Twisted' hybrid base.
def twistedReadFile(agent):
d =,
def processData(data):
print('Read data:', data)
return d
# END `hybrid.Twisted' example
The same function can be called synchronously by supplying a
synchronous-compatible `read_data_func' parameter and invoking through the
`' method:
# Example using `hybrid.Synchronous' hybrid base.
def synchronousReadFile(fd):
data =,
print('Read data:', data)
return data
# END `hybrid.Synchronous' example
Abstract base classes can define abstract hybrid logic with the expectation
that subclasses specialize them to a specific hybrid base. For example:
class BaseReader(object):
def _read(self, fd):
raise NotImplementedError()
def complexReadLogic(self, fd):
while True:
data = yield self._read(fd)
# (...)
if condition:
class TwistedReader(BaseReader, hybrid.Twisted.MixIn):
# Overrides 'BaseReader._read'
def _read(self, fd):
d = defer.Deferred()
twisted.internet.fdesc.readFromFD(fd, d.callback)
return d
class SynchronousReader(BaseReader, hybrid.Synchronous.MixIn):
# Overrides 'BaseReader._read'
def _read(self, fd):
This allows complex multipart logic to be defined in a central
backend-independent class and attached to an implementation independently.
Behind the scenes, the `hybrid' module runs `inlineCallbacks' methods through
a lightweight inner reactor that proxies between generalized hybrid functions
and the hybrid base's implementation.
For the Twisted implementation, this reactor falls through directly to
Twisted's implementation of 'inlineCallbacks' and 'returnValue'.
For the Synchronous implementation, the reactor passes through to an outer
reactor that executes the respective methods synchronously.
import types
import common.memo as memo
from functools import wraps
__all__ = (
HYBRID_CONTEXT_KWARG = '__hybrid_context'
class _HybridReturnValue(Exception):
def __init__(self, value):
super(_HybridReturnValue, self).__init__()
self.value = value
class HybridBase(object):
# Disable 'no __init__ method' warning | pylint: disable=W0232
class MixIn(object):
def _hybrid__getHybridContext():
raise NotImplementedError()
def getHybridContext(cls):
raise NotImplementedError()
def call(cls, inline_callback):
Returns: (func) a proxy function for 'inline_callback' that invokes it with
a hybrid base.
def wrapper(*args, **kwargs):
kwargs[HYBRID_CONTEXT_KWARG] = cls.getHybridContext()
return inline_callback(*args, **kwargs)
return wrapper
class Synchronous(HybridBase):
# Disable 'no __init__ method' warning | pylint: disable=W0232
"""Hybrid base that executes coroutines synchronously"""
class MixIn(HybridBase.MixIn):
# Disable 'no __init__ method' warning | pylint: disable=W0232
"""Synchronous MixIn uses the synchronous hybrid context"""
def _hybrid__getHybridContext():
return Synchronous.getHybridContext()
def getHybridContext(cls):
# Serially execute the 'inlineCallbacks' methods
def synchronousInlineCallbacks(func):
def wrap(*args, **kwargs):
result = None
gen = func(*args, **kwargs)
while True:
result = gen.send(result)
except _HybridReturnValue, e:
return e.value
except StopIteration:
return None
return wrap
def synchronousReturnValue(value):
raise _HybridReturnValue(value)
return (synchronousInlineCallbacks, synchronousReturnValue)
class Twisted(HybridBase):
"""Hybrid base that returns Twisted Deferreds."""
class MixIn(HybridBase.MixIn):
# Disable 'no __init__ method' warning | pylint: disable=W0232
def _hybrid__getHybridContext():
return Twisted.getHybridContext()
def getHybridContext(cls):
from twisted.internet import defer
return (defer.inlineCallbacks, defer.returnValue)
class InlineCallbackInvocation(object):
def __init__(self, func, *args, **kwargs):
self.func = func
self.args = args
self.kwargs = kwargs
# Get the argument-passed context, if one is defined
self.context = kwargs.pop(HYBRID_CONTEXT_KWARG, None)
def __repr__(self):
return '%s%s' % (type(self).__name__, self.func)
def _runWithContext(self, context, *args, **kwargs):
if context is None:
raise ValueError("Failed to get hybrid context for %r(*%r, **%r)" % (
self.func, args, kwargs))
result = None
g = self.func(*args, **kwargs)
if not isinstance(g, types.GeneratorType):
raise TypeError("Hybrid function %r is not a generator" % (self.func,))
while g:
result = g.send(result)
# Unwrap and forward context to nested 'inlineCallbacks'
if isinstance(result, type(self)):
result = result(context)
result = yield result
except _HybridReturnValue, e:
result = e.value
g = None
except StopIteration:
result = None
g = None
def __call__(self, context=None):
# Run our hybrid generator through the outer 'inlineCallbacks'
return context[0](self._runWithContext)(
(context or self.context),
# Standard public interface
def inlineCallbacks(func):
def wrapper(*args, **kwargs):
invocation = InlineCallbackInvocation(func, *args, **kwargs)
context = kwargs.pop(HYBRID_CONTEXT_KWARG, None)
if context is None:
if len(args) > 0:
func_self = args[0]
# Fetch the context from the MixIn, if args[0] is 'self'
if isinstance(func_self, HybridBase.MixIn):
# 'Access to protected member' warning | pylint: disable=W0212
context = func_self._hybrid__getHybridContext()
# Behave differently based on how we're being used. If a context is
# available to us, assume that we are being bootstrapped (either by a
# '' variant directly via MixIn magic). In this case,
# evaluate against that context and return the actual result.
if context:
return invocation(context)
# Otherwise, assume we are being used in the middle of a hybrid reactor.
# In this case, we want the reactor to supply the context that it's using
# to us directly. Therefore, we will return the 'InlineCallbackInvocation'
# object and let the reactor call it.
return invocation
return wrapper
def returnValue(val):
raise _HybridReturnValue(val)