blob: 0586caeeeca30e8e50c37d9c7f736d965267fda9 [file] [log] [blame]
# Copyright 2019 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.
"""This file implements a PEP302-style import hook which makes the
`RECIPE_MODULES` python module space importable.
# Accesses /$recipe_engine/recipe_modules/path/* python files
from RECIPE_MODULES.recipe_engine import path
from RECIPE_MODULES.recipe_engine.path import config
Read PEP302 for the full description of how this works, but the short version is
that an import hook implements:
* find_module(name): returns an object with `.load_module` (possibly self) if
this hook knows how to load `name`. Otherwise returns None.
* load_module(name): returns a loaded python module object for name.
There are some subtleties here like:
* python will try find_module('') when '' tries to
`import os`
* The __package__ metavar (set in load_module) is important to let python know
"this module may have submodules in it". Otherwise if this is not set then
python will assume that the module is from some .py file, and so cannot
have submodules.
* Once a module is loaded (e.g. ''), it's cached in `sys.modules`. The
import hook is only used when the module has not yet been loaded. Further
imports of the same module get the cached version.
* ... except with reload() which jacks up everything.
import imp
import importlib
import inspect
import os
import sys
from future.utils import iteritems, itervalues
from ..config_types import Path, ModuleBasePath, RepoBasePath
from ..recipe_api import BoundProperty, RecipeApi, RecipeApiPlain
from ..recipe_test_api import RecipeTestApi
from . import proto_support
class RecipeModuleImporter(object):
"""This implements both the `find_module` and `load_module` halves of the
import hook protocol.
It uses a RecipeDeps object as the source of truth for what repos and modules
are actually available.
def __init__(self, recipe_deps):
self._recipe_deps = recipe_deps
def find_module(self, fullname, path=None): # pylint: disable=unused-argument
if fullname == self.PREFIX or fullname.startswith(self.PREFIX + '.'):
toks = fullname.split('.')
if 1 <= len(toks) <= 3:
# We should definitely handle all of these.
# RECIPE_MODULES.<repo_name>
# RECIPE_MODULES.<repo_name>.<module_name>
return self
if len(toks) > 3:
# We can only be guaranteed to handle this if such a file exists.
# For example, if in a module does `import base64`, python will
# first try:
# RECIPE_MODULES.<repo_name>.<module_name>.base64
# So we should only return ourselves as a loader if we can ACTUALLY
# import the requested module.
repo_name = toks[1]
module_name = toks[2]
mod_dir = self._recipe_deps.repos[repo_name].modules[module_name].path
target = os.path.join(mod_dir, *toks[3:])
if (os.path.isdir(target) and
os.path.isfile(os.path.join(target, ''))):
# This is a module in the recipe package.
return self
elif os.path.isfile(target + '.py'):
# This is a python file in the recipe package.
return self
# Otherwise, we'll have to let python resolve this along e.g. sys.path
# and builtins.
return None
def load_module(self, fullname):
* `RECIPE_MODULES` module. This is an empty placeholder module.
* `RECIPE_MODULES.repo_name` module. This is an empty placeholder module,
but it has __path__ set to the `recipe_modules` path of the given repo.
* `RECIPE_MODULES.repo_name.mod_name` module. This is the real python
module (i.e. whatever is in this modules' Additionally
this adds extra attributes (see _patchup_module).
* `RECIPE_MODULES.repo_name.mod_name...` modules. These are the real
python modules contained within 'repo_name/mod_name' without any tweaks.
# fullname is RECIPE_MODULES[.<repo_name>[.<mod_name>[.etc.etc]]]
mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
mod.__loader__ = self
toks = fullname.split('.')
assert len(toks) > 0
if len(toks) == 1:
mod.__file__ = "<RecipeModuleImporter>"
mod.__path__ = []
mod.__package__ = fullname
return mod
repo_name = toks[1]
repo = self._recipe_deps.repos[repo_name]
if len(toks) == 2:
mod.__file__ = repo.modules_dir
mod.__path__ = [mod.__file__]
mod.__package__ = fullname
return mod
# This is essentially a regular python module loader at this point, except
# that:
# * The prefix of the module name is `RECIPE_MODULES.<repo_name>`.
# * The loader is scoped within the actual directory of the recipe module.
# * If it's exactly `RECIPE_MODULES.<repo_name>.<mod_name>`, then we also
# call _patchup_module on it.
parent_mod_name, _, to_load = fullname.rpartition('.')
# This imports the parent module. This doesn't devolve into recursive
# madness because:
# * Python always imports modules in order (i.e. 'a.b' comes before
# 'a.b.c'.
# * Python caches imports in sys.modules (so this should always turn into
# a dictionary lookup in sys.modules).
# We don't just look up stuff in sys.modules because `importlib` is the
# correct api for accessing modules.
parent_mod = importlib.import_module(parent_mod_name)
f, pathname, description = imp.find_module(to_load, parent_mod.__path__)
loaded = imp.load_module(fullname, f, pathname, description)
if f:
if len(toks) == 3: # RECIPE_MODULES.repo_name.module_name
self._patchup_module(loaded, repo.path)
return mod
def _patchup_module(mod, repo_root):
"""Adds a bunch of fields to the imported recipe module.
TODO: most of these are obsolete and could be calculated at the sites that
use them. At some point in the future we should remove _patchup_module
entirely and rely on the builtin fields like __name__, __file__, etc. to
calculate all these details.
Currently sets the additional attributes on the python module:
* `NAME`: The module's short name (e.g. 'path')
* `API`: The class derived from RecipeApiPlain in
* `TEST_API`: The class derived from RecipeTestApi in If none
exists, then this is set to RecipeTestApi.
* `PROPERTIES`: A dictionary derived from 'PROPERTIES' defined in, except that all of the Property values are 'bound' by
calling their `bind()` method.
* `MODULE_DIRECTORY`: The module's directory (as a Path object).
* `RESOURCE_DIRECTORY`: The module's ./resource directory (as a Path
* `REPO_ROOT`: A Path object for the root of the repo containing this
* `CONFIG_CTX`: The ConfigContext object (defined in for this
module, or None if no exists.
* `DEPS`: Sets a default DEPS value of `()` so that other code in the
engine can assume that there's ALWAYS a DEPS object for a module.
* `WARNINGS`: A list of warnings issued against this recipe module.
* `DISABLE_STRICT_COVERAGE`: Sets a default value of False.
* `PYTHON_VERSION_COMPATIBILITY`: "PY2", "PY2+3" or "PY3". Defaults to
* mod (python module type) - This will be the module loaded for e.g.
* repo_root (str) - Absolute path to the root of the module's repository
on disk.
_, repo_name, module_name = mod.__name__.split('.')
mod.NAME = module_name
mod.MODULE_DIRECTORY = Path(ModuleBasePath(mod))
mod.REPO_ROOT = Path(RepoBasePath(repo_name, repo_root))
mod.CONFIG_CTX = getattr(mod, 'CONFIG_CTX', None)
mod.DEPS = getattr(mod, 'DEPS', ())
mod.WARNINGS = getattr(mod, 'WARNINGS', ())
# TODO(iannucci, probably): remove DISABLE_STRICT_COVERAGE (crbug/693058).
# TODO(iannucci): do these imports on-demand at the callsites needing these.
# NOTE: late import to avoid early protobuf import
from ..config import ConfigContext
cfg_module = None
if os.path.isfile(os.path.join(mod.__path__[0], '')):
cfg_module = importlib.import_module(mod.__name__ + '.config')
if cfg_module:
for v in itervalues(cfg_module.__dict__):
if isinstance(v, ConfigContext):
assert not mod.CONFIG_CTX, (
'More than one configuration context: %s, %s' %
(cfg_module, mod.CONFIG_CTX))
mod.CONFIG_CTX = v
assert mod.CONFIG_CTX, 'Config file, but no config context?'
# The current config system relies on implicitly importing all the
# * files... ugh.
for fname in os.listdir(os.path.dirname(mod.__file__)):
if fname.endswith(''):
importlib.import_module(mod.__name__ + '.' + fname.rstrip('.py'))
# Identify the RecipeApiPlain subclass as this module's API.
mod.API = getattr(mod, 'API', None)
api_module = importlib.import_module(mod.__name__ + '.api')
for v in itervalues(api_module.__dict__):
# If the recipe has literally imported the RecipeApi, we don't want to
# consider that to be the real RecipeApi :)
if v is RecipeApiPlain or v is RecipeApi:
if inspect.isclass(v) and issubclass(v, RecipeApiPlain):
assert not mod.API, (
'%s has more than one RecipeApi subclass: %s, %s' % (
module_name, v, mod.API))
mod.API = v
assert mod.API, 'Recipe module has no api? %s' % (mod,)
# Identify the (optional) RecipeTestApi subclass as this module's test API.
test_module = None
if os.path.isfile(os.path.join(mod.__path__[0], '')):
test_module = importlib.import_module(mod.__name__ + '.test_api')
mod.TEST_API = getattr(mod, 'TEST_API', None)
if test_module:
for v in itervalues(mod.test_api.__dict__):
# If the recipe has literally imported the RecipeTestApi, we don't want
# to consider that to be the real RecipeTestApi :)
if v is RecipeTestApi:
if inspect.isclass(v) and issubclass(v, RecipeTestApi):
assert not mod.TEST_API, (
'More than one TestApi subclass: %s' % mod.api)
mod.TEST_API = v
assert mod.API, (
'Recipe module has but no TestApi subclass? %s' % (mod,))
mod.TEST_API = RecipeTestApi
properties_def = getattr(mod, 'PROPERTIES', {})
# If PROPERTIES isn't a protobuf Message, it must be a legacy Property dict.
if not proto_support.is_message_class(properties_def):
# Let each property object know about the property name.
full_decl_name = '%s::%s' % (repo_name, module_name)
prop_name: value.bind(prop_name, BoundProperty.MODULE_PROPERTY,
for prop_name, value in iteritems(properties_def)