Add versioned virtualenv
BUG=chromium:703769
TEST=None
Change-Id: I25ef7145028c17ec247441fb2ab529ef49c7d102
diff --git a/bin/create_venv b/bin/create_venv
new file mode 100755
index 0000000..fecf7a0
--- /dev/null
+++ b/bin/create_venv
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright 2017 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Create or update a virtualenv.
+#
+# $ create_venv path/to/requirements.txt
+#
+# This creates a versioned virtualenv with the given requirements
+# installed. The path to the created virtualenv is printed to stdout.
+set -eu
+basedir=$(readlink -f -- "$(dirname -- "${BASH_SOURCE[0]}" )")
+venvdir=$(cd "$basedir"; cd ..; pwd)
+PYTHONPATH="$venvdir" /usr/bin/python2.7 -m cros_venv.scripts.create_venv "$@"
diff --git a/bin/find_virtualenv.sh b/bin/find_virtualenv.sh
index 9cdd775..9df7872 100644
--- a/bin/find_virtualenv.sh
+++ b/bin/find_virtualenv.sh
@@ -9,23 +9,25 @@
# sourcing script should cd into the directory containing this script
# first. This script defines functions for performing common tasks:
#
-# exec_python -- Execute Python module inside of virtualenv
+# exec_python_module -- Execute Python module inside of virtualenv
+#
+# This script is a canonical template that can be copied to other
+# repositories. The venv_repo variable should be changed to point to
+# this repository.
set -eu
realpath() {
readlink -f -- "$1"
}
-readonly virtualenv_dir=$(realpath ..)
-readonly venv_dir=$(realpath ../venv/.venv)
-readonly reqs_file=$(realpath ../venv/requirements.txt)
-
-_create_venv() {
- "$virtualenv_dir/create_venv" "$venv_dir" "$reqs_file" >&2
-}
+# venv_repo should be changed if this script is copied to other repos.
+readonly venv_repo=$(realpath ..)
+readonly create_script=$(realpath "$venv_repo/bin/create_venv")
+readonly venv_home=$(realpath ../venv)
+readonly reqs_file=$(realpath "$venv_home/requirements.txt")
exec_python_module() {
- _create_venv
- export PYTHONPATH='../venv'
- exec "$venv_dir/bin/python" -m "$@"
+ venvdir=$("$create_script" "$reqs_file")
+ export PYTHONPATH=$venv_home
+ exec "$venvdir/bin/python" -m "$@"
}
diff --git a/create_venv b/create_venv
index d146005..ede60cd 100755
--- a/create_venv
+++ b/create_venv
@@ -5,7 +5,7 @@
#
# Create or update a virtualenv.
#
-# $ create_venv path/to/venv path/to/requirements.txt [path/to/custom.pth]
+# $ create_venv path/to/venv path/to/requirements.txt
#
# See the cros_venv Python package for details, or pass the --help
# option.
diff --git a/cros_venv/__init__.py b/cros_venv/__init__.py
index 07a8f31..62ef90d 100644
--- a/cros_venv/__init__.py
+++ b/cros_venv/__init__.py
@@ -4,13 +4,5 @@
"""This package contains modules for creating and updating virtualenvs.
-To run:
-
- $ python2 -m venv
-
-See __main__.py for details.
-
-To run the tests:
-
- $ python2 -m unittest discover venv
+To run the tests, use the script bin/run_tests.
"""
diff --git a/cros_venv/__main__.py b/cros_venv/__main__.py
index a580d36..9c361a0 100644
--- a/cros_venv/__main__.py
+++ b/cros_venv/__main__.py
@@ -4,7 +4,10 @@
"""Create or update a virtualenv.
-See __init__.py for how to run this module.
+$ python -m cros_venv path/to/venv path/to/requirements.txt
+
+This creates a virtualenv at path/to/venv with the given requirements
+installed.
"""
from __future__ import absolute_import
diff --git a/cros_venv/scripts/__init__.py b/cros_venv/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cros_venv/scripts/__init__.py
diff --git a/cros_venv/scripts/create_venv.py b/cros_venv/scripts/create_venv.py
new file mode 100644
index 0000000..2864d1b
--- /dev/null
+++ b/cros_venv/scripts/create_venv.py
@@ -0,0 +1,46 @@
+# Copyright 2017 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Create or update a virtualenv.
+
+$ python -m cros_venv.scripts.create_venv path/to/requirements.txt
+
+This creates a versioned virtualenv with the given requirements
+installed. The path to the created virtualenv is printed to stdout.
+"""
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from cros_venv import venvlib
+
+
+def main():
+ """See module docstring."""
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('reqs_file')
+ parser.add_argument('--verbose', action='store_true')
+ args = parser.parse_args()
+
+ configure_logging(args.verbose)
+
+ with open(args.reqs_file, 'r') as f:
+ spec = venvlib.make_spec(f)
+ venv = venvlib.VersionedVenv(spec)
+ print(venv.ensure())
+
+
+def configure_logging(verbose):
+ if verbose:
+ logging.basicConfig(level='DEBUG')
+ else:
+ logging.getLogger().addHandler(logging.NullHandler())
+
+
+if __name__ == '__main__':
+ main()
diff --git a/cros_venv/test_venvlib.py b/cros_venv/test_venvlib.py
index eb30643..66315d1 100644
--- a/cros_venv/test_venvlib.py
+++ b/cros_venv/test_venvlib.py
@@ -6,9 +6,14 @@
from __future__ import print_function
from __future__ import unicode_literals
+from cStringIO import StringIO
import os
import unittest
+import warnings
+import mock
+
+from cros_venv import flock
from cros_venv import testcases
from cros_venv import venvlib
@@ -17,6 +22,11 @@
"""TestCase for Venv that uses a temp directory."""
+ def run(self, result=None):
+ with warnings.catch_warnings():
+ warnings.filterwarnings('ignore')
+ super(VenvTmpDirTestCase, self).run(result)
+
def test__reqs_up_to_date(self):
"""Test _reqs_up_to_date()."""
venv = venvlib.Venv('.', 'reqs.txt')
@@ -92,6 +102,19 @@
iter(['foo', 'bar']),
))
+ def test__get_cache_dir(self):
+ got = venvlib._get_cache_dir()
+ self.assertTrue(got.endswith('/.cache/cros_venv'))
+
+ def test__get_cache_dir_should_be_absolute(self):
+ got = venvlib._get_cache_dir()
+ self.assertTrue(got.startswith('/'))
+
+ @mock.patch.object(venvlib, 'sys')
+ def test__get_python_version(self, sys):
+ sys.version_info = (2, 7, 12, 'final', 0)
+ self.assertEqual(venvlib._get_python_version(), '2.7.12')
+
class VenvlibTmpDirTestCase(testcases.TmpdirTestCase):
@@ -104,3 +127,196 @@
venvlib._makedirs_exist_ok('foo')
except OSError:
self.fail('OSError raised')
+
+ def test__log_check_call(self):
+ with open('tmp', 'w') as f:
+ venvlib._log_check_call(['echo', 'hi'], logfile=f)
+ with open('tmp', 'r') as f:
+ self.assertEqual(f.read(), "Running [u'echo', u'hi']\nhi\n")
+
+
+class VenvPathsTestCase(unittest.TestCase):
+
+ """Tests for _VenvPaths."""
+
+ def test_repr(self):
+ paths = venvlib._VenvPaths('/tmp')
+ self.assertEqual(repr(paths),
+ "_VenvPaths(u'/tmp')")
+
+ def test_venvdir(self):
+ """Test for venvdir attribute."""
+ paths = venvlib._VenvPaths('/tmp')
+ self.assertEqual(paths.venvdir, '/tmp')
+
+ def test_python(self):
+ """Test for python attribute."""
+ paths = venvlib._VenvPaths('/tmp')
+ self.assertEqual(paths.python, '/tmp/bin/python')
+
+ def test_lockfile(self):
+ """Test for lockfile attribute."""
+ paths = venvlib._VenvPaths('/tmp')
+ self.assertEqual(paths.lockfile, '/tmp/change.lock')
+
+ def test_logfile(self):
+ """Test for logfile attribute."""
+ paths = venvlib._VenvPaths('/tmp')
+ self.assertEqual(paths.logfile, '/tmp/create.log')
+
+ def test_spec(self):
+ """Test for spec attribute."""
+ paths = venvlib._VenvPaths('/tmp')
+ self.assertEqual(paths.spec, '/tmp/spec.json')
+
+
+class VersionedVenvTestCase(unittest.TestCase):
+
+ """Tests for VersionedVenv."""
+
+ def test_repr(self):
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ venv = venvlib.VersionedVenv(spec)
+ with mock.patch.object(venvlib._VenvSpec, '__repr__') as venv_repr:
+ venv_repr.return_value = 'foo'
+ self.assertEqual(repr(venv),
+ "VersionedVenv(foo)")
+
+
+class VersionedVenvTmpDirTestCase(testcases.TmpdirTestCase):
+
+ """Tests for VersionedVenv that use a temp directory."""
+
+ def setUp(self):
+ super(VersionedVenvTmpDirTestCase, self).setUp()
+ patcher = mock.patch.object(venvlib, '_get_cache_dir')
+ get_cache_dir = patcher.start()
+ self.addCleanup(patcher.stop)
+ get_cache_dir.return_value = os.getcwd()
+
+ @mock.patch.object(venvlib.VersionedVenv, '_create')
+ @mock.patch.object(venvlib.VersionedVenv, '_check')
+ def test__check_or_create_should_create_missing_dir(self, check, create):
+ """Test that _check_or_create() creates when directory is missing."""
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ venv = venvlib.VersionedVenv(spec)
+ venv._check_or_create()
+ check.assert_not_called()
+ create.assert_called_once_with()
+
+ @mock.patch.object(venvlib.VersionedVenv, '_create')
+ @mock.patch.object(venvlib.VersionedVenv, '_check')
+ def test__check_or_create_should_create_missing_spec(self, check, create):
+ """Test that _check_or_create() creates when spec file is missing."""
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ venv = venvlib.VersionedVenv(spec)
+ venvdir = 'venv-1.7.6-5442279a613e6a2c4a93c9bda10e63f4'
+ os.mkdir(venvdir)
+ venv._check_or_create()
+ check.assert_not_called()
+ create.assert_called_once_with()
+
+ @mock.patch.object(venvlib.VersionedVenv, '_create')
+ @mock.patch.object(venvlib.VersionedVenv, '_check')
+ def test__check_or_create_should_check_with_spec(self, check, create):
+ """Test that _check_or_create() creates when spec file is missing."""
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ venv = venvlib.VersionedVenv(spec)
+ venvdir = 'venv-1.7.6-5442279a613e6a2c4a93c9bda10e63f4'
+ os.mkdir(venvdir)
+ with open(venvdir + '/spec.json', 'w') as f:
+ f.write('["1.7.6", "foo==1.0.0\\n"]')
+ venv._check_or_create()
+ check.assert_called_once_with(spec)
+ create.assert_not_called()
+
+ def test__check_should_pass_with_same_spec(self):
+ """Test that _check() passes when spec matches."""
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ venv = venvlib.VersionedVenv(spec)
+ try:
+ venv._check(spec)
+ except venvlib.SpecMismatchError:
+ self.fail('Check did not pass')
+
+ def test__check_should_fail_with_different_spec(self):
+ """Test that _check() fails when spec does not match."""
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ venv = venvlib.VersionedVenv(spec)
+ with self.assertRaises(venvlib.SpecMismatchError):
+ venv._check(venvlib._VenvSpec('1.7.6', 'foo=2.0.0\n'))
+
+ def test_dump_spec_and_load_spec(self):
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ venv = venvlib.VersionedVenv(spec)
+ os.mkdir(venv._paths.venvdir)
+ venv._dump_spec()
+ got = venv._load_spec()
+ self.assertEqual(spec, got)
+
+
+class VenvSpecTestCase(unittest.TestCase):
+
+ """Tests for _VenvSpec."""
+
+ def test_eq(self):
+ self.assertEqual(
+ venvlib._VenvSpec('2.7.3', 'foo==1.0.0\n'),
+ venvlib._VenvSpec('2.7.3', 'foo==1.0.0\n'),
+ )
+
+ def test_not_eq(self):
+ self.assertNotEqual(
+ venvlib._VenvSpec('2.7.3', 'foo==1.0.0\n'),
+ venvlib._VenvSpec('2.7.4', 'foo==1.0.0\n'),
+ )
+
+ @mock.patch.object(venvlib, '_get_python_version')
+ def test_make_spec(self, get_ver):
+ f = StringIO('foo==1.0.0\n')
+ get_ver.return_value = '1.7.6'
+ self.assertEqual(
+ venvlib.make_spec(f),
+ venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ )
+
+ def test__get_reqs_hash(self):
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ self.assertEqual(venvlib._get_reqs_hash(spec),
+ '5442279a613e6a2c4a93c9bda10e63f4')
+
+ @mock.patch.object(venvlib, '_get_cache_dir')
+ def test__get_venvdir(self, get_cache_dir):
+ get_cache_dir.return_value = '/home/arland/.cache/cros_venv'
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ got = venvlib._get_venvdir(spec)
+ self.assertEqual(
+ got,
+ '/home/arland/.cache/cros_venv'
+ '/venv-1.7.6-5442279a613e6a2c4a93c9bda10e63f4')
+
+ def test__get_venvdir_should_be_absolute(self):
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ got = venvlib._get_venvdir(spec)
+ self.assertTrue(got.startswith('/'))
+
+ def test_dump_spec_and_load_spec(self):
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ f = StringIO()
+ venvlib._dump_spec(spec, f)
+ f.seek(0)
+ got = venvlib._load_spec(f)
+ self.assertEqual(spec, got)
+
+ def test__make_reqs_file_should_cleanup(self):
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ with venvlib._make_reqs_file(spec) as f:
+ self.assertTrue(os.path.exists(f.name))
+ self.assertFalse(os.path.exists(f.name))
+
+ def test__make_reqs_file_content(self):
+ spec = venvlib._VenvSpec('1.7.6', 'foo==1.0.0\n')
+ with venvlib._make_reqs_file(spec) as f, \
+ open(f.name, 'r') as rfile:
+ self.assertEqual(rfile.read(),
+ 'setuptools==28.2.0\npip==8.1.2\nfoo==1.0.0\n')
diff --git a/cros_venv/venvlib.py b/cros_venv/venvlib.py
index 04460d6..1991bc0 100644
--- a/cros_venv/venvlib.py
+++ b/cros_venv/venvlib.py
@@ -8,11 +8,17 @@
from __future__ import print_function
from __future__ import unicode_literals
+import collections
+import functools
+import hashlib
import itertools
+import json
import os
import shutil
import subprocess
import sys
+import tempfile
+import warnings
from cros_venv import constants
from cros_venv import flock
@@ -30,6 +36,7 @@
"""Wraps all operations on a virtualenv directory."""
def __init__(self, venv_dir, reqs_file):
+ warnings.warn('Venv is deprecated; use VersionedVenv instead')
self._venv_dir = venv_dir
self._reqs_file = reqs_file
@@ -100,6 +107,219 @@
shutil.copyfile(self._reqs_file, self._installed_reqs_file)
+class _VenvPaths(object):
+
+ """Wrapper defining paths inside a versioned virtualenv."""
+
+ def __init__(self, venvdir):
+ """Initialize instance.
+
+ venvdir is the absolute path to a virtualenv.
+ """
+ self.venvdir = venvdir
+
+ def __repr__(self):
+ return '{cls}({this.venvdir!r})'.format(
+ cls=type(self).__name__,
+ this=self,
+ )
+
+ @property
+ def python(self):
+ """Path to the virtualenv's Python binary."""
+ return os.path.join(self.venvdir, 'bin', 'python')
+
+ @property
+ def lockfile(self):
+ """Path to lock file for changing virtualenv."""
+ return os.path.join(self.venvdir, 'change.lock')
+
+ @property
+ def logfile(self):
+ """Path to log file for creating virtualenv."""
+ return os.path.join(self.venvdir, 'create.log')
+
+ @property
+ def spec(self):
+ """Path to spec file inside virtualenv directory."""
+ return os.path.join(self.venvdir, 'spec.json')
+
+
+class VersionedVenv(object):
+
+ """Versioned virtualenv, specified by a _VenvSpec.
+
+ This class provides a method for ensuring the versioned virtualenv
+ is created.
+ """
+
+ def __init__(self, spec):
+ """Initialize instance.
+
+ spec is a _VenvSpec.
+ """
+ self._spec = spec
+ self._paths = _VenvPaths(_get_venvdir(spec))
+
+ def __repr__(self):
+ return '{cls}({this._spec!r})'.format(
+ cls=type(self).__name__,
+ this=self,
+ )
+
+ def ensure(self):
+ """Ensure that the virtualenv exists."""
+ _makedirs_exist_ok(self._paths.venvdir)
+ with flock.FileLock(self._paths.lockfile):
+ self._check_or_create()
+ return self._paths.venvdir
+
+ def _check_or_create(self):
+ """Check virtualenv, creating it if it is not created."""
+ try:
+ existing_spec = self._load_spec()
+ except IOError:
+ self._create()
+ else:
+ self._check(existing_spec)
+
+ def _create(self):
+ """Create virtualenv."""
+ with open(self._paths.logfile, 'w') as logfile, \
+ _make_reqs_file(self._spec) as reqs_file:
+ _create_venv(venvdir=self._paths.venvdir,
+ logfile=logfile)
+ _install_reqs_file(python_path=self._paths.python,
+ reqs_path=reqs_file.name,
+ logfile=logfile)
+ self._dump_spec()
+
+ def _check(self, spec):
+ """Check if the given spec matches our spec.
+
+ Raise SpecMismatchError if check fails.
+ """
+ if spec != self._spec:
+ raise SpecMismatchError
+
+ def _dump_spec(self):
+ """Save the _VenvSpec to the virtualenv on disk."""
+ with open(self._paths.spec, 'w') as f:
+ return _dump_spec(self._spec, f)
+
+ def _load_spec(self):
+ """Return the _VenvSpec for the virtualenv on disk."""
+ with open(self._paths.spec, 'r') as f:
+ return _load_spec(f)
+
+
+class SpecMismatchError(Exception):
+ """Versioned virtualenv specs do not match."""
+
+
+_VenvSpec = collections.namedtuple('_VenvSpec', 'py_version,reqs')
+
+
+def make_spec(f):
+ """Make _VenvSpec from a requirements file object."""
+ return _VenvSpec(_get_python_version(), f.read())
+
+
+def _get_reqs_hash(spec):
+ """Return hash string for _VenvSpec requirements.
+
+ Make sure to check for collisions.
+ """
+ hasher = hashlib.md5()
+ hasher.update(spec.reqs)
+ return hasher.hexdigest()
+
+
+def _get_venvdir(spec):
+ """Return the virtualenv directory to use for the _VenvSpec.
+
+ Returns absolute path.
+ """
+ cache_dir = _get_cache_dir()
+ return os.path.join(
+ cache_dir, 'venv-%s-%s' % (spec.py_version, _get_reqs_hash(spec)))
+
+
+def _dump_spec(spec, f):
+ """Dump _VenvSpec to a file."""
+ json.dump(spec, f)
+
+
+def _load_spec(f):
+ """Load _VenvSpec from a file."""
+ return _VenvSpec._make(json.load(f))
+
+
+def _make_reqs_file(spec):
+ """Return a temporary reqs file for the virtualenv spec.
+
+ The return value is a tempfile.NamedTemporaryFile, which cleans
+ up on close. The filename is accessible via the name attribute.
+ """
+ f = tempfile.NamedTemporaryFile('w')
+ f.writelines(req + '\n' for req in _BASE_DEPENDENCIES)
+ f.write(spec.reqs)
+ f.flush()
+ return f
+
+
+def _create_venv(venvdir, logfile):
+ """Create a virtualenv at the given path."""
+ # TODO(ayatane): Ubuntu Precise ships with virtualenv 1.7, which
+ # requires specifying --setuptools, else distribute is used
+ # (distribute is deprecated). virtualenv after 1.10 uses setuptools
+ # by default. virtualenv >1.10 accepts the --setuptools option but
+ # does not document it. Once we no longer have any hosts on
+ # virtualenv 1.7, the --setuptools option can be removed.
+ command = ['virtualenv', venvdir, '-p', _VENV_PY,
+ '--extra-search-dir', _PACKAGE_DIR, '--setuptools']
+ _log_check_call(command, logfile=logfile)
+
+
+def _install_reqs_file(python_path, reqs_path, logfile):
+ """Install reqs file using pip."""
+ command = [python_path, '-m', 'pip', 'install',
+ '--no-index', '-f', 'file://' + _PACKAGE_DIR, '-r', reqs_path]
+ _log_check_call(command, logfile=logfile)
+
+
+def _add_call_logging(call_func):
+ """Wrap a subprocess-style call with logging."""
+ @functools.wraps(call_func)
+ def wrapped_command(args, logfile, **kwargs):
+ """Logging-wrapped call.
+
+ Arguments are similar to subprocess.Popen, depending on the
+ underlying call. There is an extra keyword-only parameter
+ logfile, which takes a file object.
+ """
+ logfile.write('Running %r\n' % (args,))
+ logfile.flush()
+ call_func(args, stdout=logfile, **kwargs)
+ return wrapped_command
+
+
+_log_check_call = _add_call_logging(subprocess.check_call)
+
+
+def _get_python_version():
+ """Return the version string for the current Python."""
+ return '.'.join(unicode(part) for part in sys.version_info[:3])
+
+
+def _get_cache_dir():
+ """Get cache dir to use for cros_venv.
+
+ Returns absolute path.
+ """
+ return os.path.expanduser('~/.cache/cros_venv')
+
+
def _iter_equal(first, second):
"""Return whether two iterables are equal.
