[vpython] Verify environment, named installs.
Support installation of named enviornments, which are VirtualEnv whose
name is explicitly specified instead of implicitly inferred from the
supplied hash. This can be used by systems that want to use "vpython" to
bootstrap a VirtualEnv in a known place instead of default usage.
Since the specification is no longer always derived from the name, we
add a step to verify that a given VirtualEnv actally matches the
expected VirutalEnv. If it doesn't, it's considered incomplete and
reprovisioned. This allows a path to upgrade named "vpython" roots from
one installation to another, as well as additional cheap sanity checking
to the basic "vpython" assumption that a stamped VirtualEnv is complete.
BUG=None
TEST=local
- Ran "install" and setup normally, appears to work.
R=sergeyberezin@chromium.org
Review-Url: https://codereview.chromium.org/2918623003
diff --git a/vpython/application/subcommand_delete.go b/vpython/application/subcommand_delete.go
index 36649c7..7f387d3 100644
--- a/vpython/application/subcommand_delete.go
+++ b/vpython/application/subcommand_delete.go
@@ -24,7 +24,7 @@
var cr deleteCommandRun
fs := cr.GetFlags()
- fs.BoolVar(&cr.all, "all", false, "Delete all VirtualEnv environments, rather than the current ones.")
+ fs.BoolVar(&cr.all, "all", cr.all, "Delete all VirtualEnv environments, rather than the current ones.")
return &cr
},
diff --git a/vpython/application/subcommand_install.go b/vpython/application/subcommand_install.go
index b9549f8..8e60d9a 100644
--- a/vpython/application/subcommand_install.go
+++ b/vpython/application/subcommand_install.go
@@ -21,17 +21,29 @@
LongDesc: "installs the configured VirtualEnv, but doesn't run any associated commands",
Advanced: false,
CommandRun: func() subcommands.CommandRun {
- return &installCommandRun{}
+ var cr installCommandRun
+
+ fs := cr.GetFlags()
+ fs.StringVar(&cr.name, "name", cr.name,
+ "Use this VirtualEnv name, instead of generating one via hash. This will force specification "+
+ "validation, causing any existing VirtualEnv with this name to be deleted and reprovisioned "+
+ "if it doesn't match.")
+
+ return &cr
},
}
type installCommandRun struct {
subcommands.CommandRunBase
+
+ name string
}
func (cr *installCommandRun) Run(app subcommands.Application, args []string, env subcommands.Env) int {
c := cli.GetContext(app, cr, env)
a := getApplication(c, args)
+ a.opts.EnvConfig.PruneThreshold = 0 // Don't prune on install.
+ a.opts.EnvConfig.OverrideName = cr.name
return run(c, func(c context.Context) error {
err := venv.With(c, a.opts.EnvConfig, false, func(context.Context, *venv.Env) error {
diff --git a/vpython/application/subcommand_verify.go b/vpython/application/subcommand_verify.go
index 9576237..b9c3e43 100644
--- a/vpython/application/subcommand_verify.go
+++ b/vpython/application/subcommand_verify.go
@@ -42,7 +42,10 @@
if err := a.opts.ResolveSpec(c); err != nil {
return errors.Annotate(err).Reason("failed to resolve specification").Err()
}
- if err := spec.Normalize(a.opts.EnvConfig.Spec, &a.opts.EnvConfig.Package); err != nil {
+ if a.opts.EnvConfig.Spec.Virtualenv == nil {
+ a.opts.EnvConfig.Spec.Virtualenv = &a.opts.EnvConfig.Package
+ }
+ if err := spec.NormalizeSpec(a.opts.EnvConfig.Spec); err != nil {
return errors.Annotate(err).Reason("failed to normalize specification").Err()
}
s := a.opts.EnvConfig.Spec
diff --git a/vpython/spec/spec.go b/vpython/spec/spec.go
index 6a52115..97abe5a 100644
--- a/vpython/spec/spec.go
+++ b/vpython/spec/spec.go
@@ -21,13 +21,27 @@
// Render creates a human-readable string from spec.
func Render(spec *vpython.Spec) string { return proto.MarshalTextString(spec) }
-// Normalize normalizes the specification Message such that two messages
-// with identical meaning will have identical representation.
-func Normalize(spec *vpython.Spec, defaultVENVPackage *vpython.Spec_Package) error {
- if spec.Virtualenv == nil {
- spec.Virtualenv = defaultVENVPackage
+// NormalizeEnvironment normalizes the supplied Environment such that two
+// messages with identical meaning will have identical representation.
+func NormalizeEnvironment(env *vpython.Environment) error {
+ if env.Spec == nil {
+ env.Spec = &vpython.Spec{}
+ }
+ if err := NormalizeSpec(env.Spec); err != nil {
+ return err
}
+ if env.Runtime == nil {
+ env.Runtime = &vpython.Runtime{}
+ }
+
+ sort.Sort(pep425TagSlice(env.Pep425Tag))
+ return nil
+}
+
+// NormalizeSpec normalizes the specification Message such that two messages
+// with identical meaning will have identical representation.
+func NormalizeSpec(spec *vpython.Spec) error {
sort.Sort(specPackageSlice(spec.Wheel))
// No duplicate packages. Since we're sorted, we can just check for no
@@ -87,3 +101,16 @@
func(i, j int) bool { return s[i].Version < s[j].Version },
}.Use(i, j)
}
+
+type pep425TagSlice []*vpython.Pep425Tag
+
+func (s pep425TagSlice) Len() int { return len(s) }
+func (s pep425TagSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+func (s pep425TagSlice) Less(i, j int) bool {
+ return sortby.Chain{
+ func(i, j int) bool { return s[i].Version < s[j].Version },
+ func(i, j int) bool { return s[i].Abi < s[j].Abi },
+ func(i, j int) bool { return s[i].Arch < s[j].Arch },
+ }.Use(i, j)
+}
diff --git a/vpython/spec/spec_test.go b/vpython/spec/spec_test.go
index af25053..abc17c2 100644
--- a/vpython/spec/spec_test.go
+++ b/vpython/spec/spec_test.go
@@ -21,36 +21,41 @@
pkgBaz := &vpython.Spec_Package{Name: "baz", Version: "3"}
Convey(`Test manifest generation`, t, func() {
- var spec vpython.Spec
+ var env vpython.Environment
var rt vpython.Runtime
Convey(`Will normalize an empty spec`, func() {
- So(Normalize(&spec, pkgFoo), ShouldBeNil)
- So(spec, ShouldResemble, vpython.Spec{
- Virtualenv: pkgFoo,
+ So(NormalizeEnvironment(&env), ShouldBeNil)
+ So(env, ShouldResemble, vpython.Environment{
+ Spec: &vpython.Spec{},
+ Runtime: &vpython.Runtime{},
})
})
- Convey(`Will normalize to sorted order.`, func() {
- spec.Wheel = []*vpython.Spec_Package{pkgFoo, pkgBar, pkgBaz}
- So(Normalize(&spec, nil), ShouldBeNil)
- So(spec, ShouldResemble, vpython.Spec{
- Wheel: []*vpython.Spec_Package{pkgBar, pkgBaz, pkgFoo},
+ Convey(`With a non-nil spec`, func() {
+ env.Spec = &vpython.Spec{}
+
+ Convey(`Will normalize to sorted order.`, func() {
+ env.Spec.Wheel = []*vpython.Spec_Package{pkgFoo, pkgBar, pkgBaz}
+ So(NormalizeEnvironment(&env), ShouldBeNil)
+ So(env.Spec, ShouldResemble, &vpython.Spec{
+ Wheel: []*vpython.Spec_Package{pkgBar, pkgBaz, pkgFoo},
+ })
+
+ So(Hash(env.Spec, &rt, ""), ShouldEqual, "7e80b8643051ce0d82bf44fb180687e988791cfd7f3da39861370f0a56fc80f8")
+ So(Hash(env.Spec, &rt, "extra"), ShouldEqual, "140a02bb88b011d4aceafb9533266288fd4b441c3bdb70494419b3ef76457f34")
})
- So(Hash(&spec, &rt, ""), ShouldEqual, "7e80b8643051ce0d82bf44fb180687e988791cfd7f3da39861370f0a56fc80f8")
- So(Hash(&spec, &rt, "extra"), ShouldEqual, "140a02bb88b011d4aceafb9533266288fd4b441c3bdb70494419b3ef76457f34")
- })
+ Convey(`Will fail to normalize if there are duplicate wheels.`, func() {
+ env.Spec.Wheel = []*vpython.Spec_Package{pkgFoo, pkgFoo, pkgBar, pkgBaz}
+ So(NormalizeEnvironment(&env), ShouldErrLike, "duplicate spec entries")
- Convey(`Will fail to normalize if there are duplicate wheels.`, func() {
- spec.Wheel = []*vpython.Spec_Package{pkgFoo, pkgFoo, pkgBar, pkgBaz}
- So(Normalize(&spec, nil), ShouldErrLike, "duplicate spec entries")
-
- // Even if the versions differ.
- fooClone := *pkgFoo
- fooClone.Version = "other"
- spec.Wheel = []*vpython.Spec_Package{pkgFoo, &fooClone, pkgBar, pkgBaz}
- So(Normalize(&spec, nil), ShouldErrLike, "duplicate spec entries")
+ // Even if the versions differ.
+ fooClone := *pkgFoo
+ fooClone.Version = "other"
+ env.Spec.Wheel = []*vpython.Spec_Package{pkgFoo, &fooClone, pkgBar, pkgBaz}
+ So(NormalizeEnvironment(&env), ShouldErrLike, "duplicate spec entries")
+ })
})
})
}
diff --git a/vpython/venv/config.go b/vpython/venv/config.go
index 48bb564..1a08675 100644
--- a/vpython/venv/config.go
+++ b/vpython/venv/config.go
@@ -33,6 +33,13 @@
// BaseDir is the parent directory of all VirtualEnv.
BaseDir string
+ // OverrideName overrides the name of the specified VirtualEnv.
+ //
+ // Because the name is no longer derived from the specification, this will
+ // force revalidation and deletion of any existing content if it is not a
+ // fully defined and matching VirtualEnv
+ OverrideName string
+
// Package is the VirtualEnv package to install. It must be non-nil and
// valid. It will be used if the environment specification doesn't supply an
// overriding one.
@@ -97,6 +104,7 @@
}
clone := *cfg
+ clone.OverrideName = ""
clone.Spec = clone.Spec.Clone()
clone.Spec.Wheel = nil
return &clone
@@ -145,24 +153,20 @@
}
}
- // Ensure and normalize our specification file.
- if cfg.Spec == nil {
- cfg.Spec = &vpython.Spec{}
- } else {
- cfg.Spec = cfg.Spec.Clone()
- }
- if err := spec.Normalize(cfg.Spec, &cfg.Package); err != nil {
- return nil, errors.Annotate(err).Reason("invalid specification").Err()
- }
-
- // Choose our VirtualEnv package.
- if cfg.Spec.Virtualenv == nil {
- cfg.Spec.Virtualenv = &cfg.Package
- }
-
// Construct a new, independent Environment for this Env.
e = e.Clone()
- e.Spec = cfg.Spec.Clone()
+ if cfg.Spec != nil {
+ e.Spec = cfg.Spec.Clone()
+ }
+ if err := spec.NormalizeEnvironment(e); err != nil {
+ return nil, errors.Annotate(err).Reason("invalid environment").Err()
+ }
+
+ // If the environment doesn't specify a VirtualEnv package (expected), use
+ // our default.
+ if e.Spec.Virtualenv == nil {
+ e.Spec.Virtualenv = &cfg.Package
+ }
if err := cfg.Loader.Resolve(c, e); err != nil {
return nil, errors.Annotate(err).Reason("failed to resolve packages").Err()
@@ -171,17 +175,15 @@
if err := cfg.resolvePythonInterpreter(c, e.Spec); err != nil {
return nil, errors.Annotate(err).Reason("failed to resolve system Python interpreter").Err()
}
- rt := vpython.Runtime{
- Path: cfg.si.Python,
- Version: e.Spec.PythonVersion,
- }
+ e.Runtime.Path = cfg.si.Python
+ e.Runtime.Version = e.Spec.PythonVersion
var err error
- if rt.Hash, err = cfg.si.Hash(); err != nil {
+ if e.Runtime.Hash, err = cfg.si.Hash(); err != nil {
return nil, err
}
- e.Runtime = &rt
- logging.Debugf(c, "Resolved system Python runtime (%s @ %s): %s", rt.Version, rt.Hash, rt.Path)
+ logging.Debugf(c, "Resolved system Python runtime (%s @ %s): %s",
+ e.Runtime.Version, e.Runtime.Hash, e.Runtime.Path)
// Ensure that our base directory exists.
if err := filesystem.MakeDirs(cfg.BaseDir); err != nil {
@@ -192,7 +194,12 @@
// Generate our environment name based on the deterministic hash of its
// fully-resolved specification.
- return cfg.envForName(cfg.envNameForSpec(e.Spec, e.Runtime), e), nil
+ envName := cfg.OverrideName
+ if envName == "" {
+ envName = cfg.envNameForSpec(e.Spec, e.Runtime)
+ }
+ env := cfg.envForName(envName, e)
+ return env, nil
}
// EnvName returns the VirtualEnv environment name for the environment that cfg
diff --git a/vpython/venv/venv.go b/vpython/venv/venv.go
index 34908cb..2e27fe8 100644
--- a/vpython/venv/venv.go
+++ b/vpython/venv/venv.go
@@ -20,6 +20,7 @@
"github.com/luci/luci-go/vpython/api/vpython"
"github.com/luci/luci-go/vpython/python"
+ "github.com/luci/luci-go/vpython/spec"
"github.com/luci/luci-go/vpython/wheel"
"github.com/luci/luci-go/common/clock"
@@ -394,6 +395,25 @@
D("path", e.EnvironmentStampPath).
Err()
}
+ if err := spec.NormalizeEnvironment(&environment); err != nil {
+ return errors.Annotate(err).Reason("failed to normalize stamp environment").Err()
+ }
+
+ // If we are configured with an environment, validate that it matches the
+ // the environment that we just loaded.
+ //
+ // We only consider our environment-defining fields (Spec and Runtime).
+ //
+ // Note that both environments will have been normalized at this point, so
+ // comparison should be reliable.
+ if e.Environment != nil {
+ if !proto.Equal(e.Environment.Spec, environment.Spec) {
+ return errors.New("environment stamp specification does not match")
+ }
+ if !proto.Equal(e.Environment.Runtime, environment.Runtime) {
+ return errors.New("environment stamp runtime does not match")
+ }
+ }
e.Environment = &environment
return nil
}
