| // Copyright 2017 The LUCI Authors. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package venv |
| |
| import ( |
| "archive/zip" |
| "context" |
| "crypto/sha256" |
| "encoding/hex" |
| "hash" |
| "io" |
| "io/ioutil" |
| "net/http" |
| "os" |
| "path/filepath" |
| "testing" |
| "time" |
| |
| "github.com/danjacques/gofslock/fslock" |
| |
| "go.chromium.org/luci/cipd/client/cipd" |
| "go.chromium.org/luci/cipd/common" |
| "go.chromium.org/luci/common/errors" |
| "go.chromium.org/luci/common/system/environ" |
| "go.chromium.org/luci/common/system/filesystem" |
| "go.chromium.org/luci/hardcoded/chromeinfra" |
| "go.chromium.org/luci/vpython/api/vpython" |
| "go.chromium.org/luci/vpython/python" |
| "go.chromium.org/luci/vpython/wheel" |
| ) |
| |
| const testDataDir = "test_data" |
| |
| // remoteFiles is the set of remote files to acquire. |
| var remoteFiles = []struct { |
| // install installs this remote file into the test environment. |
| install func(te *testingLoader, path string) |
| |
| // name is the name of the file. |
| name string |
| // contentHash is the SHA256 has of the content. |
| contentHash string |
| |
| // cipdPackage, if not empty, is the name of the CIPD package that contains |
| // this file. |
| cipdPackage string |
| // cipdVersion is the version string of the CIPD package. |
| cipdVersion string |
| |
| // urls, if not empty, is a set of remote URLs where this file can be |
| // downloaded from. |
| urls []string |
| }{ |
| { |
| install: func(tl *testingLoader, path string) { tl.virtualEnvZIP = path }, |
| name: "virtualenv-15.1.0.zip", |
| contentHash: "f7682a57c98a10d32474b4c1df75478dea9a0802c140335c0269a6ec3af46201", |
| cipdPackage: "infra/test-data/vpython/virtualenv", |
| cipdVersion: "version:15.1.0", |
| urls: []string{ |
| "https://github.com/pypa/virtualenv/archive/15.1.0.zip", |
| }, |
| }, |
| } |
| |
| // testingLoader is a map of a CIPD package name to the root directory |
| // that it should be loaded from. |
| type testingLoader struct { |
| PackageLoader |
| |
| cacheDir string |
| |
| virtualEnvZIP string |
| |
| pantsWheelPath string |
| shirtWheelPath string |
| } |
| |
| // loadTestEnvironment sets up the test environment for the VirtualEnv tests. |
| // |
| // This environment includes the acquisition and construction of binary data |
| // that will be used to perform the VirtualEnv test suite, namely: |
| // |
| // - Building test wheel files from source. |
| // - Downloading the testing VirtualEnv package. |
| // |
| // This online setup is preferred to actually checking these binary files into |
| // Git, as it offers more versatility and doesn't clutter Git with binary junk. |
| // |
| // To optimize repeated test re-executions, withTestEnvironment will also cache |
| // the downloaded artifacts in a cache directory. All artifacts will be verified |
| // by their SHA256 hashes, which will be baked into the source here. |
| func loadTestEnvironment(ctx context.Context, t *testing.T) (*testingLoader, error) { |
| wd, err := os.Getwd() |
| if err != nil { |
| return nil, errors.Annotate(err, "failed to get working directory").Err() |
| } |
| |
| cacheDir := filepath.Join(wd, ".venv_test_cache") |
| if err := filesystem.MakeDirs(cacheDir); err != nil { |
| return nil, errors.Annotate(err, "failed to create cache dir").Err() |
| } |
| |
| tl := testingLoader{ |
| cacheDir: cacheDir, |
| } |
| return &tl, tl.withCacheLock(t, func() error { |
| return tl.ensureRemoteFilesLocked(ctx, t) |
| }) |
| } |
| |
| func (tl *testingLoader) withCacheLock(t *testing.T, fn func() error) error { |
| lockPath := filepath.Join(tl.cacheDir, ".lock") |
| blocker := func() error { |
| t.Logf("Cache [%s] is currently locked; sleeping...", lockPath) |
| time.Sleep(1 * time.Second) |
| return nil |
| } |
| return fslock.WithBlocking(lockPath, blocker, func() error { |
| return fn() |
| }) |
| } |
| |
| func (tl *testingLoader) ensureWheels(ctx context.Context, t *testing.T, py *python.Interpreter, tdir string) error { |
| var err error |
| if tl.pantsWheelPath, err = tl.buildWheelLocked(t, py, "pants-1.2-py2.py3-none-any.whl", tdir); err != nil { |
| return err |
| } |
| if tl.shirtWheelPath, err = tl.buildWheelLocked(t, py, "shirt-3.14-py2.py3-none-any.whl", tdir); err != nil { |
| return err |
| } |
| return nil |
| } |
| |
| func (tl *testingLoader) Resolve(c context.Context, e *vpython.Environment) error { |
| |
| e.Spec.Virtualenv.Version = "resolved" |
| for _, wheel := range e.Spec.Wheel { |
| wheel.Version = "resolved" |
| } |
| return nil |
| } |
| |
| func (tl *testingLoader) Ensure(c context.Context, root string, packages []*vpython.Spec_Package) error { |
| for _, pkg := range packages { |
| if err := tl.installPackage(pkg.Name, root); err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| func (tl *testingLoader) installPackage(name, root string) error { |
| switch name { |
| case "foo/bar/virtualenv": |
| return unzip(tl.virtualEnvZIP, root) |
| case "foo/bar/shirt": |
| return copyFileIntoDir(tl.shirtWheelPath, root) |
| case "foo/bar/pants": |
| return copyFileIntoDir(tl.pantsWheelPath, root) |
| |
| default: |
| return errors.Reason("don't know how to install %q", name).Err() |
| } |
| } |
| |
| func (tl *testingLoader) buildWheelLocked(t *testing.T, py *python.Interpreter, name, outDir string) (string, error) { |
| ctx := context.Background() |
| w, err := wheel.ParseName(name) |
| if err != nil { |
| return "", errors.Annotate(err, "failed to parse wheel name %q", name).Err() |
| } |
| |
| outWheelPath := filepath.Join(outDir, w.String()) |
| switch _, err := os.Stat(outWheelPath); { |
| case err == nil: |
| t.Logf("Using cached wheel for %q: %s", name, outWheelPath) |
| return outWheelPath, nil |
| |
| case os.IsNotExist(err): |
| // Will build a new wheel. |
| break |
| |
| default: |
| return "", errors.Annotate(err, "failed to stat wheel path [%s]", outWheelPath).Err() |
| } |
| |
| srcDir := filepath.Join(testDataDir, w.Distribution+".src") |
| |
| // Create a bootstrap wheel-generating VirtualEnv! |
| cfg := Config{ |
| MaxHashLen: 1, // Only going to be 1 environment. |
| BaseDir: filepath.Join(outDir, ".env"), |
| SetupEnv: environ.System(), |
| Python: []string{py.Python}, |
| Package: vpython.Spec_Package{ |
| Name: "foo/bar/virtualenv", |
| Version: "whatever", |
| }, |
| Loader: tl, |
| Spec: &vpython.Spec{}, |
| |
| // Testing parameters for this bootstrap wheel-building environment. |
| testLeaveReadWrite: true, |
| } |
| |
| // Build the wheel in a temporary directory, then copy it into outDir. This |
| // will stop wheel builds from stepping on each other or inheriting each |
| // others' state accidentally. |
| tdir, err := ioutil.TempDir(t.TempDir(), "vpython_venv_wheel") |
| if err != nil { |
| return "", errors.Annotate(err, "failed to create tempdir").Err() |
| } |
| |
| buildDir := filepath.Join(tdir, "build") |
| if err := filesystem.MakeDirs(buildDir); err != nil { |
| return "", err |
| } |
| |
| distDir := filepath.Join(tdir, "dist") |
| if err := filesystem.MakeDirs(distDir); err != nil { |
| return "", err |
| } |
| |
| // Use an empty bootstrap VirtualEnv to build the wheel. This guarantees |
| // that we actually have "setuptools" and "wheel" packages, which are |
| // required for building wheels, and not necessarily present in their |
| // expected forms on all systems. |
| err = With(ctx, cfg, func(ctx context.Context, env *Env) error { |
| cmd := env.Interpreter().MkIsolatedCommand(ctx, |
| python.ScriptTarget{Path: "setup.py"}, |
| "--no-user-cfg", |
| "bdist_wheel", |
| "--bdist-dir", buildDir, |
| "--dist-dir", distDir) |
| defer cmd.Cleanup() |
| cmd.Dir = srcDir |
| if err := cmd.Run(); err != nil { |
| return errors.Annotate(err, "failed to build wheel").Err() |
| } |
| return nil |
| }) |
| if err != nil { |
| return "", errors.Annotate(err, "failed to build wheel").Err() |
| } |
| |
| // Assert that the expected wheel file was generated, and copy it into |
| // outDir. |
| wheelPath := filepath.Join(distDir, w.String()) |
| if _, err := os.Stat(wheelPath); err != nil { |
| return "", errors.Annotate(err, "failed to generate wheel").Err() |
| } |
| if err := copyFileIntoDir(wheelPath, outDir); err != nil { |
| return "", errors.Annotate(err, "failed to install wheel").Err() |
| } |
| |
| t.Logf("Generated wheel file %q: %s", name, outWheelPath) |
| return outWheelPath, nil |
| } |
| |
| func (tl *testingLoader) ensureRemoteFilesLocked(ctx context.Context, t *testing.T) error { |
| MainLoop: |
| for _, rf := range remoteFiles { |
| cachePath := filepath.Join(tl.cacheDir, rf.name) |
| |
| // Check if the remote file is already cached. |
| err := getCachedFileLocked(t, cachePath, rf.contentHash) |
| if err == nil { |
| t.Logf("Remote file [%s] is already cached: [%s]", rf.name, cachePath) |
| rf.install(tl, cachePath) |
| continue MainLoop |
| } |
| t.Logf("Remote file [%s] is not cached: %s", rf.name, err) |
| |
| // Download from CIPD. |
| if rf.cipdPackage != "" { |
| err := cacheFromCIPDLocked(ctx, t, cachePath, rf.name, rf.contentHash, rf.cipdPackage, rf.cipdVersion) |
| if err == nil { |
| t.Logf("Cached remote file [%s] from CIPD source: [%s]", rf.name, cachePath) |
| rf.install(tl, cachePath) |
| continue MainLoop |
| } |
| t.Logf("Failed to load from CIPD package %q @%q: %s", rf.cipdPackage, rf.cipdVersion, err) |
| } |
| |
| // Download from URL. |
| for _, url := range rf.urls { |
| err := cacheFromURLLocked(t, cachePath, rf.contentHash, url) |
| if err == nil { |
| t.Logf("Cached remote file [%s] from URL [%s]: [%s]", rf.name, url, cachePath) |
| rf.install(tl, cachePath) |
| continue MainLoop |
| } |
| t.Logf("Failed to load from URL %q: %s", url, err) |
| } |
| |
| return errors.Reason("failed to acquire remote file %q", rf.name).Err() |
| } |
| |
| return nil |
| } |
| |
| func getCachedFileLocked(t *testing.T, cachePath, hash string) error { |
| return validateHash(t, cachePath, hash, true) |
| } |
| |
| func validateHash(t *testing.T, path, hash string, deleteIfInvalid bool) error { |
| fd, err := os.Open(path) |
| if err != nil { |
| return errors.Annotate(err, "failed to open file").Err() |
| } |
| defer fd.Close() |
| |
| h := sha256.New() |
| if _, err := io.Copy(h, fd); err != nil { |
| return errors.Annotate(err, "failed to hash file").Err() |
| } |
| |
| if err := hashesEqual(h, hash); err != nil { |
| t.Logf("File [%s] has invalid hash: %s", path, err) |
| |
| if deleteIfInvalid { |
| if err := os.Remove(path); err != nil { |
| t.Logf("Failed to delete invalid hash file [%s]: %s", path, err) |
| } |
| } |
| return err |
| } |
| |
| return nil |
| } |
| |
| func hashesEqual(h hash.Hash, expected string) error { |
| if v := hex.EncodeToString(h.Sum(nil)); v != expected { |
| return errors.Reason("hash %q doesn't match expected %q", v, expected).Err() |
| } |
| return nil |
| } |
| |
| var testCIPDClientOptions = cipd.ClientOptions{ |
| ServiceURL: chromeinfra.CIPDServiceURL, |
| UserAgent: "vpython venv tests", |
| } |
| |
| func cacheFromCIPDLocked(ctx context.Context, t *testing.T, cachePath, name, hash, pkg, version string) error { |
| tdir, err := ioutil.TempDir(t.TempDir(), "vpython_venv_cipd") |
| if err != nil { |
| return errors.Annotate(err, "failed to create tempdir").Err() |
| } |
| |
| opts := testCIPDClientOptions |
| opts.Root = tdir |
| |
| client, err := cipd.NewClient(opts) |
| if err != nil { |
| return errors.Annotate(err, "failed to create CIPD client").Err() |
| } |
| defer client.Close(ctx) |
| |
| pin, err := client.ResolveVersion(ctx, pkg, version) |
| if err != nil { |
| return errors.Annotate(err, "failed to resolve CIPD version for %s @%s", pkg, version).Err() |
| } |
| |
| _, err = client.EnsurePackages(ctx, common.PinSliceBySubdir{"": {pin}}, nil) |
| if err != nil { |
| return errors.Annotate(err, "failed to fetch/deploy CIPD package").Err() |
| } |
| |
| path := filepath.Join(opts.Root, name) |
| if err := validateHash(t, path, hash, false); err != nil { |
| // Do not export the invalid path. |
| return err |
| } |
| |
| if err := copyFile(path, cachePath, nil); err != nil { |
| return errors.Annotate(err, "failed to install CIPD package file").Err() |
| } |
| |
| return nil |
| } |
| |
| func cacheFromURLLocked(t *testing.T, cachePath, hash, url string) (err error) { |
| resp, err := http.Get(url) |
| if err != nil { |
| t.Logf("Failed to GET file from URL [%s]: %s", url, err) |
| } |
| defer resp.Body.Close() |
| |
| fd, err := os.Create(cachePath) |
| if err != nil { |
| t.Logf("Failed to create output file [%s]: %s", cachePath, err) |
| } |
| defer func() { |
| if closeErr := fd.Close(); closeErr != nil && err == nil { |
| err = errors.Annotate(closeErr, "failed to close file").Err() |
| } |
| }() |
| |
| h := sha256.New() |
| tr := io.TeeReader(resp.Body, h) |
| if _, err := io.Copy(fd, tr); err != nil { |
| return errors.Annotate(err, "failed to download").Err() |
| } |
| |
| if err = hashesEqual(h, hash); err != nil { |
| return |
| } |
| return nil |
| } |
| |
| func unzip(src, dst string) error { |
| fd, err := zip.OpenReader(src) |
| if err != nil { |
| return errors.Annotate(err, "failed to open ZIP reader").Err() |
| } |
| defer fd.Close() |
| |
| for _, f := range fd.File { |
| path := filepath.Join(dst, filepath.FromSlash(f.Name)) |
| fi := f.FileInfo() |
| |
| // Unzip this entry. |
| if fi.IsDir() { |
| if err := os.MkdirAll(path, 0755); err != nil { |
| return errors.Annotate(err, "failed to mkdir").Err() |
| } |
| } else { |
| if err := copyFileOpener(f.Open, path, fi); err != nil { |
| return err |
| } |
| } |
| } |
| return nil |
| } |
| |
| func copyFileIntoDir(src, dstDir string) error { |
| return copyFile(src, filepath.Join(dstDir, filepath.Base(src)), nil) |
| } |
| |
| func copyFile(src, dst string, fi os.FileInfo) error { |
| opener := func() (io.ReadCloser, error) { return os.Open(src) } |
| return copyFileOpener(opener, dst, fi) |
| } |
| |
| func copyFileOpener(opener func() (io.ReadCloser, error), dst string, fi os.FileInfo) (err error) { |
| sfd, err := opener() |
| if err != nil { |
| return errors.Annotate(err, "failed to open source").Err() |
| } |
| defer sfd.Close() |
| |
| dfd, err := os.Create(dst) |
| if err != nil { |
| return errors.Annotate(err, "failed to create destination").Err() |
| } |
| defer func() { |
| if closeErr := dfd.Close(); closeErr != nil && err == nil { |
| err = errors.Annotate(closeErr, "failed to close destination").Err() |
| } |
| }() |
| |
| if _, err := io.Copy(dfd, sfd); err != nil { |
| return errors.Annotate(err, "failed to copy file").Err() |
| } |
| if fi != nil { |
| if err := os.Chmod(dst, fi.Mode()); err != nil { |
| return errors.Annotate(err, "failed to chmod").Err() |
| } |
| } |
| return nil |
| } |