| // Copyright 2022 The LUCI Authors. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| // Package python includes generating python venv and parsing python command | |
| // line arguments. | |
| package python | |
| import ( | |
| "context" | |
| "embed" | |
| "fmt" | |
| "io" | |
| "io/fs" | |
| "os" | |
| "path/filepath" | |
| "runtime" | |
| "strings" | |
| "google.golang.org/protobuf/proto" | |
| "go.chromium.org/luci/cipd/client/cipd/ensure" | |
| "go.chromium.org/luci/cipkg/base/generators" | |
| "go.chromium.org/luci/cipkg/base/workflow" | |
| "go.chromium.org/luci/cipkg/core" | |
| "go.chromium.org/luci/common/errors" | |
| "go.chromium.org/luci/common/system/environ" | |
| "go.chromium.org/luci/vpython/common" | |
| "go.chromium.org/luci/vpython/standard" | |
| ) | |
| // passthroughEnvVars is a list of environment variables forwarded from the | |
| // host environment to virtualenv generator subprocesses at execution time so | |
| // network tools (pip/uv) can authenticate SSL certs and navigate proxies. | |
| var passthroughEnvVars = []string{ | |
| // Proxy settings | |
| "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", | |
| "http_proxy", "https_proxy", "all_proxy", "no_proxy", | |
| // TLS / CA certificate bundles | |
| "SSL_CERT_FILE", "SSL_CERT_DIR", | |
| "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", | |
| "PIP_CERT", "UV_CERT", "UV_NATIVE_TLS", | |
| } | |
| func venvEnv() environ.Env { | |
| env := environ.New(nil) | |
| arURL := os.Getenv(common.EnvVpythonArUrl) | |
| if arURL == "" { | |
| arURL = common.DefaultARURL | |
| } | |
| env.Set(common.EnvVpythonArUrl, arURL) | |
| return env | |
| } | |
| // passthroughEnv returns host proxy, TLS certificate, and network retry | |
| // timeout environment variables to pass to virtualenv builder subprocesses at | |
| // execution time without hashing them into the derivation ID. | |
| func passthroughEnv() environ.Env { | |
| env := environ.New(nil) | |
| for _, k := range passthroughEnvVars { | |
| if _, ok := env.Lookup(k); !ok { | |
| if v := os.Getenv(k); v != "" { | |
| env.Set(k, v) | |
| } | |
| } | |
| } | |
| if timeout := os.Getenv(common.EnvVpythonNetworkRetryTimeout); timeout != "" { | |
| env.Set(common.EnvVpythonNetworkRetryTimeout, timeout) | |
| } | |
| return env | |
| } | |
| // WithPassthroughEnv returns a copy of drv with passthroughEnv merged into Env | |
| // when drv is a python_venv or uv_venv derivation. | |
| func WithPassthroughEnv(drv *core.Derivation) *core.Derivation { | |
| if drv.Name != "python_venv" && drv.Name != "uv_venv" { | |
| return drv | |
| } | |
| env := passthroughEnv() | |
| if env.Len() == 0 { | |
| return drv | |
| } | |
| env.Update(environ.New(drv.Env)) | |
| cloned := proto.CloneOf(drv) | |
| cloned.Env = env.Sorted() | |
| return cloned | |
| } | |
| type Environment struct { | |
| Executable string | |
| CPython generators.Generator | |
| Virtualenv generators.Generator | |
| } | |
| func CPythonFromCIPD(version string) generators.Generator { | |
| return &generators.CIPDExport{ | |
| Name: "cpython", | |
| Ensure: ensure.File{ | |
| PackagesBySubdir: map[string]ensure.PackageSlice{ | |
| "": { | |
| {PackageTemplate: "infra/3pp/tools/cpython/${platform}", UnresolvedVersion: version}, | |
| }, | |
| }, | |
| }, | |
| } | |
| } | |
| func CPython3FromCIPD(version string) generators.Generator { | |
| return &generators.CIPDExport{ | |
| Name: "cpython", | |
| Ensure: ensure.File{ | |
| PackagesBySubdir: map[string]ensure.PackageSlice{ | |
| "": { | |
| {PackageTemplate: "infra/3pp/tools/cpython3/${platform}", UnresolvedVersion: version}, | |
| }, | |
| }, | |
| }, | |
| } | |
| } | |
| func VirtualenvFromCIPD(version string) generators.Generator { | |
| return &generators.CIPDExport{ | |
| Name: "virtualenv", | |
| Ensure: ensure.File{ | |
| PackagesBySubdir: map[string]ensure.PackageSlice{ | |
| "": { | |
| {PackageTemplate: "infra/3pp/tools/virtualenv", UnresolvedVersion: version}, | |
| }, | |
| }, | |
| }, | |
| } | |
| } | |
| // UVFromCIPD returns a cipkg Generator that exports the prebuilt uv CIPD | |
| // package ("infra/3pp/tools/uv/${platform}") at the given version tag. | |
| func UVFromCIPD(version string) generators.Generator { | |
| return &generators.CIPDExport{ | |
| Name: "uv", | |
| Ensure: ensure.File{ | |
| PackagesBySubdir: map[string]ensure.PackageSlice{ | |
| "": { | |
| {PackageTemplate: "infra/3pp/tools/uv/${platform}", UnresolvedVersion: version}, | |
| }, | |
| }, | |
| }, | |
| } | |
| } | |
| //go:embed bootstrap.py pep425tags.py uv_bootstrap.py | |
| var bootstrapEmbed embed.FS | |
| var bootstrapGen = generators.InitEmbeddedFS("bootstrap", bootstrapEmbed) | |
| func (e *Environment) Pep425Tags() generators.Generator { | |
| // Generate an empty virtual environment to probe the pep425tags | |
| empty := &workflow.Generator{ | |
| Name: "python_empty_venv", | |
| Args: []string{ | |
| common.Python("{{.cpython}}", e.Executable), | |
| "-BsE", // -B: no .pyc files; -s: no user site-packages; -E: ignore environment overrides | |
| filepath.Join("{{.bootstrap}}", "bootstrap.py"), | |
| }, | |
| Dependencies: []generators.Dependency{ | |
| {Type: generators.DepsHostTarget, Generator: e.CPython, Runtime: true}, | |
| {Type: generators.DepsHostTarget, Generator: e.Virtualenv}, | |
| {Type: generators.DepsHostTarget, Generator: bootstrapGen}, | |
| }, | |
| } | |
| return &workflow.Generator{ | |
| Name: "python_pep425tags", | |
| Args: []string{ | |
| common.PythonVENV("{{.python_empty_venv}}", e.Executable), | |
| "-BsE", // -B: no .pyc files; -s: no user site-packages; -E: ignore environment overrides | |
| filepath.Join("{{.bootstrap}}", "pep425tags.py"), | |
| }, | |
| Dependencies: []generators.Dependency{ | |
| {Type: generators.DepsHostTarget, Generator: empty}, | |
| {Type: generators.DepsHostTarget, Generator: bootstrapGen}, | |
| }, | |
| } | |
| } | |
| func (e *Environment) WithWheels(wheels generators.Generator) generators.Generator { | |
| return &workflow.Generator{ | |
| Name: "python_venv", | |
| Args: []string{ | |
| common.Python("{{.cpython}}", e.Executable), | |
| "-BsE", // -B: no .pyc files; -s: no user site-packages; -E: ignore environment overrides | |
| filepath.Join("{{.bootstrap}}", "bootstrap.py"), | |
| }, | |
| Env: venvEnv(), | |
| Dependencies: []generators.Dependency{ | |
| {Type: generators.DepsHostTarget, Generator: e.CPython, Runtime: true}, | |
| {Type: generators.DepsHostTarget, Generator: e.Virtualenv}, | |
| {Type: generators.DepsHostTarget, Generator: wheels}, | |
| {Type: generators.DepsHostTarget, Generator: bootstrapGen}, | |
| }, | |
| } | |
| } | |
| func CPythonFromPath(dir, cipdName string) (generators.Generator, error) { | |
| cpythonDir := dir | |
| if !filepath.IsAbs(dir) { | |
| execDir, err := FindExecutableDir() | |
| if err != nil { | |
| return nil, err | |
| } | |
| cpythonDir = filepath.Join(execDir, dir) | |
| } | |
| version := dir // Default to folder name for offline local developer test packaging. | |
| if v, err := os.Open(filepath.Join(cpythonDir, ".versions", fmt.Sprintf("%s.cipd_version", cipdName))); err == nil { | |
| defer v.Close() | |
| if fb, readErr := io.ReadAll(v); readErr == nil { | |
| version = string(fb) | |
| } | |
| } | |
| return &generators.ImportTargets{ | |
| Name: "cpython", | |
| Targets: map[string]generators.ImportTarget{ | |
| ".": {Source: cpythonDir, Version: string(version), Mode: fs.ModeDir, FollowSymlinks: true}, | |
| }, | |
| }, nil | |
| } | |
| // FindExecutableDir returns the absolute directory path of the current running executable, | |
| // resolving any symlinks under POSIX platforms (non-Windows) for path safety. | |
| func FindExecutableDir() (string, error) { | |
| path, err := os.Executable() | |
| if err != nil { | |
| return "", errors.Fmt("failed to get current executable path: %w", err) | |
| } | |
| if runtime.GOOS != "windows" { | |
| if path, err = filepath.EvalSymlinks(path); err != nil { | |
| return "", errors.Fmt("failed to resolve executable symlinks: %w", err) | |
| } | |
| } | |
| return filepath.Dir(path), nil | |
| } | |
| // UVFromPath imports the pre-packaged uv directory next to the binary as a standard cipkg Generator. | |
| func UVFromPath(dir string) (generators.Generator, error) { | |
| if !filepath.IsAbs(dir) { | |
| execDir, err := FindExecutableDir() | |
| if err != nil { | |
| return nil, err | |
| } | |
| dir = filepath.Join(execDir, dir) | |
| } | |
| versionFile := filepath.Join(dir, ".versions", "uv.cipd_version") | |
| version := filepath.Base(dir) // Default to folder name for offline local developer test packaging. | |
| if fb, err := os.ReadFile(versionFile); err == nil { | |
| version = strings.TrimSpace(string(fb)) | |
| } | |
| return &generators.ImportTargets{ | |
| Name: "uv", | |
| Targets: map[string]generators.ImportTarget{ | |
| ".": {Source: dir, Version: version, Mode: fs.ModeDir, FollowSymlinks: true}, | |
| }, | |
| }, nil | |
| } | |
| // SpecRequirementsGenerator writes ProjectSpec dependencies into a requirements.txt file. | |
| type SpecRequirementsGenerator struct { | |
| Spec *standard.ProjectSpec | |
| } | |
| // Generate returns the copy Action for the requirements file. | |
| func (s *SpecRequirementsGenerator) Generate(_ context.Context, plats generators.Platforms) (*core.Action, error) { | |
| var sb strings.Builder | |
| for _, req := range s.Spec.Dependencies { | |
| sb.WriteString(req) | |
| sb.WriteByte('\n') | |
| } | |
| return &core.Action{ | |
| Name: "vpython_requirements", | |
| Spec: &core.Action_Copy{ | |
| Copy: &core.ActionFilesCopy{ | |
| Files: map[string]*core.ActionFilesCopy_Source{ | |
| "requirements.txt": { | |
| Content: &core.ActionFilesCopy_Source_Raw{ | |
| Raw: []byte(sb.String()), | |
| }, | |
| Mode: 0o444, | |
| }, | |
| }, | |
| }, | |
| }, | |
| }, nil | |
| } | |
| // WithUV returns a workflow.Generator for the UV virtualenv. | |
| func (e *Environment) WithUV(uv generators.Generator, spec *standard.ProjectSpec) generators.Generator { | |
| reqGen := &SpecRequirementsGenerator{Spec: spec} | |
| uvBin := filepath.Join("{{.uv}}", "uv") | |
| if runtime.GOOS == "windows" { | |
| uvBin += ".exe" | |
| } | |
| return &workflow.Generator{ | |
| Name: "uv_venv", | |
| Args: []string{ | |
| common.Python("{{.cpython}}", e.Executable), | |
| "-BsE", // -B: no .pyc files; -s: no user site-packages; -E: ignore environment overrides | |
| filepath.Join("{{.bootstrap}}", "uv_bootstrap.py"), | |
| "--uv-bin", uvBin, | |
| "--python-bin", common.Python("{{.cpython}}", e.Executable), | |
| "--req-file", filepath.Join("{{.vpython_requirements}}", "requirements.txt"), | |
| }, | |
| Env: venvEnv(), | |
| Dependencies: []generators.Dependency{ | |
| {Type: generators.DepsHostTarget, Generator: e.CPython, Runtime: true}, | |
| {Type: generators.DepsHostTarget, Generator: uv}, | |
| {Type: generators.DepsHostTarget, Generator: reqGen}, | |
| {Type: generators.DepsHostTarget, Generator: bootstrapGen}, | |
| }, | |
| } | |
| } |