| """ |
| Repository rule to manage hermetic Python interpreter under Bazel. |
| |
| Version can be set via build parameter "--repo_env=HERMETIC_PYTHON_VERSION=3.11" |
| |
| To set wheel name, add "--repo_env=WHEEL_NAME=tensorflow_cpu" |
| """ |
| |
| DEFAULT_VERSION = "3.11" |
| |
| def _python_repository_impl(ctx): |
| version, py_kind = _get_python_version(ctx) |
| version_and_kind = "%s-%s" % (version, py_kind) if py_kind else version |
| |
| ctx.file("BUILD", "") |
| wheel_name = ctx.os.environ.get("WHEEL_NAME", "tensorflow") |
| wheel_collab = ctx.os.environ.get("WHEEL_COLLAB", False) |
| macos_deployment_target = ctx.os.environ.get("MACOSX_DEPLOYMENT_TARGET", "") |
| hermetic_url = ctx.os.environ.get("HERMETIC_PYTHON_URL", "") |
| hermetic_sha256 = ctx.os.environ.get("HERMETIC_PYTHON_SHA256", "") |
| hermetic_prefix = ctx.os.environ.get("HERMETIC_PYTHON_PREFIX", "python") |
| custom_requirements = ctx.os.environ.get("HERMETIC_REQUIREMENTS_LOCK", None) |
| |
| if not (hermetic_url + hermetic_sha256) and (hermetic_url or hermetic_sha256): |
| fail(""" |
| Please either specify both HERMETIC_PYTHON_URL and HERMETIC_PYTHON_SHA256 |
| to set up a custom python interpreter, or none of them to rely on default ones. |
| """) |
| requirements = None |
| if not requirements: |
| for i in range(0, len(ctx.attr.requirements_locks)): |
| if ctx.attr.requirements_versions[i] == version_and_kind: |
| requirements = ctx.attr.requirements_locks[i] |
| break |
| |
| if not requirements and not custom_requirements: |
| fail(""" |
| Could not find requirements_lock.txt file matching specified Python version. |
| Specified python version: {version} |
| Python versions with available requirement_lock.txt files: {versions} |
| Please check python_init_repositories() in your WORKSPACE file. |
| """.format( |
| version = version, |
| versions = ", ".join(ctx.attr.requirements_versions), |
| )) |
| |
| if custom_requirements: |
| custom_requirements_path = ctx.path(custom_requirements) |
| requirements_with_local_wheels = "@{repo}//:{label}".format( |
| repo = ctx.name, |
| label = custom_requirements_path.basename, |
| ) |
| ctx.file( |
| custom_requirements_path.basename, |
| ctx.read(custom_requirements_path), |
| ) |
| elif ctx.attr.local_wheel_workspaces: |
| base_requirements = ctx.read(requirements) |
| local_wheel_requirements = _get_injected_local_wheels( |
| ctx, |
| version, |
| ctx.attr.local_wheel_workspaces, |
| base_requirements, |
| ) |
| requirements_content = [base_requirements] + local_wheel_requirements |
| merged_requirements_content = "\n".join(requirements_content) |
| |
| requirements_with_local_wheels = "@{repo}//:{label}".format( |
| repo = ctx.name, |
| label = requirements.name, |
| ) |
| ctx.file( |
| requirements.name, |
| merged_requirements_content, |
| ) |
| else: |
| requirements_with_local_wheels = str(requirements) |
| |
| use_pywrap_rules = bool( |
| ctx.os.environ.get("USE_PYWRAP_RULES", False), |
| ) |
| |
| if use_pywrap_rules: |
| print("!!!Using pywrap rules instead of directly creating .so objects!!!") # buildifier: disable=print |
| |
| interpreter_type = "\"default\" (provided by rules_python)" |
| if hermetic_url: |
| interpreter_type = "\"custom\" (pulled from %s)" % hermetic_url |
| print( |
| """ |
| ============================= |
| Hermetic Python configuration: |
| Version: "{version}" |
| Kind: "{py_kind}" |
| Interpreter: {interpreter_type} |
| Requirements_lock label: "{requirements_lock_label}" |
| ===================================== |
| """.format( |
| version = version, |
| py_kind = py_kind, |
| interpreter_type = interpreter_type, |
| requirements_lock_label = requirements_with_local_wheels, |
| ), |
| ) # buildifier: disable=print |
| |
| ctx.file( |
| "py_version.bzl", |
| """ |
| TF_PYTHON_VERSION = "{version}" |
| HERMETIC_PYTHON_VERSION = "{version}" |
| HERMETIC_PYTHON_VERSION_KIND = "{py_kind}" |
| WHEEL_NAME = "{wheel_name}" |
| WHEEL_COLLAB = "{wheel_collab}" |
| REQUIREMENTS = "{requirements}" |
| REQUIREMENTS_WITH_LOCAL_WHEELS = "{requirements_with_local_wheels}" |
| USE_PYWRAP_RULES = {use_pywrap_rules} |
| MACOSX_DEPLOYMENT_TARGET = "{macos_deployment_target}" |
| HERMETIC_PYTHON_URL = "{hermetic_url}" |
| HERMETIC_PYTHON_SHA256 = "{hermetic_sha256}" |
| HERMETIC_PYTHON_PREFIX = "{hermetic_prefix}" |
| """.format( |
| version = version, |
| py_kind = py_kind, |
| wheel_name = wheel_name, |
| wheel_collab = wheel_collab, |
| requirements = str(requirements), |
| requirements_with_local_wheels = requirements_with_local_wheels, |
| use_pywrap_rules = use_pywrap_rules, |
| macos_deployment_target = macos_deployment_target, |
| hermetic_url = hermetic_url, |
| hermetic_sha256 = hermetic_sha256, |
| hermetic_prefix = hermetic_prefix, |
| ), |
| ) |
| |
| def _get_python_version(ctx): |
| print_warning = False |
| |
| version = ctx.os.environ.get("HERMETIC_PYTHON_VERSION", "") |
| if not version: |
| version = ctx.os.environ.get("TF_PYTHON_VERSION", "") |
| if not version: |
| print_warning = True |
| if ctx.attr.default_python_version == "system": |
| python_version_result = ctx.execute(["python3", "--version"]) |
| if python_version_result.return_code == 0: |
| version = python_version_result.stdout |
| else: |
| fail(""" |
| Cannot match hermetic Python version to system Python version. |
| System Python was not found.""") |
| else: |
| version = ctx.attr.default_python_version |
| |
| version, kind = _parse_python_version(version) |
| |
| if print_warning: |
| print(""" |
| HERMETIC_PYTHON_VERSION variable was not set correctly, using default version. |
| Python {} will be used. |
| To select Python version, either set HERMETIC_PYTHON_VERSION env variable in |
| your shell: |
| export HERMETIC_PYTHON_VERSION=3.12 |
| OR pass it as an argument to bazel command directly or inside your .bazelrc |
| file: |
| --repo_env=HERMETIC_PYTHON_VERSION=3.12 |
| """.format(version)) # buildifier: disable=print |
| return version, kind |
| |
| def _parse_python_version(version_str): |
| if version_str.startswith("Python "): |
| py_ver_chunks = version_str[7:].split(".") |
| return "%s.%s" % (py_ver_chunks[0], py_ver_chunks[1]), "" |
| elif "-" in version_str: |
| return version_str.split("-") |
| return version_str, "" |
| |
| def _get_injected_local_wheels( |
| ctx, |
| py_version, |
| local_wheel_workspaces, |
| base_requirements): |
| os_name = ctx.os.name |
| is_windows = "windows" in os_name.lower() |
| local_file_path_prefix = "file:" if is_windows else "file://" |
| |
| local_wheel_requirements = [] |
| py_ver_marker = "-cp%s-" % py_version.replace(".", "") |
| py_major_ver_marker = "-py%s-" % py_version.split(".")[0] |
| wheels = {} |
| |
| if local_wheel_workspaces: |
| for local_wheel_workspace in local_wheel_workspaces: |
| local_wheel_workspace_path = ctx.path(local_wheel_workspace) |
| dist_folder = ctx.attr.local_wheel_dist_folder |
| dist_folder_path = local_wheel_workspace_path.dirname.get_child(dist_folder) |
| if dist_folder_path.exists: |
| dist_wheels = dist_folder_path.readdir() |
| _process_dist_wheels( |
| dist_wheels, |
| wheels, |
| py_ver_marker, |
| py_major_ver_marker, |
| ctx.attr.local_wheel_inclusion_list, |
| ctx.attr.local_wheel_exclusion_list, |
| ) |
| |
| for wheel_name, wheel_path in wheels.items(): |
| # Normalize `foo_bar` to `foo-bar`. We assume that, if `foo_bar` |
| # isn't present in requirements, it must be named `foo-bar`. The |
| # exact same distribution name needs to be used to ensure it is |
| # correctly overridden. |
| if "_" in wheel_name and wheel_name not in base_requirements: |
| local_package_name = wheel_name.replace("_", "-") |
| else: |
| local_package_name = wheel_name |
| local_wheel_requirements.append( |
| "{pypi_package_name} @ {local_file_path_prefix}{wheel_path}".format( |
| local_file_path_prefix = local_file_path_prefix, |
| pypi_package_name = local_package_name, |
| wheel_path = wheel_path.realpath, |
| ), |
| ) |
| |
| return local_wheel_requirements |
| |
| python_repository = repository_rule( |
| implementation = _python_repository_impl, |
| attrs = { |
| "requirements_versions": attr.string_list( |
| mandatory = False, |
| default = [], |
| ), |
| "requirements_locks": attr.label_list( |
| mandatory = False, |
| default = [], |
| ), |
| "local_wheel_workspaces": attr.label_list( |
| mandatory = False, |
| default = [], |
| ), |
| "local_wheel_dist_folder": attr.string( |
| mandatory = False, |
| default = "dist", |
| ), |
| "default_python_version": attr.string( |
| mandatory = False, |
| default = DEFAULT_VERSION, |
| ), |
| "local_wheel_inclusion_list": attr.string_list( |
| mandatory = False, |
| default = ["*"], |
| ), |
| "local_wheel_exclusion_list": attr.string_list( |
| mandatory = False, |
| default = [], |
| ), |
| }, |
| environ = [ |
| "TF_PYTHON_VERSION", |
| "HERMETIC_PYTHON_VERSION", |
| "HERMETIC_PYTHON_URL", |
| "HERMETIC_PYTHON_SHA256", |
| "HERMETIC_REQUIREMENTS_LOCK", |
| "HERMETIC_PYTHON_PREFIX", |
| "WHEEL_NAME", |
| "WHEEL_COLLAB", |
| "USE_PYWRAP_RULES", |
| "MACOSX_DEPLOYMENT_TARGET", |
| ], |
| local = True, |
| ) |
| |
| def _process_dist_wheels( |
| dist_wheels, |
| wheels, |
| py_ver_marker, |
| py_major_ver_marker, |
| local_wheel_inclusion_list, |
| local_wheel_exclusion_list): |
| for wheel in dist_wheels: |
| bn = wheel.basename |
| if not bn.endswith(".whl") or (bn.find(py_ver_marker) < 0 and bn.find(py_major_ver_marker) < 0): |
| continue |
| if not _basic_wildcard_match(bn, local_wheel_inclusion_list, True, False): |
| continue |
| if not _basic_wildcard_match(bn, local_wheel_exclusion_list, False, True): |
| continue |
| |
| name_components = bn.split("-") |
| package_name = name_components[0] |
| for name_component in name_components[1:]: |
| if name_component[0].isdigit(): |
| break |
| package_name += "-" + name_component |
| |
| latest_wheel = wheels.get(package_name, None) |
| |
| if not latest_wheel or latest_wheel.basename < wheel.basename: |
| wheels[package_name] = wheel |
| |
| def _basic_wildcard_match(name, patterns, expected_match_result, match_all): |
| match = False |
| for pattern in patterns: |
| match = False |
| if pattern.startswith("*") and pattern.endswith("*"): |
| match = name.find(pattern[1:-1]) >= 0 |
| elif pattern.startswith("*"): |
| match = name.endswith(pattern[1:]) |
| elif pattern.endswith("*"): |
| match = name.startswith(pattern[:-1]) |
| else: |
| match = name == pattern |
| |
| if match_all: |
| if match != expected_match_result: |
| return False |
| elif match == expected_match_result: |
| return True |
| |
| return match == expected_match_result |
| |
| def _custom_python_interpreter_impl(ctx): |
| version = ctx.attr.version |
| version_variant = ctx.attr.version_variant |
| strip_prefix = ctx.attr.strip_prefix.format( |
| version = version, |
| version_variant = version_variant, |
| ) |
| urls = [url.format(version = version, version_variant = version_variant) for url in ctx.attr.urls] |
| binary_name = ctx.attr.binary_name |
| if not binary_name: |
| ver_chunks = version.split(".") |
| binary_name = "python%s.%s" % (ver_chunks[0], ver_chunks[1]) |
| |
| install_dir = "{name}-{version}".format(name = ctx.attr.name, version = version) |
| _exec_and_check(ctx, ["mkdir", install_dir]) |
| install_path = ctx.path(install_dir) |
| srcs_dir = "srcs" |
| ctx.download_and_extract( |
| url = urls, |
| stripPrefix = strip_prefix, |
| output = srcs_dir, |
| ) |
| |
| configure_params = list(ctx.attr.configure_params) |
| if "CC" in ctx.os.environ: |
| configure_params.append("CC={}".format(ctx.os.environ["CC"])) |
| if "CXX" in ctx.os.environ: |
| configure_params.append("CXX={}".format(ctx.os.environ["CXX"])) |
| |
| configure_params.append("--prefix=%s" % install_path.realpath) |
| _exec_and_check( |
| ctx, |
| ["./configure"] + configure_params, |
| working_directory = srcs_dir, |
| quiet = False, |
| ) |
| res = _exec_and_check(ctx, ["nproc"]) |
| cores = 12 if res.return_code != 0 else max(1, int(res.stdout.strip()) - 1) |
| _exec_and_check(ctx, ["make", "-j%s" % cores], working_directory = srcs_dir) |
| _exec_and_check(ctx, ["make", "altinstall"], working_directory = srcs_dir) |
| _exec_and_check(ctx, ["ln", "-s", binary_name, "python3"], working_directory = install_dir + "/bin") |
| tar = "{install_dir}.tgz".format(install_dir = install_dir) |
| _exec_and_check(ctx, ["tar", "czpf", tar, install_dir]) |
| _exec_and_check(ctx, ["rm", "-rf", srcs_dir]) |
| res = _exec_and_check(ctx, ["sha256sum", tar]) |
| |
| sha256 = res.stdout.split(" ")[0].strip() |
| tar_path = ctx.path(tar) |
| |
| example = """\n\n |
| To use newly built Python interpreter add the following code snippet RIGHT AFTER |
| python_init_toolchains() in your WORKSPACE file. The code sample should work as |
| is but it may need some tuning, if you have special requirements. |
| |
| ``` |
| load("@rules_python//python:repositories.bzl", "python_register_toolchains") |
| python_register_toolchains( |
| name = "python", |
| # By default assume the interpreter is on the local file system, replace |
| # with proper URL if it is not the case. |
| base_url = "file://", |
| ignore_root_user_error = True, |
| python_version = "{version}", |
| tool_versions = {{ |
| "{version}": {{ |
| # Path to .tar.gz with Python binary. By default it points to .tgz |
| # file in cache where it was built originally; replace with proper |
| # file location, if you moved it somewhere else. |
| "url": "{tar_path}", |
| "sha256": {{ |
| # By default we assume Linux x86_64 architecture, eplace with |
| # proper architecture if you were building on a different platform. |
| "x86_64-unknown-linux-gnu": "{sha256}", |
| }}, |
| "strip_prefix": "{install_dir}", |
| }}, |
| }}, |
| ) |
| ``` |
| \n\n""".format(version = version, tar_path = tar_path, sha256 = sha256, install_dir = install_dir) |
| |
| instructions = "INSTRUCTIONS-{version}.md".format(version = version) |
| ctx.file(instructions + ".tmpl", example, executable = False) |
| ctx.file( |
| "BUILD.bazel", |
| """ |
| genrule( |
| name = "{name}", |
| srcs = ["{tar}", "{instructions}.tmpl"], |
| outs = ["{install_dir}.tar.gz", "{instructions}"], |
| cmd = "cp $(location {tar}) $(location {install_dir}.tar.gz); cp $(location {instructions}.tmpl) $(location {instructions})", |
| visibility = ["//visibility:public"], |
| ) |
| """.format( |
| name = ctx.attr.name, |
| tar = tar, |
| install_dir = install_dir, |
| instructions = instructions, |
| ), |
| executable = False, |
| ) |
| |
| print(example) # buildifier: disable=print |
| |
| custom_python_interpreter = repository_rule( |
| implementation = _custom_python_interpreter_impl, |
| attrs = { |
| "urls": attr.string_list(), |
| "strip_prefix": attr.string(), |
| "binary_name": attr.string(mandatory = False), |
| "version": attr.string(), |
| "version_variant": attr.string(), |
| "configure_params": attr.string_list( |
| mandatory = False, |
| default = ["--enable-optimizations"], |
| ), |
| }, |
| ) |
| |
| def _exec_and_check(ctx, command, fail_on_error = True, quiet = False, **kwargs): |
| res = ctx.execute(command, quiet = quiet, **kwargs) |
| if fail_on_error and res.return_code != 0: |
| fail(""" |
| Failed to execute command: `{command}` |
| Exit Code: {code} |
| STDERR: {stderr} |
| """.format( |
| command = command, |
| code = res.return_code, |
| stderr = res.stderr, |
| )) |
| return res |
