1# Copyright 2024 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""This file contains repository rules and macros to support toolchain registration. 16""" 17 18load("//python:versions.bzl", "PLATFORMS") 19load(":auth.bzl", "get_auth") 20load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") 21load(":text_util.bzl", "render") 22 23STANDALONE_INTERPRETER_FILENAME = "STANDALONE_INTERPRETER" 24 25def is_standalone_interpreter(rctx, python_interpreter_path, *, logger = None): 26 """Query a python interpreter target for whether or not it's a rules_rust provided toolchain 27 28 Args: 29 rctx: {type}`repository_ctx` The repository rule's context object. 30 python_interpreter_path: {type}`path` A path representing the interpreter. 31 logger: Optional logger to use for operations. 32 33 Returns: 34 {type}`bool` Whether or not the target is from a rules_python generated toolchain. 35 """ 36 37 # Only update the location when using a hermetic toolchain. 38 if not python_interpreter_path: 39 return False 40 41 # This is a rules_python provided toolchain. 42 return repo_utils.execute_unchecked( 43 rctx, 44 op = "IsStandaloneInterpreter", 45 arguments = [ 46 "ls", 47 "{}/{}".format( 48 python_interpreter_path.dirname, 49 STANDALONE_INTERPRETER_FILENAME, 50 ), 51 ], 52 logger = logger, 53 ).return_code == 0 54 55def _python_repository_impl(rctx): 56 if rctx.attr.distutils and rctx.attr.distutils_content: 57 fail("Only one of (distutils, distutils_content) should be set.") 58 if bool(rctx.attr.url) == bool(rctx.attr.urls): 59 fail("Exactly one of (url, urls) must be set.") 60 61 logger = repo_utils.logger(rctx) 62 63 platform = rctx.attr.platform 64 python_version = rctx.attr.python_version 65 python_version_info = python_version.split(".") 66 python_short_version = "{0}.{1}".format(*python_version_info) 67 release_filename = rctx.attr.release_filename 68 urls = rctx.attr.urls or [rctx.attr.url] 69 auth = get_auth(rctx, urls) 70 71 if release_filename.endswith(".zst"): 72 rctx.download( 73 url = urls, 74 sha256 = rctx.attr.sha256, 75 output = release_filename, 76 auth = auth, 77 ) 78 unzstd = rctx.which("unzstd") 79 if not unzstd: 80 url = rctx.attr.zstd_url.format(version = rctx.attr.zstd_version) 81 rctx.download_and_extract( 82 url = url, 83 sha256 = rctx.attr.zstd_sha256, 84 auth = auth, 85 ) 86 working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version) 87 88 repo_utils.execute_checked( 89 rctx, 90 op = "python_repository.MakeZstd", 91 arguments = [ 92 repo_utils.which_checked(rctx, "make"), 93 "--jobs=4", 94 ], 95 timeout = 600, 96 quiet = True, 97 working_directory = working_directory, 98 logger = logger, 99 ) 100 zstd = "{working_directory}/zstd".format(working_directory = working_directory) 101 unzstd = "./unzstd" 102 rctx.symlink(zstd, unzstd) 103 104 repo_utils.execute_checked( 105 rctx, 106 op = "python_repository.ExtractRuntime", 107 arguments = [ 108 repo_utils.which_checked(rctx, "tar"), 109 "--extract", 110 "--strip-components=2", 111 "--use-compress-program={unzstd}".format(unzstd = unzstd), 112 "--file={}".format(release_filename), 113 ], 114 logger = logger, 115 ) 116 else: 117 rctx.download_and_extract( 118 url = urls, 119 sha256 = rctx.attr.sha256, 120 stripPrefix = rctx.attr.strip_prefix, 121 auth = auth, 122 ) 123 124 patches = rctx.attr.patches 125 if patches: 126 for patch in patches: 127 rctx.patch(patch, strip = rctx.attr.patch_strip) 128 129 # Write distutils.cfg to the Python installation. 130 if "windows" in platform: 131 distutils_path = "Lib/distutils/distutils.cfg" 132 else: 133 distutils_path = "lib/python{}/distutils/distutils.cfg".format(python_short_version) 134 if rctx.attr.distutils: 135 rctx.file(distutils_path, rctx.read(rctx.attr.distutils)) 136 elif rctx.attr.distutils_content: 137 rctx.file(distutils_path, rctx.attr.distutils_content) 138 139 if "darwin" in platform and "osx" == repo_utils.get_platforms_os_name(rctx): 140 # Fix up the Python distribution's LC_ID_DYLIB field. 141 # It points to a build directory local to the GitHub Actions 142 # host machine used in the Python standalone build, which causes 143 # dyld lookup errors. To fix, set the full path to the dylib as 144 # it appears in the Bazel workspace as its LC_ID_DYLIB using 145 # the `install_name_tool` bundled with macOS. 146 dylib = "lib/libpython{}.dylib".format(python_short_version) 147 full_dylib_path = rctx.path(dylib) 148 repo_utils.execute_checked( 149 rctx, 150 op = "python_repository.FixUpDyldIdPath", 151 arguments = [repo_utils.which_checked(rctx, "install_name_tool"), "-id", full_dylib_path, dylib], 152 logger = logger, 153 ) 154 155 # Make the Python installation read-only. This is to prevent issues due to 156 # pycs being generated at runtime: 157 # * The pycs are not deterministic (they contain timestamps) 158 # * Multiple processes trying to write the same pycs can result in errors. 159 if not rctx.attr.ignore_root_user_error: 160 if "windows" not in platform: 161 lib_dir = "lib" if "windows" not in platform else "Lib" 162 163 repo_utils.execute_checked( 164 rctx, 165 op = "python_repository.MakeReadOnly", 166 arguments = [repo_utils.which_checked(rctx, "chmod"), "-R", "ugo-w", lib_dir], 167 logger = logger, 168 ) 169 exec_result = repo_utils.execute_unchecked( 170 rctx, 171 op = "python_repository.TestReadOnly", 172 arguments = [repo_utils.which_checked(rctx, "touch"), "{}/.test".format(lib_dir)], 173 logger = logger, 174 ) 175 176 # The issue with running as root is the installation is no longer 177 # read-only, so the problems due to pyc can resurface. 178 if exec_result.return_code == 0: 179 stdout = repo_utils.execute_checked_stdout( 180 rctx, 181 op = "python_repository.GetUserId", 182 arguments = [repo_utils.which_checked(rctx, "id"), "-u"], 183 logger = logger, 184 ) 185 uid = int(stdout.strip()) 186 if uid == 0: 187 fail("The current user is root, please run as non-root when using the hermetic Python interpreter. See https://github.com/bazelbuild/rules_python/pull/713.") 188 else: 189 fail("The current user has CAP_DAC_OVERRIDE set, please drop this capability when using the hermetic Python interpreter. See https://github.com/bazelbuild/rules_python/pull/713.") 190 191 python_bin = "python.exe" if ("windows" in platform) else "bin/python3" 192 193 if "linux" in platform: 194 # Workaround around https://github.com/indygreg/python-build-standalone/issues/231 195 for url in urls: 196 head_and_release, _, _ = url.rpartition("/") 197 _, _, release = head_and_release.rpartition("/") 198 if not release.isdigit(): 199 # Maybe this is some custom toolchain, so skip this 200 break 201 202 if int(release) >= 20240224: 203 # Starting with this release the Linux toolchains have infinite symlink loop 204 # on host platforms that are not Linux. Delete the files no 205 # matter the host platform so that the cross-built artifacts 206 # are the same irrespective of the host platform we are 207 # building on. 208 # 209 # Link to the first affected release: 210 # https://github.com/indygreg/python-build-standalone/releases/tag/20240224 211 rctx.delete("share/terminfo") 212 break 213 214 glob_include = [] 215 glob_exclude = [] 216 if rctx.attr.ignore_root_user_error or "windows" in platform: 217 glob_exclude += [ 218 # These pycache files are created on first use of the associated python files. 219 # Exclude them from the glob because otherwise between the first time and second time a python toolchain is used," 220 # the definition of this filegroup will change, and depending rules will get invalidated." 221 # See https://github.com/bazelbuild/rules_python/issues/1008 for unconditionally adding these to toolchains so we can stop ignoring them." 222 "**/__pycache__/*.pyc", 223 "**/__pycache__/*.pyo", 224 ] 225 226 if "windows" in platform: 227 glob_include += [ 228 "*.exe", 229 "*.dll", 230 "DLLs/**", 231 "Lib/**", 232 "Scripts/**", 233 "tcl/**", 234 ] 235 else: 236 glob_include.append( 237 "lib/**", 238 ) 239 240 if "windows" in platform: 241 coverage_tool = None 242 else: 243 coverage_tool = rctx.attr.coverage_tool 244 245 build_content = """\ 246# Generated by python/private/python_repositories.bzl 247 248load("@rules_python//python/private:hermetic_runtime_repo_setup.bzl", "define_hermetic_runtime_toolchain_impl") 249 250package(default_visibility = ["//visibility:public"]) 251 252define_hermetic_runtime_toolchain_impl( 253 name = "define_runtime", 254 extra_files_glob_include = {extra_files_glob_include}, 255 extra_files_glob_exclude = {extra_files_glob_exclude}, 256 python_version = {python_version}, 257 python_bin = {python_bin}, 258 coverage_tool = {coverage_tool}, 259) 260""".format( 261 extra_files_glob_exclude = render.list(glob_exclude), 262 extra_files_glob_include = render.list(glob_include), 263 python_bin = render.str(python_bin), 264 python_version = render.str(rctx.attr.python_version), 265 coverage_tool = render.str(coverage_tool), 266 ) 267 rctx.delete("python") 268 rctx.symlink(python_bin, "python") 269 rctx.file(STANDALONE_INTERPRETER_FILENAME, "# File intentionally left blank. Indicates that this is an interpreter repo created by rules_python.") 270 rctx.file("BUILD.bazel", build_content) 271 272 attrs = { 273 "auth_patterns": rctx.attr.auth_patterns, 274 "coverage_tool": rctx.attr.coverage_tool, 275 "distutils": rctx.attr.distutils, 276 "distutils_content": rctx.attr.distutils_content, 277 "ignore_root_user_error": rctx.attr.ignore_root_user_error, 278 "name": rctx.attr.name, 279 "netrc": rctx.attr.netrc, 280 "patch_strip": rctx.attr.patch_strip, 281 "patches": rctx.attr.patches, 282 "platform": platform, 283 "python_version": python_version, 284 "release_filename": release_filename, 285 "sha256": rctx.attr.sha256, 286 "strip_prefix": rctx.attr.strip_prefix, 287 } 288 289 if rctx.attr.url: 290 attrs["url"] = rctx.attr.url 291 else: 292 attrs["urls"] = urls 293 294 return attrs 295 296python_repository = repository_rule( 297 _python_repository_impl, 298 doc = "Fetches the external tools needed for the Python toolchain.", 299 attrs = { 300 "auth_patterns": attr.string_dict( 301 doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", 302 ), 303 "coverage_tool": attr.string( 304 doc = """ 305This is a target to use for collecting code coverage information from {rule}`py_binary` 306and {rule}`py_test` targets. 307 308The target is accepted as a string by the python_repository and evaluated within 309the context of the toolchain repository. 310 311For more information see {attr}`py_runtime.coverage_tool`. 312""", 313 ), 314 "distutils": attr.label( 315 allow_single_file = True, 316 doc = "A distutils.cfg file to be included in the Python installation. " + 317 "Either distutils or distutils_content can be specified, but not both.", 318 mandatory = False, 319 ), 320 "distutils_content": attr.string( 321 doc = "A distutils.cfg file content to be included in the Python installation. " + 322 "Either distutils or distutils_content can be specified, but not both.", 323 mandatory = False, 324 ), 325 "ignore_root_user_error": attr.bool( 326 default = False, 327 doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", 328 mandatory = False, 329 ), 330 "netrc": attr.string( 331 doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", 332 ), 333 "patch_strip": attr.int( 334 doc = """ 335Same as the --strip argument of Unix patch. 336 337:::{note} 338In the future the default value will be set to `0`, to mimic the well known 339function defaults (e.g. `single_version_override` for `MODULE.bazel` files. 340::: 341 342:::{versionadded} 0.36.0 343::: 344""", 345 default = 1, 346 mandatory = False, 347 ), 348 "patches": attr.label_list( 349 doc = "A list of patch files to apply to the unpacked interpreter", 350 mandatory = False, 351 ), 352 "platform": attr.string( 353 doc = "The platform name for the Python interpreter tarball.", 354 mandatory = True, 355 values = PLATFORMS.keys(), 356 ), 357 "python_version": attr.string( 358 doc = "The Python version.", 359 mandatory = True, 360 ), 361 "release_filename": attr.string( 362 doc = "The filename of the interpreter to be downloaded", 363 mandatory = True, 364 ), 365 "sha256": attr.string( 366 doc = "The SHA256 integrity hash for the Python interpreter tarball.", 367 mandatory = True, 368 ), 369 "strip_prefix": attr.string( 370 doc = "A directory prefix to strip from the extracted files.", 371 ), 372 "url": attr.string( 373 doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", 374 ), 375 "urls": attr.string_list( 376 doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", 377 ), 378 "zstd_sha256": attr.string( 379 default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0", 380 ), 381 "zstd_url": attr.string( 382 default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz", 383 ), 384 "zstd_version": attr.string( 385 default = "1.5.2", 386 ), 387 "_rule_name": attr.string(default = "python_repository"), 388 }, 389 environ = [REPO_DEBUG_ENV_VAR], 390) 391