1# Copyright 2022 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Implementation for Bazel Python executable.""" 15 16load("@bazel_skylib//lib:dicts.bzl", "dicts") 17load("@bazel_skylib//lib:paths.bzl", "paths") 18load("//python/private:flags.bzl", "BootstrapImplFlag") 19load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") 20load(":attributes_bazel.bzl", "IMPORTS_ATTRS") 21load( 22 ":common.bzl", 23 "create_binary_semantics_struct", 24 "create_cc_details_struct", 25 "create_executable_result_struct", 26 "target_platform_has_any_constraint", 27 "union_attrs", 28) 29load(":common_bazel.bzl", "collect_cc_info", "get_imports", "maybe_precompile") 30load(":providers.bzl", "DEFAULT_STUB_SHEBANG") 31load( 32 ":py_executable.bzl", 33 "create_base_executable_rule", 34 "py_executable_base_impl", 35) 36load(":py_internal.bzl", "py_internal") 37 38_py_builtins = py_internal 39_EXTERNAL_PATH_PREFIX = "external" 40_ZIP_RUNFILES_DIRECTORY_NAME = "runfiles" 41 42BAZEL_EXECUTABLE_ATTRS = union_attrs( 43 IMPORTS_ATTRS, 44 { 45 "legacy_create_init": attr.int( 46 default = -1, 47 values = [-1, 0, 1], 48 doc = """\ 49Whether to implicitly create empty `__init__.py` files in the runfiles tree. 50These are created in every directory containing Python source code or shared 51libraries, and every parent directory of those directories, excluding the repo 52root directory. The default, `-1` (auto), means true unless 53`--incompatible_default_to_explicit_init_py` is used. If false, the user is 54responsible for creating (possibly empty) `__init__.py` files and adding them to 55the `srcs` of Python targets as required. 56 """, 57 ), 58 "_bootstrap_template": attr.label( 59 allow_single_file = True, 60 default = "@bazel_tools//tools/python:python_bootstrap_template.txt", 61 ), 62 "_launcher": attr.label( 63 cfg = "target", 64 # NOTE: This is an executable, but is only used for Windows. It 65 # can't have executable=True because the backing target is an 66 # empty target for other platforms. 67 default = "//tools/launcher:launcher", 68 ), 69 "_py_interpreter": attr.label( 70 # The configuration_field args are validated when called; 71 # we use the precense of py_internal to indicate this Bazel 72 # build has that fragment and name. 73 default = configuration_field( 74 fragment = "bazel_py", 75 name = "python_top", 76 ) if py_internal else None, 77 ), 78 # TODO: This appears to be vestigial. It's only added because 79 # GraphlessQueryTest.testLabelsOperator relies on it to test for 80 # query behavior of implicit dependencies. 81 "_py_toolchain_type": attr.label( 82 default = TARGET_TOOLCHAIN_TYPE, 83 ), 84 "_windows_launcher_maker": attr.label( 85 default = "@bazel_tools//tools/launcher:launcher_maker", 86 cfg = "exec", 87 executable = True, 88 ), 89 "_zipper": attr.label( 90 cfg = "exec", 91 executable = True, 92 default = "@bazel_tools//tools/zip:zipper", 93 ), 94 }, 95) 96 97def create_executable_rule(*, attrs, **kwargs): 98 return create_base_executable_rule( 99 attrs = dicts.add(BAZEL_EXECUTABLE_ATTRS, attrs), 100 fragments = ["py", "bazel_py"], 101 **kwargs 102 ) 103 104def py_executable_bazel_impl(ctx, *, is_test, inherited_environment): 105 """Common code for executables for Bazel.""" 106 return py_executable_base_impl( 107 ctx = ctx, 108 semantics = create_binary_semantics_bazel(), 109 is_test = is_test, 110 inherited_environment = inherited_environment, 111 ) 112 113def create_binary_semantics_bazel(): 114 return create_binary_semantics_struct( 115 # keep-sorted start 116 create_executable = _create_executable, 117 get_cc_details_for_binary = _get_cc_details_for_binary, 118 get_central_uncachable_version_file = lambda ctx: None, 119 get_coverage_deps = _get_coverage_deps, 120 get_debugger_deps = _get_debugger_deps, 121 get_extra_common_runfiles_for_binary = lambda ctx: ctx.runfiles(), 122 get_extra_providers = _get_extra_providers, 123 get_extra_write_build_data_env = lambda ctx: {}, 124 get_imports = get_imports, 125 get_interpreter_path = _get_interpreter_path, 126 get_native_deps_dso_name = _get_native_deps_dso_name, 127 get_native_deps_user_link_flags = _get_native_deps_user_link_flags, 128 get_stamp_flag = _get_stamp_flag, 129 maybe_precompile = maybe_precompile, 130 should_build_native_deps_dso = lambda ctx: False, 131 should_create_init_files = _should_create_init_files, 132 should_include_build_data = lambda ctx: False, 133 # keep-sorted end 134 ) 135 136def _get_coverage_deps(ctx, runtime_details): 137 _ = ctx, runtime_details # @unused 138 return [] 139 140def _get_debugger_deps(ctx, runtime_details): 141 _ = ctx, runtime_details # @unused 142 return [] 143 144def _get_extra_providers(ctx, main_py, runtime_details): 145 _ = ctx, main_py, runtime_details # @unused 146 return [] 147 148def _get_stamp_flag(ctx): 149 # NOTE: Undocumented API; private to builtins 150 return ctx.configuration.stamp_binaries 151 152def _should_create_init_files(ctx): 153 if ctx.attr.legacy_create_init == -1: 154 return not ctx.fragments.py.default_to_explicit_init_py 155 else: 156 return bool(ctx.attr.legacy_create_init) 157 158def _create_executable( 159 ctx, 160 *, 161 executable, 162 main_py, 163 imports, 164 is_test, 165 runtime_details, 166 cc_details, 167 native_deps_details, 168 runfiles_details): 169 _ = is_test, cc_details, native_deps_details # @unused 170 171 is_windows = target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints) 172 173 if is_windows: 174 if not executable.extension == "exe": 175 fail("Should not happen: somehow we are generating a non-.exe file on windows") 176 base_executable_name = executable.basename[0:-4] 177 else: 178 base_executable_name = executable.basename 179 180 # The check for stage2_bootstrap_template is to support legacy 181 # BuiltinPyRuntimeInfo providers, which is likely to come from 182 # @bazel_tools//tools/python:autodetecting_toolchain, the toolchain used 183 # for workspace builds when no rules_python toolchain is configured. 184 if (BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT and 185 runtime_details.effective_runtime and 186 hasattr(runtime_details.effective_runtime, "stage2_bootstrap_template")): 187 stage2_bootstrap = _create_stage2_bootstrap( 188 ctx, 189 output_prefix = base_executable_name, 190 output_sibling = executable, 191 main_py = main_py, 192 imports = imports, 193 runtime_details = runtime_details, 194 ) 195 extra_runfiles = ctx.runfiles([stage2_bootstrap]) 196 zip_main = _create_zip_main( 197 ctx, 198 stage2_bootstrap = stage2_bootstrap, 199 runtime_details = runtime_details, 200 ) 201 else: 202 stage2_bootstrap = None 203 extra_runfiles = ctx.runfiles() 204 zip_main = ctx.actions.declare_file(base_executable_name + ".temp", sibling = executable) 205 _create_stage1_bootstrap( 206 ctx, 207 output = zip_main, 208 main_py = main_py, 209 imports = imports, 210 is_for_zip = True, 211 runtime_details = runtime_details, 212 ) 213 214 zip_file = ctx.actions.declare_file(base_executable_name + ".zip", sibling = executable) 215 _create_zip_file( 216 ctx, 217 output = zip_file, 218 original_nonzip_executable = executable, 219 zip_main = zip_main, 220 runfiles = runfiles_details.default_runfiles.merge(extra_runfiles), 221 ) 222 223 extra_files_to_build = [] 224 225 # NOTE: --build_python_zip defaults to true on Windows 226 build_zip_enabled = ctx.fragments.py.build_python_zip 227 228 # When --build_python_zip is enabled, then the zip file becomes 229 # one of the default outputs. 230 if build_zip_enabled: 231 extra_files_to_build.append(zip_file) 232 233 # The logic here is a bit convoluted. Essentially, there are 3 types of 234 # executables produced: 235 # 1. (non-Windows) A bootstrap template based program. 236 # 2. (non-Windows) A self-executable zip file of a bootstrap template based program. 237 # 3. (Windows) A native Windows executable that finds and launches 238 # the actual underlying Bazel program (one of the above). Note that 239 # it implicitly assumes one of the above is located next to it, and 240 # that --build_python_zip defaults to true for Windows. 241 242 should_create_executable_zip = False 243 bootstrap_output = None 244 if not is_windows: 245 if build_zip_enabled: 246 should_create_executable_zip = True 247 else: 248 bootstrap_output = executable 249 else: 250 _create_windows_exe_launcher( 251 ctx, 252 output = executable, 253 use_zip_file = build_zip_enabled, 254 python_binary_path = runtime_details.executable_interpreter_path, 255 ) 256 if not build_zip_enabled: 257 # On Windows, the main executable has an "exe" extension, so 258 # here we re-use the un-extensioned name for the bootstrap output. 259 bootstrap_output = ctx.actions.declare_file(base_executable_name) 260 261 # The launcher looks for the non-zip executable next to 262 # itself, so add it to the default outputs. 263 extra_files_to_build.append(bootstrap_output) 264 265 if should_create_executable_zip: 266 if bootstrap_output != None: 267 fail("Should not occur: bootstrap_output should not be used " + 268 "when creating an executable zip") 269 _create_executable_zip_file( 270 ctx, 271 output = executable, 272 zip_file = zip_file, 273 stage2_bootstrap = stage2_bootstrap, 274 runtime_details = runtime_details, 275 ) 276 elif bootstrap_output: 277 _create_stage1_bootstrap( 278 ctx, 279 output = bootstrap_output, 280 stage2_bootstrap = stage2_bootstrap, 281 runtime_details = runtime_details, 282 is_for_zip = False, 283 imports = imports, 284 main_py = main_py, 285 ) 286 else: 287 # Otherwise, this should be the Windows case of launcher + zip. 288 # Double check this just to make sure. 289 if not is_windows or not build_zip_enabled: 290 fail(("Should not occur: The non-executable-zip and " + 291 "non-bootstrap-template case should have windows and zip " + 292 "both true, but got " + 293 "is_windows={is_windows} " + 294 "build_zip_enabled={build_zip_enabled}").format( 295 is_windows = is_windows, 296 build_zip_enabled = build_zip_enabled, 297 )) 298 299 return create_executable_result_struct( 300 extra_files_to_build = depset(extra_files_to_build), 301 output_groups = {"python_zip_file": depset([zip_file])}, 302 extra_runfiles = extra_runfiles, 303 ) 304 305def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details): 306 # The location of this file doesn't really matter. It's added to 307 # the zip file as the top-level __main__.py file and not included 308 # elsewhere. 309 output = ctx.actions.declare_file(ctx.label.name + "_zip__main__.py") 310 ctx.actions.expand_template( 311 template = runtime_details.effective_runtime.zip_main_template, 312 output = output, 313 substitutions = { 314 "%python_binary%": runtime_details.executable_interpreter_path, 315 "%stage2_bootstrap%": "{}/{}".format( 316 ctx.workspace_name, 317 stage2_bootstrap.short_path, 318 ), 319 "%workspace_name%": ctx.workspace_name, 320 }, 321 ) 322 return output 323 324def _create_stage2_bootstrap( 325 ctx, 326 *, 327 output_prefix, 328 output_sibling, 329 main_py, 330 imports, 331 runtime_details): 332 output = ctx.actions.declare_file( 333 # Prepend with underscore to prevent pytest from trying to 334 # process the bootstrap for files starting with `test_` 335 "_{}_stage2_bootstrap.py".format(output_prefix), 336 sibling = output_sibling, 337 ) 338 runtime = runtime_details.effective_runtime 339 if (ctx.configuration.coverage_enabled and 340 runtime and 341 runtime.coverage_tool): 342 coverage_tool_runfiles_path = "{}/{}".format( 343 ctx.workspace_name, 344 runtime.coverage_tool.short_path, 345 ) 346 else: 347 coverage_tool_runfiles_path = "" 348 349 template = runtime.stage2_bootstrap_template 350 351 ctx.actions.expand_template( 352 template = template, 353 output = output, 354 substitutions = { 355 "%coverage_tool%": coverage_tool_runfiles_path, 356 "%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False", 357 "%imports%": ":".join(imports.to_list()), 358 "%main%": "{}/{}".format(ctx.workspace_name, main_py.short_path), 359 "%target%": str(ctx.label), 360 "%workspace_name%": ctx.workspace_name, 361 }, 362 is_executable = True, 363 ) 364 return output 365 366def _create_stage1_bootstrap( 367 ctx, 368 *, 369 output, 370 main_py = None, 371 stage2_bootstrap = None, 372 imports = None, 373 is_for_zip, 374 runtime_details): 375 runtime = runtime_details.effective_runtime 376 377 subs = { 378 "%is_zipfile%": "1" if is_for_zip else "0", 379 "%python_binary%": runtime_details.executable_interpreter_path, 380 "%target%": str(ctx.label), 381 "%workspace_name%": ctx.workspace_name, 382 } 383 384 if stage2_bootstrap: 385 subs["%stage2_bootstrap%"] = "{}/{}".format( 386 ctx.workspace_name, 387 stage2_bootstrap.short_path, 388 ) 389 template = runtime.bootstrap_template 390 subs["%shebang%"] = runtime.stub_shebang 391 else: 392 if (ctx.configuration.coverage_enabled and 393 runtime and 394 runtime.coverage_tool): 395 coverage_tool_runfiles_path = "{}/{}".format( 396 ctx.workspace_name, 397 runtime.coverage_tool.short_path, 398 ) 399 else: 400 coverage_tool_runfiles_path = "" 401 if runtime: 402 subs["%shebang%"] = runtime.stub_shebang 403 template = runtime.bootstrap_template 404 else: 405 subs["%shebang%"] = DEFAULT_STUB_SHEBANG 406 template = ctx.file._bootstrap_template 407 408 subs["%coverage_tool%"] = coverage_tool_runfiles_path 409 subs["%import_all%"] = ("True" if ctx.fragments.bazel_py.python_import_all_repositories else "False") 410 subs["%imports%"] = ":".join(imports.to_list()) 411 subs["%main%"] = "{}/{}".format(ctx.workspace_name, main_py.short_path) 412 413 ctx.actions.expand_template( 414 template = template, 415 output = output, 416 substitutions = subs, 417 ) 418 419def _create_windows_exe_launcher( 420 ctx, 421 *, 422 output, 423 python_binary_path, 424 use_zip_file): 425 launch_info = ctx.actions.args() 426 launch_info.use_param_file("%s", use_always = True) 427 launch_info.set_param_file_format("multiline") 428 launch_info.add("binary_type=Python") 429 launch_info.add(ctx.workspace_name, format = "workspace_name=%s") 430 launch_info.add( 431 "1" if py_internal.runfiles_enabled(ctx) else "0", 432 format = "symlink_runfiles_enabled=%s", 433 ) 434 launch_info.add(python_binary_path, format = "python_bin_path=%s") 435 launch_info.add("1" if use_zip_file else "0", format = "use_zip_file=%s") 436 437 launcher = ctx.attr._launcher[DefaultInfo].files_to_run.executable 438 ctx.actions.run( 439 executable = ctx.executable._windows_launcher_maker, 440 arguments = [launcher.path, launch_info, output.path], 441 inputs = [launcher], 442 outputs = [output], 443 mnemonic = "PyBuildLauncher", 444 progress_message = "Creating launcher for %{label}", 445 # Needed to inherit PATH when using non-MSVC compilers like MinGW 446 use_default_shell_env = True, 447 ) 448 449def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfiles): 450 workspace_name = ctx.workspace_name 451 legacy_external_runfiles = _py_builtins.get_legacy_external_runfiles(ctx) 452 453 manifest = ctx.actions.args() 454 manifest.use_param_file("@%s", use_always = True) 455 manifest.set_param_file_format("multiline") 456 457 manifest.add("__main__.py={}".format(zip_main.path)) 458 manifest.add("__init__.py=") 459 manifest.add( 460 "{}=".format( 461 _get_zip_runfiles_path("__init__.py", workspace_name, legacy_external_runfiles), 462 ), 463 ) 464 for path in runfiles.empty_filenames.to_list(): 465 manifest.add("{}=".format(_get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles))) 466 467 def map_zip_runfiles(file): 468 if file != original_nonzip_executable and file != output: 469 return "{}={}".format( 470 _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles), 471 file.path, 472 ) 473 else: 474 return None 475 476 manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) 477 478 inputs = [zip_main] 479 if _py_builtins.is_bzlmod_enabled(ctx): 480 zip_repo_mapping_manifest = ctx.actions.declare_file( 481 output.basename + ".repo_mapping", 482 sibling = output, 483 ) 484 _py_builtins.create_repo_mapping_manifest( 485 ctx = ctx, 486 runfiles = runfiles, 487 output = zip_repo_mapping_manifest, 488 ) 489 manifest.add("{}/_repo_mapping={}".format( 490 _ZIP_RUNFILES_DIRECTORY_NAME, 491 zip_repo_mapping_manifest.path, 492 )) 493 inputs.append(zip_repo_mapping_manifest) 494 495 for artifact in runfiles.files.to_list(): 496 # Don't include the original executable because it isn't used by the 497 # zip file, so no need to build it for the action. 498 # Don't include the zipfile itself because it's an output. 499 if artifact != original_nonzip_executable and artifact != output: 500 inputs.append(artifact) 501 502 zip_cli_args = ctx.actions.args() 503 zip_cli_args.add("cC") 504 zip_cli_args.add(output) 505 506 ctx.actions.run( 507 executable = ctx.executable._zipper, 508 arguments = [zip_cli_args, manifest], 509 inputs = depset(inputs), 510 outputs = [output], 511 use_default_shell_env = True, 512 mnemonic = "PythonZipper", 513 progress_message = "Building Python zip: %{label}", 514 ) 515 516def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): 517 if legacy_external_runfiles and path.startswith(_EXTERNAL_PATH_PREFIX): 518 zip_runfiles_path = paths.relativize(path, _EXTERNAL_PATH_PREFIX) 519 else: 520 # NOTE: External runfiles (artifacts in other repos) will have a leading 521 # path component of "../" so that they refer outside the main workspace 522 # directory and into the runfiles root. By normalizing, we simplify e.g. 523 # "workspace/../foo/bar" to simply "foo/bar". 524 zip_runfiles_path = paths.normalize("{}/{}".format(workspace_name, path)) 525 return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path) 526 527def _create_executable_zip_file(ctx, *, output, zip_file, stage2_bootstrap, runtime_details): 528 prelude = ctx.actions.declare_file( 529 "{}_zip_prelude.sh".format(output.basename), 530 sibling = output, 531 ) 532 if stage2_bootstrap: 533 _create_stage1_bootstrap( 534 ctx, 535 output = prelude, 536 stage2_bootstrap = stage2_bootstrap, 537 runtime_details = runtime_details, 538 is_for_zip = True, 539 ) 540 else: 541 ctx.actions.write(prelude, "#!/usr/bin/env python3\n") 542 543 ctx.actions.run_shell( 544 command = "cat {prelude} {zip} > {output}".format( 545 prelude = prelude.path, 546 zip = zip_file.path, 547 output = output.path, 548 ), 549 inputs = [prelude, zip_file], 550 outputs = [output], 551 use_default_shell_env = True, 552 mnemonic = "PyBuildExecutableZip", 553 progress_message = "Build Python zip executable: %{label}", 554 ) 555 556def _get_cc_details_for_binary(ctx, extra_deps): 557 cc_info = collect_cc_info(ctx, extra_deps = extra_deps) 558 return create_cc_details_struct( 559 cc_info_for_propagating = cc_info, 560 cc_info_for_self_link = cc_info, 561 cc_info_with_extra_link_time_libraries = None, 562 extra_runfiles = ctx.runfiles(), 563 # Though the rules require the CcToolchain, it isn't actually used. 564 cc_toolchain = None, 565 feature_config = None, 566 ) 567 568def _get_interpreter_path(ctx, *, runtime, flag_interpreter_path): 569 if runtime: 570 if runtime.interpreter_path: 571 interpreter_path = runtime.interpreter_path 572 else: 573 interpreter_path = "{}/{}".format( 574 ctx.workspace_name, 575 runtime.interpreter.short_path, 576 ) 577 578 # NOTE: External runfiles (artifacts in other repos) will have a 579 # leading path component of "../" so that they refer outside the 580 # main workspace directory and into the runfiles root. By 581 # normalizing, we simplify e.g. "workspace/../foo/bar" to simply 582 # "foo/bar" 583 interpreter_path = paths.normalize(interpreter_path) 584 585 elif flag_interpreter_path: 586 interpreter_path = flag_interpreter_path 587 else: 588 fail("Unable to determine interpreter path") 589 590 return interpreter_path 591 592def _get_native_deps_dso_name(ctx): 593 _ = ctx # @unused 594 fail("Building native deps DSO not supported.") 595 596def _get_native_deps_user_link_flags(ctx): 597 _ = ctx # @unused 598 fail("Building native deps DSO not supported.") 599