diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b858c7d89..40d1affe0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,8 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-changed} ### Changed * (binaries/tests) The `PYTHONBREAKPOINT` environment variable is automatically inherited +* (binaries/tests) The {obj}`stamp` attribute now transitions the Bazel builtin + {obj}`--stamp` flag. {#v0-0-0-fixed} ### Fixed @@ -69,6 +71,9 @@ END_UNRELEASED_TEMPLATE ### Added * (binaries/tests) {obj}`--debugger`: allows specifying an extra dependency to add to binaries/tests for custom debuggers. +* (binaries/tests) Build information is now included in binaries and tests. + Use the `bazel_binary_info` module to access it. The {flag}`--stamp` flag will + add {flag}`--workspace_status` information. {#v1-8-0} ## [1.8.0] - 2025-12-19 diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 708b3f7000..efc1dd7319 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -21,6 +21,7 @@ load(":print_toolchain_checksums.bzl", "print_toolchains_checksums") load(":py_exec_tools_toolchain.bzl", "current_interpreter_executable") load(":sentinel.bzl", "sentinel") load(":stamp.bzl", "stamp_build_setting") +load(":uncachable_version_file.bzl", "define_uncachable_version_file") package( default_visibility = ["//:__subpackages__"], @@ -99,6 +100,14 @@ bzl_library( ], ) +alias( + name = "build_data_writer", + actual = select({ + "@platforms//os:windows": ":build_data_writer.ps1", + "//conditions:default": ":build_data_writer.sh", + }), +) + bzl_library( name = "builders_bzl", srcs = ["builders.bzl"], @@ -683,6 +692,10 @@ bzl_library( ], ) +define_uncachable_version_file( + name = "uncachable_version_file", +) + bzl_library( name = "version_bzl", srcs = ["version.bzl"], diff --git a/python/private/attributes.bzl b/python/private/attributes.bzl index 0e0872fbf5..4687693cb7 100644 --- a/python/private/attributes.bzl +++ b/python/private/attributes.bzl @@ -450,8 +450,20 @@ Whether to encode build information into the binary. Possible values: Stamped binaries are not rebuilt unless their dependencies change. -WARNING: Stamping can harm build performance by reducing cache hits and should +Stamped build information can accessed using the `bazel_binary_info` module. +See the [Accessing build information docs] for more information. + +:::{warning} +Stamping can harm build performance by reducing cache hits and should be avoided if possible. + +In addition, this transitions the {obj}`--stamp` flag, which can additional +config state overhead. +::: + +:::{note} +Stamping of build data output is always disabled for the exec config. +::: """, default = -1, ), diff --git a/python/private/build_data_writer.ps1 b/python/private/build_data_writer.ps1 new file mode 100644 index 0000000000..384d1ce539 --- /dev/null +++ b/python/private/build_data_writer.ps1 @@ -0,0 +1,18 @@ +$OutputPath = $env:OUTPUT + +Add-Content -Path $OutputPath -Value "TARGET $env:TARGET" +Add-Content -Path $OutputPath -Value "CONFIG_ID $env:CONFIG_ID" +Add-Content -Path $OutputPath -Value "CONFIG_MODE $env:CONFIG_MODE" +Add-Content -Path $OutputPath -Value "STAMPED $env:STAMPED" + +$VersionFilePath = $env:VERSION_FILE +if (-not [string]::IsNullOrEmpty($VersionFilePath)) { + Get-Content -Path $VersionFilePath | Add-Content -Path $OutputPath +} + +$InfoFilePath = $env:INFO_FILE +if (-not [string]::IsNullOrEmpty($InfoFilePath)) { + Get-Content -Path $InfoFilePath | Add-Content -Path $OutputPath +} + +exit 0 diff --git a/python/private/build_data_writer.sh b/python/private/build_data_writer.sh new file mode 100755 index 0000000000..7b88a582f5 --- /dev/null +++ b/python/private/build_data_writer.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +echo "TARGET $TARGET" >> $OUTPUT +echo "CONFIG_MODE $CONFIG_MODE" >> $OUTPUT +echo "STAMPED $STAMPED" >> $OUTPUT +if [ -n "$VERSION_FILE" ]; then + cat "$VERSION_FILE" >> "$OUTPUT" +fi +if [ -n "$INFO_FILE" ]; then + cat "$INFO_FILE" >> "$OUTPUT" +fi +exit 0 diff --git a/python/private/common.bzl b/python/private/common.bzl index c31aeb383f..2d4afca3f5 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -43,34 +43,25 @@ PYTHON_FILE_EXTENSIONS = [ def create_binary_semantics_struct( *, - get_central_uncachable_version_file, get_native_deps_dso_name, - should_build_native_deps_dso, - should_include_build_data): + should_build_native_deps_dso): """Helper to ensure a semantics struct has all necessary fields. Call this instead of a raw call to `struct(...)`; it'll help ensure all the necessary functions are being correctly provided. Args: - get_central_uncachable_version_file: Callable that returns an optional - Artifact; this artifact is special: it is never cached and is a copy - of `ctx.version_file`; see py_builtins.copy_without_caching get_native_deps_dso_name: Callable that returns a string, which is the basename (with extension) of the native deps DSO library. should_build_native_deps_dso: Callable that returns bool; True if building a native deps DSO is supported, False if not. - should_include_build_data: Callable that returns bool; True if - build data should be generated, False if not. Returns: A "BinarySemantics" struct. """ return struct( # keep-sorted - get_central_uncachable_version_file = get_central_uncachable_version_file, get_native_deps_dso_name = get_native_deps_dso_name, should_build_native_deps_dso = should_build_native_deps_dso, - should_include_build_data = should_include_build_data, ) def create_cc_details_struct( diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 73d0c8399f..1b884e9b3b 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -206,6 +206,11 @@ accepting arbitrary Python versions. allow_single_file = True, default = "@bazel_tools//tools/python:python_bootstrap_template.txt", ), + "_build_data_writer": lambda: attrb.Label( + default = "//python/private:build_data_writer", + allow_files = True, + cfg = "exec", + ), "_debugger_flag": lambda: attrb.Label( default = "//python/private:debugger_if_target_config", providers = [PyInfo], @@ -226,6 +231,10 @@ accepting arbitrary Python versions. "_python_version_flag": lambda: attrb.Label( default = labels.PYTHON_VERSION, ), + "_uncachable_version_file": lambda: attrb.Label( + default = "//python/private:uncachable_version_file", + allow_files = True, + ), "_venvs_use_declare_symlink_flag": lambda: attrb.Label( default = labels.VENVS_USE_DECLARE_SYMLINK, providers = [BuildSettingInfo], @@ -271,10 +280,8 @@ def py_executable_impl(ctx, *, is_test, inherited_environment): def create_binary_semantics(): return create_binary_semantics_struct( # keep-sorted start - get_central_uncachable_version_file = lambda ctx: None, get_native_deps_dso_name = _get_native_deps_dso_name, should_build_native_deps_dso = lambda ctx: False, - should_include_build_data = lambda ctx: False, # keep-sorted end ) @@ -336,6 +343,7 @@ def _create_executable( imports = imports, runtime_details = runtime_details, venv = venv, + build_data_file = runfiles_details.build_data_file, ) extra_runfiles = ctx.runfiles( [stage2_bootstrap] + ( @@ -648,6 +656,7 @@ def _create_stage2_bootstrap( main_py, imports, runtime_details, + build_data_file, venv): output = ctx.actions.declare_file( # Prepend with underscore to prevent pytest from trying to @@ -668,6 +677,7 @@ def _create_stage2_bootstrap( template = template, output = output, substitutions = { + "%build_data_file%": runfiles_root_path(ctx, build_data_file.short_path), "%coverage_instrumented%": str(int(ctx.configuration.coverage_enabled and ctx.coverage_instrumented())), "%coverage_tool%": _get_coverage_tool_runfiles_path(ctx, runtime), "%import_all%": "True" if read_possibly_native_flag(ctx, "python_import_all_repositories") else "False", @@ -1053,7 +1063,6 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = cc_details.extra_runfiles, native_deps_details.runfiles, ], - semantics = semantics, ) exec_result = _create_executable( ctx, @@ -1242,8 +1251,7 @@ def _get_base_runfiles_for_binary( required_pyc_files, implicit_pyc_files, implicit_pyc_source_files, - extra_common_runfiles, - semantics): + extra_common_runfiles): """Returns the set of runfiles necessary prior to executable creation. NOTE: The term "common runfiles" refers to the runfiles that are common to @@ -1265,7 +1273,6 @@ def _get_base_runfiles_for_binary( files that are used when the implicit pyc files are not. extra_common_runfiles: List of runfiles; additional runfiles that will be added to the common runfiles. - semantics: A `BinarySemantics` struct; see `create_binary_semantics_struct`. Returns: struct with attributes: @@ -1306,6 +1313,9 @@ def _get_base_runfiles_for_binary( common_runfiles.add_targets(extra_deps) common_runfiles.add(extra_common_runfiles) + build_data_file = _write_build_data(ctx) + common_runfiles.add(build_data_file) + common_runfiles = common_runfiles.build(ctx) if _should_create_init_files(ctx): @@ -1314,25 +1324,10 @@ def _get_base_runfiles_for_binary( runfiles = common_runfiles, ) - # Don't include build_data.txt in the non-exe runfiles. The build data - # may contain program-specific content (e.g. target name). runfiles_with_exe = common_runfiles.merge(ctx.runfiles([executable])) - # Don't include build_data.txt in data runfiles. This allows binaries to - # contain other binaries while still using the same fixed location symlink - # for the build_data.txt file. Really, the fixed location symlink should be - # removed and another way found to locate the underlying build data file. data_runfiles = runfiles_with_exe - - if is_stamping_enabled(ctx) and semantics.should_include_build_data(ctx): - build_data_file, build_data_runfiles = _create_runfiles_with_build_data( - ctx, - semantics.get_central_uncachable_version_file(ctx), - ) - default_runfiles = runfiles_with_exe.merge(build_data_runfiles) - else: - build_data_file = None - default_runfiles = runfiles_with_exe + default_runfiles = runfiles_with_exe return struct( runfiles_without_exe = common_runfiles, @@ -1341,31 +1336,18 @@ def _get_base_runfiles_for_binary( data_runfiles = data_runfiles, ) -def _create_runfiles_with_build_data( - ctx, - central_uncachable_version_file): - build_data_file = _write_build_data( - ctx, - central_uncachable_version_file, - ) - build_data_runfiles = ctx.runfiles(files = [ - build_data_file, - ]) - return build_data_file, build_data_runfiles - -def _write_build_data(ctx, central_uncachable_version_file): - # TODO: Remove this logic when a central file is always available - if not central_uncachable_version_file: - version_file = ctx.actions.declare_file(ctx.label.name + "-uncachable_version_file.txt") - _py_builtins.copy_without_caching( - ctx = ctx, - read_from = ctx.version_file, - write_to = version_file, - ) +def _write_build_data(ctx): + inputs = builders.DepsetBuilder() + if is_stamping_enabled(ctx): + # NOTE: ctx.info_file is undocumented; see + # https://github.com/bazelbuild/bazel/issues/9363 + info_file = ctx.info_file + version_file = ctx.files._uncachable_version_file[0] + inputs.add(info_file) + inputs.add(version_file) else: - version_file = central_uncachable_version_file - - direct_inputs = [ctx.info_file, version_file] + info_file = None + version_file = None # A "constant metadata" file is basically a special file that doesn't # support change detection logic and reports that it is unchanged. i.e., it @@ -1397,23 +1379,36 @@ def _write_build_data(ctx, central_uncachable_version_file): root = ctx.bin_dir, ) + action_args = ctx.actions.args() + writer_file = ctx.files._build_data_writer[0] + if writer_file.path.endswith(".ps1"): + action_exe = "pwsh.exe" + action_args.add("-File") + action_args.add(writer_file) + inputs.add(writer_file) + else: + action_exe = ctx.attr._build_data_writer[DefaultInfo].files_to_run + ctx.actions.run( - executable = ctx.executable._build_data_gen, + executable = action_exe, + arguments = [action_args], env = { - # NOTE: ctx.info_file is undocumented; see - # https://github.com/bazelbuild/bazel/issues/9363 - "INFO_FILE": ctx.info_file.path, + # Include config mode so that binaries can detect if they're + # being used as a build tool or not, allowing for runtime optimizations. + "CONFIG_MODE": "EXEC" if _is_tool_config(ctx) else "TARGET", + "INFO_FILE": info_file.path if info_file else "", "OUTPUT": build_data.path, - "PLATFORM": cc_helper.find_cpp_toolchain(ctx).toolchain_id, + # Include this so it's explicit, otherwise, one has to detect + # this by looking for the absense of info_file keys. + "STAMPED": "TRUE" if is_stamping_enabled(ctx) else "FALSE", "TARGET": str(ctx.label), - "VERSION_FILE": version_file.path, + "VERSION_FILE": version_file.path if version_file else "", }, - inputs = depset( - direct = direct_inputs, - ), + inputs = inputs.build(), outputs = [build_data], mnemonic = "PyWriteBuildData", - progress_message = "Generating %{label} build_data.txt", + progress_message = "Reticulating %{label} build data", + toolchain = None, ) return build_data @@ -1608,6 +1603,9 @@ def is_stamping_enabled(ctx): Returns: bool; True if stamping is enabled, False if not. """ + + # Always ignore stamping for exec config. This mitigates stamping + # invalidating build action caching. if _is_tool_config(ctx): return False @@ -1617,8 +1615,9 @@ def is_stamping_enabled(ctx): elif stamp == 0: return False elif stamp == -1: - # NOTE: Undocumented API; private to builtins - return ctx.configuration.stamp_binaries + # NOTE: ctx.configuration.stamp_binaries() exposes this, but that's + # a private API. To workaround, it'd been eposed via py_internal. + return py_internal.stamp_binaries(ctx) else: fail("Unsupported `stamp` value: {}".format(stamp)) @@ -1771,6 +1770,9 @@ def _transition_executable_impl(settings, attr): if attr.python_version and attr.python_version not in ("PY2", "PY3"): settings[labels.PYTHON_VERSION] = attr.python_version + + if attr.stamp != -1: + settings["//command_line_option:stamp"] = str(attr.stamp) return settings def create_executable_rule(*, attrs, **kwargs): @@ -1821,8 +1823,14 @@ def create_executable_rule_builder(implementation, **kwargs): ] + ([ruleb.ToolchainType(_LAUNCHER_MAKER_TOOLCHAIN_TYPE)] if rp_config.bazel_9_or_later else []), cfg = dict( implementation = _transition_executable_impl, - inputs = TRANSITION_LABELS + [labels.PYTHON_VERSION], - outputs = TRANSITION_LABELS + [labels.PYTHON_VERSION], + inputs = TRANSITION_LABELS + [ + labels.PYTHON_VERSION, + "//command_line_option:stamp", + ], + outputs = TRANSITION_LABELS + [ + labels.PYTHON_VERSION, + "//command_line_option:stamp", + ], ), **kwargs ) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index e3e303b3b1..3595a43110 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -20,7 +20,9 @@ import os import re import runpy +import types import uuid +from functools import cache # ===== Template substitutions start ===== # We just put them in one place so its easy to tell which are used. @@ -42,11 +44,34 @@ VENV_SITE_PACKAGES = "%venv_rel_site_packages%" # Whether we should generate coverage data. +# string, 1 or 0 COVERAGE_INSTRUMENTED = "%coverage_instrumented%" == "1" +# runfiles-root-relative path to a file with binary-specific build information +BUILD_DATA_FILE = "%build_data_file%" + # ===== Template substitutions end ===== +class BazelBinaryInfoModule(types.ModuleType): + BUILD_DATA_FILE = BUILD_DATA_FILE + + @cache + def get_build_data(self): + """Returns a string of the raw build data.""" + try: + # Prefer dep via pypi + import runfiles + except ImportError: + from python.runfiles import runfiles + path = runfiles.Create().Rlocation(self.BUILD_DATA_FILE) + with open(path) as fp: + return fp.read() + + +sys.modules["bazel_binary_info"] = BazelBinaryInfoModule("bazel_binary_info") + + # Return True if running on Windows def is_windows(): return os.name == "nt" @@ -89,17 +114,6 @@ def get_windows_path_with_unc_prefix(path): return unicode_prefix + os.path.abspath(path) -def search_path(name): - """Finds a file in a given search path.""" - search_path = os.getenv("PATH", os.defpath).split(os.pathsep) - for directory in search_path: - if directory: - path = os.path.join(directory, name) - if os.path.isfile(path) and os.access(path, os.X_OK): - return path - return None - - def is_verbose(): return bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")) @@ -322,8 +336,11 @@ def _maybe_collect_coverage(enable): # We need for coveragepy to use relative paths. This can only be configured # using an rc file. rcfile_name = os.path.join(coverage_dir, ".coveragerc_{}".format(unique_id)) - disable_warnings = ('disable_warnings = module-not-imported, no-data-collected' - if COVERAGE_INSTRUMENTED else '') + disable_warnings = ( + "disable_warnings = module-not-imported, no-data-collected" + if COVERAGE_INSTRUMENTED + else "" + ) print_verbose_coverage("coveragerc file:", rcfile_name) with open(rcfile_name, "w") as rcfile: rcfile.write( diff --git a/python/private/uncachable_version_file.bzl b/python/private/uncachable_version_file.bzl new file mode 100644 index 0000000000..9b1d65a469 --- /dev/null +++ b/python/private/uncachable_version_file.bzl @@ -0,0 +1,39 @@ +"""Implementation of uncachable_version_file.""" + +load(":py_internal.bzl", "py_internal") + +def _uncachable_version_file_impl(ctx): + version_file = ctx.actions.declare_file("uncachable_version_file.txt") + py_internal.copy_without_caching( + ctx = ctx, + # NOTE: ctx.version_file is undocumented; see + # https://github.com/bazelbuild/bazel/issues/9363 + # NOTE: Even though the version file changes every build (it contains + # the build timestamp), it is ignored when computing what inputs + # changed. See https://bazel.build/docs/user-manual#workspace-status + read_from = ctx.version_file, + write_to = version_file, + ) + return [DefaultInfo( + files = depset([version_file]), + )] + +uncachable_version_file = rule( + doc = """ +Creates a copy of `ctx.version_file`, except it isn't ignored by +Bazel's change-detecting logic. In fact, it's the opposite: +caching is disabled for the action generating this file, so any +actions depending on this file will always re-run. +""", + implementation = _uncachable_version_file_impl, +) + +def define_uncachable_version_file(name): + native.alias( + name = name, + actual = select({ + ":stamp_detect": ":uncachable_version_file_impl", + "//conditions:default": ":sentinel", + }), + ) + uncachable_version_file(name = "uncachable_version_file_impl") diff --git a/sphinxdocs/inventories/bazel_inventory.txt b/sphinxdocs/inventories/bazel_inventory.txt index e14ea76067..e704d20d73 100644 --- a/sphinxdocs/inventories/bazel_inventory.txt +++ b/sphinxdocs/inventories/bazel_inventory.txt @@ -157,6 +157,7 @@ runfiles.merge bzl:type 1 rules/lib/builtins/runfiles#merge - runfiles.merge_all bzl:type 1 rules/lib/builtins/runfiles#merge_all - runfiles.root_symlinks bzl:type 1 rules/lib/builtins/runfiles#root_symlinks - runfiles.symlinks bzl:type 1 rules/lib/builtins/runfiles#symlinks - +stamp bzl:flag 1 reference/command-line-reference#flag--stamp - str bzl:type 1 rules/lib/string - struct bzl:type 1 rules/lib/builtins/struct - target_compatible_with bzl:attr 1 reference/be/common-definitions#common.target_compatible_with - @@ -171,3 +172,4 @@ toolchain.target_settings bzl:attr 1 reference/be/platforms-and-toolchains#toolc toolchain_type bzl:type 1 rules/lib/builtins/toolchain_type.html - transition bzl:type 1 rules/lib/builtins/transition - tuple bzl:type 1 rules/lib/core/tuple - +workspace_status bzl:flag 1 reference/command-line-reference#build-flag--workspace_status_command - diff --git a/tests/build_data/BUILD.bazel b/tests/build_data/BUILD.bazel new file mode 100644 index 0000000000..64db005f51 --- /dev/null +++ b/tests/build_data/BUILD.bazel @@ -0,0 +1,25 @@ +load("//python:py_binary.bzl", "py_binary") +load("//python:py_test.bzl", "py_test") + +py_test( + name = "build_data_test", + srcs = ["build_data_test.py"], + data = [ + ":tool_build_data.txt", + ], + stamp = 1, + deps = ["//python/runfiles"], +) + +py_binary( + name = "print_build_data", + srcs = ["print_build_data.py"], + deps = ["//python/runfiles"], +) + +genrule( + name = "tool_build_data", + outs = ["tool_build_data.txt"], + cmd = "$(location :print_build_data) > $(OUTS)", + tools = [":print_build_data"], +) diff --git a/tests/build_data/build_data_test.py b/tests/build_data/build_data_test.py new file mode 100644 index 0000000000..e4ff81a634 --- /dev/null +++ b/tests/build_data/build_data_test.py @@ -0,0 +1,32 @@ +import unittest + +from python.runfiles import runfiles + + +class BuildDataTest(unittest.TestCase): + + def test_target_build_data(self): + import bazel_binary_info + + self.assertIn("build_data.txt", bazel_binary_info.BUILD_DATA_FILE) + + build_data = bazel_binary_info.get_build_data() + self.assertIn("TARGET ", build_data) + self.assertIn("BUILD_HOST ", build_data) + self.assertIn("BUILD_USER ", build_data) + self.assertIn("BUILD_TIMESTAMP ", build_data) + self.assertIn("FORMATTED_DATE ", build_data) + self.assertIn("CONFIG_MODE TARGET", build_data) + self.assertIn("STAMPED TRUE", build_data) + + def test_tool_build_data(self): + rf = runfiles.Create() + path = rf.Rlocation("rules_python/tests/build_data/tool_build_data.txt") + with open(path) as fp: + build_data = fp.read() + + self.assertIn("STAMPED FALSE", build_data) + self.assertIn("CONFIG_MODE EXEC", build_data) + + +unittest.main() diff --git a/tests/build_data/print_build_data.py b/tests/build_data/print_build_data.py new file mode 100644 index 0000000000..0af77d72be --- /dev/null +++ b/tests/build_data/print_build_data.py @@ -0,0 +1,3 @@ +import bazel_binary_info + +print(bazel_binary_info.get_build_data())