From e735e65e04bc4ba8d619df481d7d00d075328a6c Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:10 -0400 Subject: [PATCH 01/14] Build xcode-locator as a universal platform-aware binary --- src/MODULE.tools | 1 + src/test/shell/bazel/apple/bazel_objc_test.sh | 4 ++ tools/osx/BUILD | 49 +++++++++++++------ 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/MODULE.tools b/src/MODULE.tools index 3c278c6278b0c7..507fd1ebf88dd6 100644 --- a/src/MODULE.tools +++ b/src/MODULE.tools @@ -6,6 +6,7 @@ module(name = "bazel_tools") bazel_dep(name = "rules_license", version = "1.0.0") +bazel_dep(name = "bazel_skylib", version = "1.8.2") bazel_dep(name = "buildozer", version = "8.5.1") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "zlib", version = "1.3.1.bcr.5") diff --git a/src/test/shell/bazel/apple/bazel_objc_test.sh b/src/test/shell/bazel/apple/bazel_objc_test.sh index 08525bbe6d1a80..d1abef5439417b 100755 --- a/src/test/shell/bazel/apple/bazel_objc_test.sh +++ b/src/test/shell/bazel/apple/bazel_objc_test.sh @@ -25,6 +25,10 @@ if ! is_darwin; then fi function test_xcodelocator_embedded_tool() { + # Declare apple_support directly so its Apple C++ toolchain is registered + # before the generic rules_cc toolchain inherited through @bazel_tools. + add_bazel_dep "apple_support" MODULE.bazel + rm -rf ios mkdir -p ios diff --git a/tools/osx/BUILD b/tools/osx/BUILD index 7e572962870694..42bb0470437eac 100644 --- a/tools/osx/BUILD +++ b/tools/osx/BUILD @@ -1,3 +1,7 @@ +load("@apple_support//rules:universal_binary.bzl", "universal_binary") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:objc_library.bzl", "objc_library") load( "//tools/osx:xcode_version_flag.bzl", "ios_sdk_version_flag", @@ -26,24 +30,37 @@ exports_files([ "xcode_configure.bzl", ]) -DARWIN_XCODE_LOCATOR_COMPILE_COMMAND = """ - /usr/bin/xcrun --sdk macosx clang -mmacosx-version-min=10.13 -fobjc-arc -framework CoreServices \ - -framework Foundation -arch arm64 -arch x86_64 -o $@ $< -""" +objc_library( + name = "xcode-locator-lib", + srcs = ["xcode_locator.m"], + sdk_frameworks = [ + "CoreServices", + "Foundation", + ], + target_compatible_with = ["@platforms//os:macos"], +) -genrule( - name = "xcode-locator-genrule", - srcs = select({ - "//src/conditions:darwin": ["xcode_locator.m"], - "//conditions:default": ["xcode_locator_stub.sh"], - }), - outs = ["xcode-locator"], - cmd = select({ - "//src/conditions:darwin": DARWIN_XCODE_LOCATOR_COMPILE_COMMAND, - "//conditions:default": "cp $< $@", +cc_binary( + name = "xcode-locator-native", + target_compatible_with = ["@platforms//os:macos"], + deps = [":xcode-locator-lib"], +) + +universal_binary( + name = "xcode-locator-universal", + binary = ":xcode-locator-native", + target_compatible_with = ["@platforms//os:macos"], +) + +copy_file( + name = "xcode-locator-copy", + src = select({ + "//src/conditions:darwin": ":xcode-locator-universal", + "//conditions:default": "xcode_locator_stub.sh", }), - local = 1, - output_to_bindir = 1, + out = "xcode-locator", + allow_symlink = True, + is_executable = True, ) # TODO(cparsons): Consolidate with config_settings under //src From f370f64f75a84cc262ffd9d59008d57cedeb5018 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:48 -0400 Subject: [PATCH 02/14] Register hermetic LLVM toolchains and macOS frameworks --- MODULE.bazel | 27 +++++++++++ MODULE.bazel.lock | 13 ++++++ third_party/llvm_0.8.6_hermetic_glibc.patch | 50 +++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 third_party/llvm_0.8.6_hermetic_glibc.patch diff --git a/MODULE.bazel b/MODULE.bazel index 0399e607da1354..f8669e1018fba9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -23,6 +23,7 @@ bazel_dep(name = "googleapis", version = "0.0.0-20250604-de157ca3") bazel_dep(name = "googletest", version = "1.17.0.bcr.2", repo_name = "com_google_googletest") bazel_dep(name = "grpc-java", version = "1.71.0") bazel_dep(name = "grpc", version = "1.76.0.bcr.1", repo_name = "com_github_grpc_grpc") +bazel_dep(name = "llvm", version = "0.8.6") bazel_dep(name = "platforms", version = "1.1.0") bazel_dep(name = "protobuf", version = "35.1", repo_name = "com_google_protobuf") bazel_dep(name = "re2", version = "2025-11-05.bcr.1") @@ -45,6 +46,25 @@ bazel_dep(name = "zstd-jni", version = "1.5.6-9.bcr.1") bazel_dep(name = "apple_support", version = "2.5.2") bazel_dep(name = "rules_cc", version = "0.2.19") +osx = use_extension("@llvm//extensions:osx.bzl", "osx") +osx.frameworks( + names = [ + "CFNetwork", + "CoreFoundation", + "CoreGraphics", + "CoreServices", + "DiskArbitration", + "Foundation", + "IOKit", + "Kernel", + "OSLog", + "Security", + "SystemConfiguration", + ], +) + +register_toolchains("@llvm//toolchain:all") + # The starlark rules in @rules_cc are hidden behind macros but docgen needs to # load the rule class directly, so we need to expose the cc_compatibility_proxy # repo for access to the symbols @@ -75,6 +95,13 @@ single_version_override( version = "1.34.5.bcr.4", ) +single_version_override( + module_name = "llvm", + patch_strip = 1, + patches = ["//third_party:llvm_0.8.6_hermetic_glibc.patch"], + version = "0.8.6", +) + single_version_override( module_name = "rules_jvm_external", patch_strip = 1, diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 28d4bff80a8879..1a90823043b3dc 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -49,16 +49,21 @@ "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", + "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", "https://bcr.bazel.build/modules/bazel_features/1.39.0/MODULE.bazel": "28739425c1fc283c91931619749c832b555e60bcd1010b40d8441ce0a5cf726d", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", "https://bcr.bazel.build/modules/bazel_features/1.47.0/MODULE.bazel": "e34df3cb35b1684cfa69923a61ae3803595babd3942cd306a488d51400886b30", "https://bcr.bazel.build/modules/bazel_features/1.47.0/source.json": "4ba0b5138327f2d73352a51547a4e49a0a828ef400e046b15334d8905bf6b7ff", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", "https://bcr.bazel.build/modules/bazel_lib/3.1.0/MODULE.bazel": "6809765c14e3c766a9b9286c7b0ec56ed87a73326e48fe01749f0c0fdcfe3287", "https://bcr.bazel.build/modules/bazel_lib/3.1.0/source.json": "aaf7c2dc816219f4cb356c9d65f2555fb7f9543e537199f74a921f7877d23dfb", + "https://bcr.bazel.build/modules/bazel_lib/3.2.2/MODULE.bazel": "e2c890c8a515d6bca9c66d47718aa9e44b458fde64ec7204b8030bf2d349058c", + "https://bcr.bazel.build/modules/bazel_lib/3.2.2/source.json": "9e84e115c20e14652c5c21401ae85ff4daa8702e265b5c0b3bf89353f17aa212", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", @@ -104,6 +109,8 @@ "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/MODULE.bazel": "1fe72489212c530086e3ffb0e018b2bfef4663200ca03571570f9f006bef1d75", "https://bcr.bazel.build/modules/envoy_api/0.0.0-20251105-4a2b9a3/MODULE.bazel": "b66e87a0e0c2207f07e35c321388eb1feb036344565977444b52912c53a84466", "https://bcr.bazel.build/modules/envoy_api/0.0.0-20251105-4a2b9a3/source.json": "c4780edf780977f2ab7d00a189432c5b0b2fa08c6e4e2e09d2950499364a687d", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", "https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996", "https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b", "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", @@ -161,6 +168,8 @@ "https://bcr.bazel.build/modules/libpfm/4.11.0.bcr.1/MODULE.bazel": "e5362dadc90aab6724c83a2cc1e67cbed9c89a05d97fb1f90053c8deb1e445c8", "https://bcr.bazel.build/modules/libpfm/4.11.0.bcr.1/source.json": "0646414d9037f8aad148781dd760bec90b0b25ac12fda5e03f8aadbd6b9c61e6", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/llvm/0.8.6/MODULE.bazel": "1f878674a99e40f17a6b4c49613d017613d6d9de581338a9499761809d6b1c5c", + "https://bcr.bazel.build/modules/llvm/0.8.6/source.json": "090fd2f8eefa82427bc09891a7c934482ae251a3a0c179c0433c8b9fe35c8148", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", @@ -261,6 +270,7 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", "https://bcr.bazel.build/modules/rules_cc/0.2.19/MODULE.bazel": "d5e0f05b63273281a16654eb6b1a8742a75ec153ac8b4f0419949d6e401e46f0", "https://bcr.bazel.build/modules/rules_cc/0.2.19/source.json": "1ef48cdbd7aa6238015189b582d3d74ef0cbea3cb3e2cb259d782463f570c14a", + "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", @@ -378,11 +388,14 @@ "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/MODULE.bazel": "e8f9ff79199e8d9eaad7f1b0a77ad74b30bb82d794b87d8ca942bead5de83ae9", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/source.json": "20143442376c03426f6135292ba02d825cb75308aa47e6bf42dd4cc5a435c2ff", "https://bcr.bazel.build/modules/upb/0.0.0-20211020-160625a/MODULE.bazel": "6cced416be2dc5b9c05efd5b997049ba795e5e4e6fafbe1624f4587767638928", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", "https://bcr.bazel.build/modules/upb/0.0.0-20230907-e7430e6/MODULE.bazel": "3a7dedadf70346e678dc059dbe44d05cbf3ab17f1ce43a1c7a42edc7cbf93fd9", "https://bcr.bazel.build/modules/wabt/1.0.37/MODULE.bazel": "c3907ead6f98348df098f380061c5c9d8f4ac9a60aefdcf83919e29c30777da0", "https://bcr.bazel.build/modules/wabt/1.0.37/source.json": "d86a91e1a6e4b548d2d80938144340d124defc35a3d59c1c2f02960debf97f32", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.13.0/MODULE.bazel": "5bba14f21fa3353465660f96aaa4882b125690e62598f2caaef5651bed868c14", "https://bcr.bazel.build/modules/with_cfg.bzl/0.13.0/source.json": "617358d9dc53040b6d9d286396123b66a0c0293d329501b83c3256ae989d212c", "https://bcr.bazel.build/modules/xds/0.0.0-20240423-555b57e/MODULE.bazel": "cea509976a77e34131411684ef05a1d6ad194dd71a8d5816643bc5b0af16dc0f", diff --git a/third_party/llvm_0.8.6_hermetic_glibc.patch b/third_party/llvm_0.8.6_hermetic_glibc.patch new file mode 100644 index 00000000000000..117243a3e4f586 --- /dev/null +++ b/third_party/llvm_0.8.6_hermetic_glibc.patch @@ -0,0 +1,50 @@ +diff --git a/3rd_party/libc/glibc/glibc.BUILD.bazel b/3rd_party/libc/glibc/glibc.BUILD.bazel +index 1e82cbd..847935a 100644 +--- a/3rd_party/libc/glibc/glibc.BUILD.bazel ++++ b/3rd_party/libc/glibc/glibc.BUILD.bazel +@@ -181,6 +181,7 @@ copy_file( + name = "glibc_crt1.object", + src = "Scrt1.o", + out = "crt1.o", ++ allow_symlink = True, + visibility = ["//visibility:public"], + ) + +diff --git a/runtimes/mingw/BUILD.bazel b/runtimes/mingw/BUILD.bazel +index 87a10bc..7b1f831 100644 +--- a/runtimes/mingw/BUILD.bazel ++++ b/runtimes/mingw/BUILD.bazel +@@ -1,7 +1,7 @@ + load("@bazel_lib//:bzl_library.bzl", "bzl_library") +-load("@bazel_lib//lib:copy_file.bzl", "copy_file") + load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") + load("@bazel_lib//lib:run_binary.bzl", "run_binary") ++load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + load("@llvm//toolchain/runtimes:cc_runtime_library.bzl", "cc_runtime_stage0_library") + load("//runtimes:defs.bzl", "stub_library") + load(":crt_object.bzl", "crt_object") +@@ -314,6 +314,7 @@ copy_file( + name = "pthread", + src = ":winpthread", + out = "libpthread.a", ++ allow_symlink = True, + visibility = ["//visibility:public"], + ) + +diff --git a/runtimes/mingw/crt_object.bzl b/runtimes/mingw/crt_object.bzl +index 1b43f4d..1d53fe0 100644 +--- a/runtimes/mingw/crt_object.bzl ++++ b/runtimes/mingw/crt_object.bzl +@@ -1,4 +1,4 @@ +-load("@bazel_lib//lib:copy_file.bzl", "copy_file") ++load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + load("@llvm//toolchain/runtimes:cc_runtime_library.bzl", "cc_runtime_stage0_library") + + def crt_object(name, out, visibility, **kwargs): +@@ -17,5 +17,6 @@ def crt_object(name, out, visibility, **kwargs): + name = name, + src = name + "_file", + out = out, ++ allow_symlink = True, + visibility = visibility, + ) From 30193d4d751f49b73d3b91379630bad4e3793aed Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:48 -0400 Subject: [PATCH 03/14] Use hermetic zlib header copies --- MODULE.bazel | 2 +- MODULE.bazel.lock | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index f8669e1018fba9..d846bcb7e34d29 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -38,7 +38,7 @@ bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_testing", version = "0.9.0") bazel_dep(name = "stardoc", version = "0.8.0", repo_name = "io_bazel_skydoc") bazel_dep(name = "with_cfg.bzl", version = "0.13.0") -bazel_dep(name = "zlib", version = "1.3.1.bcr.7") +bazel_dep(name = "zlib", version = "1.3.1.bcr.8") bazel_dep(name = "zstd-jni", version = "1.5.6-9.bcr.1") # Depend on apple_support first and then rules_cc so that the Xcode toolchain diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 1a90823043b3dc..ed3431a2eee26f 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -406,7 +406,8 @@ "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.7/MODULE.bazel": "26a6764cda2bfa720e5ea6bea9e6aa4282b69f96d3b9cfcfbce1ef596ce30e43", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.7/source.json": "086122bc43f9108094fed21aaace4c0affd5abd8364af0520dbacdb76cc0546d", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.8/MODULE.bazel": "772c674bb78a0342b8caf32ab5c25085c493ca4ff08398208dcbe4375fe9f776", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.8/source.json": "cf377d76800dfc3d3b71e9dd4a8c53a62837cbce37cc4f25e6207b15fc1e8f2b", "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72", "https://bcr.bazel.build/modules/zstd-jni/1.5.6-9.bcr.1/MODULE.bazel": "c6696745628cdc909b39adc10d79548815b32d4504bf225be98cad6bfabd4def", From c74ad63f75b9bcd36d1d9553448c43cce76cce75 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:48 -0400 Subject: [PATCH 04/14] Run Java and Python generated-file actions without host shell tools --- MODULE.bazel | 16 ++++- .../rules_java_9.5.0_hermetic_jni.patch | 66 +++++++++++++++++++ .../rules_python_1.7.0_hermetic_zip.patch | 51 ++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 third_party/rules_java_9.5.0_hermetic_jni.patch create mode 100644 third_party/rules_python_1.7.0_hermetic_zip.patch diff --git a/MODULE.bazel b/MODULE.bazel index d846bcb7e34d29..13b5b41d9a43e7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,6 +15,7 @@ module( bazel_dep(name = "abseil-cpp", version = "20250814.1") bazel_dep(name = "bazel_features", version = "1.47.0") +bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "blake3", version = "1.8.2.bcr.1") bazel_dep(name = "googleapis-grpc-java", version = "1.0.0") @@ -81,7 +82,6 @@ bazel_dep(name = "rules_apple", version = "4.3.1", repo_name = None) bazel_dep(name = "google_benchmark", version = "1.9.4", repo_name = None) bazel_dep(name = "onetbb", version = "2022.2.0", repo_name = None) bazel_dep(name = "protoc-gen-validate", version = "1.3.0", repo_name = None) -bazel_dep(name = "bazel_lib", version = "3.1.0", repo_name = None) single_version_override( module_name = "grpc", @@ -95,6 +95,13 @@ single_version_override( version = "1.34.5.bcr.4", ) +single_version_override( + module_name = "rules_java", + patch_strip = 1, + patches = ["//third_party:rules_java_9.5.0_hermetic_jni.patch"], + version = "9.7.2", +) + single_version_override( module_name = "llvm", patch_strip = 1, @@ -113,6 +120,13 @@ single_version_override( version = "6.8", ) +single_version_override( + module_name = "rules_python", + patch_strip = 1, + patches = ["//third_party:rules_python_1.7.0_hermetic_zip.patch"], + version = "1.7.0", +) + # TODO(pcloudy): Remove once the next version of rules_graalvm is available. single_version_override( module_name = "rules_graalvm", diff --git a/third_party/rules_java_9.5.0_hermetic_jni.patch b/third_party/rules_java_9.5.0_hermetic_jni.patch new file mode 100644 index 00000000000000..738cb8dc5372a3 --- /dev/null +++ b/third_party/rules_java_9.5.0_hermetic_jni.patch @@ -0,0 +1,66 @@ +diff --git a/toolchains/default_java_toolchain.bzl b/toolchains/default_java_toolchain.bzl +index f4f83f6..5229efd 100644 +--- a/toolchains/default_java_toolchain.bzl ++++ b/toolchains/default_java_toolchain.bzl +@@ -198,6 +198,33 @@ def default_java_toolchain(name, configuration = DEFAULT_TOOLCHAIN_CONFIGURATION + target_compatible_with = target_compatible_with, + ) + ++def _java_runtime_file_impl(ctx): ++ java_runtime = ctx.toolchains["@bazel_tools//tools/jdk:runtime_toolchain_type"].java_runtime ++ coreutils = ctx.toolchains["@bazel_lib//lib:coreutils_toolchain_type"].coreutils_info ++ args = ctx.actions.args() ++ args.add("cp") ++ args.add(java_runtime.java_home + "/" + ctx.attr.src) ++ args.add(ctx.outputs.out) ++ ctx.actions.run( ++ executable = coreutils.bin, ++ arguments = [args], ++ inputs = java_runtime.files, ++ outputs = [ctx.outputs.out], ++ toolchain = "@bazel_lib//lib:coreutils_toolchain_type", ++ ) ++ ++_java_runtime_file = rule( ++ implementation = _java_runtime_file_impl, ++ attrs = { ++ "src": attr.string(mandatory = True), ++ "out": attr.output(mandatory = True), ++ }, ++ toolchains = [ ++ "@bazel_tools//tools/jdk:runtime_toolchain_type", ++ "@bazel_lib//lib:coreutils_toolchain_type", ++ ], ++) ++ + def java_runtime_files(name, srcs): + """Copies the given sources out of the current Java runtime.""" + +@@ -207,12 +234,10 @@ def java_runtime_files(name, srcs): + tags = ["manual"], + ) + for src in srcs: +- native.genrule( ++ _java_runtime_file( + name = "gen_%s" % src, +- srcs = [Label("//toolchains:current_java_runtime")], +- toolchains = [Label("//toolchains:current_java_runtime")], +- cmd = "cp $(JAVABASE)/%s $@" % src, +- outs = [src], ++ src = src, ++ out = src, + tags = ["manual"], + ) + +diff --git a/MODULE.bazel b/MODULE.bazel +index 70e975f..fe16975 100644 +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -8,6 +8,7 @@ + bazel_dep(name = "platforms", version = "0.0.11") + bazel_dep(name = "rules_cc", version = "0.2.17") + bazel_dep(name = "bazel_features", version = "1.30.0") ++bazel_dep(name = "bazel_lib", version = "3.2.2") + bazel_dep(name = "bazel_skylib", version = "1.6.1") + bazel_dep(name = "protobuf", version = "32.1", repo_name = "com_google_protobuf") + bazel_dep(name = "zlib", version = "1.3.1.bcr.5") diff --git a/third_party/rules_python_1.7.0_hermetic_zip.patch b/third_party/rules_python_1.7.0_hermetic_zip.patch new file mode 100644 index 00000000000000..c881d308310860 --- /dev/null +++ b/third_party/rules_python_1.7.0_hermetic_zip.patch @@ -0,0 +1,51 @@ +diff -ruN a/MODULE.bazel b/MODULE.bazel +--- a/MODULE.bazel 2026-06-07 09:44:22 ++++ b/MODULE.bazel 2026-06-07 09:44:22 +@@ -5,6 +5,7 @@ + ) + + bazel_dep(name = "bazel_features", version = "1.21.0") ++bazel_dep(name = "bazel_lib", version = "3.2.2") + bazel_dep(name = "bazel_skylib", version = "1.8.2") + bazel_dep(name = "rules_cc", version = "0.1.5") + bazel_dep(name = "platforms", version = "0.0.11") +diff -ruN a/python/private/py_executable.bzl b/python/private/py_executable.bzl +--- a/python/private/py_executable.bzl 2026-06-07 09:44:22 ++++ b/python/private/py_executable.bzl 2026-06-07 09:44:22 +@@ -68,6 +68,7 @@ + + _py_builtins = py_internal + _EXTERNAL_PATH_PREFIX = "external" ++_COREUTILS_TOOLCHAIN = Label("@bazel_lib//lib:coreutils_toolchain_type") + _ZIP_RUNFILES_DIRECTORY_NAME = "runfiles" + + # Non-Google-specific attributes for executables +@@ -906,15 +907,14 @@ + else: + ctx.actions.write(prelude, "#!/usr/bin/env python3\n") + ++ coreutils = ctx.toolchains[_COREUTILS_TOOLCHAIN].coreutils_info + ctx.actions.run_shell( +- command = "cat {prelude} {zip} > {output}".format( +- prelude = prelude.path, +- zip = zip_file.path, +- output = output.path, +- ), ++ command = '"$1" cat "$2" "$3" > "$4"', ++ arguments = [coreutils.bin.path, prelude.path, zip_file.path, output.path], + inputs = [prelude, zip_file], + outputs = [output], +- use_default_shell_env = True, ++ tools = [coreutils.bin], ++ toolchain = _COREUTILS_TOOLCHAIN, + mnemonic = "PyBuildExecutableZip", + progress_message = "Build Python zip executable: %{label}", + ) +@@ -1823,6 +1823,7 @@ + toolchains = [ + ruleb.ToolchainType(TOOLCHAIN_TYPE), + ruleb.ToolchainType(EXEC_TOOLS_TOOLCHAIN_TYPE, mandatory = False), ++ ruleb.ToolchainType(_COREUTILS_TOOLCHAIN), + ruleb.ToolchainType("@bazel_tools//tools/cpp:toolchain_type", mandatory = False), + ], + cfg = dict( From 55c7c796b5b878ae68716c433571626e4fe026a0 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 14 Aug 2026 13:10:52 -0400 Subject: [PATCH 05/14] Update zstd-jni to 1.5.6-9.bcr.2 --- MODULE.bazel | 2 +- MODULE.bazel.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 13b5b41d9a43e7..2162346582b838 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -40,7 +40,7 @@ bazel_dep(name = "rules_testing", version = "0.9.0") bazel_dep(name = "stardoc", version = "0.8.0", repo_name = "io_bazel_skydoc") bazel_dep(name = "with_cfg.bzl", version = "0.13.0") bazel_dep(name = "zlib", version = "1.3.1.bcr.8") -bazel_dep(name = "zstd-jni", version = "1.5.6-9.bcr.1") +bazel_dep(name = "zstd-jni", version = "1.5.6-9.bcr.2") # Depend on apple_support first and then rules_cc so that the Xcode toolchain # from apple_support wins over the generic Unix toolchain from rules_cc. diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ed3431a2eee26f..9736e7fa968c23 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -410,8 +410,8 @@ "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.8/source.json": "cf377d76800dfc3d3b71e9dd4a8c53a62837cbce37cc4f25e6207b15fc1e8f2b", "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72", - "https://bcr.bazel.build/modules/zstd-jni/1.5.6-9.bcr.1/MODULE.bazel": "c6696745628cdc909b39adc10d79548815b32d4504bf225be98cad6bfabd4def", - "https://bcr.bazel.build/modules/zstd-jni/1.5.6-9.bcr.1/source.json": "889fe50d0f7cf72ef35b692ad32a6b0873732463a396a47ec917023da353bc81" + "https://bcr.bazel.build/modules/zstd-jni/1.5.6-9.bcr.2/MODULE.bazel": "867ee1f4864de7f29545db4c5ea7675723a704f941e47c129ed89c782f462eff", + "https://bcr.bazel.build/modules/zstd-jni/1.5.6-9.bcr.2/source.json": "ff6e5c953dc3a44a59c38a9eb361540276dc72e8032b7746decd406a6a99c0e7" }, "selectedYankedVersions": {}, "moduleExtensions": { From 5ce2c987cff6e5a751ad6ce970814deb51013ddf Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:48 -0400 Subject: [PATCH 06/14] Select protoc path separators from the execution platform --- third_party/grpc-java.patch | 18 ++++++++++ third_party/protobuf.patch | 65 +++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/third_party/grpc-java.patch b/third_party/grpc-java.patch index b3d2410a4455ee..d58ac92a5a7d24 100644 --- a/third_party/grpc-java.patch +++ b/third_party/grpc-java.patch @@ -38,3 +38,21 @@ index 630ced383..e7d133a27 100644 arguments = [args], use_default_shell_env = True, toolchain = None, +diff --git a/java_grpc_library.bzl b/java_grpc_library.bzl +--- a/java_grpc_library.bzl ++++ b/java_grpc_library.bzl +@@ -94,7 +94,13 @@ + args = ctx.actions.args() + args.add(toolchain.plugin[DefaultInfo].files_to_run.executable, format = "--plugin=protoc-gen-rpc-plugin=%s") + args.add("--rpc-plugin_out={0}:{1}".format(toolchain.plugin_arg, srcjar.path)) +- args.add_joined("--descriptor_set_in", descriptor_set_in, join_with = ctx.configuration.host_path_separator) ++ protoc = toolchain.protoc[DefaultInfo].files_to_run.executable ++ path_separator = ";" if protoc.extension == "exe" else ":" ++ args.add_joined( ++ "--descriptor_set_in", ++ descriptor_set_in, ++ join_with = path_separator, ++ ) + args.add_all(srcs, map_each = _path_ignoring_repository) + + ctx.actions.run( diff --git a/third_party/protobuf.patch b/third_party/protobuf.patch index 8fa92db4ea591c..03dc6c438583ec 100644 --- a/third_party/protobuf.patch +++ b/third_party/protobuf.patch @@ -246,3 +246,68 @@ index 54f8490ce4..801c58a8b2 100644 output("$0,\n", FieldInitializer(pools, ext)); output(" $0,\n", GetSub(ext, true, options)); output(" &$0,\n", MessageVarName(ext.containing_type())); +diff --git a/upb_generator/bootstrap_compiler.bzl b/upb_generator/bootstrap_compiler.bzl +--- a/upb_generator/bootstrap_compiler.bzl ++++ b/upb_generator/bootstrap_compiler.bzl +@@ -18,7 +18,8 @@ + _stages = ["_stage0", "_stage1", ""] + _protoc = "//src/google/protobuf/compiler/release:protoc_minimal" + +-_extra_proto_path = "-I$$(dirname $(location //:descriptor_proto_srcs))/../.. " ++_descriptor_proto = "$(location //:descriptor_proto_srcs)" ++_extra_proto_path = "-I$${DESCRIPTOR_PROTO%/google/protobuf/descriptor.proto} " + + # This visibility is used automatically for anything used by the bootstrapping process. + _bootstrap_visibility = [ +@@ -111,6 +112,7 @@ + outs = ["bootstrap_generated_sources/" + f.replace("third_party", "3rd_party") for f in _generated_hdrs_and_srcs(src_files, "stage0", "upb")], + tools = [_protoc, _upbc("upb", 0)], + cmd = ++ "DESCRIPTOR_PROTO=" + _descriptor_proto + "; " + + "$(location " + _protoc + ") " + + "-I. -I$(GENDIR)/" + strip_prefix + " " + _extra_proto_path + + "--plugin=protoc-gen-upb=$(location " + _upbc("upb", 0) + ") " + +@@ -137,7 +139,8 @@ + name = "gen_{}_{}_stage1".format(name, generator), + srcs = src_rules, + outs = _generated_hdrs_and_srcs(src_files, "stage1", generator), +- cmd = "$(location " + _protoc + ") " + ++ cmd = "DESCRIPTOR_PROTO=" + _descriptor_proto + "; " + ++ "$(location " + _protoc + ") " + + "--plugin=protoc-gen-" + generator + + "=$(location " + _upbc(generator, 0) + ") " + _extra_proto_path + + "--" + generator + "_out=bootstrap_stage=1:$(RULEDIR)/stage1 " + +diff --git a/protobuf.bzl b/protobuf.bzl +--- a/protobuf.bzl ++++ b/protobuf.bzl +@@ -92,6 +92,12 @@ + return toolchain.proto.proto_compiler + else: + return ctx.attr._proto_compiler[DefaultInfo].files_to_run ++ ++def _protoc_path_separator(ctx): ++ protoc = _protoc_files_to_run(ctx).executable ++ if protoc and protoc.extension == "exe": ++ return ";" ++ return ":" + + ProtoGenInfo = provider( + fields = ["srcs", "import_flags", "deps"], +@@ -320,7 +326,7 @@ + args.add_joined( + "--descriptor_set_in", + descriptors, +- join_with = ctx.configuration.host_path_separator, ++ join_with = _protoc_path_separator(ctx), + ) + + for dep in deps: +@@ -380,7 +386,7 @@ + args.add_joined( + "--descriptor_set_in", + descriptors, +- join_with = ctx.configuration.host_path_separator, ++ join_with = _protoc_path_separator(ctx), + ) + + for dep in deps: From 5657c26b103751d467336a11b46b81c26d44b965 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:49 -0400 Subject: [PATCH 07/14] Support Clang and MinGW Windows dependencies --- .bazelrc | 8 --- MODULE.bazel | 8 ++- src/main/cpp/BUILD | 20 +++++++ src/tools/launcher/java_launcher.cc | 4 +- third_party/c-ares_1.34.5.bcr.2.patch | 36 ++++++++++++ .../grpc_1.76.0.bcr.1_hermetic_plugin.patch | 56 +++++++++++++++++++ 6 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 third_party/c-ares_1.34.5.bcr.2.patch create mode 100644 third_party/grpc_1.76.0.bcr.1_hermetic_plugin.patch diff --git a/.bazelrc b/.bazelrc index 82470c2c2f835c..d2f2d421076648 100644 --- a/.bazelrc +++ b/.bazelrc @@ -49,14 +49,6 @@ common:linux --per_file_copt=external/.*@-w common:linux --host_per_file_copt=external/.*@-w common:macos --per_file_copt=external/.*@-w common:macos --host_per_file_copt=external/.*@-w -# Silence specific noisy warnings for external repos -# Note that we silence individual warnings here. This is tedious and brittle, -# but avoids the enormous log spam from passing /w or /W0 to MSVC, since that -# overrides the default /W3 warning level. MSVC creates even more warnings for -# this behavior. -common:windows --per_file_copt=external/.*@/wd4047,/wd4624,/wd4267,/wd4244,/wd4715,/wd4312,/wd4805,/wd4700,/wd4722 -common:windows --host_per_file_copt=external/.*@/wd4047,/wd4624,/wd4267,/wd4244,/wd4715,/wd4312,/wd4805,/wd4700,/wd4722 - # Enable Protobuf MSVC support on Windows common:windows --define=protobuf_allow_msvc=true diff --git a/MODULE.bazel b/MODULE.bazel index 2162346582b838..7d24dd38989abb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -86,12 +86,18 @@ bazel_dep(name = "protoc-gen-validate", version = "1.3.0", repo_name = None) single_version_override( module_name = "grpc", patch_strip = 1, - patches = ["//third_party:grpc-load-fixes.patch"], + patches = [ + "//third_party:grpc-load-fixes.patch", + "//third_party:grpc_1.76.0.bcr.1_hermetic_plugin.patch", + ], + version = "1.76.0.bcr.1", ) # bazel_dep overrides single_version_override( module_name = "c-ares", + patch_strip = 1, + patches = ["//third_party:c-ares_1.34.5.bcr.2.patch"], version = "1.34.5.bcr.4", ) diff --git a/src/main/cpp/BUILD b/src/main/cpp/BUILD index 6de3db58ff1e96..b8cc42112e4b87 100644 --- a/src/main/cpp/BUILD +++ b/src/main/cpp/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//lib:selects.bzl", "selects") load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_cc//cc:cc_library.bzl", "cc_library") @@ -14,6 +15,14 @@ WIN_LINK_OPTS = [ "-DEFAULTLIB:ws2_32.lib", # grpc ] +selects.config_setting_group( + name = "windows_clang", + match_all = [ + "//src/conditions:windows", + "@rules_cc//cc/compiler:clang", + ], +) + cc_library( name = "blaze_util", srcs = [ @@ -58,6 +67,17 @@ cc_library( "//conditions:default": [ "-lrt", ], + }) + select({ + ":windows_clang": [ + "-ladvapi32", + "-lbcrypt", + "-lole32", + "-lshell32", + "-luuid", + "-lws2_32", + "-municode", + ], + "//conditions:default": [], }), deps = [ "//src/main/cpp/util", diff --git a/src/tools/launcher/java_launcher.cc b/src/tools/launcher/java_launcher.cc index 42fc50263823a9..8c2437bb1e50ac 100644 --- a/src/tools/launcher/java_launcher.cc +++ b/src/tools/launcher/java_launcher.cc @@ -253,7 +253,7 @@ wstring JavaBinaryLauncher::CreateClasspathJar(const wstring& classpath) { wstring jar_manifest_file_path = binary_base_path + rand_id + L".jar_manifest"; blaze_util::AddUncPrefixMaybe(&jar_manifest_file_path); - wofstream jar_manifest_file{jar_manifest_file_path}; + wofstream jar_manifest_file{jar_manifest_file_path.c_str()}; jar_manifest_file << L"Manifest-Version: 1.0\n"; // No line in the MANIFEST.MF file may be longer than 72 bytes. // A space prefix indicates the line is still the content of the last @@ -313,7 +313,7 @@ wstring JavaBinaryLauncher::CreateClasspathFlagfile(const wstring& classpath) { wstring flagfile_path = binary_base_path + L"-" + rand_id_ + L"-classpath.txt"; blaze_util::AddUncPrefixMaybe(&flagfile_path); - ofstream flagfile{flagfile_path}; + ofstream flagfile{flagfile_path.c_str()}; flagfile << contents_utf8; flagfile.close(); if (flagfile.fail()) { diff --git a/third_party/c-ares_1.34.5.bcr.2.patch b/third_party/c-ares_1.34.5.bcr.2.patch new file mode 100644 index 00000000000000..f0c12dbd1477cd --- /dev/null +++ b/third_party/c-ares_1.34.5.bcr.2.patch @@ -0,0 +1,36 @@ +diff --git a/BUILD.bazel b/BUILD.bazel +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -1,3 +1,4 @@ ++load("@bazel_skylib//lib:selects.bzl", "selects") + load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + load("@rules_cc//cc:cc_library.bzl", "cc_library") + +@@ -11,6 +12,14 @@ + constraint_values = ["@platforms//os:windows"], + ) + ++selects.config_setting_group( ++ name = "windows_clang", ++ match_all = [ ++ ":windows", ++ "@rules_cc//cc/compiler:clang", ++ ], ++) ++ + # Android is not officially supported through C++. + # This just helps with the build for now. + config_setting( +@@ -236,6 +245,12 @@ + "-defaultlib:iphlpapi.lib", + ], + "//conditions:default": [], ++ }) + select({ ++ ":windows_clang": [ ++ "-lws2_32", ++ "-liphlpapi", ++ ], ++ "//conditions:default": [], + }), + linkstatic = 1, + visibility = [ diff --git a/third_party/grpc_1.76.0.bcr.1_hermetic_plugin.patch b/third_party/grpc_1.76.0.bcr.1_hermetic_plugin.patch new file mode 100644 index 00000000000000..045d402d90efdc --- /dev/null +++ b/third_party/grpc_1.76.0.bcr.1_hermetic_plugin.patch @@ -0,0 +1,56 @@ +diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl +--- a/bazel/grpc_build_system.bzl ++++ b/bazel/grpc_build_system.bzl +@@ -27,6 +27,7 @@ + Contains macros used throughout the repo. + """ + ++load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + load("@build_bazel_apple_support//rules:universal_binary.bzl", "universal_binary") + load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") + load("@build_bazel_rules_apple//apple/testing/default_runner:ios_test_runner.bzl", "ios_test_runner") +@@ -207,15 +208,15 @@ + ) + + # In order to avoid warnings from Bazel, names of the rule and its output file must differ. +- native.genrule( ++ copy_file( + name = name, +- srcs = select({ +- "@platforms//os:macos": [name + "_universal"], +- "//conditions:default": [name + "_native"], ++ src = select({ ++ "@platforms//os:macos": name + "_universal", ++ "//conditions:default": name + "_native", + }), +- outs = [name + "_binary"], +- cmd = "cp $< $@", +- executable = True, ++ out = name + "_binary", ++ allow_symlink = True, ++ is_executable = True, + ) + + def grpc_internal_proto_library( +diff --git a/BUILD b/BUILD +--- a/BUILD ++++ b/BUILD +@@ -4274,6 +4274,7 @@ + "//src/core:error", + "//src/core:json", + "//src/core:load_file", ++ "//src/core:slice", + "//src/core:useful", + ], + ) +diff --git a/src/core/BUILD b/src/core/BUILD +--- a/src/core/BUILD ++++ b/src/core/BUILD +@@ -2931,6 +2931,7 @@ + "grpc_check", + "iomgr_port", + "posix_event_engine_timer_manager", ++ "resource_quota", + "sync", + "time", + "windows_endpoint", From 5d7d1eeead26d13d7bcacbfd697be6c47154f774 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:49 -0400 Subject: [PATCH 08/14] Add ZIP entry filtering to zipper --- third_party/ijar/test/zip_test.sh | 28 ++++++ third_party/ijar/zip_main.cc | 136 ++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) diff --git a/third_party/ijar/test/zip_test.sh b/third_party/ijar/test/zip_test.sh index 45009f65d8194e..08c7f15f74d9ea 100755 --- a/third_party/ijar/test/zip_test.sh +++ b/third_party/ijar/test/zip_test.sh @@ -97,6 +97,34 @@ function test_zipper() { expect_not_log "path" } +function test_zipper_delete_entries() { + mkdir -p ${TEST_TMPDIR}/filter/META-INF/license + mkdir -p ${TEST_TMPDIR}/filter/META-INF/native + echo "Manifest-Version: 1.0" >${TEST_TMPDIR}/filter/META-INF/MANIFEST.MF + echo "license" >${TEST_TMPDIR}/filter/META-INF/license/LICENSE.txt + echo "linux" >${TEST_TMPDIR}/filter/META-INF/native/library.so + echo "windows" >${TEST_TMPDIR}/filter/META-INF/native/library.dll + + (cd ${TEST_TMPDIR}/filter && \ + $ZIPPER c ${TEST_TMPDIR}/input.zip \ + META-INF/MANIFEST.MF \ + META-INF/license/LICENSE.txt \ + META-INF/native/library.so \ + META-INF/native/library.dll) + $ZIPPER d ${TEST_TMPDIR}/input.zip ${TEST_TMPDIR}/output.zip \ + "*/license/*" "*.dll" + + $ZIPPER v ${TEST_TMPDIR}/output.zip >$TEST_log + expect_log "META-INF/MANIFEST.MF" + expect_log "META-INF/native/library.so" + expect_not_log "license" + expect_not_log "library.dll" + + $ZIPPER d ${TEST_TMPDIR}/input.zip ${TEST_TMPDIR}/empty.zip "*" + $ZIPPER v ${TEST_TMPDIR}/empty.zip >$TEST_log + [ ! -s $TEST_log ] || fail "The '*' pattern did not remove every entry" +} + function test_zipper_junk_paths() { mkdir -p ${TEST_TMPDIR}/test/path/to/some mkdir -p ${TEST_TMPDIR}/test/some/other/path diff --git a/third_party/ijar/zip_main.cc b/third_party/ijar/zip_main.cc index e05447c42ad4cb..068de76b2918a5 100644 --- a/third_party/ijar/zip_main.cc +++ b/third_party/ijar/zip_main.cc @@ -27,9 +27,11 @@ #include #include +#include #include #include #include +#include #include "third_party/ijar/platform_utils.h" #include "third_party/ijar/zip.h" @@ -79,6 +81,102 @@ class UnzipProcessor : public ZipExtractorProcessor { std::set file_names; }; +class FilterProcessor : public ZipExtractorProcessor { + public: + explicit FilterProcessor(char **patterns) : patterns_(patterns) {} + + void SetBuilder(ZipBuilder *builder) { builder_ = builder; } + + bool Failed() const { return failed_; } + + void WriteAll() { + std::sort(entries_.begin(), entries_.end(), + [](const Entry &left, const Entry &right) { + return left.filename < right.filename; + }); + for (const Entry &entry : entries_) { + u4 attr = entry.attr; + if (attr == 0) { + bool is_directory = !entry.filename.empty() && + entry.filename[entry.filename.size() - 1] == '/'; + mode_t mode = is_directory ? 0755 : 0644; + attr = stat_to_zipattr(Stat{entry.data.size(), mode, is_directory}); + } + u1 *buffer = builder_->NewFile(entry.filename.c_str(), attr); + if (buffer == nullptr) { + failed_ = true; + return; + } + if (!entry.data.empty()) { + memcpy(buffer, entry.data.data(), entry.data.size()); + } + if (builder_->FinishFile(entry.data.size(), /*compress=*/true, + /*compute_crc=*/true) < 0) { + failed_ = true; + return; + } + } + } + + bool Accept(const char *filename, const u4 /*attr*/) override { + for (int i = 0; patterns_ != nullptr && patterns_[i] != nullptr; ++i) { + if (Matches(filename, patterns_[i])) { + return false; + } + } + return true; + } + + void Process(const char *filename, u4 attr, const u1 *data, + size_t size) override { + entries_.push_back( + Entry{filename, attr, std::vector(data, data + size)}); + } + + private: + struct Entry { + std::string filename; + u4 attr; + std::vector data; + }; + + static bool Matches(const char *filename, const char *pattern) { + size_t filename_length = strlen(filename); + size_t pattern_length = strlen(pattern); + bool leading_wildcard = pattern_length > 0 && pattern[0] == '*'; + bool trailing_wildcard = + pattern_length > 0 && pattern[pattern_length - 1] == '*'; + if (pattern_length == 1 && leading_wildcard) { + return true; + } + const char *literal = pattern + (leading_wildcard ? 1 : 0); + size_t literal_length = + pattern_length - (leading_wildcard ? 1 : 0) - + (trailing_wildcard ? 1 : 0); + + if (leading_wildcard && trailing_wildcard) { + return std::string(filename).find(std::string(literal, literal_length)) != + std::string::npos; + } + if (leading_wildcard) { + return filename_length >= literal_length && + memcmp(filename + filename_length - literal_length, literal, + literal_length) == 0; + } + if (trailing_wildcard) { + return filename_length >= literal_length && + memcmp(filename, literal, literal_length) == 0; + } + return filename_length == literal_length && + memcmp(filename, literal, literal_length) == 0; + } + + char **patterns_; + ZipBuilder *builder_ = nullptr; + std::vector entries_; + bool failed_ = false; +}; + // Concatene 2 path, path1 and path2, using / as a directory separator and // putting the result in "out". "size" specify the size of the output buffer. If // the result would overflow the output buffer, print an error message and @@ -197,6 +295,39 @@ int extract(char *zipfile, char *exdir, char **files, bool verbose, return 0; } +int filter(char *input_zip, char *output_zip, char **patterns) { + FilterProcessor processor(patterns); + std::unique_ptr extractor( + ZipExtractor::Create(input_zip, &processor)); + if (extractor == nullptr) { + fprintf(stderr, "Unable to open zip file %s: %s.\n", input_zip, + strerror(errno)); + return -1; + } + std::unique_ptr builder( + ZipBuilder::Create(output_zip, extractor->CalculateOutputLength())); + if (builder == nullptr) { + fprintf(stderr, "Unable to create zip file %s: %s.\n", output_zip, + strerror(errno)); + return -1; + } + processor.SetBuilder(builder.get()); + if (extractor->ProcessAll() < 0) { + fprintf(stderr, "%s.\n", extractor->GetError()); + return -1; + } + processor.WriteAll(); + if (processor.Failed()) { + fprintf(stderr, "%s\n", builder->GetError()); + return -1; + } + if (builder->Finish() < 0) { + fprintf(stderr, "%s\n", builder->GetError()); + return -1; + } + return 0; +} + // add a file to the zip int add_file(std::unique_ptr const &builder, char *file, char *zip_path, bool flatten, bool verbose, bool compress) { @@ -400,6 +531,8 @@ static void usage(char *progname) { fprintf(stderr, " C compress - compress files when using the create operation\n"); fprintf(stderr, "x and c cannot be used in the same command-line.\n"); + fprintf(stderr, + " d input.zip output.zip patterns... # Delete matching entries.\n"); fprintf(stderr, "\nFor every file, a path in the zip can be specified. Examples:\n"); fprintf(stderr, @@ -415,6 +548,9 @@ static void usage(char *progname) { } int main(int argc, char **argv) { + if (argc >= 5 && strcmp(argv[1], "d") == 0) { + return devtools_ijar::filter(argv[2], argv[3], argv + 4); + } bool extract = false; bool verbose = false; bool create = false; From c78e2dca581feb45bbbac0a4513af2c188a4887b Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:49 -0400 Subject: [PATCH 09/14] Process Netty and ProGuard JARs with declared actions --- third_party/BUILD | 88 ++++++++++++++++++-------- third_party/filter_zip.bzl | 32 ++++++++++ tools/build_defs/proguard/BUILD.bazel | 6 -- tools/build_defs/proguard/proguard.bzl | 21 +++++- tools/build_defs/proguard/wrapper.py | 68 +++++++------------- 5 files changed, 138 insertions(+), 77 deletions(-) create mode 100644 third_party/filter_zip.bzl diff --git a/third_party/BUILD b/third_party/BUILD index 5281d4ff899da6..a432a8977d2e76 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -3,6 +3,7 @@ load("@rules_license//rules:license.bzl", "license") load("//src/tools/bzlmod:utils.bzl", "get_repo_root") load("//tools/build_defs/proguard:proguard.bzl", "proguard_jar") load("//tools/distributions:distribution_rules.bzl", "distrib_jar_filegroup", "distrib_java_import") +load(":filter_zip.bzl", "filter_zip") package(default_visibility = ["//visibility:public"]) @@ -493,40 +494,75 @@ alias( ) UNNECESSARY_DYNAMIC_LIBRARIES = select({ - "//src/conditions:windows": "*.so *.jnilib", - "//src/conditions:darwin": "*.so *.dll *.pyd", - "//src/conditions:linux_x86_64": "*.jnilib *.dll *.pyd", - "//src/conditions:linux_s390x": "*.jnilib *.dll *.pyd", + "//src/conditions:windows": [ + "*.so", + "*.jnilib", + ], + "//src/conditions:darwin": [ + "*.so", + "*.dll", + "*.pyd", + ], + "//src/conditions:linux_x86_64": [ + "*.jnilib", + "*.dll", + "*.pyd", + ], + "//src/conditions:linux_s390x": [ + "*.jnilib", + "*.dll", + "*.pyd", + ], # The .so file is an x86/s390x one, so we can just remove it if the CPU is not x86/s390x - "//src/conditions:arm": "*.so *.jnilib *.dll *.pyd", - "//src/conditions:linux_aarch64": "*.so *.jnilib *.dll *.pyd", - "//src/conditions:linux_ppc": "*.so *.jnilib *.dll *.pyd", - "//src/conditions:freebsd": "*.so *.jnilib *.dll *.pyd", - "//src/conditions:openbsd": "*.so *.jnilib *.dll *.pyd", + "//src/conditions:arm": [ + "*.so", + "*.jnilib", + "*.dll", + "*.pyd", + ], + "//src/conditions:linux_aarch64": [ + "*.so", + "*.jnilib", + "*.dll", + "*.pyd", + ], + "//src/conditions:linux_ppc": [ + "*.so", + "*.jnilib", + "*.dll", + "*.pyd", + ], + "//src/conditions:freebsd": [ + "*.so", + "*.jnilib", + "*.dll", + "*.pyd", + ], + "//src/conditions:openbsd": [ + "*.so", + "*.jnilib", + "*.dll", + "*.pyd", + ], # Default is to play it safe -- better have a big binary than a slow binary - # The empty string means nothing is to be removed from the library; - # the rule command tests for the empty string explictly to avoid - # zip erroring when it finds nothing to remove. - "//conditions:default": "", + "//conditions:default": [], }) # Remove native libraries that are for a platform different from the one we are # building Bazel for. -genrule( +filter_zip( name = "filter_netty_dynamic_libs", - srcs = select({ - "//src/conditions:darwin_arm64": ["@maven//:io_netty_netty_tcnative_boringssl_static_osx_aarch_64_file"], - "//src/conditions:darwin_x86_64": ["@maven//:io_netty_netty_tcnative_boringssl_static_osx_x86_64_file"], - "//src/conditions:linux_aarch64": ["@maven//:io_netty_netty_tcnative_boringssl_static_linux_aarch_64_file"], - "//src/conditions:linux_x86_64": ["@maven//:io_netty_netty_tcnative_boringssl_static_linux_x86_64_file"], - "//src/conditions:windows": ["@maven//:io_netty_netty_tcnative_boringssl_static_windows_x86_64_file"], - "//conditions:default": [], + src = select({ + "//src/conditions:darwin_arm64": "@maven//:io_netty_netty_tcnative_boringssl_static_osx_aarch_64_file", + "//src/conditions:darwin_x86_64": "@maven//:io_netty_netty_tcnative_boringssl_static_osx_x86_64_file", + "//src/conditions:linux_aarch64": "@maven//:io_netty_netty_tcnative_boringssl_static_linux_aarch_64_file", + "//src/conditions:linux_x86_64": "@maven//:io_netty_netty_tcnative_boringssl_static_linux_x86_64_file", + "//src/conditions:windows": "@maven//:io_netty_netty_tcnative_boringssl_static_windows_x86_64_file", + "//conditions:default": "@maven//:io_netty_netty_tcnative_boringssl_static_linux_x86_64_file", }), - outs = ["netty_tcnative/netty-tcnative-filtered.jar"], - cmd = "cp $< $@ && " + - # Make sure we can write the output file, even if the input isn't writable. - "chmod +w $@ && " + - "zip -qd $@ */license/* " + UNNECESSARY_DYNAMIC_LIBRARIES, + out = "netty_tcnative/netty-tcnative-filtered.jar", + exclude_patterns = ["*/license/*"] + UNNECESSARY_DYNAMIC_LIBRARIES, + zipper = "//third_party/ijar:zipper", ) distrib_java_import( diff --git a/third_party/filter_zip.bzl b/third_party/filter_zip.bzl new file mode 100644 index 00000000000000..5f9125b6c9ff30 --- /dev/null +++ b/third_party/filter_zip.bzl @@ -0,0 +1,32 @@ +"""Hermetic ZIP filtering rule.""" + +def _filter_zip_impl(ctx): + args = ctx.actions.args() + args.add("d") + args.add(ctx.file.src) + args.add(ctx.outputs.out) + args.add_all(ctx.attr.exclude_patterns) + + ctx.actions.run( + executable = ctx.executable.zipper, + arguments = [args], + inputs = [ctx.file.src], + outputs = [ctx.outputs.out], + mnemonic = "FilterZip", + progress_message = "Filtering %{input}", + tools = [ctx.executable.zipper], + ) + +filter_zip = rule( + implementation = _filter_zip_impl, + attrs = { + "src": attr.label(allow_single_file = True, mandatory = True), + "exclude_patterns": attr.string_list(), + "out": attr.output(mandatory = True), + "zipper": attr.label( + executable = True, + cfg = "exec", + mandatory = True, + ), + }, +) diff --git a/tools/build_defs/proguard/BUILD.bazel b/tools/build_defs/proguard/BUILD.bazel index 0efe89865c534d..ae217b156c1141 100644 --- a/tools/build_defs/proguard/BUILD.bazel +++ b/tools/build_defs/proguard/BUILD.bazel @@ -16,14 +16,8 @@ exports_files( py_binary( name = "wrapper_private", srcs = ["wrapper.py"], - data = [ - ":proguard_private", - ], main = "wrapper.py", visibility = ["//visibility:private"], - deps = [ - "@rules_python//python/runfiles", - ], ) java_binary( diff --git a/tools/build_defs/proguard/proguard.bzl b/tools/build_defs/proguard/proguard.bzl index 9052e63ac1e575..9f24191dedc831 100644 --- a/tools/build_defs/proguard/proguard.bzl +++ b/tools/build_defs/proguard/proguard.bzl @@ -20,11 +20,19 @@ """Apply proguard rules to a JAR file.""" +load("@rules_java//java:defs.bzl", "java_common") + def _proguard_jar_impl(ctx): - inputs = ctx.files.srcs + ctx.files.deps + [ctx.file.proguard_spec] + java_runtime = ctx.attr._java_runtime[java_common.JavaRuntimeInfo] + inputs = ctx.files.srcs + ctx.files.deps + [ + ctx.file.proguard_spec, + ctx.file._proguard, + ] output = ctx.outputs.out args = ctx.actions.args() + args.add("--java_executable", java_runtime.java_executable_exec_path) + args.add("--proguard_jar", ctx.file._proguard) args.add_joined("--srcs", ctx.files.srcs, join_with = ",") args.add_joined("--deps", ctx.files.deps, join_with = ",") args.add("--proguard_spec", ctx.file.proguard_spec) @@ -37,6 +45,7 @@ def _proguard_jar_impl(ctx): outputs = [output], executable = ctx.executable._wrapper, arguments = [args], + tools = java_runtime.files, ) return DefaultInfo(files = depset([output])) @@ -48,6 +57,16 @@ proguard_jar = rule( "deps": attr.label_list(), "proguard_spec": attr.label(allow_single_file = True), "out": attr.output(), + "_java_runtime": attr.label( + cfg = "exec", + default = "@bazel_tools//tools/jdk:current_java_runtime", + providers = [java_common.JavaRuntimeInfo], + ), + "_proguard": attr.label( + allow_single_file = True, + cfg = "exec", + default = ":proguard_private_deploy.jar", + ), "_wrapper": attr.label( cfg = "exec", default = ":wrapper_private", diff --git a/tools/build_defs/proguard/wrapper.py b/tools/build_defs/proguard/wrapper.py index f3a2ae4a027aa2..fa4eaf8d864e22 100644 --- a/tools/build_defs/proguard/wrapper.py +++ b/tools/build_defs/proguard/wrapper.py @@ -17,48 +17,18 @@ import argparse import datetime import os -import platform import subprocess import tempfile import zipfile -from python.runfiles import Runfiles - -_PROGUARD_PATH = "_main/tools/build_defs/proguard/proguard_private" - - -def lookup_binary(r, path): - """Lookup the runfiles-adjusted path to a binary. - - Args: - r: The Runfiles object to use for the lookup. - path: The path of the binary being found. - - Returns: - The full path to the binary. - - Raises: - RuntimeError: If the path is not present in the runfiles, or if the adjusted - path does not - exist on the filesystem. - """ - - if platform.system() == "Windows": - path = path + ".exe" - binary = r.Rlocation(path) - if not binary: - raise RuntimeError(f"Runfiles failed to resolve {path}") - elif not os.path.exists(binary): - raise RuntimeError( - f"Runfiles resolved {path} to {binary} but the file does not exist" - ) - return binary - - -def apply_proguard(srcs, deps, proguard_spec, output_jar): +def apply_proguard( + java_executable, proguard_jar, srcs, deps, proguard_spec, output_jar +): """Call proguard on the given source jars with the spec. Args: + java_executable: The execution-platform Java executable. + proguard_jar: The ProGuard deploy JAR. srcs: The source jars to be modified. deps: Dependency jars needed to resolve the source jars. proguard_spec: The path to the proguard spec file describing what @@ -70,12 +40,11 @@ def apply_proguard(srcs, deps, proguard_spec, output_jar): stderr. """ - # Set up runfiles and call the proguard binary. - r = Runfiles.Create() - proguard_path = lookup_binary(r, _PROGUARD_PATH) - command = [ - proguard_path, + java_executable, + "-Dlog4j.rootLogger=OFF", + "-jar", + proguard_jar, "-injars", srcs, "-libraryjars", @@ -85,10 +54,8 @@ def apply_proguard(srcs, deps, proguard_spec, output_jar): "@" + proguard_spec, ] - env = os.environ.copy() - env.update(r.EnvVars()) # print("Running proguard: %s" % " ".join(command)) - p = subprocess.run(command, capture_output=True, env=env, check=False) + p = subprocess.run(command, capture_output=True, check=False) if p.returncode != 0: message = f"Proguard failed ({p.returncode})" @@ -126,6 +93,12 @@ def main() -> None: parser = argparse.ArgumentParser( description="Resets timestamps in ZIP files", fromfile_prefix_chars="@" ) + parser.add_argument( + "--java_executable", required=True, help="Execution-platform Java." + ) + parser.add_argument( + "--proguard_jar", required=True, help="ProGuard deploy JAR." + ) parser.add_argument( "--srcs", required=True, help="Input jar files, mandatory." ) @@ -146,7 +119,14 @@ def main() -> None: with tempfile.TemporaryDirectory() as wdir: output_jar = os.path.join(wdir, "stripped.jar") - apply_proguard(opts.srcs, opts.deps, opts.proguard_spec, output_jar) + apply_proguard( + opts.java_executable, + opts.proguard_jar, + opts.srcs, + opts.deps, + opts.proguard_spec, + output_jar, + ) reset_timestamps(output_jar, opts.output, opts.timestamp) From 2b08ea668ea5201331b7b2daf042af33385eb75d Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:49 -0400 Subject: [PATCH 10/14] Select JDK minimization tools for the execution platform --- src/BUILD | 22 ++++++++++++-- src/minimize_jdk.sh | 72 +++++++++++++++++++++++++++++++++------------ 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/src/BUILD b/src/BUILD index 7baad3330f387f..196f8f8818fb57 100644 --- a/src/BUILD +++ b/src/BUILD @@ -181,6 +181,13 @@ copy_file( allow_symlink = True, ) +copy_file( + name = "objcopy_for_jlink", + src = "@llvm//tools:llvm-objcopy", + out = "objcopy", + allow_symlink = True, +) + # Reads the app manifest of a Windows executable. cc_binary( name = "read_manifest", @@ -212,7 +219,10 @@ cc_binary( sh_binary( name = "minimize_jdk", srcs = ["minimize_jdk.sh"], - data = select({ + data = [ + ":objcopy_for_jlink", + "//third_party/ijar:zipper", + ] + select({ "@platforms//os:windows": [ ":read_manifest", ":write_manifest", @@ -248,7 +258,10 @@ genrule( ":jdeps_modules.golden", ], outs = ["minimal_jdk.zip"], - cmd = "$(location :minimize_jdk) " + _JLINK_TOOL_LOCATION + " $(location :embedded_jdk_vanilla) $(location :jdeps_modules.golden) $@", + cmd = "$(location :minimize_jdk)" + select({ + "//src/conditions:windows": " --target_windows", + "//conditions:default": "", + }) + " " + _JLINK_TOOL_LOCATION + " $(location :embedded_jdk_vanilla) $(location :jdeps_modules.golden) $@", tools = [":minimize_jdk"] + _JLINK_TOOL, visibility = ["//src/test/shell/bazel:__pkg__"], ) @@ -260,7 +273,10 @@ genrule( ":jdeps_modules.golden", ], outs = ["allmodules_jdk.zip"], - cmd = "$(location :minimize_jdk) --allmodules " + _JLINK_TOOL_LOCATION + " $(location :embedded_jdk_vanilla) $(location :jdeps_modules.golden) $@", + cmd = "$(location :minimize_jdk) --allmodules" + select({ + "//src/conditions:windows": " --target_windows", + "//conditions:default": "", + }) + " " + _JLINK_TOOL_LOCATION + " $(location :embedded_jdk_vanilla) $(location :jdeps_modules.golden) $@", tools = [":minimize_jdk"] + _JLINK_TOOL, visibility = ["//src/test/shell/bazel:__pkg__"], ) diff --git a/src/minimize_jdk.sh b/src/minimize_jdk.sh index dc533b539aa719..2b1a59804410c1 100755 --- a/src/minimize_jdk.sh +++ b/src/minimize_jdk.sh @@ -42,14 +42,25 @@ if [ "$1" == "--allmodules" ]; then shift modules="ALL-MODULE-PATH" else - modules=$(cat "$3" | paste -sd "," - | tr -d '\r') - # We have to add this module explicitly because jdeps doesn't find the - # dependency on it but it is still necessary for TLSv1.3. - modules="$modules,jdk.crypto.ec" + modules="" +fi +target_windows=false +if [ "$1" == "--target_windows" ]; then + target_windows=true + shift fi tooljdk=$1 fulljdk=$2 out=$4 +if [ -z "$modules" ]; then + while IFS= read -r module; do + module=${module%$'\r'} + modules=${modules:+$modules,}$module + done < "$3" + # We have to add this module explicitly because jdeps doesn't find the + # dependency on it but it is still necessary for TLSv1.3. + modules="$modules,jdk.crypto.ec" +fi # Optional 5th argument: a separate jmods archive for JDKs that don't ship # with jmods (e.g. Adoptium Temurin with JEP 493 enabled). jmods_archive=${5:-} @@ -58,7 +69,11 @@ if [ -n "$jmods_archive" ]; then jmods_archive=$(cd "$(dirname "$jmods_archive")" && echo "$(pwd)/$(basename "$jmods_archive")") fi -UNAME=$(uname -s | tr 'A-Z' 'a-z') +ZIPPER=$(rlocation io_bazel/third_party/ijar/zipper) +ZIPPER=$(cd "${ZIPPER%/*}" && echo "$PWD/${ZIPPER##*/}") +OBJCOPY=$(rlocation io_bazel/src/objcopy) +OBJCOPY=$(cd "${OBJCOPY%/*}" && echo "$PWD/${OBJCOPY##*/}") +export PATH="${OBJCOPY%/*}:$PATH" # Options for the JVM that runs the Bazel server, which are either required or # recommended when using the embedded JDK on platforms that use a minified JDK. # Setting these options here rather than in blaze.cc avoids the need to detect @@ -90,10 +105,30 @@ strip_to_jdk_home() { fi } -if [[ "$UNAME" =~ msys_nt* ]]; then - unzip -q "$tooljdk" -d "tool_jdk.$$" +extract_archive() { + local archive=$1 + local dir=$2 + mkdir "$dir" + if [[ "$archive" == *.zip ]]; then + "$ZIPPER" x "$archive" -d "$dir" + else + tar xf "$archive" --no-same-owner -C "$dir" + fi +} + +create_zip() { + local archive=$1 + local dir=$2 + ( + cd "$dir" + "$ZIPPER" cC "$archive" $(find reduced -print) + ) +} + +if $target_windows; then + extract_archive "$tooljdk" "tool_jdk.$$" strip_to_jdk_home "tool_jdk.$$" - unzip -q "$fulljdk" -d "full_jdk.$$" + extract_archive "$fulljdk" "full_jdk.$$" strip_to_jdk_home "full_jdk.$$" tool_jdk_home=$(cd "tool_jdk.$$" && pwd) cd "full_jdk.$$" @@ -101,7 +136,7 @@ if [[ "$UNAME" =~ msys_nt* ]]; then # provided jmods archive. if [ ! -f jmods/java.base.jmod ]; then if [ -n "$jmods_archive" ]; then - unzip -q "$jmods_archive" -d jmods_tmp + "$ZIPPER" x "$jmods_archive" -d jmods_tmp # The archive contains a single top-level directory with jmod files. mv jmods_tmp/*/* jmods_tmp/ 2>/dev/null || true # Move all .jmod files into the jmods directory. @@ -129,13 +164,16 @@ if [[ "$UNAME" =~ msys_nt* ]]; then # support of Unicode characters outside the system code page. # The JDK currently (as of JDK 23) doesn't support this natively: # https://mail.openjdk.org/pipermail/core-libs-dev/2024-November/133773.html - "$(rlocation io_bazel/src/read_manifest.exe)" reduced/bin/java.exe \ - | sed 's||UTF-8&|' \ - | "$(rlocation io_bazel/src/write_manifest.exe)" reduced/bin/java.exe + if [[ -n "${RUNFILES_MANIFEST_FILE:-}" ]] && \ + grep -q 'io_bazel/src/read_manifest.exe ' "$RUNFILES_MANIFEST_FILE"; then + "$(rlocation io_bazel/src/read_manifest.exe)" reduced/bin/java.exe \ + | sed 's||UTF-8&|' \ + | "$(rlocation io_bazel/src/write_manifest.exe)" reduced/bin/java.exe + fi for f in DISCLAIMER readme.txt legal/java.base/ASSEMBLY_EXCEPTION; do [ -f "$f" ] && cp "$f" reduced/; done # These are necessary for --host_jvm_debug to work. cp bin/dt_socket.dll bin/jdwp.dll reduced/bin - zip -q -X -r ../reduced.zip reduced/ + create_zip "$(pwd)/../reduced.zip" "$(pwd)" cd .. mv reduced.zip "$out" rm -rf "full_jdk.$$" "tool_jdk.$$" @@ -143,11 +181,9 @@ else # The --no-same-owner flag instructs tar to not try to chown extracted files # to the owner stored in the archive - it will try to do that when running as # root, but fail when running inside Docker, so we explicitly disable it. - mkdir "tool_jdk.$$" - tar xf "$tooljdk" --no-same-owner -C "tool_jdk.$$" + extract_archive "$tooljdk" "tool_jdk.$$" strip_to_jdk_home "tool_jdk.$$" - mkdir "target_jdk.$$" - tar xf "$fulljdk" --no-same-owner -C "target_jdk.$$" + extract_archive "$fulljdk" "target_jdk.$$" strip_to_jdk_home "target_jdk.$$" cd "target_jdk.$$" # If the full JDK doesn't ship with jmods (e.g. JEP 493), use the separately @@ -172,7 +208,7 @@ else # These are necessary for --host_jvm_debug to work. cp lib/libdt_socket.* lib/libjdwp.* reduced/lib find reduced -exec touch -ht 198001010000 {} + - zip -q -X -r ../reduced.zip reduced/ + create_zip "$(pwd)/../reduced.zip" "$(pwd)" cd .. mv reduced.zip "$out" rm -rf "target_jdk.$$" "tool_jdk.$$" From 0e3ccc1ec373642a4a1134eca308fb0288529065 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 6 Aug 2026 01:11:04 -0400 Subject: [PATCH 11/14] Run Bazel archive packaging with the execution-platform Java runtime --- src/BUILD | 11 ----- src/package-bazel.sh | 106 ------------------------------------------- src/package_zip.bzl | 20 ++++++-- 3 files changed, 15 insertions(+), 122 deletions(-) delete mode 100755 src/package-bazel.sh diff --git a/src/BUILD b/src/BUILD index 196f8f8818fb57..28ff27794a9263 100644 --- a/src/BUILD +++ b/src/BUILD @@ -332,17 +332,6 @@ genrule( ("_nojdk", True, False), ]] -genrule( - name = "package_bazel_on_host_platform", - srcs = ["package-bazel.sh"], - outs = ["package-bazel-on-host-platform.sh"], - cmd = select({ - "//src/conditions:freebsd": "cat $(SRCS) | sed -e 's@#!/bin/bash@#!/usr/local/bin/bash@' > $@", - "//src/conditions:openbsd": "cat $(SRCS) | sed -e 's@#!/bin/bash@#!/usr/local/bin/bash@' > $@", - "//conditions:default": "cp $(SRCS) $@", - }), -) - [bazel_binary( name = "bazel-bin" + suffix, out = "bazel" + suffix, diff --git a/src/package-bazel.sh b/src/package-bazel.sh deleted file mode 100755 index fb85b8de40a130..00000000000000 --- a/src/package-bazel.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -# This script creates the Bazel archive that Bazel client unpacks and then -# starts the server from. - -WORKDIR="$(pwd)" -OUT=$1; shift -EMBEDDED_TOOLS=$1; shift -DEPLOY_JAR=$1; shift -INSTALL_BASE_KEY=$1; shift -PLATFORMS_ARCHIVE=$1; shift - -if [[ "$OUT" == *jdk_allmodules.zip ]]; then - DEV_BUILD=1 -else - DEV_BUILD=0 -fi - -TMP_DIR=${TMPDIR:-/tmp} -ROOT="$(mktemp -d ${TMP_DIR%%/}/bazel.XXXXXXXX)" -RECOMP="$ROOT/recomp" -PACKAGE_DIR="$ROOT/pkg" -DEPLOY_UNCOMP="$ROOT/deploy-uncompressed.jar" -FILE_LIST="$ROOT/file.list" -mkdir -p "${PACKAGE_DIR}" -trap "rm -fr ${ROOT}" EXIT - -cp $* ${PACKAGE_DIR} - -if [[ $DEV_BUILD -eq 0 ]]; then - # Unpack the deploy jar for postprocessing and for "re-compressing" to save - # ~10% of final binary size. - mkdir -p $RECOMP - unzip -q -d $RECOMP ${DEPLOY_JAR} - cd $RECOMP - - # Zero out timestamps and sort the entries to ensure determinism. - find . -type f -print0 | xargs -0 touch -t 198001010000.00 - find . -type f | sort | zip -q0DX@ "$DEPLOY_UNCOMP" - - # While we're in the deploy jar, grab the label and pack it into the final - # packaged distribution zip where it can be used to quickly determine version - # info. - bazel_label="$(\ - (grep '^build.label=' build-data.properties | cut -d'=' -f2- | tr -d '\n') \ - || echo -n 'no_version')" - - cd "$WORKDIR" - - DEPLOY_JAR="$DEPLOY_UNCOMP" -fi -echo -n "${bazel_label:-no_version}" > "${PACKAGE_DIR}/build-label.txt" - -if [ -n "${EMBEDDED_TOOLS}" ]; then - mkdir ${PACKAGE_DIR}/embedded_tools - (cd ${PACKAGE_DIR}/embedded_tools && unzip -q "${WORKDIR}/${EMBEDDED_TOOLS}") -fi - -( - cd $PACKAGE_DIR - tar -xf "$WORKDIR/$PLATFORMS_ARCHIVE" -C . - # "platforms" is a well-known module, so no need to tamper with anything here. -) - -# Make a list of the files in the order we want them inside the final zip. -( - cd $PACKAGE_DIR - # The server jar needs to be the first binary we extract. - # This is how the Bazel client knows which .jar to pass to the JVM. - echo A-server.jar - find . -type f | sort - # And install_base_key must be last. - echo install_base_key -) > $FILE_LIST - -# Move these after the 'find' above. -cp $DEPLOY_JAR $PACKAGE_DIR/A-server.jar -cp $INSTALL_BASE_KEY $PACKAGE_DIR/install_base_key - -# Zero timestamps. -(cd $PACKAGE_DIR; xargs touch -t 198001010000.00) < $FILE_LIST - -if [[ "$DEV_BUILD" -eq 1 ]]; then - # Create output zip with lowest compression, but fast. - ZIP_ARGS="-q1DX@" -else - # Create output zip with highest compression, but slow. - ZIP_ARGS="-q9DX@" -fi -(cd $PACKAGE_DIR; zip $ZIP_ARGS "$WORKDIR/$OUT") < $FILE_LIST diff --git a/src/package_zip.bzl b/src/package_zip.bzl index 81ef7ef13e309a..d66760fa882adf 100644 --- a/src/package_zip.bzl +++ b/src/package_zip.bzl @@ -14,7 +14,11 @@ """Hermetic Starlark rule for building Bazel package.zip archives.""" +load("@rules_java//java:defs.bzl", "java_common") + def _package_zip_impl(ctx): + package_zipper = ctx.file._package_zipper + java_runtime = ctx.attr._java_runtime[java_common.JavaRuntimeInfo] args = ctx.actions.args() if ctx.attr.dev_build: args.add("--fast") @@ -24,10 +28,11 @@ def _package_zip_impl(ctx): args.add_all(ctx.files.srcs) ctx.actions.run( - executable = ctx.executable._package_zipper, - arguments = [args], - inputs = [ctx.file.server_jar, ctx.file.install_base_key] + ctx.files.srcs, + executable = java_runtime.java_executable_exec_path, + arguments = ["-jar", package_zipper.path, args], + inputs = [package_zipper, ctx.file.server_jar, ctx.file.install_base_key] + ctx.files.srcs, outputs = [ctx.outputs.out], + tools = java_runtime.files, mnemonic = "PackageZip", progress_message = "Building %{output}", ) @@ -44,9 +49,14 @@ package_zip = rule( doc = "If True, use fast compression (level 1) for developer builds.", ), "_package_zipper": attr.label( - default = "//src/java_tools/singlejar/java/com/google/devtools/build/zip:package_zipper", - executable = True, + default = "//src/java_tools/singlejar/java/com/google/devtools/build/zip:package_zipper_deploy.jar", + allow_single_file = True, + cfg = "exec", + ), + "_java_runtime": attr.label( cfg = "exec", + default = "@bazel_tools//tools/jdk:current_java_runtime", + providers = [java_common.JavaRuntimeInfo], ), }, ) From 3646980e5eed766bd2e8862b5cdeabc1dc943ee3 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:49 -0400 Subject: [PATCH 12/14] Run AdjustSfx with the execution-platform Java runtime --- src/bazel_binary.bzl | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/bazel_binary.bzl b/src/bazel_binary.bzl index 95c0c05023b321..df61fdfe3c3b21 100644 --- a/src/bazel_binary.bzl +++ b/src/bazel_binary.bzl @@ -14,11 +14,14 @@ """Custom rule for packaging Bazel binary.""" +load("@rules_java//java:defs.bzl", "java_common") + def _bazel_binary_impl(ctx): client = ctx.file.client package_zip = ctx.file.package_zip cat_binary = ctx.executable._cat_binary - adjust_sfx = ctx.executable._adjust_sfx + adjust_sfx = ctx.file._adjust_sfx + java_runtime = ctx.attr._java_runtime[java_common.JavaRuntimeInfo] output = ctx.outputs.out unaligned = ctx.actions.declare_file(output.basename + ".unaligned") @@ -34,10 +37,11 @@ def _bazel_binary_impl(ctx): # Adjust SFX ctx.actions.run( - inputs = [unaligned], + inputs = [unaligned, adjust_sfx], outputs = [output], - executable = adjust_sfx, - arguments = [unaligned.path, output.path], + executable = java_runtime.java_executable_exec_path, + arguments = ["-jar", adjust_sfx.path, unaligned.path, output.path], + tools = java_runtime.files, mnemonic = "AdjustSfx", ) @@ -58,9 +62,14 @@ bazel_binary = rule( cfg = "exec", ), "_adjust_sfx": attr.label( - default = Label("//src/java_tools/singlejar/java/com/google/devtools/build/zip:adjust_sfx"), - executable = True, + default = Label("//src/java_tools/singlejar/java/com/google/devtools/build/zip:adjust_sfx_deploy.jar"), + allow_single_file = True, + cfg = "exec", + ), + "_java_runtime": attr.label( cfg = "exec", + default = "@bazel_tools//tools/jdk:current_java_runtime", + providers = [java_common.JavaRuntimeInfo], ), }, executable = True, From a81dabfebfb8c6925b47a1c8ab5953d4638cd665 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:49 -0400 Subject: [PATCH 13/14] Generate test XML with the execution-platform shell --- .../lib/analysis/test/TestRunnerAction.java | 2 +- .../build/lib/analysis/test/TestStrategy.java | 6 + .../com/google/devtools/build/lib/exec/BUILD | 1 + .../lib/exec/StandaloneTestStrategy.java | 36 +++-- src/test/shell/bazel/generate_xml_test.sh | 5 + src/test/shell/remote_exec/BUILD | 6 + src/test/shell/remote_exec/no_xml_test.sh | 5 + tools/test/generate-xml.sh | 125 +++++++++--------- 8 files changed, 113 insertions(+), 73 deletions(-) create mode 100644 src/test/shell/remote_exec/BUILD create mode 100755 src/test/shell/remote_exec/no_xml_test.sh diff --git a/src/main/java/com/google/devtools/build/lib/analysis/test/TestRunnerAction.java b/src/main/java/com/google/devtools/build/lib/analysis/test/TestRunnerAction.java index d7b71c983c0970..9c70587a5cc38d 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/test/TestRunnerAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/test/TestRunnerAction.java @@ -1091,7 +1091,7 @@ public FilesToRunProvider getCollectCoverageScript() { } @Nullable - PathFragment getShExecutableMaybe() { + public PathFragment getShExecutableMaybe() { return shExecutable; } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/test/TestStrategy.java b/src/main/java/com/google/devtools/build/lib/analysis/test/TestStrategy.java index 4fb6ffccb017a8..dcad314ae3f24d 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/test/TestStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/test/TestStrategy.java @@ -216,6 +216,12 @@ public static ImmutableList expandedArgsFromAction(TestRunnerAction test List args = new ArrayList<>(); OS executionOs = testAction.getExecutionSettings().getExecutionOs(); + if (executionOs != OS.WINDOWS) { + args.add( + Preconditions.checkNotNull(testAction.getShExecutableMaybe()) + .getCallablePathStringForOs(executionOs)); + } + Artifact testSetup = testAction.getTestSetupScript(); args.add(testSetup.getExecPath().getCallablePathStringForOs(executionOs)); diff --git a/src/main/java/com/google/devtools/build/lib/exec/BUILD b/src/main/java/com/google/devtools/build/lib/exec/BUILD index 895b033a0f388f..3e5449b0d3a229 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/BUILD +++ b/src/main/java/com/google/devtools/build/lib/exec/BUILD @@ -419,6 +419,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/collect/nestedset", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/runtime:test_summary_options", + "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/util/io", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", diff --git a/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java index 51558c98a3d629..750f3f6db87a8a 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java @@ -56,6 +56,7 @@ import com.google.devtools.build.lib.server.FailureDetails.Execution.Code; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.TestAction; +import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.FileStatus; import com.google.devtools.build.lib.vfs.FileSystemUtils; @@ -450,16 +451,21 @@ private static Duration toProtoDuration(int timeInMs) { */ private static Spawn createXmlGeneratingSpawn( TestRunnerAction action, ImmutableMap testEnv, SpawnResult result) { - ImmutableList args = - ImmutableList.of( - action - .getTestXmlGeneratorScript() - .getExecPath() - .getCallablePathStringForOs(action.getExecutionSettings().getExecutionOs()), - action.getTestLog().getExecPathString(), - action.getTestXml().getExecPathString(), - Integer.toString(result.getWallTimeInMs() / 1000), - Integer.toString(result.exitCode())); + ImmutableList.Builder args = ImmutableList.builder(); + if (action.getExecutionSettings().getExecutionOs() != OS.WINDOWS) { + args.add( + Preconditions.checkNotNull(action.getShExecutableMaybe()) + .getCallablePathStringForOs(action.getExecutionSettings().getExecutionOs())); + } + args.add( + action + .getTestXmlGeneratorScript() + .getExecPath() + .getCallablePathStringForOs(action.getExecutionSettings().getExecutionOs()), + action.getTestLog().getExecPathString(), + action.getTestXml().getExecPathString(), + Integer.toString(result.getWallTimeInMs() / 1000), + Integer.toString(result.exitCode())); ImmutableMap.Builder envBuilder = ImmutableMap.builder(); // "PATH" and "TEST_BINARY" are also required, they should always be set in testEnv. Preconditions.checkArgument(testEnv.containsKey("PATH")); @@ -473,7 +479,7 @@ private static Spawn createXmlGeneratingSpawn( } return new SimpleSpawn( action, - args, + args.build(), envBuilder.buildOrThrow(), // Pass the execution info of the action which is identical to the supported tags set on the // test target. In particular, this does not set the test timeout on the spawn. @@ -890,7 +896,13 @@ private TestAttemptResult runTestAttempt( .addAll(xmlSpawnResults) .build(); } catch (InterruptedException | ExecException e) { - closeSuppressed(e, xmlSpawnOutErr); + try { + xmlSpawnOutErr.close(); + writeOutFile(xmlSpawnOutErr.getOutputPath(), fileOutErr.getOutputPath()); + writeOutFile(xmlSpawnOutErr.getErrorPath(), fileOutErr.getOutputPath()); + } catch (IOException closeException) { + e.addSuppressed(closeException); + } throw e; } } diff --git a/src/test/shell/bazel/generate_xml_test.sh b/src/test/shell/bazel/generate_xml_test.sh index 85a5aedceef446..be36eacae68e4e 100755 --- a/src/test/shell/bazel/generate_xml_test.sh +++ b/src/test/shell/bazel/generate_xml_test.sh @@ -88,4 +88,9 @@ function test_generated_testcase_has_classname() { "$TEST_TMPDIR/test.xml" } +function test_without_sed_on_path() { + assert_equals 'Simple ascii' \ + "$(echo 'Simple ascii' | PATH=/does-not-exist "$BASH" "$GENERATE_XML" - - - -)" +} + run_suite "generate-xml.sh tests" diff --git a/src/test/shell/remote_exec/BUILD b/src/test/shell/remote_exec/BUILD new file mode 100644 index 00000000000000..deee8ebfc0c15f --- /dev/null +++ b/src/test/shell/remote_exec/BUILD @@ -0,0 +1,6 @@ +load("@rules_shell//shell:sh_test.bzl", "sh_test") + +sh_test( + name = "no_xml_test", + srcs = ["no_xml_test.sh"], +) diff --git a/src/test/shell/remote_exec/no_xml_test.sh b/src/test/shell/remote_exec/no_xml_test.sh new file mode 100755 index 00000000000000..782e2721dbe79e --- /dev/null +++ b/src/test/shell/remote_exec/no_xml_test.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +echo "This test intentionally does not write to XML_OUTPUT_FILE." diff --git a/tools/test/generate-xml.sh b/tools/test/generate-xml.sh index 5829aca05120d9..ebee28787f53e8 100755 --- a/tools/test/generate-xml.sh +++ b/tools/test/generate-xml.sh @@ -22,58 +22,10 @@ DURATION_IN_SECONDS="$3" EXIT_CODE="$4" function encode_stream { - # Replace invalid XML characters and invalid sequence in CDATA - # We do this in four steps: - # - # 1. Add a single whitespace character to the end of every line - # - # 2. Replace every sequence of legal characters followed by an illegal - # character *or* followed by a legal character at the end of the line with - # the same sequence of legal characters followed by a question mark - # character (replacing the illegal or last character). Since this will - # always replace the last character in a line with a question mark, we - # make sure to append a whitespace in step #1. - # - # A character is legal if it is a valid UTF-8 character that is allowed in - # an XML file (this excludes a few control codes, but otherwise allows - # most UTF-8 characters). - # - # We can't use sed in UTF-8 mode, because it would fail on the first - # illegal character. Instead, we have to match legal characters by their - # 8-bit binary sequences, and also switch sed to an 8-bit mode. - # - # The legal UTF codepoint ranges are 9,a,d,20-d7ff,e000-fffd,10000-10ffff, - # which results in the following 8-bit binary UTF-8 matchers: - # [\x9\xa\xd\x20-\x7f] <--- (9,A,D,20-7F) - # [\xc0-\xdf][\x80-\xbf] <--- (0080-07FF) - # [\xe0-\xec][\x80-\xbf][\x80-\xbf] <--- (0800-CFFF) - # [\xed][\x80-\x9f][\x80-\xbf] <--- (D000-D7FF) - # [\xee][\x80-\xbf][\x80-\xbf] <--- (E000-EFFF) - # [\xef][\x80-\xbe][\x80-\xbf] <--- (F000-FFEF) - # [\xef][\xbf][\x80-\xbd] <--- (FFF0-FFFD) - # [\xf0-\xf7][\x80-\xbf][\x80-\xbf][\x80-\xbf] <--- (010000-10FFFF) - # - # We omit \xa and \xd below since sed already splits the input into lines. - # - # 3. Remove the last character in the line, which we expect to be a - # question mark (that was originally added as a whitespace in step #1). - # - # 4. Replace the string ']]>' with ']]>]]' to prevent escaping the - # surrounding CDATA block. - # - # Sed supports the necessary operations as of version 4.4, but not in all - # earlier versions. Specifically, we have found that sed 4.1.5 is not 8-bit - # safe even when set to an 8-bit locale. - # - # OSX sed does not support escape sequences (\xhh), use echo as workaround. - # - # Alternatives considered: - # Perl - We originally used Perl, but wanted to avoid the dependency. - # Recent versions of Perl now error on invalid utf-8 characters. - # tr - tr only replaces single-byte sequences, so cannot handle utf-8. - LC_ALL=C sed -E \ - -e 's/.*/& /g' \ - -e 's/(('\ + if command -v sed >/dev/null; then + LC_ALL=C sed -E \ + -e 's/.*/& /g' \ + -e 's/(('\ "$(echo -e '[\x9\x20-\x7f]')|"\ "$(echo -e '[\xc0-\xdf][\x80-\xbf]')|"\ "$(echo -e '[\xe0-\xec][\x80-\xbf][\x80-\xbf]')|"\ @@ -81,13 +33,67 @@ function encode_stream { "$(echo -e '[\xee-\xef][\x80-\xbf][\x80-\xbf]')|"\ "$(echo -e '[\xf0][\x80-\x8f][\x80-\xbf][\x80-\xbf]')"\ ')*)./\1?/g' \ - -e 's/(.*)\?/\1/g' \ - -e 's|]]>|]]>]]|g' + -e 's/(.*)\?/\1/g' \ + -e 's|]]>|]]>]]|g' + return + fi + + # Use only Bash builtins so this action does not depend on utilities installed + # on the execution platform. Match the byte sequences accepted by the former + # sed implementation and replace every other byte with a question mark. + local LC_ALL=C + local line + while IFS= read -r line || [[ -n "$line" ]]; do + local encoded="" + local i=0 + local length=${#line} + while ((i < length)); do + local b0 b1=-1 b2=-1 b3=-1 sequence_length=0 + printf -v b0 '%d' "'${line:i:1}" + if ((i + 1 < length)); then + printf -v b1 '%d' "'${line:i+1:1}" + fi + if ((i + 2 < length)); then + printf -v b2 '%d' "'${line:i+2:1}" + fi + if ((i + 3 < length)); then + printf -v b3 '%d' "'${line:i+3:1}" + fi + + if ((b0 == 9 || b0 == 13 || (b0 >= 32 && b0 <= 127))); then + sequence_length=1 + elif ((b0 >= 192 && b0 <= 223 && b1 >= 128 && b1 <= 191)); then + sequence_length=2 + elif (( + ((b0 >= 224 && b0 <= 236) || (b0 >= 238 && b0 <= 239)) && + b1 >= 128 && b1 <= 191 && b2 >= 128 && b2 <= 191 + )); then + sequence_length=3 + elif ((b0 == 237 && b1 >= 128 && b1 <= 159 && b2 >= 128 && b2 <= 191)); then + sequence_length=3 + elif (( + b0 == 240 && b1 >= 128 && b1 <= 143 && + b2 >= 128 && b2 <= 191 && b3 >= 128 && b3 <= 191 + )); then + sequence_length=4 + fi + + if ((sequence_length > 0)); then + encoded+="${line:i:sequence_length}" + ((i += sequence_length)) + else + encoded+='?' + ((i += 1)) + fi + done + encoded="${encoded//]]>/]]>]]}" + printf '%s\n' "$encoded" + done } function encode_as_xml { if [ -f "$1" ]; then - cat "$1" | encode_stream + encode_stream <"$1" fi } @@ -115,16 +121,15 @@ fi FAILED=0 ENCODED_LOG="$(encode_as_xml "${TEST_LOG}")" || FAILED=$? -cat >"${XML_OUTPUT_FILE}" < +printf '%s' " - - ${error_msg} + + ${error_msg} Generated test.log (if the file is not UTF-8, then this may be unreadable): -EOF +" >"${XML_OUTPUT_FILE}" exit "$FAILED" From 34983c258a613064dc3676a1e2fee22d1cca8332 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 22:54:50 -0400 Subject: [PATCH 14/14] Run Windows-hosted Linux remote execution in GitHub Actions --- .bazelrc | 18 +++++++++-- .github/workflows/windows-remote-exec.yml | 39 +++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/windows-remote-exec.yml diff --git a/.bazelrc b/.bazelrc index d2f2d421076648..625d8edae20a11 100644 --- a/.bazelrc +++ b/.bazelrc @@ -20,8 +20,22 @@ common:ubuntu2404 --host_platform=//:rbe_ubuntu2404_platform common:ubuntu2404 --platforms=//:rbe_ubuntu2404_platform common:ubuntu2404 --config=remote_shared -# Alias -common:remote --config=ubuntu2404 +# BuildBuddy remote execution with a hermetic Linux execution platform. Pass +# --remote_header=x-buildbuddy-api-key=... separately when authentication is +# required. +common:remote --bes_results_url=https://app.buildbuddy.io/invocation/ +common:remote --bes_backend=grpcs://remote.buildbuddy.io +common:remote --experimental_remote_downloader=grpcs://remote.buildbuddy.io +common:remote --remote_cache=grpcs://remote.buildbuddy.io +common:remote --remote_cache_compression +common:remote --remote_download_toplevel +common:remote --remote_executor=grpcs://remote.buildbuddy.io +common:remote --remote_timeout=120 +common:remote --jobs=800 +common:remote --extra_execution_platforms=@llvm//:rbe_platform +common:remote --shell_executable=/bin/bash +common:remote --action_env=PATH=/bin:/usr/bin:/usr/local/bin +common:remote --@rules_python//python/config_settings:build_python_zip=false common:macos --host_macos_minimum_os=10.13 common:macos --macos_minimum_os=10.13 diff --git a/.github/workflows/windows-remote-exec.yml b/.github/workflows/windows-remote-exec.yml new file mode 100644 index 00000000000000..4261dce1a2a7f3 --- /dev/null +++ b/.github/workflows/windows-remote-exec.yml @@ -0,0 +1,39 @@ +name: Windows remote execution XML reproducer + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + windows-remote-exec: + runs-on: windows-2025 + env: + BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + steps: + - uses: actions/checkout@v4 + + - name: Build Bazel with Linux remote execution + shell: pwsh + run: | + bazelisk build --config=remote ` + --remote_header=x-buildbuddy-api-key=$env:BUILDBUDDY_API_KEY ` + //src:bazel + + - name: Run no-XML shell test with the built Bazel + shell: pwsh + run: | + $bazel = Join-Path $env:RUNNER_TEMP bazel-under-test.exe + Copy-Item bazel-bin/src/bazel $bazel + & $bazel --output_base="$env:RUNNER_TEMP/bazel-remote-test" test ` + --config=remote ` + --platforms=@llvm//platforms:linux_amd64 ` + --remote_header=x-buildbuddy-api-key=$env:BUILDBUDDY_API_KEY ` + --test_output=all ` + //src/test/shell/remote_exec:no_xml_test + $exitCode = $LASTEXITCODE + Get-ChildItem -Recurse "$env:RUNNER_TEMP/bazel-remote-test" -Filter test.log | + ForEach-Object { Get-Content $_ } + exit $exitCode