From 57b9588fe11f128357af212dfa61c19c896c043a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BE=9A=E6=8C=AF?= Date: Tue, 11 Aug 2026 17:52:24 +0800 Subject: [PATCH 1/3] Reencode non-ASCII option values and build metadata for the BEP Bazel keeps strings internally as raw bytes in Latin-1 `String`s, whereas protobuf `string` fields hold Unicode. `CommandLineEvent` and `BuildMetadataEvent` wrote internal strings into the BEP protos as is, so a non-ASCII option value, residue entry or build metadata value came out double encoded. Route these values through `StringEncoding#internalToUnicode`, as is already done for other proto fields, e.g. in `BuildLanguageInfoItem`. ASCII values are unaffected. The values are the ones a consumer of the BEP is most likely to display: an invocation started with a non-ASCII `--build_metadata`, `--client_env` or target pattern showed mojibake in the UI rather than what the user had passed in, and there was no way to recover the original bytes from the stream. --- .../google/devtools/build/lib/runtime/BUILD | 2 + .../build/lib/runtime/BuildMetadataEvent.java | 5 +- .../build/lib/runtime/CommandLineEvent.java | 19 +++--- .../lib/runtime/CommandLineEventTest.java | 61 +++++++++++++++++++ 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BUILD b/src/main/java/com/google/devtools/build/lib/runtime/BUILD index cc651e39206912..471435bd6a08fa 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BUILD +++ b/src/main/java/com/google/devtools/build/lib/runtime/BUILD @@ -106,6 +106,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/buildeventstream", "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto", "//src/main/java/com/google/devtools/build/lib/util:pair", + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:command_line_java_proto", "//src/main/protobuf:option_filters_java_proto", @@ -326,6 +327,7 @@ java_library( deps = [ "//src/main/java/com/google/devtools/build/lib/buildeventstream", "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto", + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//third_party:guava", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BuildMetadataEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/BuildMetadataEvent.java index 099571f9acbc73..1e248d54a33fd5 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BuildMetadataEvent.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BuildMetadataEvent.java @@ -21,6 +21,7 @@ import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildEventId; import com.google.devtools.build.lib.buildeventstream.BuildEventWithOrderConstraint; import com.google.devtools.build.lib.buildeventstream.GenericBuildEvent; +import com.google.devtools.build.lib.util.StringEncoding; import java.util.Collection; import java.util.Map; @@ -56,7 +57,9 @@ public BuildEventStreamProtos.BuildEvent asStreamProto(BuildEventContext convert BuildEventStreamProtos.BuildMetadata.Builder metadataBuilder = BuildEventStreamProtos.BuildMetadata.newBuilder(); for (Map.Entry entry : buildMetadata.entrySet()) { - metadataBuilder.putMetadata(entry.getKey(), entry.getValue()); + metadataBuilder.putMetadata( + StringEncoding.internalToUnicode(entry.getKey()), + StringEncoding.internalToUnicode(entry.getValue())); } return GenericBuildEvent.protoChaining(this).setBuildMetadata(metadataBuilder.build()).build(); } diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandLineEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandLineEvent.java index 92234a9cf44b5b..2be92bfb4a7604 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/CommandLineEvent.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandLineEvent.java @@ -16,6 +16,7 @@ import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import com.google.common.io.BaseEncoding; import com.google.devtools.build.lib.buildeventstream.BuildEventContext; import com.google.devtools.build.lib.buildeventstream.BuildEventIdUtil; @@ -31,6 +32,7 @@ import com.google.devtools.build.lib.runtime.proto.CommandLineOuterClass.Option; import com.google.devtools.build.lib.runtime.proto.CommandLineOuterClass.OptionList; import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.StringEncoding; import com.google.devtools.common.options.OptionDefinition; import com.google.devtools.common.options.OptionEffectTag; import com.google.devtools.common.options.OptionMetadataTag; @@ -165,15 +167,15 @@ private Option createOption( String combinedForm, @Nullable String value) { Option.Builder option = Option.newBuilder(); - option.setCombinedForm(combinedForm); + option.setCombinedForm(StringEncoding.internalToUnicode(combinedForm)); option.setOptionName(optionDefinition.getOptionName()); if (value != null) { - option.setOptionValue(value); + option.setOptionValue(StringEncoding.internalToUnicode(value)); } option.addAllEffectTags(getProtoEffectTags(optionDefinition.getOptionEffectTags())); option.addAllMetadataTags(getProtoMetadataTags(optionDefinition.getOptionMetadataTags())); if (source != null) { - option.setSource(source); + option.setSource(StringEncoding.internalToUnicode(source)); } return option.build(); } @@ -205,10 +207,10 @@ Option createSingleStarlarkOption(String starlarkFlag, @Nullable Object value) { } } Option.Builder option = Option.newBuilder(); - option.setCombinedForm(sb.toString()); + option.setCombinedForm(StringEncoding.internalToUnicode(sb.toString())); option.setOptionName(starlarkFlag); if (value != null) { - option.setOptionValue(String.valueOf(value)); + option.setOptionValue(StringEncoding.internalToUnicode(String.valueOf(value))); } return option.build(); } @@ -244,13 +246,16 @@ CommandLineSection getResidual() { CommandLineSection.newBuilder().setSectionLabel("residual"); if (commandName.equals("run") && !includeResidueInRunBepEvent && !residue.isEmpty()) { String target = residue.get(0); - ChunkList.Builder residual = ChunkList.newBuilder().addChunk(target); + ChunkList.Builder residual = + ChunkList.newBuilder().addChunk(StringEncoding.internalToUnicode(target)); if (residue.size() > 1) { residual.addChunk("REDACTED"); } builder.setChunkList(residual); } else { - builder.setChunkList(ChunkList.newBuilder().addAllChunk(residue)); + builder.setChunkList( + ChunkList.newBuilder() + .addAllChunk(Iterables.transform(residue, StringEncoding::internalToUnicode))); } return builder.build(); } diff --git a/src/test/java/com/google/devtools/build/lib/runtime/CommandLineEventTest.java b/src/test/java/com/google/devtools/build/lib/runtime/CommandLineEventTest.java index 93af3c657922a0..730828d3bc59a7 100644 --- a/src/test/java/com/google/devtools/build/lib/runtime/CommandLineEventTest.java +++ b/src/test/java/com/google/devtools/build/lib/runtime/CommandLineEventTest.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.runtime.proto.CommandLineOuterClass.CommandLineSection.SectionTypeCase; import com.google.devtools.build.lib.runtime.proto.CommandLineOuterClass.OptionList; import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.StringEncoding; import com.google.devtools.common.options.OptionPriority.PriorityCategory; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingException; @@ -342,6 +343,66 @@ public void testOptionsAtVariousPriorities_canonicalCommandLine() throws Options assertThat(line.getSections(4).getChunkList().getChunkCount()).isEqualTo(0); } + @Test + public void testNonAsciiOptionValue_canonicalCommandLine() throws OptionsParsingException { + OptionsParser fakeStartupOptions = + OptionsParser.builder().optionsClasses(BlazeServerStartupOptions.class).build(); + OptionsParser fakeCommandOptions = + OptionsParser.builder().optionsClasses(TestOptions.class).build(); + // Option values reach the server as internal strings, i.e. raw bytes stored in a Latin-1 + // String. Simulate a UTF-8 encoded value as it is received from the client. + fakeCommandOptions.parse( + PriorityCategory.COMMAND_LINE, + "command line", + ImmutableList.of("--test_string=" + StringEncoding.unicodeToInternal("¡Buenos días!"))); + + CommandLine line = + new CanonicalCommandLineEvent( + "testblaze", + fakeStartupOptions, + "someCommandName", + fakeCommandOptions.getResidue(), + false, + ImmutableSortedMap.of(), + ImmutableSortedMap.of(), + ImmutableSet.of(), + fakeCommandOptions.asListOfCanonicalOptions(), + /* replaceable= */ false) + .asStreamProto(null) + .getStructuredCommandLine(); + + // The proto fields are Unicode strings, so the value has to be reencoded. + assertThat(line.getSections(3).getOptionList().getOption(0).getOptionValue()) + .isEqualTo("¡Buenos días!"); + assertThat(line.getSections(3).getOptionList().getOption(0).getCombinedForm()) + .isEqualTo("--test_string=¡Buenos días!"); + } + + @Test + public void testNonAsciiResidue_canonicalCommandLine() throws OptionsParsingException { + OptionsParser fakeStartupOptions = + OptionsParser.builder().optionsClasses(BlazeServerStartupOptions.class).build(); + OptionsParser fakeCommandOptions = + OptionsParser.builder().optionsClasses(TestOptions.class).build(); + + CommandLine line = + new CanonicalCommandLineEvent( + "testblaze", + fakeStartupOptions, + "someCommandName", + ImmutableList.of(StringEncoding.unicodeToInternal("//foo:bär")), + false, + ImmutableSortedMap.of(), + ImmutableSortedMap.of(), + ImmutableSet.of(), + fakeCommandOptions.asListOfCanonicalOptions(), + /* replaceable= */ false) + .asStreamProto(null) + .getStructuredCommandLine(); + + assertThat(line.getSections(4).getChunkList().getChunk(0)).isEqualTo("//foo:bär"); + } + @Test public void testExpansionOption_originalCommandLine() throws OptionsParsingException { OptionsParser fakeStartupOptions = From 847a9be099b64f98c00cff5b64016e015d340163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BE=9A=E6=8C=AF?= Date: Tue, 11 Aug 2026 20:34:34 +0800 Subject: [PATCH 2/3] Keep workspace status values in the internal encoding The workspace status command output was decoded as UTF-8, which made its entries the only Unicode strings in the status maps, and it was reencoded as UTF-8 on the way into the status files. `WorkspaceStatusAction#parseValues` reads those files as Latin-1, so a non-ASCII value did not survive the round trip and reached `BuildInfoFileWriteAction` mangled. Decode the output as Latin-1 so that it stays in the internal encoding like every other entry, and write the status files without reencoding. The files still hold the exact bytes the command emitted. Convert to Unicode in `BuildInfoEvent`, where the BEP proto is formed and it is actually needed. --- .../google/devtools/build/lib/analysis/BUILD | 1 + .../build/lib/analysis/BuildInfoEvent.java | 5 +++-- .../lib/bazel/BazelWorkspaceStatusModule.java | 12 ++++++---- .../google/devtools/build/lib/runtime/BUILD | 1 + .../integration/workspace_status_test.sh | 22 +++++++++++++++++++ 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BUILD b/src/main/java/com/google/devtools/build/lib/analysis/BUILD index fe2c9c40f6bc83..d2417ebd714e8b 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BUILD +++ b/src/main/java/com/google/devtools/build/lib/analysis/BUILD @@ -473,6 +473,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/buildeventstream", "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto", "//src/main/java/com/google/devtools/build/lib/events", + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//third_party:guava", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java index 3d7680a4974def..21ddb3e6b2f984 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.buildeventstream.BuildEventWithOrderConstraint; import com.google.devtools.build.lib.buildeventstream.GenericBuildEvent; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.util.StringEncoding; import java.util.Collection; import java.util.Map; @@ -67,8 +68,8 @@ public BuildEventStreamProtos.BuildEvent asStreamProto(BuildEventContext convert for (Map.Entry entry : getBuildInfoMap().entrySet()) { status.addItem( BuildEventStreamProtos.WorkspaceStatus.Item.newBuilder() - .setKey(entry.getKey()) - .setValue(entry.getValue()) + .setKey(StringEncoding.internalToUnicode(entry.getKey())) + .setValue(StringEncoding.internalToUnicode(entry.getValue())) .build()); } return GenericBuildEvent.protoChaining(this).setWorkspaceStatus(status.build()).build(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java index 0506100ac5d7f3..9ca0dba81a3fe6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java @@ -14,7 +14,7 @@ package com.google.devtools.build.lib.bazel; import static com.google.common.base.StandardSystemProperty.USER_NAME; -import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.charset.StandardCharsets.ISO_8859_1; import static java.util.stream.Collectors.joining; import com.google.common.collect.ImmutableList; @@ -58,7 +58,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -117,7 +116,10 @@ private String getAdditionalWorkspaceStatus( } catch (IOException e) { throw createExecutionException(e, Code.STDERR_IO_EXCEPTION); } - return stdoutStream.toString(UTF_8); + // Keep the output as an internal string, i.e. raw bytes in a Latin-1 String, so that it + // matches the encoding of the other entries in the status maps and of the bytes written + // to the status files. It is converted to Unicode only when the BEP proto is formed. + return stdoutStream.toString(ISO_8859_1); } } catch (BadExitStatusException e) { throw createExecutionException(e, Code.NON_ZERO_EXIT); @@ -154,7 +156,9 @@ private static byte[] printStatusMap(Map map) { .map(entry -> entry.getKey() + " " + entry.getValue()) .collect(joining("\n")); s += "\n"; - return s.getBytes(StandardCharsets.UTF_8); + // The map values are internal strings, i.e. raw bytes in a Latin-1 String, so write them out + // as is rather than reencoding them. + return s.getBytes(ISO_8859_1); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/runtime/BUILD b/src/test/java/com/google/devtools/build/lib/runtime/BUILD index da89842814771b..fe1a9397429491 100644 --- a/src/test/java/com/google/devtools/build/lib/runtime/BUILD +++ b/src/test/java/com/google/devtools/build/lib/runtime/BUILD @@ -150,6 +150,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code", "//src/main/java/com/google/devtools/build/lib/util:exit_code", "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//src/main/java/com/google/devtools/build/lib/util/io:io-proto", "//src/main/java/com/google/devtools/build/lib/util/io:out-err", "//src/main/java/com/google/devtools/build/lib/vfs", diff --git a/src/test/shell/integration/workspace_status_test.sh b/src/test/shell/integration/workspace_status_test.sh index c665a8cf6e459b..edf00f4b7aba51 100755 --- a/src/test/shell/integration/workspace_status_test.sh +++ b/src/test/shell/integration/workspace_status_test.sh @@ -123,4 +123,26 @@ function test_embed_label_must_be_single_line() { expect_log "Value must not contain multiple lines" } +function test_non_ascii_status_value() { + local script="$TEST_TMPDIR/non_ascii.sh" + cat > "$script" <<'EOF' +#!/usr/bin/env bash +echo "STABLE_GREETING ¡Buenos días!" +EOF + chmod +x "$script" + + bazel build --stamp --workspace_status_command="$script" \ + --build_event_json_file="$TEST_TMPDIR/bep.json" >& "$TEST_log" \ + || fail "Build failed" + + # The status file holds the raw bytes emitted by the script. + local status_file="$(bazel info output_path)/stable-status.txt" + grep -q "STABLE_GREETING ¡Buenos días!" "$status_file" \ + || fail "Expected UTF-8 bytes in stable-status.txt, got: $(cat "$status_file")" + + # The BEP is a proto with Unicode string fields, so the value must be reencoded. + grep -q '¡Buenos días!' "$TEST_TMPDIR/bep.json" \ + || fail "Expected correctly encoded value in BEP" +} + run_suite "${PRODUCT_NAME} workspace status command tests" From 9a6bdb67fc5e36df862da0a82757447951c2d517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BE=9A=E6=8C=AF?= Date: Tue, 11 Aug 2026 21:19:23 +0800 Subject: [PATCH 3/3] Reference StringEncoding in the encoding comments Rather than restating how the internal encoding represents bytes, name it and point at `StringEncoding`, which documents it. --- .../build/lib/bazel/BazelWorkspaceStatusModule.java | 9 ++++----- .../devtools/build/lib/runtime/CommandLineEventTest.java | 3 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java index 9ca0dba81a3fe6..e687414c83e422 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java @@ -116,9 +116,8 @@ private String getAdditionalWorkspaceStatus( } catch (IOException e) { throw createExecutionException(e, Code.STDERR_IO_EXCEPTION); } - // Keep the output as an internal string, i.e. raw bytes in a Latin-1 String, so that it - // matches the encoding of the other entries in the status maps and of the bytes written - // to the status files. It is converted to Unicode only when the BEP proto is formed. + // Keep the output in Bazel's internal string encoding (see StringEncoding), like the + // other entries in the status maps. return stdoutStream.toString(ISO_8859_1); } } catch (BadExitStatusException e) { @@ -156,8 +155,8 @@ private static byte[] printStatusMap(Map map) { .map(entry -> entry.getKey() + " " + entry.getValue()) .collect(joining("\n")); s += "\n"; - // The map values are internal strings, i.e. raw bytes in a Latin-1 String, so write them out - // as is rather than reencoding them. + // The values are in Bazel's internal string encoding (see StringEncoding), write them out as + // is. return s.getBytes(ISO_8859_1); } diff --git a/src/test/java/com/google/devtools/build/lib/runtime/CommandLineEventTest.java b/src/test/java/com/google/devtools/build/lib/runtime/CommandLineEventTest.java index 730828d3bc59a7..0cb2e5bb78265b 100644 --- a/src/test/java/com/google/devtools/build/lib/runtime/CommandLineEventTest.java +++ b/src/test/java/com/google/devtools/build/lib/runtime/CommandLineEventTest.java @@ -349,8 +349,7 @@ public void testNonAsciiOptionValue_canonicalCommandLine() throws OptionsParsing OptionsParser.builder().optionsClasses(BlazeServerStartupOptions.class).build(); OptionsParser fakeCommandOptions = OptionsParser.builder().optionsClasses(TestOptions.class).build(); - // Option values reach the server as internal strings, i.e. raw bytes stored in a Latin-1 - // String. Simulate a UTF-8 encoded value as it is received from the client. + // Option values reach the server in Bazel's internal string encoding (see StringEncoding). fakeCommandOptions.parse( PriorityCategory.COMMAND_LINE, "command line",