[7.1.0] Implement a new execution log conversion tool. (#21187)

For now, it knows how to convert between the binary and JSON formats.
Ability to convert from the new compact format will be added in a
followup.

Tested by running manually. I will add end-to-end test coverage once the
converter has been fully implemented.

PiperOrigin-RevId: 603684017
Change-Id: Idf0d851feb9ea22e7021a1b62f003f0978bcd378

src/tools/execlog/BUILD

@@ -20,3 +20,10 @@ java_binary(
     visibility = ["//visibility:public"],
     runtime_deps = ["//src/tools/execlog/src/main/java/com/google/devtools/build/execlog:parser"],
 )
+
+java_binary(
+    name = "converter",
+    main_class = "com.google.devtools.build.execlog.ExecLogConverter",
+    visibility = ["//visibility:public"],
+    runtime_deps = ["//src/tools/execlog/src/main/java/com/google/devtools/build/execlog:converter"],
+)

src/tools/execlog/README.md

@@ -8,7 +8,7 @@ To generate the execution log, run e.g.:
 
 Then build the parser and run it.
 
-        bazel build src/tools/execlog:all
+        bazel build src/tools/execlog:parser
         bazel-bin/src/tools/execlog/parser --log_path=/tmp/exec.log
 
 This will simply print the log contents to stdout in text form.
@@ -44,3 +44,16 @@ are put at the end of `/tmp/exec2.log.txt`.
 Note that this reordering makes it easier to see differences using text-based
 diffing tools, but may break the logical sequence of actions in
 `/tmp/exec2.log.txt`.
+
+# Execution Log Converter
+
+This tool is used to convert between Bazel execution log formats.
+
+For example, to convert from the binary format to the JSON format:
+
+        bazel build src/tools/execlog:converter
+        bazel-bin/src/tools/execlog/converter \
+            --input binary:/tmp/binary.log --output json:/tmp/json.log
+
+By default, the output will be in the same order as the input. To sort in a
+deterministic order, use --sort.

src/tools/execlog/src/main/java/com/google/devtools/build/execlog/BUILD

@@ -13,10 +13,29 @@ filegroup(
 
 java_library(
     name = "parser",
-    srcs = glob(["*.java"]),
+    srcs = [
+        "ExecLogParser.java",
+        "ParserOptions.java",
+    ],
+    deps = [
+        "//src/main/java/com/google/devtools/common/options",
+        "//src/main/protobuf:spawn_java_proto",
+        "//third_party:guava",
+    ],
+)
+
+java_library(
+    name = "converter",
+    srcs = [
+        "ConverterOptions.java",
+        "ExecLogConverter.java",
+    ],
     deps = [
+        "//src/main/java/com/google/devtools/build/lib/exec:spawn_log_context_utils",
+        "//src/main/java/com/google/devtools/build/lib/util/io:io-proto",
         "//src/main/java/com/google/devtools/common/options",
         "//src/main/protobuf:spawn_java_proto",
+        "//third_party:auto_value",
         "//third_party:guava",
     ],
 )

src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ConverterOptions.java

@@ -0,0 +1,101 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.execlog;
+
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionDocumentationCategory;
+import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParsingException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+/** Options for execution log converter. */
+public class ConverterOptions extends OptionsBase {
+  private static final Splitter COLON_SPLITTER = Splitter.on(':').limit(2);
+
+  @Option(
+      name = "input",
+      defaultValue = "null",
+      converter = FormatAndPathConverter.class,
+      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      help = "Input log format and path.")
+  public FormatAndPath input;
+
+  @Option(
+      name = "output",
+      defaultValue = "null",
+      converter = FormatAndPathConverter.class,
+      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      help = "Output log format and path.")
+  public FormatAndPath output;
+
+  @Option(
+      name = "sort",
+      defaultValue = "false",
+      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      help = "Whether to sort the output in a deterministic order.")
+  public boolean sort;
+
+  enum Format {
+    BINARY,
+    JSON
+  }
+
+  private static final ImmutableMap<String, Format> FORMAT_BY_NAME =
+      Arrays.stream(Format.values())
+          .collect(toImmutableMap(f -> f.name().toLowerCase(Locale.US), f -> f));
+
+  @AutoValue
+  abstract static class FormatAndPath {
+    public abstract Format format();
+
+    public abstract Path path();
+
+    public static FormatAndPath of(Format format, Path path) {
+      return new AutoValue_ConverterOptions_FormatAndPath(format, path);
+    }
+  }
+
+  private static class FormatAndPathConverter extends Converter.Contextless<FormatAndPath> {
+
+    @Override
+    public FormatAndPath convert(String input) throws OptionsParsingException {
+      List<String> parts = COLON_SPLITTER.splitToList(input);
+      if (parts.size() != 2
+          || !FORMAT_BY_NAME.containsKey(parts.get(0))
+          || parts.get(1).isEmpty()) {
+        throw new OptionsParsingException("'" + input + "' is not a valid log format and path.");
+      }
+      return FormatAndPath.of(FORMAT_BY_NAME.get(parts.get(0)), Path.of(parts.get(1)));
+    }
+
+    @Override
+    public String getTypeDescription() {
+      return "type:path, where type is one of " + String.join(" ", FORMAT_BY_NAME.keySet());
+    }
+  }
+}

src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ExecLogConverter.java

@@ -0,0 +1,87 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.execlog;
+
+import com.google.devtools.build.execlog.ConverterOptions.FormatAndPath;
+import com.google.devtools.build.lib.exec.Protos.SpawnExec;
+import com.google.devtools.build.lib.exec.StableSort;
+import com.google.devtools.build.lib.util.io.MessageInputStream;
+import com.google.devtools.build.lib.util.io.MessageInputStreamWrapper.BinaryInputStreamWrapper;
+import com.google.devtools.build.lib.util.io.MessageInputStreamWrapper.JsonInputStreamWrapper;
+import com.google.devtools.build.lib.util.io.MessageOutputStream;
+import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.BinaryOutputStreamWrapper;
+import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.JsonOutputStreamWrapper;
+import com.google.devtools.common.options.OptionsParser;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+
+/** A tool to convert between Bazel execution log formats. */
+final class ExecLogConverter {
+  private ExecLogConverter() {}
+
+  private static MessageInputStream<SpawnExec> getMessageInputStream(FormatAndPath log)
+      throws IOException {
+    InputStream in = Files.newInputStream(log.path());
+    switch (log.format()) {
+      case BINARY:
+        return new BinaryInputStreamWrapper<>(in, SpawnExec.getDefaultInstance());
+      case JSON:
+        return new JsonInputStreamWrapper<>(in, SpawnExec.getDefaultInstance());
+    }
+    throw new AssertionError("unsupported input format");
+  }
+
+  private static MessageOutputStream<SpawnExec> getMessageOutputStream(FormatAndPath log)
+      throws IOException {
+    OutputStream out = Files.newOutputStream(log.path());
+    switch (log.format()) {
+      case BINARY:
+        return new BinaryOutputStreamWrapper<>(out);
+      case JSON:
+        return new JsonOutputStreamWrapper<>(out);
+    }
+    throw new AssertionError("unsupported output format");
+  }
+
+  public static void main(String[] args) throws Exception {
+    OptionsParser op = OptionsParser.builder().optionsClasses(ConverterOptions.class).build();
+    op.parseAndExitUponError(args);
+
+    ConverterOptions options = op.getOptions(ConverterOptions.class);
+
+    if (options.input == null) {
+      System.err.println("--input must be specified.");
+      System.exit(1);
+    }
+
+    if (options.output == null) {
+      System.err.println("--output must be specified.");
+      System.exit(1);
+    }
+
+    try (MessageInputStream<SpawnExec> in = getMessageInputStream(options.input);
+        MessageOutputStream<SpawnExec> out = getMessageOutputStream(options.output)) {
+      if (options.sort) {
+        StableSort.stableSort(in, out);
+      } else {
+        SpawnExec ex;
+        while ((ex = in.read()) != null) {
+          out.write(ex);
+        }
+      }
+    }
+  }
+}