Convert worker to use json instead of protobuf

This was complicated by the difficulty in reading from stdin since it's
populated asynchronously from a file while the worker is running.

Fixes #516

swift/internal/actions.bzl

@@ -230,6 +230,7 @@ def run_toolchain_action(
             tool_config.use_param_file
         ):
             execution_requirements["supports-workers"] = "1"
+            execution_requirements["requires-worker-protocol"] = "json"
 
         executable = swift_toolchain.swift_worker
         tool_executable_args.add(tool_config.executable)

tools/worker/BUILD

@@ -30,17 +30,17 @@ cc_library(
     name = "compile_with_worker",
     srcs = [
         "compile_with_worker.cc",
+        "models.h",
         "work_processor.cc",
         "work_processor.h",
     ],
     hdrs = ["compile_with_worker.h"],
     deps = [
         ":swift_runner",
-        "//third_party/bazel_protos:worker_protocol_cc_proto",
         "//tools/common:file_system",
         "//tools/common:path_utils",
         "//tools/common:temp_file",
-        "@com_google_protobuf//:protobuf",
+        "@com_github_nlohmann_json//:json",
     ],
 )
 

tools/worker/compile_with_worker.cc

@@ -14,13 +14,11 @@
 
 #include "tools/worker/compile_with_worker.h"
 
-#include <google/protobuf/io/zero_copy_stream_impl.h>
-#include <google/protobuf/util/delimited_message_util.h>
+#include <nlohmann/json.hpp>
 #include <unistd.h>
 
 #include <iostream>
 
-#include "third_party/bazel_protos/worker_protocol.pb.h"
 #include "tools/worker/work_processor.h"
 
 // How Swift Incremental Compilation Works
@@ -76,40 +74,34 @@
 // it can find them as well.
 
 int CompileWithWorker(const std::vector<std::string> &args) {
-  // Set up the input and output streams used to communicate with Bazel over
-  // stdin and stdout.
-  google::protobuf::io::FileInputStream file_input_stream(STDIN_FILENO);
-  file_input_stream.SetCloseOnDelete(false);
-  google::protobuf::io::FileOutputStream file_output_stream(STDOUT_FILENO);
-  file_output_stream.SetCloseOnDelete(false);
-
   // Pass the "universal arguments" to the Swift work processor. They will be
   // rewritten to replace any placeholders if necessary, and then passed at the
   // beginning of any process invocation. Note that these arguments include the
   // tool itself (i.e., "swiftc").
   WorkProcessor swift_worker(args);
+  int offset = 0;
 
   while (true) {
-    blaze::worker::WorkRequest request;
-    if (!google::protobuf::util::ParseDelimitedFromZeroCopyStream(
-            &request, &file_input_stream, nullptr)) {
-      std::cerr << "Could not read WorkRequest from stdin. Killing worker "
-                << "process.\n";
-      return 254;
+    std::string line;
+    int result;
+    do {
+      char buffer[1024];
+      lseek(STDIN_FILENO, offset, SEEK_SET);
+      result = read(STDIN_FILENO, buffer, 1024);
+      buffer[result] = '\0';
+      offset += result;
+      line.append(buffer);
+    } while (result == 1024);
+
+    if (line == "") {
+      continue;
     }
 
-    blaze::worker::WorkResponse response;
+    auto request_json = nlohmann::json::parse(line);
+    WorkRequest request(0, request_json["arguments"]);
+    WorkResponse response;
     swift_worker.ProcessWorkRequest(request, &response);
-
-    if (!google::protobuf::util::SerializeDelimitedToZeroCopyStream(
-            response, &file_output_stream)) {
-      std::cerr << "Could not write WorkResponse to stdout. Killing worker "
-                << "process.\n";
-      return 254;
-    }
-    // Flush stdout after writing to ensure that Bazel doesn't hang waiting for
-    // the response due to buffering.
-    file_output_stream.Flush();
+    std::cout << response.to_json().dump() << std::flush;
   }
 
   return 0;

tools/worker/models.h

@@ -0,0 +1,50 @@
+#include <nlohmann/json.hpp>
+
+class WorkRequest {
+  public:
+    WorkRequest(std::int64_t request_id, const std::vector<std::string> &args) :
+      request_id_(request_id), arguments_(args) {};
+
+    const std::vector<std::string> arguments() const {
+      return arguments_;
+    };
+
+    const int64_t request_id() const {
+      return request_id_;
+    }
+
+  private:
+    int64_t request_id_;
+    std::vector<std::string> arguments_;
+};
+
+class WorkResponse {
+  public:
+    WorkResponse() {};
+
+    nlohmann::json to_json() {
+      return nlohmann::json{
+        {"exitCode", this->exit_code_},
+        {"output", this->output_},
+        {"requestId", this->request_id_},
+      };
+    }
+
+    void set_exit_code(int exit_code) {
+      exit_code_ = exit_code;
+    }
+
+    void set_output(std::string output) {
+      output_ = output;
+    }
+
+    void set_request_id(int64_t request_id) {
+      request_id_ = request_id;
+    }
+
+  private:
+    int exit_code_;
+    std::string output_;
+    std::int64_t request_id_;
+};
+

tools/worker/work_processor.cc

@@ -14,7 +14,6 @@
 
 #include "tools/worker/work_processor.h"
 
-#include <google/protobuf/text_format.h>
 #include <sys/stat.h>
 
 #include <fstream>
@@ -31,10 +30,11 @@
 
 namespace {
 
-static void FinalizeWorkRequest(const blaze::worker::WorkRequest &request,
-                                blaze::worker::WorkResponse *response,
+static void FinalizeWorkRequest(const WorkRequest &request,
+                                WorkResponse *response,
                                 int exit_code,
                                 const std::ostringstream &output) {
+
   response->set_exit_code(exit_code);
   response->set_output(output.str());
   response->set_request_id(request.request_id());
@@ -47,8 +47,8 @@ WorkProcessor::WorkProcessor(const std::vector<std::string> &args) {
 }
 
 void WorkProcessor::ProcessWorkRequest(
-    const blaze::worker::WorkRequest &request,
-    blaze::worker::WorkResponse *response) {
+    const WorkRequest &request,
+    WorkResponse *response) {
   std::vector<std::string> processed_args(universal_args_);
 
   // Bazel's worker spawning strategy reads the arguments from the params file

tools/worker/work_processor.h

@@ -19,7 +19,7 @@
 #include <string>
 #include <vector>
 
-#include "third_party/bazel_protos/worker_protocol.pb.h"
+#include "tools/worker/models.h"
 
 // Manages persistent global state for the Swift worker and processes individual
 // work requests.
@@ -31,8 +31,7 @@ class WorkProcessor {
 
   // Processes the given work request and writes its exit code and stderr output
   // (if any) into the given response.
-  void ProcessWorkRequest(const blaze::worker::WorkRequest &request,
-                          blaze::worker::WorkResponse *response);
+  void ProcessWorkRequest(const WorkRequest &request, WorkResponse *response);
 
  private:
   std::vector<std::string> universal_args_;