chore: split emulator into core without deps and a higher level wrapp…

…er with grpc helpers Currently the emulator exists in a single artifact with optional deps. The reason for this is that bigtable-hbase needs the emulator w/o grpc. However this is causing issues in graalvm packaging in #1234. This PR makes this easier to manage: a -core artifact without dependencies that just wraps the golang binary that bigtable-hbase can use and a wrapper that has a hard dep on grpc & gax. This is technically a breaking change but the emulator artifact is pre-GA an dis marked with BetaApi
googleapis · May 26, 2022 · 9f93918 · 9f93918
1 parent 6304d88
commit 9f93918
Show file tree

Hide file tree

Showing 6 changed files with 373 additions and 225 deletions.
diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml
@@ -69,6 +69,11 @@
           <artifactId>google-cloud-bigtable-emulator</artifactId>
           <version>0.144.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigtable-emulator:current} -->
         </dependency>
+        <dependency>
+          <groupId>com.google.cloud</groupId>
+          <artifactId>google-cloud-bigtable-emulator-core</artifactId>
+          <version>0.144.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigtable-emulator:current} -->
+        </dependency>
         <dependency>
           <groupId>com.google.api.grpc</groupId>
           <artifactId>grpc-google-cloud-bigtable-admin-v2</artifactId>

diff --git a/google-cloud-bigtable-emulator-core/pom.xml b/google-cloud-bigtable-emulator-core/pom.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <artifactId>google-cloud-bigtable-parent</artifactId>
+    <groupId>com.google.cloud</groupId>
+    <version>2.7.1-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>google-cloud-bigtable-emulator-core</artifactId>
+  <version>0.144.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigtable-emulator:current} -->
+
+  <description>
+    A Java wrapper for the Cloud Bigtable emulator.
+  </description>
+
+  <url>https://github.com/googleapis/java-bigtable</url>
+  <scm>
+    <connection>scm:git:[email protected]:googleapis/java-bigtable.git</connection>
+    <developerConnection>scm:git:[email protected]:googleapis/java-bigtable.git</developerConnection>
+    <url>https://github.com/googleapis/java-bigtable</url>
+    <tag>HEAD</tag>
+  </scm>
+  <developers>
+    <developer>
+      <id>igorberstein</id>
+      <name>Igor Bernstein</name>
+      <email>[email protected]</email>
+      <organization>Google</organization>
+      <roles>
+        <role>Developer</role>
+      </roles>
+    </developer>
+  </developers>
+
+  <properties>
+    <maven.compiler.source>8</maven.compiler.source>
+    <maven.compiler.target>8</maven.compiler.target>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <!-- https://github.com/googleapis/java-gcloud-maven-plugin -->
+        <groupId>com.google.cloud</groupId>
+        <artifactId>google-cloud-gcloud-maven-plugin</artifactId>
+        <version>0.1.5</version>
+
+        <executions>
+          <execution>
+            <id>gen-sources</id>
+            <phase>generate-resources</phase>
+            <goals>
+              <goal>download</goal>
+            </goals>
+            <configuration>
+              <componentNames>
+                <componentName>bigtable-darwin-arm</componentName>
+                <componentName>bigtable-darwin-x86_64</componentName>
+                <componentName>bigtable-linux-arm</componentName>
+                <componentName>bigtable-linux-x86</componentName>
+                <componentName>bigtable-linux-x86_64</componentName>
+                <componentName>bigtable-windows-x86</componentName>
+                <componentName>bigtable-windows-x86_64</componentName>
+              </componentNames>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/...ulator-core/src/main/java/com/google/cloud/bigtable/emulator/core/EmulatorController.java b/...ulator-core/src/main/java/com/google/cloud/bigtable/emulator/core/EmulatorController.java
@@ -0,0 +1,251 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.emulator.core;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.file.Path;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Wraps the Bigtable emulator in a java api.
+ *
+ * <p>This class will use the golang binaries embedded in this jar to launch the emulator as an
+ * external process and redirect its output to a {@link Logger}.
+ */
+public class EmulatorController {
+  private static final Logger LOGGER = Logger.getLogger(EmulatorController.class.getName());
+
+  private final Path executable;
+  private Process process;
+  private boolean isStopped = true;
+  private Thread shutdownHook;
+
+  private int port;
+
+  public static EmulatorController createFromPath(Path path) {
+    return new EmulatorController(path);
+  }
+  /**
+   * Create a new instance of emulator. The emulator will use the bundled binaries in this jar.
+   * Please note that the emulator is created in a stopped state, please use {@link #start()} after
+   * creating it.
+   */
+  public static EmulatorController createBundled() throws IOException {
+    String resourcePath = getBundledResourcePath();
+
+    File tmpEmulator = File.createTempFile("cbtemulator", "");
+    tmpEmulator.deleteOnExit();
+
+    try (InputStream is = EmulatorController.class.getResourceAsStream(resourcePath);
+        FileOutputStream os = new FileOutputStream(tmpEmulator)) {
+
+      if (is == null) {
+        throw new FileNotFoundException(
+            "Failed to find the bundled emulator binary: " + resourcePath);
+      }
+
+      byte[] buff = new byte[2048];
+      int length;
+
+      while ((length = is.read(buff)) != -1) {
+        os.write(buff, 0, length);
+      }
+    }
+    tmpEmulator.setExecutable(true);
+
+    return new EmulatorController(tmpEmulator.toPath());
+  }
+
+  private EmulatorController(Path executable) {
+    this.executable = executable;
+  }
+
+  public synchronized boolean isRunning() {
+    return !isStopped;
+  }
+  /** Starts the emulator process and waits for it to be ready. */
+  public synchronized void start() throws IOException, TimeoutException, InterruptedException {
+    if (!isStopped) {
+      throw new IllegalStateException("Emulator is already started");
+    }
+    this.port = getAvailablePort();
+
+    // Try to align the localhost address across java & golang emulator
+    // This should fix issues on systems that default to ipv4 but the jvm is started with
+    // -Djava.net.preferIPv6Addresses=true
+    Optional<String> localhostAddress = Optional.empty();
+    try {
+      localhostAddress = Optional.of(InetAddress.getByName(null).getHostAddress());
+    } catch (UnknownHostException e) {
+    }
+
+    // Workaround https://bugs.openjdk.java.net/browse/JDK-8068370
+    for (int attemptsLeft = 3; process == null; attemptsLeft--) {
+      try {
+        String cmd = executable.toString();
+        if (localhostAddress.isPresent()) {
+          cmd += String.format(" -host [%s]", localhostAddress.get());
+        }
+        cmd += String.format(" -port %d", port);
+        process = Runtime.getRuntime().exec(cmd);
+      } catch (IOException e) {
+        if (attemptsLeft > 0) {
+          Thread.sleep(1000);
+          continue;
+        }
+        throw e;
+      }
+    }
+    pipeStreamToLog(process.getInputStream(), Level.INFO);
+    pipeStreamToLog(process.getErrorStream(), Level.WARNING);
+    isStopped = false;
+
+    shutdownHook =
+        new Thread(
+            () -> {
+              if (!isStopped) {
+                isStopped = true;
+                process.destroy();
+              }
+            });
+
+    Runtime.getRuntime().addShutdownHook(shutdownHook);
+
+    waitForPort(port);
+  }
+
+  /** Stops the emulator process. */
+  public synchronized void stop() {
+    if (isStopped) {
+      throw new IllegalStateException("Emulator already stopped");
+    }
+
+    try {
+      Runtime.getRuntime().removeShutdownHook(shutdownHook);
+      shutdownHook = null;
+    } finally {
+      isStopped = true;
+      process.destroy();
+    }
+  }
+
+  public synchronized int getPort() {
+    if (isStopped) {
+      throw new IllegalStateException("Emulator is not running");
+    }
+    return port;
+  }
+  // <editor-fold desc="Helpers">
+
+  /** Gets the current platform, which will be used to select the appropriate emulator binary. */
+  private static String getBundledResourcePath() {
+    String unformattedOs = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH);
+    String os;
+    String suffix = "";
+
+    if (unformattedOs.contains("mac") || unformattedOs.contains("darwin")) {
+      os = "darwin";
+    } else if (unformattedOs.contains("win")) {
+      os = "windows";
+      suffix = ".exe";
+    } else if (unformattedOs.contains("linux")) {
+      os = "linux";
+    } else {
+      throw new UnsupportedOperationException(
+          "Emulator is not supported on your platform: " + unformattedOs);
+    }
+
+    String unformattedArch = System.getProperty("os.arch");
+    String arch;
+
+    switch (unformattedArch) {
+      case "x86":
+        arch = "x86";
+        break;
+      case "x86_64":
+      case "amd64":
+        arch = "x86_64";
+        break;
+      case "aarch64":
+        arch = "arm";
+        break;
+      default:
+        throw new UnsupportedOperationException("Unsupported architecture: " + unformattedArch);
+    }
+
+    return String.format(
+        "/gcloud/bigtable-%s-%s/platform/bigtable-emulator/cbtemulator%s", os, arch, suffix);
+  }
+
+  /** Gets a random open port number. */
+  private static int getAvailablePort() {
+    try (ServerSocket serverSocket = new ServerSocket(0)) {
+      return serverSocket.getLocalPort();
+    } catch (IOException e) {
+      throw new RuntimeException("Failed to find open port");
+    }
+  }
+
+  /** Waits for a port to open. It's used to wait for the emulator's gRPC server to be ready. */
+  private static void waitForPort(int port) throws InterruptedException, TimeoutException {
+    for (int i = 0; i < 100; i++) {
+      try (Socket ignored = new Socket("localhost", port)) {
+        return;
+      } catch (IOException e) {
+        Thread.sleep(200);
+      }
+    }
+
+    throw new TimeoutException("Timed out waiting for server to start");
+  }
+
+  /** Creates a thread that will pipe an {@link InputStream} to this class' Logger. */
+  private static void pipeStreamToLog(final InputStream stream, final Level level) {
+    final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
+
+    Thread thread =
+        new Thread(
+            () -> {
+              try {
+                String line;
+                while ((line = reader.readLine()) != null) {
+                  LOGGER.log(level, line);
+                }
+              } catch (IOException e) {
+                if (!"Stream closed".equals(e.getMessage())) {
+                  LOGGER.log(Level.WARNING, "Failed to read process stream", e);
+                }
+              }
+            });
+    thread.setDaemon(true);
+    thread.start();
+  }
+  // </editor-fold>
+}
diff --git a/google-cloud-bigtable-emulator/pom.xml b/google-cloud-bigtable-emulator/pom.xml
@@ -69,8 +69,8 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
         <configuration>
-          <!-- grpc-netty-shaded is used at test runtime -->
-          <usedDependencies>io.grpc:grpc-netty-shaded</usedDependencies>
+          <!-- needd gax-grpc for graalvm configs and grpc-netty -->
+          <usedDependencies>com.google.api:gax-grpc</usedDependencies>
         </configuration>
       </plugin>
     </plugins>
@@ -96,16 +96,28 @@
   </dependencyManagement>
 
   <dependencies>
-    <!-- Compile deps in alphabetical order -->
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-bigtable-emulator-core</artifactId>
+      <version>0.144.1-SNAPSHOT</version>
+    </dependency>
+
     <dependency>
       <groupId>com.google.api</groupId>
       <artifactId>api-common</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.grpc</groupId>
       <artifactId>grpc-api</artifactId>
-      <!-- gRPC deps are provided by the client -->
-      <scope>provided</scope>
+    </dependency>
+    <!-- declare dependency on gax for graalvm configs -->
+    <dependency>
+      <groupId>com.google.api</groupId>
+      <artifactId>gax-grpc</artifactId>
     </dependency>
 
     <dependency>
@@ -160,11 +172,5 @@
         </exclusion>
       </exclusions>
     </dependency>
-
-    <dependency>
-      <groupId>io.grpc</groupId>
-      <artifactId>grpc-netty-shaded</artifactId>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 </project>