Skip to content

Commit

Permalink
chore: split emulator into core without deps and a higher level wrapp…
Browse files Browse the repository at this point in the history 
…er with grpc helpers

Currently the emulator exists in a single artifact with optional deps. The reason for this is
that bigtable-hbase needs the emulator w/o grpc. However this is causing issues in graalvm
packaging in #1234. This PR makes this easier to manage: a -core artifact without dependencies
that just wraps the golang binary that bigtable-hbase can use and a wrapper that has a hard dep
on grpc & gax.

This is technically a breaking change but the emulator artifact is pre-GA an is marked with BetaApi
  • Loading branch information
@igorbernstein2
igorbernstein2 committed May 26, 2022
1 parent 6304d88 commit c3e99cd
Show file tree
Hide file tree
Showing 6 changed files with 373 additions and 225 deletions.
5 changes: 5 additions & 0 deletions google-cloud-bigtable-bom/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@
<artifactId>google-cloud-bigtable-emulator</artifactId>
<version>0.144.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigtable-emulator:current} -->
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable-emulator-core</artifactId>
<version>0.144.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigtable-emulator:current} -->
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>grpc-google-cloud-bigtable-admin-v2</artifactId>
Expand Down
75 changes: 75 additions & 0 deletions google-cloud-bigtable-emulator-core/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<artifactId>google-cloud-bigtable-parent</artifactId>
<groupId>com.google.cloud</groupId>
<version>2.7.1-SNAPSHOT</version>
</parent>

<artifactId>google-cloud-bigtable-emulator-core</artifactId>
<version>0.144.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigtable-emulator:current} -->

<description>
A Java wrapper for the Cloud Bigtable emulator.
</description>

<url>https://github.com/googleapis/java-bigtable</url>
<scm>
<connection>scm:git:[email protected]:googleapis/java-bigtable.git</connection>
<developerConnection>scm:git:[email protected]:googleapis/java-bigtable.git</developerConnection>
<url>https://github.com/googleapis/java-bigtable</url>
<tag>HEAD</tag>
</scm>
<developers>
<developer>
<id>igorberstein</id>
<name>Igor Bernstein</name>
<email>[email protected]</email>
<organization>Google</organization>
<roles>
<role>Developer</role>
</roles>
</developer>
</developers>

<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>

<build>
<plugins>
<plugin>
<!-- https://github.com/googleapis/java-gcloud-maven-plugin -->
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-gcloud-maven-plugin</artifactId>
<version>0.1.5</version>

<executions>
<execution>
<id>gen-sources</id>
<phase>generate-resources</phase>
<goals>
<goal>download</goal>
</goals>
<configuration>
<componentNames>
<componentName>bigtable-darwin-arm</componentName>
<componentName>bigtable-darwin-x86_64</componentName>
<componentName>bigtable-linux-arm</componentName>
<componentName>bigtable-linux-x86</componentName>
<componentName>bigtable-linux-x86_64</componentName>
<componentName>bigtable-windows-x86</componentName>
<componentName>bigtable-windows-x86_64</componentName>
</componentNames>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
251 changes: 251 additions & 0 deletions ...ulator-core/src/main/java/com/google/cloud/bigtable/emulator/core/EmulatorController.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.cloud.bigtable.emulator.core;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.UnknownHostException;
import java.nio.file.Path;
import java.util.Locale;
import java.util.Optional;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
* Wraps the Bigtable emulator in a java api.
*
* <p>This class will use the golang binaries embedded in this jar to launch the emulator as an
* external process and redirect its output to a {@link Logger}.
*/
public class EmulatorController {
private static final Logger LOGGER = Logger.getLogger(EmulatorController.class.getName());

private final Path executable;
private Process process;
private boolean isStopped = true;
private Thread shutdownHook;

private int port;

public static EmulatorController createFromPath(Path path) {
return new EmulatorController(path);
}
/**
* Create a new instance of emulator. The emulator will use the bundled binaries in this jar.
* Please note that the emulator is created in a stopped state, please use {@link #start()} after
* creating it.
*/
public static EmulatorController createBundled() throws IOException {
String resourcePath = getBundledResourcePath();

File tmpEmulator = File.createTempFile("cbtemulator", "");
tmpEmulator.deleteOnExit();

try (InputStream is = EmulatorController.class.getResourceAsStream(resourcePath);
FileOutputStream os = new FileOutputStream(tmpEmulator)) {

if (is == null) {
throw new FileNotFoundException(
"Failed to find the bundled emulator binary: " + resourcePath);
}

byte[] buff = new byte[2048];
int length;

while ((length = is.read(buff)) != -1) {
os.write(buff, 0, length);
}
}
tmpEmulator.setExecutable(true);

return new EmulatorController(tmpEmulator.toPath());
}

private EmulatorController(Path executable) {
this.executable = executable;
}

public synchronized boolean isRunning() {
return !isStopped;
}
/** Starts the emulator process and waits for it to be ready. */
public synchronized void start() throws IOException, TimeoutException, InterruptedException {
if (!isStopped) {
throw new IllegalStateException("Emulator is already started");
}
this.port = getAvailablePort();

// Try to align the localhost address across java & golang emulator
// This should fix issues on systems that default to ipv4 but the jvm is started with
// -Djava.net.preferIPv6Addresses=true
Optional<String> localhostAddress = Optional.empty();
try {
localhostAddress = Optional.of(InetAddress.getByName(null).getHostAddress());
} catch (UnknownHostException e) {
}

// Workaround https://bugs.openjdk.java.net/browse/JDK-8068370
for (int attemptsLeft = 3; process == null; attemptsLeft--) {
try {
String cmd = executable.toString();
if (localhostAddress.isPresent()) {
cmd += String.format(" -host [%s]", localhostAddress.get());
}
cmd += String.format(" -port %d", port);
process = Runtime.getRuntime().exec(cmd);
} catch (IOException e) {
if (attemptsLeft > 0) {
Thread.sleep(1000);
continue;
}
throw e;
}
}
pipeStreamToLog(process.getInputStream(), Level.INFO);
pipeStreamToLog(process.getErrorStream(), Level.WARNING);
isStopped = false;

shutdownHook =
new Thread(
() -> {
if (!isStopped) {
isStopped = true;
process.destroy();
}
});

Runtime.getRuntime().addShutdownHook(shutdownHook);

waitForPort(port);
}

/** Stops the emulator process. */
public synchronized void stop() {
if (isStopped) {
throw new IllegalStateException("Emulator already stopped");
}

try {
Runtime.getRuntime().removeShutdownHook(shutdownHook);
shutdownHook = null;
} finally {
isStopped = true;
process.destroy();
}
}

public synchronized int getPort() {
if (isStopped) {
throw new IllegalStateException("Emulator is not running");
}
return port;
}
// <editor-fold desc="Helpers">

/** Gets the current platform, which will be used to select the appropriate emulator binary. */
private static String getBundledResourcePath() {
String unformattedOs = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH);
String os;
String suffix = "";

if (unformattedOs.contains("mac") || unformattedOs.contains("darwin")) {
os = "darwin";
} else if (unformattedOs.contains("win")) {
os = "windows";
suffix = ".exe";
} else if (unformattedOs.contains("linux")) {
os = "linux";
} else {
throw new UnsupportedOperationException(
"Emulator is not supported on your platform: " + unformattedOs);
}

String unformattedArch = System.getProperty("os.arch");
String arch;

switch (unformattedArch) {
case "x86":
arch = "x86";
break;
case "x86_64":
case "amd64":
arch = "x86_64";
break;
case "aarch64":
arch = "arm";
break;
default:
throw new UnsupportedOperationException("Unsupported architecture: " + unformattedArch);
}

return String.format(
"/gcloud/bigtable-%s-%s/platform/bigtable-emulator/cbtemulator%s", os, arch, suffix);
}

/** Gets a random open port number. */
private static int getAvailablePort() {
try (ServerSocket serverSocket = new ServerSocket(0)) {
return serverSocket.getLocalPort();
} catch (IOException e) {
throw new RuntimeException("Failed to find open port");
}
}

/** Waits for a port to open. It's used to wait for the emulator's gRPC server to be ready. */
private static void waitForPort(int port) throws InterruptedException, TimeoutException {
for (int i = 0; i < 100; i++) {
try (Socket ignored = new Socket("localhost", port)) {
return;
} catch (IOException e) {
Thread.sleep(200);
}
}

throw new TimeoutException("Timed out waiting for server to start");
}

/** Creates a thread that will pipe an {@link InputStream} to this class' Logger. */
private static void pipeStreamToLog(final InputStream stream, final Level level) {
final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));

Thread thread =
new Thread(
() -> {
try {
String line;
while ((line = reader.readLine()) != null) {
LOGGER.log(level, line);
}
} catch (IOException e) {
if (!"Stream closed".equals(e.getMessage())) {
LOGGER.log(Level.WARNING, "Failed to read process stream", e);
}
}
});
thread.setDaemon(true);
thread.start();
}
// </editor-fold>
}
28 changes: 17 additions & 11 deletions google-cloud-bigtable-emulator/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<!-- grpc-netty-shaded is used at test runtime -->
<usedDependencies>io.grpc:grpc-netty-shaded</usedDependencies>
<!-- needd gax-grpc for graalvm configs and grpc-netty -->
<usedDependencies>com.google.api:gax-grpc</usedDependencies>
</configuration>
</plugin>
</plugins>
Expand All @@ -96,16 +96,28 @@
</dependencyManagement>

<dependencies>
<!-- Compile deps in alphabetical order -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable-emulator-core</artifactId>
<version>0.144.1-SNAPSHOT</version>
</dependency>

<dependency>
<groupId>com.google.api</groupId>
<artifactId>api-common</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-api</artifactId>
<!-- gRPC deps are provided by the client -->
<scope>provided</scope>
</dependency>
<!-- declare dependency on gax for graalvm configs -->
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax-grpc</artifactId>
</dependency>

<dependency>
Expand Down Expand Up @@ -160,11 +172,5 @@
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>
Loading

0 comments on commit c3e99cd

Please sign in to comment.