Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make Snappy optional in JVM mode #34722

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import org.eclipse.microprofile.config.ConfigProvider;
import org.jboss.jandex.ClassInfo;
import org.jboss.jandex.DotName;
import org.xerial.snappy.OSInfo;

import io.quarkus.arc.deployment.AdditionalBeanBuildItem;
import io.quarkus.arc.deployment.UnremovableBeanBuildItem;
Expand Down Expand Up @@ -95,6 +94,7 @@
import io.quarkus.kafka.client.runtime.KafkaBindingConverter;
import io.quarkus.kafka.client.runtime.KafkaRecorder;
import io.quarkus.kafka.client.runtime.KafkaRuntimeConfigProducer;
import io.quarkus.kafka.client.runtime.SnappyRecorder;
import io.quarkus.kafka.client.runtime.ui.KafkaTopicClient;
import io.quarkus.kafka.client.runtime.ui.KafkaUiRecorder;
import io.quarkus.kafka.client.runtime.ui.KafkaUiUtils;
Expand Down Expand Up @@ -292,7 +292,7 @@ public void build(

}

@BuildStep(onlyIf = { IsSnappy.class, NativeOrNativeSourcesBuild.class })
@BuildStep(onlyIf = { HasSnappy.class, NativeOrNativeSourcesBuild.class })
public void handleSnappyInNative(NativeImageRunnerBuildItem nativeImageRunner,
BuildProducer<ReflectiveClassBuildItem> reflectiveClass,
BuildProducer<NativeImageResourceBuildItem> nativeLibs) {
Expand All @@ -307,19 +307,17 @@ public void handleSnappyInNative(NativeImageRunnerBuildItem nativeImageRunner,
String path = root + dir + "/" + snappyNativeLibraryName;
nativeLibs.produce(new NativeImageResourceBuildItem(path));
} else { // otherwise the native lib of the platform this build runs on
String dir = OSInfo.getNativeLibFolderPathForCurrentOS();
String dir = SnappyUtils.getNativeLibFolderPathForCurrentOS();
String snappyNativeLibraryName = System.mapLibraryName("snappyjava");
String path = root + dir + "/" + snappyNativeLibraryName;
nativeLibs.produce(new NativeImageResourceBuildItem(path));
}
}

@BuildStep
@BuildStep(onlyIf = HasSnappy.class)
@Record(ExecutionTime.RUNTIME_INIT)
void loadSnappyIfEnabled(KafkaRecorder recorder, KafkaBuildTimeConfig config) {
if (config.snappyEnabled) {
recorder.loadSnappy();
}
void loadSnappyIfEnabled(SnappyRecorder recorder, KafkaBuildTimeConfig config) {
recorder.loadSnappy();
}

@Consume(RuntimeConfigSetupCompleteBuildItem.class)
Expand Down Expand Up @@ -585,17 +583,17 @@ public DevConsoleWebjarBuildItem setupWebJar(LaunchModeBuildItem launchModeBuild
.build();
}

public static final class IsSnappy implements BooleanSupplier {
public static final class HasSnappy implements BooleanSupplier {

private final KafkaBuildTimeConfig config;

public IsSnappy(KafkaBuildTimeConfig config) {
public HasSnappy(KafkaBuildTimeConfig config) {
this.config = config;
}

@Override
public boolean getAsBoolean() {
return config.snappyEnabled;
return QuarkusClassLoader.isClassPresentAtRuntime("org.xerial.snappy.OSInfo") && config.snappyEnabled;
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package io.quarkus.kafka.client.deployment;

import org.xerial.snappy.OSInfo;

/**
* This class should only be used if Snappy is available on the classpath.
*/
public class SnappyUtils {

private SnappyUtils() {
// Avoid direct instantiation
}

public static String getNativeLibFolderPathForCurrentOS() {
return OSInfo.getNativeLibFolderPathForCurrentOS();
}
}
Original file line number Diff line number Diff line change
@@ -1,69 +1,15 @@
package io.quarkus.kafka.client.runtime;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URL;
import java.util.Optional;

import org.eclipse.microprofile.config.Config;
import org.eclipse.microprofile.config.ConfigProvider;
import org.xerial.snappy.OSInfo;
import org.xerial.snappy.SnappyError;
import org.xerial.snappy.SnappyErrorCode;
import org.xerial.snappy.SnappyLoader;

import io.quarkus.runtime.annotations.Recorder;

@Recorder
public class KafkaRecorder {

public void loadSnappy() {
// Resolve the library file name with a suffix (e.g., dll, .so, etc.)
String snappyNativeLibraryName = System.mapLibraryName("snappyjava");
String snappyNativeLibraryPath = "/org/xerial/snappy/native/" + OSInfo.getNativeLibFolderPathForCurrentOS();
boolean hasNativeLib = hasResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName);

if (!hasNativeLib) {
String errorMessage = String.format("no native library is found for os.name=%s and os.arch=%s", OSInfo.getOSName(),
OSInfo.getArchName());
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, errorMessage);
}

File out = extractLibraryFile(
SnappyLoader.class.getResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName),
snappyNativeLibraryName);

System.load(out.getAbsolutePath());
}

private static boolean hasResource(String path) {
return SnappyLoader.class.getResource(path) != null;
}

private static File extractLibraryFile(URL library, String name) {
String tmp = System.getProperty("java.io.tmpdir");
File extractedLibFile = new File(tmp, name);

try (BufferedInputStream inputStream = new BufferedInputStream(library.openStream());
FileOutputStream fileOS = new FileOutputStream(extractedLibFile)) {
byte[] data = new byte[8192];
int byteContent;
while ((byteContent = inputStream.read(data, 0, 8192)) != -1) {
fileOS.write(data, 0, byteContent);
}
} catch (IOException e) {
throw new UncheckedIOException(
"Unable to extract native library " + name + " to " + extractedLibFile.getAbsolutePath(), e);
}

extractedLibFile.deleteOnExit();

return extractedLibFile;
}

public void checkBoostrapServers() {
Config config = ConfigProvider.getConfig();
Boolean serviceBindingEnabled = config.getValue("quarkus.kubernetes-service-binding.enabled", Boolean.class);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package io.quarkus.kafka.client.runtime;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URL;

import org.xerial.snappy.OSInfo;
import org.xerial.snappy.SnappyLoader;

import io.quarkus.runtime.annotations.Recorder;

@Recorder
public class SnappyRecorder {

public void loadSnappy() {
// Resolve the library file name with a suffix (e.g., dll, .so, etc.)
String snappyNativeLibraryName = System.mapLibraryName("snappyjava");
String snappyNativeLibraryPath = "/org/xerial/snappy/native/" + OSInfo.getNativeLibFolderPathForCurrentOS();
boolean hasNativeLib = hasResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName);

if (!hasNativeLib) {
String errorMessage = String.format("no native library is found for os.name=%s and os.arch=%s", OSInfo.getOSName(),
OSInfo.getArchName());
throw new RuntimeException(errorMessage);
}

File out = extractLibraryFile(
SnappyLoader.class.getResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName),
snappyNativeLibraryName);

System.load(out.getAbsolutePath());
}

private static boolean hasResource(String path) {
return SnappyLoader.class.getResource(path) != null;
}

private static File extractLibraryFile(URL library, String name) {
String tmp = System.getProperty("java.io.tmpdir");
File extractedLibFile = new File(tmp, name);

try (BufferedInputStream inputStream = new BufferedInputStream(library.openStream());
FileOutputStream fileOS = new FileOutputStream(extractedLibFile)) {
byte[] data = new byte[8192];
int byteContent;
while ((byteContent = inputStream.read(data, 0, 8192)) != -1) {
fileOS.write(data, 0, byteContent);
}
} catch (IOException e) {
throw new UncheckedIOException(
"Unable to extract native library " + name + " to " + extractedLibFile.getAbsolutePath(), e);
}

extractedLibFile.deleteOnExit();

return extractedLibFile;
}
}