From 3ffe39898f743063f7dacaabffcf34585b0275f3 Mon Sep 17 00:00:00 2001 From: Clement Escoffier Date: Fri, 26 Apr 2024 11:41:46 +0200 Subject: [PATCH] Allow Snappy to be loaded from a shared classloader This commit introduces a solution for loading the Snappy native library across multiple test profiles. Due to the constraint that native libraries can only be loaded from a single classloader, a shared classloader is now utilized for loading Snappy in test mode. Please note, this feature is exclusively applicable when running tests. Fixes: https://github.com/quarkusio/quarkus/issues/39767 --- .../deployment/KafkaBuildTimeConfig.java | 8 +++++ .../client/deployment/KafkaProcessor.java | 9 ++++-- .../kafka/client/runtime/SnappyLoader.java | 18 +++++++++++ .../kafka/client/runtime/SnappyRecorder.java | 30 ++++++++++++++----- 4 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 extensions/kafka-client/runtime/src/main/java/io/quarkus/kafka/client/runtime/SnappyLoader.java diff --git a/extensions/kafka-client/deployment/src/main/java/io/quarkus/kafka/client/deployment/KafkaBuildTimeConfig.java b/extensions/kafka-client/deployment/src/main/java/io/quarkus/kafka/client/deployment/KafkaBuildTimeConfig.java index 420d8bd85a453..a4fd0b9f2951b 100644 --- a/extensions/kafka-client/deployment/src/main/java/io/quarkus/kafka/client/deployment/KafkaBuildTimeConfig.java +++ b/extensions/kafka-client/deployment/src/main/java/io/quarkus/kafka/client/deployment/KafkaBuildTimeConfig.java @@ -23,6 +23,14 @@ public class KafkaBuildTimeConfig { @ConfigItem(name = "snappy.enabled", defaultValue = "false") public boolean snappyEnabled; + /** + * Whether to load the Snappy native library from the shared classloader. + * This setting is only used in tests if the tests are using different profiles, which would lead to + * unsatisfied link errors when loading Snappy. + */ + @ConfigItem(name = "snappy.load-from-shared-classloader", defaultValue = "false") + public boolean snappyLoadFromSharedClassLoader; + /** * Configuration for DevServices. DevServices allows Quarkus to automatically start Kafka in dev and test mode. */ diff --git a/extensions/kafka-client/deployment/src/main/java/io/quarkus/kafka/client/deployment/KafkaProcessor.java b/extensions/kafka-client/deployment/src/main/java/io/quarkus/kafka/client/deployment/KafkaProcessor.java index 298fbfd6de13f..393c85fdf3cd2 100644 --- a/extensions/kafka-client/deployment/src/main/java/io/quarkus/kafka/client/deployment/KafkaProcessor.java +++ b/extensions/kafka-client/deployment/src/main/java/io/quarkus/kafka/client/deployment/KafkaProcessor.java @@ -72,6 +72,7 @@ import io.quarkus.deployment.builditem.ExtensionSslNativeSupportBuildItem; import io.quarkus.deployment.builditem.FeatureBuildItem; import io.quarkus.deployment.builditem.IndexDependencyBuildItem; +import io.quarkus.deployment.builditem.LaunchModeBuildItem; import io.quarkus.deployment.builditem.LogCategoryBuildItem; import io.quarkus.deployment.builditem.RunTimeConfigurationDefaultBuildItem; import io.quarkus.deployment.builditem.RuntimeConfigSetupCompleteBuildItem; @@ -304,8 +305,12 @@ public void handleSnappyInNative(NativeImageRunnerBuildItem nativeImageRunner, @BuildStep(onlyIf = HasSnappy.class) @Record(ExecutionTime.RUNTIME_INIT) - void loadSnappyIfEnabled(SnappyRecorder recorder, KafkaBuildTimeConfig config) { - recorder.loadSnappy(); + void loadSnappyIfEnabled(LaunchModeBuildItem launch, SnappyRecorder recorder, KafkaBuildTimeConfig config) { + boolean loadFromSharedClassLoader = false; + if (launch.isTest()) { + loadFromSharedClassLoader = config.snappyLoadFromSharedClassLoader; + } + recorder.loadSnappy(loadFromSharedClassLoader); } @Consume(RuntimeConfigSetupCompleteBuildItem.class) diff --git a/extensions/kafka-client/runtime/src/main/java/io/quarkus/kafka/client/runtime/SnappyLoader.java b/extensions/kafka-client/runtime/src/main/java/io/quarkus/kafka/client/runtime/SnappyLoader.java new file mode 100644 index 0000000000000..c1ce89c65ec0f --- /dev/null +++ b/extensions/kafka-client/runtime/src/main/java/io/quarkus/kafka/client/runtime/SnappyLoader.java @@ -0,0 +1,18 @@ +package io.quarkus.kafka.client.runtime; + +import java.io.File; + +public class SnappyLoader { + + /* + * This class is intended to be loaded from a shared classloader (e.g., the system classloader) to avoid + * unsatisfied link errors when the native library is loaded from a different classloader. + * See https://github.com/quarkusio/quarkus/issues/39767. + * + * This class is only used in tests if the `quarkus.kafka.snappy.load-from-shared-classloader=true` is set. + */ + static { + File out = SnappyRecorder.getLibraryFile(); + System.load(out.getAbsolutePath()); + } +} diff --git a/extensions/kafka-client/runtime/src/main/java/io/quarkus/kafka/client/runtime/SnappyRecorder.java b/extensions/kafka-client/runtime/src/main/java/io/quarkus/kafka/client/runtime/SnappyRecorder.java index e06726204230e..68107576d82b3 100644 --- a/extensions/kafka-client/runtime/src/main/java/io/quarkus/kafka/client/runtime/SnappyRecorder.java +++ b/extensions/kafka-client/runtime/src/main/java/io/quarkus/kafka/client/runtime/SnappyRecorder.java @@ -8,14 +8,32 @@ import java.net.URL; import org.xerial.snappy.OSInfo; -import org.xerial.snappy.SnappyLoader; +import io.quarkus.runtime.Application; import io.quarkus.runtime.annotations.Recorder; @Recorder public class SnappyRecorder { - public void loadSnappy() { + public void loadSnappy(boolean loadFromSharedClassLoader) { + if (loadFromSharedClassLoader) { + try { + Application.class.getClassLoader().loadClass(SnappyLoader.class.getName()); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } else { + File out = getLibraryFile(); + try { + System.load(out.getAbsolutePath()); + } catch (UnsatisfiedLinkError e) { + // Try to load the library from the system library path + throw new RuntimeException("Failed to load Snappy native library", e); + } + } + } + + static File getLibraryFile() { // Resolve the library file name with a suffix (e.g., dll, .so, etc.) String snappyNativeLibraryName = System.mapLibraryName("snappyjava"); String snappyNativeLibraryPath = "/org/xerial/snappy/native/" + OSInfo.getNativeLibFolderPathForCurrentOS(); @@ -27,18 +45,16 @@ public void loadSnappy() { throw new RuntimeException(errorMessage); } - File out = extractLibraryFile( + return extractLibraryFile( SnappyLoader.class.getResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName), snappyNativeLibraryName); - - System.load(out.getAbsolutePath()); } - private static boolean hasResource(String path) { + static boolean hasResource(String path) { return SnappyLoader.class.getResource(path) != null; } - private static File extractLibraryFile(URL library, String name) { + static File extractLibraryFile(URL library, String name) { String tmp = System.getProperty("java.io.tmpdir"); File extractedLibFile = new File(tmp, name);