From 37fdbfa0b95d84857a31906bfda577b50f9092ff Mon Sep 17 00:00:00 2001 From: xerial-bot Date: Thu, 21 Mar 2024 10:06:48 -0700 Subject: [PATCH] Update hadoop-common to 3.4.0 (#568) * Update hadoop-common to 3.4.0 * Fix tests * Fix for arm --------- Co-authored-by: Taro L. Saito --- build.sbt | 2 +- .../snappy/SnappyHadoopCompatibleOutputStreamTest.java | 5 ++--- src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java | 3 +++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 24b85abb..0464f511 100644 --- a/build.sbt +++ b/build.sbt @@ -78,7 +78,7 @@ libraryDependencies ++= Seq( "org.wvlet.airframe" %% "airframe-log" % "24.3.0" % "test", "org.osgi" % "org.osgi.core" % "6.0.0" % "provided", "com.github.sbt" % "junit-interface" % "0.13.3" % "test", - "org.apache.hadoop" % "hadoop-common" % "2.10.2" % "test" exclude ("org.xerial.snappy", "snappy-java") + "org.apache.hadoop" % "hadoop-common" % "3.4.0" % "test" exclude ("org.xerial.snappy", "snappy-java") ) enablePlugins(SbtOsgi) diff --git a/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java b/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java index 72422bc3..e9c66bf9 100644 --- a/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java +++ b/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java @@ -2,7 +2,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.SystemUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.SnappyCodec; import org.junit.AfterClass; @@ -25,13 +24,13 @@ public static void loadHadoopNativeLibrary() throws Exception { final String libResourceFolder; Map libraryNames = new LinkedHashMap<>(); - if (SystemUtils.IS_OS_LINUX) { + if (OSInfo.getOSName() == "Linux") { libResourceFolder = "/lib/Linux"; libraryNames.put("libhadoop.so", "libhadoop.so"); // certain Linux systems need these shared library be copied before the JVM started, see build.sbt libraryNames.put("libsnappy.so", "libsnappy.so"); libraryNames.put("libsnappy.so.1", "libsnappy.so"); - } else if (SystemUtils.IS_OS_MAC_OSX) { + } else if (OSInfo.getOSName() == "Mac") { libResourceFolder = "/lib/MacOSX"; libraryNames.put("libhadoop.dylib", "libhadoop.dylib"); libraryNames.put("libsnappy.dylib", "libsnappy.dylib"); diff --git a/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java b/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java index d7831cbe..36ef552e 100755 --- a/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java +++ b/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java @@ -179,6 +179,9 @@ public void batchingOfWritesShouldNotAffectCompressedDataSize() // compression quality: if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) assertEquals(90992, expectedCompressedData.length); + else if(OSInfo.getArchName() == "aarch64") + // Arm has a better compression ratio + assertEquals(91051, expectedCompressedData.length); else assertEquals(91080, expectedCompressedData.length); // The chunk size should not affect the size of the compressed output: