diff --git a/airframe-parquet/src/test/scala/wvlet/airframe/parquet/ParquetTest.scala b/airframe-parquet/src/test/scala/wvlet/airframe/parquet/ParquetTest.scala index 3f1c00d9fe..51cc028a91 100644 --- a/airframe-parquet/src/test/scala/wvlet/airframe/parquet/ParquetTest.scala +++ b/airframe-parquet/src/test/scala/wvlet/airframe/parquet/ParquetTest.scala @@ -13,7 +13,6 @@ */ package wvlet.airframe.parquet -import com.amazonaws.auth.profile.ProfileCredentialsProvider import org.apache.hadoop.conf.Configuration import org.apache.parquet.filter2.compat.FilterCompat import org.apache.parquet.filter2.predicate.FilterApi @@ -178,27 +177,4 @@ object ParquetTest extends AirSpec { } } } - - test("write to S3") { - skip("Requires a real S3 bucket to test this code") - - val conf = new Configuration() - conf.set("fs.s3a.aws.credentials.provider", "wvlet.airframe.parquet.CustomCredentialProvider") - val path = "s3a://leo-prizm-dev/test.parquet" - withResource( - Parquet.newWriter[SampleRecord](path = path, hadoopConf = conf) - ) { writer => - writer.write(r1) - writer.write(r2) - } - - withResource(Parquet.newReader[SampleRecord](path = path, hadoopConf = conf)) { reader => - reader.read() shouldBe r1 - reader.read() shouldBe r2 - reader.read() shouldBe null - } - Parquet.readSchema(path, conf) - } } - -class CustomCredentialProvider extends ProfileCredentialsProvider("engineering") diff --git a/build.sbt b/build.sbt index f88d7c16bd..c7faba3f43 100644 --- a/build.sbt +++ b/build.sbt @@ -920,9 +920,9 @@ lazy val parquet = description := "Parquet columnar format reader/writer support", libraryDependencies ++= Seq( "org.apache.parquet" % "parquet-hadoop" % PARQUET_VERSION, - "org.apache.hadoop" % "hadoop-client" % "3.3.6" % Provided, + "org.apache.hadoop" % "hadoop-client" % "3.4.0" % Provided, // For S3 support - "org.apache.hadoop" % "hadoop-aws" % "3.3.6" % Provided, + "org.apache.hadoop" % "hadoop-aws" % "3.4.0" % Provided, "software.amazon.awssdk" % "auth" % "2.25.1" % Provided, // For Apple Silicon (M1) "org.xerial.snappy" % "snappy-java" % "1.1.10.5", diff --git a/docs/airframe-parquet.md b/docs/airframe-parquet.md index 0e649bdce7..1c1742bae4 100644 --- a/docs/airframe-parquet.md +++ b/docs/airframe-parquet.md @@ -14,11 +14,11 @@ airframe-parquet is a library for reading and writing for Scala objects using Pa libraryDependencies ++= Seq( "org.wvlet.airframe" %% "airframe-parquet" % "(version)" // Use your own hadoop version - "org.apache.hadoop" % "hadoop-client" % "3.3.1", + "org.apache.hadoop" % "hadoop-client" % "3.4.0", // [Optional] For supporting S3 - "org.apache.hadoop" % "hadoop-aws" % "3.3.1", + "org.apache.hadoop" % "hadoop-aws" % "3.4.0", // [Optional] For using custom AWS credential provider - "software.amazon.awssdk" % "auth" % "2.17.18" + "software.amazon.awssdk" % "auth" % "2.25.13" ) ```