From 6506461ce4af0b3409904fe7b764aa0a098f6daa Mon Sep 17 00:00:00 2001 From: jgprogramming Date: Mon, 17 Jun 2024 12:38:49 +0200 Subject: [PATCH 1/2] Support v2 iceberg tables with gzip --- .../catalog/service/impl/ImportSnapshotWorker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog/service/impl/src/main/java/org/projectnessie/catalog/service/impl/ImportSnapshotWorker.java b/catalog/service/impl/src/main/java/org/projectnessie/catalog/service/impl/ImportSnapshotWorker.java index 8af527de0b9..b70169de969 100644 --- a/catalog/service/impl/src/main/java/org/projectnessie/catalog/service/impl/ImportSnapshotWorker.java +++ b/catalog/service/impl/src/main/java/org/projectnessie/catalog/service/impl/ImportSnapshotWorker.java @@ -109,7 +109,7 @@ private EntitySnapshotObj.Builder importIcebergTable( StorageUri metadataLocation = StorageUri.of(content.getMetadataLocation()); try { InputStream input = taskRequest.objectIO().readObject(metadataLocation); - if (metadataLocation.requiredPath().endsWith(".gz")) { + if (metadataLocation.requiredPath().endsWith("metadata.json.gz") || metadataLocation.requiredPath().endsWith(".gz.metadata.json")) { input = new GZIPInputStream(input); } tableMetadata = IcebergJson.objectMapper().readValue(input, IcebergTableMetadata.class); From c9d83557a890a620f9361dc9ffcbe2bfd2a10da0 Mon Sep 17 00:00:00 2001 From: jgprogramming Date: Tue, 18 Jun 2024 16:50:25 +0000 Subject: [PATCH 2/2] add tests for gzip metadata --- .../fixtures/IcebergGenerateFixtures.java | 17 +++++++++++++---- .../service/impl/ImportSnapshotWorker.java | 3 ++- .../catalog/service/impl/TestIcebergStuff.java | 4 +++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergGenerateFixtures.java b/catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergGenerateFixtures.java index 636ad4060e9..3a64ba765ba 100644 --- a/catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergGenerateFixtures.java +++ b/catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergGenerateFixtures.java @@ -76,10 +76,19 @@ public static String generateCompressedMetadata(ObjectWriter writer, int iceberg gzip.flush(); data = bytes.toByteArray(); } - return writer.write( - URI.create( - "table-metadata-simple-no-manifest/table-metadata-simple-compressed-no-manifest.json.gz"), - data); + String metadataPath = "table-metadata-simple-no-manifest/"; + switch (icebergSpecVersion) { + case 1: + metadataPath += "table-metadata-simple-compressed-no-manifest.metadata.json.gz"; + break; + case 2: + metadataPath += "table-metadata-simple-compressed-no-manifest.gz.metadata.json"; + break; + default: + metadataPath += "table-metadata-simple-compressed-no-manifest.json.gz"; + break; + } + return writer.write(URI.create(metadataPath), data); } public static String generateSimpleMetadata(ObjectWriter writer, int icebergSpecVersion) diff --git a/catalog/service/impl/src/main/java/org/projectnessie/catalog/service/impl/ImportSnapshotWorker.java b/catalog/service/impl/src/main/java/org/projectnessie/catalog/service/impl/ImportSnapshotWorker.java index b70169de969..9c5649d79d1 100644 --- a/catalog/service/impl/src/main/java/org/projectnessie/catalog/service/impl/ImportSnapshotWorker.java +++ b/catalog/service/impl/src/main/java/org/projectnessie/catalog/service/impl/ImportSnapshotWorker.java @@ -109,7 +109,8 @@ private EntitySnapshotObj.Builder importIcebergTable( StorageUri metadataLocation = StorageUri.of(content.getMetadataLocation()); try { InputStream input = taskRequest.objectIO().readObject(metadataLocation); - if (metadataLocation.requiredPath().endsWith("metadata.json.gz") || metadataLocation.requiredPath().endsWith(".gz.metadata.json")) { + if (metadataLocation.requiredPath().endsWith(".gz") + || metadataLocation.requiredPath().endsWith(".gz.metadata.json")) { input = new GZIPInputStream(input); } tableMetadata = IcebergJson.objectMapper().readValue(input, IcebergTableMetadata.class); diff --git a/catalog/service/impl/src/test/java/org/projectnessie/catalog/service/impl/TestIcebergStuff.java b/catalog/service/impl/src/test/java/org/projectnessie/catalog/service/impl/TestIcebergStuff.java index 46d72d9204e..364a42060b0 100644 --- a/catalog/service/impl/src/test/java/org/projectnessie/catalog/service/impl/TestIcebergStuff.java +++ b/catalog/service/impl/src/test/java/org/projectnessie/catalog/service/impl/TestIcebergStuff.java @@ -122,7 +122,9 @@ public void icebergTableImports( static Stream icebergTableImports() throws Exception { IcebergGenerateFixtures.ObjectWriter objectWriter = objectWriterForPath(tempDir); return Stream.of( - arguments("compressed table-metadata", generateCompressedMetadata(objectWriter, 2)), + arguments("compressed table-metadata generic", generateCompressedMetadata(objectWriter, 0)), + arguments("compressed table-metadata v1", generateCompressedMetadata(objectWriter, 1)), + arguments("compressed table-metadata v2", generateCompressedMetadata(objectWriter, 2)), arguments("simple table-metadata", generateSimpleMetadata(objectWriter, 2))); } }