Add Indexes and Stats format reader and writer

Add reader and writer for the Index and Statistics File Format.

build.gradle

@@ -217,6 +217,7 @@ project(':iceberg-core') {
       exclude group: 'org.tukaani' // xz compression is not supported
     }
 
+    implementation 'io.airlift:aircompressor'
     implementation 'org.apache.httpcomponents.client5:httpclient5'
     implementation "com.fasterxml.jackson.core:jackson-databind"
     implementation "com.fasterxml.jackson.core:jackson-core"

bundled-guava/src/main/java/org/apache/iceberg/GuavaClasses.java

@@ -33,6 +33,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableTable;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.ListMultimap;
@@ -73,6 +74,7 @@ public class GuavaClasses {
     ImmutableList.class.getName();
     ImmutableMap.class.getName();
     ImmutableSet.class.getName();
+    ImmutableTable.class.getName();
     Iterables.class.getName();
     Iterators.class.getName();
     ListMultimap.class.getName();

core/src/main/java/org/apache/iceberg/stats/Blob.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.stats;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+
+public final class Blob {
+  private final String type;
+  private final List<Integer> columnsCovered;
+  private final ByteBuffer blobData;
+  private final StatsCompressionCodec requestedCompression;
+
+  public Blob(String type, List<Integer> columnsCovered, ByteBuffer blobData) {
+    this(type, columnsCovered, blobData, null);
+  }
+
+  public Blob(
+      String type, List<Integer> columnsCovered, ByteBuffer blobData,
+      @Nullable StatsCompressionCodec requestedCompression) {
+    Preconditions.checkNotNull(type, "type is null");
+    Preconditions.checkNotNull(columnsCovered, "columnsCovered is null");
+    Preconditions.checkNotNull(blobData, "blobData is null");
+    this.type = type;
+    this.columnsCovered = ImmutableList.copyOf(columnsCovered);
+    this.blobData = blobData;
+    this.requestedCompression = requestedCompression;
+  }
+
+  public String type() {
+    return type;
+  }
+
+  public List<Integer> columnsCovered() {
+    return columnsCovered;
+  }
+
+  public ByteBuffer blobData() {
+    return blobData;
+  }
+
+  @Nullable
+  public StatsCompressionCodec requestedCompression() {
+    return requestedCompression;
+  }
+}

core/src/main/java/org/apache/iceberg/stats/BlobMetadata.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.stats;
+
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+
+public class BlobMetadata {
+  private final String type;
+  private final List<Integer> columns;
+  private final long offset;
+  private final long length;
+  private final String compressionCodec;
+
+  public BlobMetadata(String type, List<Integer> columns, long offset, long length, @Nullable String compressionCodec) {
+    Preconditions.checkNotNull(type, "type is null");
+    Preconditions.checkNotNull(columns, "columns is null");
+    this.type = type;
+    this.columns = ImmutableList.copyOf(columns);
+    this.offset = offset;
+    this.length = length;
+    this.compressionCodec = compressionCodec;
+  }
+
+  public String type() {
+    return type;
+  }
+
+  public List<Integer> columns() {
+    return columns;
+  }
+
+  /**
+   * Offset in the file
+   */
+  public long offset() {
+    return offset;
+  }
+
+  /**
+   * Length in the file
+   */
+  public long length() {
+    return length;
+  }
+
+  @Nullable
+  public String compressionCodec() {
+    return compressionCodec;
+  }
+}

core/src/main/java/org/apache/iceberg/stats/FileMetadata.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.stats;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+
+public class FileMetadata {
+  private final List<BlobMetadata> blobs;
+  private final Map<String, String> properties;
+
+  public FileMetadata(List<BlobMetadata> blobs, Map<String, String> properties) {
+    Preconditions.checkNotNull(blobs, "blobs is null");
+    Preconditions.checkNotNull(properties, "properties is null");
+    this.blobs = ImmutableList.copyOf(blobs);
+    this.properties = ImmutableMap.copyOf(properties);
+  }
+
+  public List<BlobMetadata> blobs() {
+    return blobs;
+  }
+
+  public Map<String, String> properties() {
+    return properties;
+  }
+}

core/src/main/java/org/apache/iceberg/stats/FileMetadataParser.java

@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.stats;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.util.JsonUtil;
+
+public final class FileMetadataParser {
+
+  private FileMetadataParser() {
+  }
+
+  private static final String BLOBS = "blobs";
+  private static final String PROPERTIES = "properties";
+
+  private static final String TYPE = "type";
+  private static final String COLUMNS = "columns";
+  private static final String OFFSET = "offset";
+  private static final String LENGTH = "length";
+  private static final String COMPRESSION_CODEC = "compression-codec";
+
+  public static String toJson(FileMetadata fileMetadata) {
+    try {
+      StringWriter writer = new StringWriter();
+      JsonGenerator generator = JsonUtil.factory().createGenerator(writer);
+      generator.useDefaultPrettyPrinter();
+      toJson(fileMetadata, generator);
+      generator.flush();
+      return writer.toString();
+    } catch (IOException e) {
+      throw new UncheckedIOException("Failed to write json for: " + fileMetadata, e);
+    }
+  }
+
+  public static FileMetadata fromJson(String json) {
+    try {
+      return fromJson(JsonUtil.mapper().readValue(json, JsonNode.class));
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  static FileMetadata fromJson(JsonNode json) {
+    return fileMetadataFromJson(json);
+  }
+
+  static void toJson(FileMetadata fileMetadata, JsonGenerator generator) throws IOException {
+    generator.writeStartObject();
+
+    generator.writeArrayFieldStart(BLOBS);
+    for (BlobMetadata blobMetadata : fileMetadata.blobs()) {
+      toJson(blobMetadata, generator);
+    }
+    generator.writeEndArray();
+
+    generator.writeObjectFieldStart(PROPERTIES);
+    for (Map.Entry<String, String> entry : fileMetadata.properties().entrySet()) {
+      generator.writeStringField(entry.getKey(), entry.getValue());
+    }
+    generator.writeEndObject();
+
+    generator.writeEndObject();
+  }
+
+  static FileMetadata fileMetadataFromJson(JsonNode json) {
+
+    ImmutableList.Builder<BlobMetadata> blobs = ImmutableList.builder();
+    JsonNode blobsJson = json.get(BLOBS);
+    Preconditions.checkArgument(blobsJson != null && blobsJson.isArray(),
+        "Cannot parse blobs from non-array: %s", blobsJson);
+    for (JsonNode blobJson : blobsJson) {
+      blobs.add(blobMetadataFromJson(blobJson));
+    }
+
+    Map<String, String> properties = ImmutableMap.of();
+    JsonNode propertiesJson = json.get(PROPERTIES);
+    if (propertiesJson != null) {
+      properties = JsonUtil.getStringMap(PROPERTIES, json);
+    }
+
+    return new FileMetadata(
+        blobs.build(),
+        properties);
+  }
+
+  static void toJson(BlobMetadata blobMetadata, JsonGenerator generator) throws IOException {
+    generator.writeStartObject();
+
+    generator.writeStringField(TYPE, blobMetadata.type());
+
+    generator.writeArrayFieldStart(COLUMNS);
+    for (int column : blobMetadata.columns()) {
+      generator.writeNumber(column);
+    }
+    generator.writeEndArray();
+
+    generator.writeNumberField(OFFSET, blobMetadata.offset());
+    generator.writeNumberField(LENGTH, blobMetadata.length());
+
+    if (blobMetadata.compressionCodec() != null) {
+      generator.writeStringField(COMPRESSION_CODEC, blobMetadata.compressionCodec());
+    }
+
+    generator.writeEndObject();
+  }
+
+  static BlobMetadata blobMetadataFromJson(JsonNode json) {
+    String type = JsonUtil.getString(TYPE, json);
+    List<Integer> columns = JsonUtil.getIntegerList(COLUMNS, json);
+    long offset = JsonUtil.getLong(OFFSET, json);
+    long length = JsonUtil.getLong(LENGTH, json);
+    String compressionCodec = JsonUtil.getStringOrNull(COMPRESSION_CODEC, json);
+
+    return new BlobMetadata(
+        type,
+        columns,
+        offset,
+        length,
+        compressionCodec);
+  }
+}

core/src/main/java/org/apache/iceberg/stats/StandardBlobTypes.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.stats;
+
+public final class StandardBlobTypes {
+  private StandardBlobTypes() {
+  }
+
+  /**
+   * 8-bytes integer stored little-endian and representing number of distinct values
+   */
+  public static final String NDV_LONG_LITTLE_ENDIAN = "ndv-long-little-endian";
+
+  /**
+   * A serialized form of a "compact" Theta sketch produced by the <a href="https://datasketches.apache.org/">Apache DataSketches</a> library
+   */
+  public static final String APACHE_DATASKETCHES_THETA_V1 = "apache-datasketches-theta-v1";
+}