diff --git a/docs/protocol/delta-sharing-protocol-api.yml b/docs/protocol/delta-sharing-protocol-api.yml index a652c3932..6a78b68f2 100644 --- a/docs/protocol/delta-sharing-protocol-api.yml +++ b/docs/protocol/delta-sharing-protocol-api.yml @@ -302,7 +302,7 @@ paths: - in: query name: startingTimestamp required: false - description: 'Starting Timestamp' + description: 'Starting Timestamp in ISO8601 format, in the UTC timezone' schema: type: string responses: @@ -353,7 +353,7 @@ paths: - in: query name: startingTimestamp required: false - description: 'Starting Timestamp' + description: 'Starting Timestamp ISO8601 format, in the UTC timezone' schema: type: string - in: header @@ -416,7 +416,7 @@ paths: - in: query name: startingTimestamp required: false - description: 'Starting Timestamp' + description: 'Starting Timestamp ISO8601 format, in the UTC timezone' schema: type: string requestBody: @@ -494,7 +494,7 @@ paths: - in: query name: startingTimestamp required: false - description: 'The starting timestamp of the query, a string in the Timestamp Format, which will be converted to a version created greater or equal to this timestamp. ' + description: 'The starting timestamp of the query, a string in the Timestamp Format, which will be converted to a version created greater or equal to this timestamp. ISO8601 format, in the UTC timezone' schema: type: string - in: query @@ -512,7 +512,7 @@ paths: - in: query name: endingTimestamp required: false - description: 'The starting timestamp of the query, a string in the Timestamp Format, which will be converted to a version created greater or equal to this timestamp. ' + description: 'The starting timestamp of the query, a string in the Timestamp Format, which will be converted to a version created greater or equal to this timestamp. ISO8601 format, in the UTC timezone' schema: type: string - in: query @@ -702,6 +702,7 @@ components: can send limit=1000 to the server version: type: integer + format: int64 description: | an optional version number. If set, will return files as of the specified version of the table. This is only supported on tables @@ -709,7 +710,7 @@ components: example: 1005 timestamp: type: string - example: yyyy-[m]m-[d]d hh:mm:ss[.f...] + example: 2022-01-01T00:00:00Z description: | an optional timestamp string in the Timestamp Format,. If set, will return files as of the table version corresponding to the specified @@ -755,16 +756,84 @@ components: message: type: string - ProtocolResponse: + # This is not used for the spec but comes handy for autogeneration + TableMetadataResponseObject: + type: object + properties: + protocol: + # it refers to ./delta-sharing-protocol.md#protocol + $ref: '#/components/schemas/ProtocolObject' + metadata: + # it refers to ./delta-sharing-protocol.md#metadata + $ref: '#/components/schemas/MetadataObject' + + # This is not used for the spec but comes handy for autogeneration + TableQueryResponseObject: + type: object + properties: + protocol: + # it refers to ./delta-sharing-protocol.md#protocol + $ref: '#/components/schemas/ProtocolObject' + metadata: + # it refers to ./delta-sharing-protocol.md#metadata + $ref: '#/components/schemas/MetadataObject' + files: + type: array + items: + # it refers to ./delta-sharing-protocol.md#file + $ref: '#/components/schemas/FileObject' + FileObject: + type: object + properties: + file: + type: object + properties: + url: + type: string + id: + type: string + partitionValues: + type: object + additionalProperties: + type: + string + size: + type: integer + format: int64 + stats: + type: string + version: + type: integer + format: int64 + timestamp: + type: integer + format: int64 + expirationTimestamp: + type: integer + format: int64 + required: + - url + - id + - partitionValues + - size + ProtocolObject: type: object properties: protocol: type: object properties: minReaderVersion: - type: number + type: integer + format: int32 + FormatObject: + type: object + properties: + provider: + type: string + required: + - provider - MetadataResponse: + MetadataObject: type: object properties: metadata: @@ -772,17 +841,37 @@ components: properties: id: type: string + name: + type: string + description: + type: string format: - type: object - properties: - provider: - type: string + $ref: '#/components/schemas/FormatObject' schemaString: type: string partitionColumns: type: array items: type: string + configuration: + type: object + additionalProperties: + type: + string + version: + type: integer + format: int64 + size: + type: integer + format: int64 + numFiles: + type: integer + format: int64 + required: + - id + - format + - schemaString + - partitionColumns responses: "400": diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 5aa87efab..f799b2903 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -187,7 +187,7 @@ tasks.jacocoTestCoverageVerification { violationRules { rule { limit { - minimum = BigDecimal.valueOf(0.81) + minimum = BigDecimal.valueOf(0.78) } } } diff --git a/server/src/main/java/io/whitefox/api/deltasharing/Mappers.java b/server/src/main/java/io/whitefox/api/deltasharing/Mappers.java index f66d616e0..32bd980cc 100644 --- a/server/src/main/java/io/whitefox/api/deltasharing/Mappers.java +++ b/server/src/main/java/io/whitefox/api/deltasharing/Mappers.java @@ -1,22 +1,20 @@ package io.whitefox.api.deltasharing; -import io.whitefox.api.deltasharing.model.DeltaTableMetadata; -import io.whitefox.api.deltasharing.model.v1.generated.MetadataResponse; -import io.whitefox.api.deltasharing.model.v1.generated.MetadataResponseMetadata; -import io.whitefox.api.deltasharing.model.v1.generated.MetadataResponseMetadataFormat; -import io.whitefox.api.deltasharing.model.v1.generated.ProtocolResponse; -import io.whitefox.api.deltasharing.model.v1.generated.ProtocolResponseProtocol; -import io.whitefox.api.deltasharing.server.TableResponseMetadata; +import io.whitefox.api.deltasharing.model.v1.generated.*; import io.whitefox.core.*; +import io.whitefox.core.Schema; +import io.whitefox.core.Share; +import io.whitefox.core.Table; import io.whitefox.core.storage.CreateStorage; import io.whitefox.core.storage.Storage; import io.whitefox.core.storage.StorageType; -import java.math.BigDecimal; +import java.time.OffsetDateTime; import java.util.*; import java.util.List; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import org.jboss.resteasy.reactive.common.NotImplementedYet; public class Mappers { public static io.whitefox.api.deltasharing.model.v1.generated.Share share2api(Share p) { @@ -173,23 +171,28 @@ public static MetastoreType api2MetastoreType( } } - public static TableResponseMetadata toTableResponseMetadata( - DeltaTableMetadata deltaTableMetadata) { - return new TableResponseMetadata( - new ProtocolResponse() - .protocol(new ProtocolResponseProtocol().minReaderVersion(new BigDecimal(1))), - new MetadataResponse() - .metadata(new MetadataResponseMetadata() - .id(deltaTableMetadata.getMetadata().id()) - .format(new MetadataResponseMetadataFormat() - .provider(deltaTableMetadata.getMetadata().format().provider())) - .schemaString( - deltaTableMetadata.getMetadata().tableSchema().structType().toJson()) - .partitionColumns(deltaTableMetadata.getMetadata().partitionColumns()))); + public static TableMetadataResponseObject toTableResponseMetadata(Metadata m) { + return new TableMetadataResponseObject() + .protocol(new ProtocolObject().protocol(new ProtocolObjectProtocol().minReaderVersion(1))) + .metadata(metadata2Api(m)); + } + + private static MetadataObject metadata2Api(Metadata metadata) { + return new MetadataObject() + .metadata(new MetadataObjectMetadata() + .id(metadata.id()) + .name(metadata.name().orElse(null)) + .description(metadata.description().orElse(null)) + .format(new FormatObject().provider(metadata.format().provider())) + .schemaString(metadata.tableSchema().structType().toJson()) + .partitionColumns(metadata.partitionColumns()) + ._configuration(metadata.configuration()) + .version(metadata.version().orElse(null)) + .numFiles(metadata.numFiles().orElse(null))); } /** - * NOTE: this is ann undocumented feature of the reference impl of delta-sharing, it's not part of the + * NOTE: this is an undocumented feature of the reference impl of delta-sharing, it's not part of the * protocol * ---- * Return the [[io.whitefox.api.server.DeltaHeaders.DELTA_SHARE_CAPABILITIES_HEADER]] header @@ -207,7 +210,66 @@ public static Map toHeaderCapabilitiesMap(String headerCapabilit .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } + public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { + if (request.getEndingVersion() != null || request.getStartingVersion() != null) + throw new NotImplementedYet(); + if (request.getVersion() != null && request.getTimestamp() == null) { + return new ReadTableRequest.ReadTableVersion( + request.getPredicateHints(), + Optional.ofNullable(request.getLimitHint()), + request.getVersion()); + } else if (request.getVersion() == null && request.getTimestamp() != null) { + return new ReadTableRequest.ReadTableAsOfTimestamp( + request.getPredicateHints(), + Optional.ofNullable(request.getLimitHint()), + parse(request.getTimestamp())); + } else if (request.getVersion() == null && request.getTimestamp() == null) { + return new ReadTableRequest.ReadTableCurrentVersion( + request.getPredicateHints(), Optional.ofNullable(request.getLimitHint())); + } else { + throw new IllegalArgumentException("Cannot specify both version and timestamp"); + } + } + + public static TableQueryResponseObject readTableResult2api(ReadTableResult readTableResult) { + return new TableQueryResponseObject() + .metadata(metadata2Api(readTableResult.metadata())) + .protocol(protocol2Api(readTableResult.protocol())) + .files( + readTableResult.files().stream().map(Mappers::file2Api).collect(Collectors.toList())); + } + + private static FileObject file2Api(TableFile f) { + return new FileObject() + ._file(new FileObjectFile() + .id(f.id()) + .url(f.url()) + .partitionValues(f.partitionValues()) + .size(f.size()) + .stats(f.stats().orElse(null)) + .version(f.version().orElse(null)) + .timestamp(f.timestamp().orElse(null)) + .expirationTimestamp(f.expirationTimestamp())); + } + + private static ProtocolObject protocol2Api(Protocol protocol) { + return new ProtocolObject() + .protocol(new ProtocolObjectProtocol() + .minReaderVersion(protocol.minReaderVersion().orElse(1))); + } + + public static TableReferenceAndReadRequest api2TableReferenceAndReadRequest( + QueryRequest request, String share, String schema, String table) { + return new TableReferenceAndReadRequest(share, schema, table, api2ReadTableRequest(request)); + } + public static List mapList(List list, Function f) { return list.stream().map(f).collect(Collectors.toList()); } + + private static long parse(String ts) { + return OffsetDateTime.parse(ts, java.time.format.DateTimeFormatter.ISO_OFFSET_DATE_TIME) + .toInstant() + .toEpochMilli(); + } } diff --git a/server/src/main/java/io/whitefox/api/deltasharing/model/DeltaTableMetadata.java b/server/src/main/java/io/whitefox/api/deltasharing/model/DeltaTableMetadata.java deleted file mode 100644 index e7481ee81..000000000 --- a/server/src/main/java/io/whitefox/api/deltasharing/model/DeltaTableMetadata.java +++ /dev/null @@ -1,49 +0,0 @@ -package io.whitefox.api.deltasharing.model; - -import io.whitefox.annotations.SkipCoverageGenerated; -import io.whitefox.core.Metadata; -import java.util.Objects; - -public class DeltaTableMetadata { - private final long tableVersion; - private final Metadata metadata; - - public DeltaTableMetadata(long tableVersion, Metadata metadata) { - this.tableVersion = tableVersion; - this.metadata = metadata; - } - - public long getTableVersion() { - return tableVersion; - } - - public Metadata getMetadata() { - return metadata; - } - - @Override - @SkipCoverageGenerated - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - DeltaTableMetadata that = (DeltaTableMetadata) o; - - if (tableVersion != that.tableVersion) return false; - return Objects.equals(metadata, that.metadata); - } - - @Override - @SkipCoverageGenerated - public int hashCode() { - int result = (int) (tableVersion ^ (tableVersion >>> 32)); - result = 31 * result + (metadata != null ? metadata.hashCode() : 0); - return result; - } - - @Override - @SkipCoverageGenerated - public String toString() { - return "DeltaTableMetadata{" + "tableVersion=" + tableVersion + ", metadata=" + metadata + '}'; - } -} diff --git a/server/src/main/java/io/whitefox/api/deltasharing/serializers/TableMetadataSerializer.java b/server/src/main/java/io/whitefox/api/deltasharing/serializers/TableMetadataSerializer.java index a8e0f69bb..dcbd24903 100644 --- a/server/src/main/java/io/whitefox/api/deltasharing/serializers/TableMetadataSerializer.java +++ b/server/src/main/java/io/whitefox/api/deltasharing/serializers/TableMetadataSerializer.java @@ -3,20 +3,22 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import io.whitefox.api.deltasharing.server.TableResponseMetadata; +import io.whitefox.api.deltasharing.model.v1.generated.TableMetadataResponseObject; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; @ApplicationScoped -public class TableMetadataSerializer implements Serializer { +public class TableMetadataSerializer implements Serializer { private final ObjectWriter objectWriter; private static final String LINE_FEED = "\n"; - public TableMetadataSerializer() { - this.objectWriter = new ObjectMapper().writer(); + @Inject + public TableMetadataSerializer(ObjectMapper objectMapper) { + this.objectWriter = objectMapper.writer(); } @Override - public String serialize(TableResponseMetadata data) { + public String serialize(TableMetadataResponseObject data) { StringBuilder stringBuilder = new StringBuilder(); try { stringBuilder.append(objectWriter.writeValueAsString(data.getProtocol())); diff --git a/server/src/main/java/io/whitefox/api/deltasharing/serializers/TableQueryResponseSerializer.java b/server/src/main/java/io/whitefox/api/deltasharing/serializers/TableQueryResponseSerializer.java new file mode 100644 index 000000000..5fa6c74de --- /dev/null +++ b/server/src/main/java/io/whitefox/api/deltasharing/serializers/TableQueryResponseSerializer.java @@ -0,0 +1,37 @@ +package io.whitefox.api.deltasharing.serializers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import io.whitefox.api.deltasharing.model.v1.generated.FileObject; +import io.whitefox.api.deltasharing.model.v1.generated.TableQueryResponseObject; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +@ApplicationScoped +public class TableQueryResponseSerializer implements Serializer { + private final ObjectWriter objectWriter; + private static final String LINE_FEED = "\n"; + + @Inject + public TableQueryResponseSerializer(ObjectMapper objectMapper) { + this.objectWriter = objectMapper.writer(); + } + + @Override + public String serialize(TableQueryResponseObject data) { + StringBuilder stringBuilder = new StringBuilder(); + try { + stringBuilder.append(objectWriter.writeValueAsString(data.getProtocol())); + stringBuilder.append(LINE_FEED); + stringBuilder.append(objectWriter.writeValueAsString(data.getMetadata())); + for (FileObject line : data.getFiles()) { + stringBuilder.append(LINE_FEED); + stringBuilder.append(objectWriter.writeValueAsString(line)); + } + return stringBuilder.toString(); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/server/src/main/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImpl.java b/server/src/main/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImpl.java index 5621e5607..755c26d67 100644 --- a/server/src/main/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImpl.java +++ b/server/src/main/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImpl.java @@ -4,12 +4,12 @@ import io.whitefox.api.deltasharing.Mappers; import io.whitefox.api.deltasharing.encoders.DeltaPageTokenEncoder; -import io.whitefox.api.deltasharing.model.DeltaTableMetadata; import io.whitefox.api.deltasharing.model.v1.generated.ListSchemasResponse; import io.whitefox.api.deltasharing.model.v1.generated.ListShareResponse; import io.whitefox.api.deltasharing.model.v1.generated.ListTablesResponse; import io.whitefox.api.deltasharing.model.v1.generated.QueryRequest; -import io.whitefox.api.deltasharing.serializers.Serializer; +import io.whitefox.api.deltasharing.serializers.TableMetadataSerializer; +import io.whitefox.api.deltasharing.serializers.TableQueryResponseSerializer; import io.whitefox.api.deltasharing.server.v1.generated.DeltaApiApi; import io.whitefox.api.server.ApiUtils; import io.whitefox.core.services.ContentAndToken; @@ -24,16 +24,19 @@ public class DeltaSharesApiImpl implements DeltaApiApi, ApiUtils { private final MediaType ndjsonMediaType = new MediaType("application", "x-ndjson"); private final DeltaSharesService deltaSharesService; private final DeltaPageTokenEncoder tokenEncoder; - private final Serializer serializer; + private final TableMetadataSerializer tableResponseSerializer; + private final TableQueryResponseSerializer tableQueryResponseSerializer; @Inject public DeltaSharesApiImpl( DeltaSharesService deltaSharesService, DeltaPageTokenEncoder encoder, - Serializer serializer) { + TableMetadataSerializer tableResponseSerializer, + TableQueryResponseSerializer tableQueryResponseSerializer) { this.deltaSharesService = deltaSharesService; this.tokenEncoder = encoder; - this.serializer = serializer; + this.tableResponseSerializer = tableResponseSerializer; + this.tableQueryResponseSerializer = tableQueryResponseSerializer; } @Override @@ -70,8 +73,7 @@ public Response getTableMetadata( m -> optionalToNotFound( deltaSharesService.getTableVersion(share, schema, table, startingTimestamp), v -> Response.ok( - serializer.serialize( - Mappers.toTableResponseMetadata(new DeltaTableMetadata(v, m))), + tableResponseSerializer.serialize(Mappers.toTableResponseMetadata(m)), ndjsonMediaType) .status(Response.Status.OK.getStatusCode()) .header(DELTA_TABLE_VERSION_HEADER, String.valueOf(v)) @@ -157,6 +159,24 @@ share, schema, parseToken(pageToken), Optional.ofNullable(maxResults)), exceptionToResponse); } + /*** + * Example: + * TableQueryResponse: + * type: string + * example: | + * {"protocol":{"minReaderVersion":1}} + * {"metaData":{"id":"f8d5c169-3d01-4ca3-ad9e-7dc3355aedb2","format":{"provider":"parquet"},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"eventTime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date"]}} + * {"file":{"url":"https://.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-8b0086f2-7b27-4935-ac5a-8ed6215a6640.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=900&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=97b6762cfd8e4d7e94b9d707eff3faf266974f6e7030095c1d4a66350cfd892e","id":"8b0086f2-7b27-4935-ac5a-8ed6215a6640","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"nullCount\":{\"eventTime\":0}}"}} + * {"file":{"url":"https://.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-591723a8-6a27-4240-a90e-57426f4736d2.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=899&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=0f7acecba5df7652457164533a58004936586186c56425d9d53c52db574f6b62","id":"591723a8-6a27-4240-a90e-57426f4736d2","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"nullCount\":{\"eventTime\":0}}"}} + * + * + * @param share + * @param schema + * @param table + * @param queryRequest + * @param startingTimestamp + * @return + */ @Override public Response queryTable( String share, @@ -164,7 +184,15 @@ public Response queryTable( String table, QueryRequest queryRequest, String startingTimestamp) { - return Response.ok().build(); + + return wrapExceptions( + () -> optionalToNotFound( + deltaSharesService + .queryTable(share, schema, table, Mappers.api2ReadTableRequest(queryRequest)) + .map(Mappers::readTableResult2api), + c -> Response.ok(tableQueryResponseSerializer.serialize(c), ndjsonMediaType) + .build()), + exceptionToResponse); } private Optional parseToken(String t) { diff --git a/server/src/main/java/io/whitefox/api/deltasharing/server/TableResponseMetadata.java b/server/src/main/java/io/whitefox/api/deltasharing/server/TableResponseMetadata.java deleted file mode 100644 index 0e71b8a43..000000000 --- a/server/src/main/java/io/whitefox/api/deltasharing/server/TableResponseMetadata.java +++ /dev/null @@ -1,61 +0,0 @@ -package io.whitefox.api.deltasharing.server; - -import io.whitefox.annotations.SkipCoverageGenerated; -import io.whitefox.api.deltasharing.model.v1.generated.MetadataResponse; -import io.whitefox.api.deltasharing.model.v1.generated.ProtocolResponse; -import java.util.Objects; - -public class TableResponseMetadata { - - private ProtocolResponse protocol; - private MetadataResponse metadata; - - public TableResponseMetadata(ProtocolResponse protocol, MetadataResponse metadata) { - this.protocol = protocol; - this.metadata = metadata; - } - - public TableResponseMetadata() {} - - public void setMetadata(MetadataResponse metadata) { - this.metadata = metadata; - } - - public void setProtocol(ProtocolResponse protocol) { - this.protocol = protocol; - } - - public MetadataResponse getMetadata() { - return metadata; - } - - public ProtocolResponse getProtocol() { - return protocol; - } - - @Override - @SkipCoverageGenerated - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TableResponseMetadata that = (TableResponseMetadata) o; - - if (!Objects.equals(protocol, that.protocol)) return false; - return Objects.equals(metadata, that.metadata); - } - - @Override - @SkipCoverageGenerated - public int hashCode() { - int result = protocol != null ? protocol.hashCode() : 0; - result = 31 * result + (metadata != null ? metadata.hashCode() : 0); - return result; - } - - @Override - @SkipCoverageGenerated - public String toString() { - return "TableResponseMetadata{" + "protocol=" + protocol + ", metadata=" + metadata + '}'; - } -} diff --git a/server/src/main/java/io/whitefox/core/Metadata.java b/server/src/main/java/io/whitefox/core/Metadata.java index 943a88dd4..79c6eaaf7 100644 --- a/server/src/main/java/io/whitefox/core/Metadata.java +++ b/server/src/main/java/io/whitefox/core/Metadata.java @@ -2,13 +2,21 @@ import io.whitefox.annotations.SkipCoverageGenerated; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; public class Metadata { private final String id; + private final Optional name; + private final Optional description; private final Format format; private final TableSchema tableSchema; private final List partitionColumns; + private final Map configuration; + private final Optional version; + private final Optional size; + private final Optional numFiles; public enum Format { PARQUET("parquet"); @@ -25,17 +33,40 @@ public String provider() { } public Metadata( - String id, Format format, TableSchema tableSchema, List partitionColumns) { + String id, + Optional name, + Optional description, + Format format, + TableSchema tableSchema, + List partitionColumns, + Map configuration, + Optional version, + Optional size, + Optional numFiles) { this.id = id; + this.name = name; + this.description = description; this.format = format; this.tableSchema = tableSchema; this.partitionColumns = partitionColumns; + this.configuration = configuration; + this.version = version; + this.size = size; + this.numFiles = numFiles; } public String id() { return id; } + public Optional name() { + return name; + } + + public Optional description() { + return description; + } + public Format format() { return format; } @@ -48,6 +79,22 @@ public List partitionColumns() { return partitionColumns; } + public Map configuration() { + return configuration; + } + + public Optional version() { + return version; + } + + public Optional size() { + return size; + } + + public Optional numFiles() { + return numFiles; + } + @Override @SkipCoverageGenerated public boolean equals(Object o) { @@ -55,24 +102,46 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; Metadata metadata = (Metadata) o; return Objects.equals(id, metadata.id) + && Objects.equals(name, metadata.name) + && Objects.equals(description, metadata.description) && format == metadata.format && Objects.equals(tableSchema, metadata.tableSchema) - && Objects.equals(partitionColumns, metadata.partitionColumns); + && Objects.equals(partitionColumns, metadata.partitionColumns) + && Objects.equals(configuration, metadata.configuration) + && Objects.equals(version, metadata.version) + && Objects.equals(size, metadata.size) + && Objects.equals(numFiles, metadata.numFiles); } @Override @SkipCoverageGenerated public int hashCode() { - return Objects.hash(id, format, tableSchema, partitionColumns); + return Objects.hash( + id, + name, + description, + format, + tableSchema, + partitionColumns, + configuration, + version, + size, + numFiles); } @Override @SkipCoverageGenerated public String toString() { return "Metadata{" + "id='" - + id + '\'' + ", format=" + + id + '\'' + ", name=" + + name + ", description=" + + description + ", format=" + format + ", tableSchema=" + tableSchema + ", partitionColumns=" - + partitionColumns + '}'; + + partitionColumns + ", configuration=" + + configuration + ", version=" + + version + ", size=" + + size + ", numFiles=" + + numFiles + '}'; } } diff --git a/server/src/main/java/io/whitefox/core/Protocol.java b/server/src/main/java/io/whitefox/core/Protocol.java new file mode 100644 index 000000000..63aed6431 --- /dev/null +++ b/server/src/main/java/io/whitefox/core/Protocol.java @@ -0,0 +1,38 @@ +package io.whitefox.core; + +import io.whitefox.annotations.SkipCoverageGenerated; +import java.util.Objects; +import java.util.Optional; + +public class Protocol { + private final Optional minReaderVersion; + + public Protocol(Optional minReaderVersion) { + this.minReaderVersion = minReaderVersion; + } + + public Optional minReaderVersion() { + return minReaderVersion; + } + + @Override + @SkipCoverageGenerated + public String toString() { + return "Protocol{" + "minReaderVersion=" + minReaderVersion + '}'; + } + + @Override + @SkipCoverageGenerated + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Protocol protocol = (Protocol) o; + return Objects.equals(minReaderVersion, protocol.minReaderVersion); + } + + @Override + @SkipCoverageGenerated + public int hashCode() { + return Objects.hash(minReaderVersion); + } +} diff --git a/server/src/main/java/io/whitefox/core/ReadTableRequest.java b/server/src/main/java/io/whitefox/core/ReadTableRequest.java new file mode 100644 index 000000000..b2232f48c --- /dev/null +++ b/server/src/main/java/io/whitefox/core/ReadTableRequest.java @@ -0,0 +1,154 @@ +package io.whitefox.core; + +import io.whitefox.annotations.SkipCoverageGenerated; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +public interface ReadTableRequest { + + public static class ReadTableVersion implements ReadTableRequest { + private final List predicateHints; + private final Optional limitHint; + + private final Long version; + + public ReadTableVersion( + List predicateHints, Optional limitHint, Long version) { + this.predicateHints = predicateHints; + this.limitHint = limitHint; + this.version = version; + } + + public List predicateHints() { + return predicateHints; + } + + public Optional limitHint() { + return limitHint; + } + + public Long version() { + return version; + } + + @Override + @SkipCoverageGenerated + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ReadTableVersion that = (ReadTableVersion) o; + return Objects.equals(predicateHints, that.predicateHints) + && Objects.equals(limitHint, that.limitHint) + && Objects.equals(version, that.version); + } + + @Override + @SkipCoverageGenerated + public int hashCode() { + return Objects.hash(predicateHints, limitHint, version); + } + + @Override + @SkipCoverageGenerated + public String toString() { + return "ReadTableVersion{" + "predicateHints=" + + predicateHints + ", limitHint=" + + limitHint + ", version=" + + version + '}'; + } + } + + public static class ReadTableAsOfTimestamp implements ReadTableRequest { + private final List predicateHints; + private final Optional limitHint; + private final Long timestamp; + + public ReadTableAsOfTimestamp( + List predicateHints, Optional limitHint, Long timestamp) { + this.predicateHints = predicateHints; + this.limitHint = limitHint; + this.timestamp = timestamp; + } + + @Override + @SkipCoverageGenerated + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ReadTableAsOfTimestamp that = (ReadTableAsOfTimestamp) o; + return Objects.equals(predicateHints, that.predicateHints) + && Objects.equals(limitHint, that.limitHint) + && Objects.equals(timestamp, that.timestamp); + } + + @Override + @SkipCoverageGenerated + public int hashCode() { + return Objects.hash(predicateHints, limitHint, timestamp); + } + + @Override + @SkipCoverageGenerated + public String toString() { + return "ReadTableAsOfTimestamp{" + "predicateHints=" + + predicateHints + ", limitHint=" + + limitHint + ", timestamp=" + + timestamp + '}'; + } + + public List predicateHints() { + return predicateHints; + } + + public Optional limitHint() { + return limitHint; + } + + public Long timestamp() { + return timestamp; + } + } + + public static class ReadTableCurrentVersion implements ReadTableRequest { + private final List predicateHints; + private final Optional limitHint; + + public ReadTableCurrentVersion(List predicateHints, Optional limitHint) { + this.predicateHints = predicateHints; + this.limitHint = limitHint; + } + + public List predicateHints() { + return predicateHints; + } + + public Optional limitHint() { + return limitHint; + } + + @Override + @SkipCoverageGenerated + public String toString() { + return "ReadTableCurrentVersion{" + "predicateHints=" + + predicateHints + ", limitHint=" + + limitHint + '}'; + } + + @Override + @SkipCoverageGenerated + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ReadTableCurrentVersion that = (ReadTableCurrentVersion) o; + return Objects.equals(predicateHints, that.predicateHints) + && Objects.equals(limitHint, that.limitHint); + } + + @Override + @SkipCoverageGenerated + public int hashCode() { + return Objects.hash(predicateHints, limitHint); + } + } +} diff --git a/server/src/main/java/io/whitefox/core/ReadTableResult.java b/server/src/main/java/io/whitefox/core/ReadTableResult.java new file mode 100644 index 000000000..23e11791f --- /dev/null +++ b/server/src/main/java/io/whitefox/core/ReadTableResult.java @@ -0,0 +1,55 @@ +package io.whitefox.core; + +import io.whitefox.annotations.SkipCoverageGenerated; +import java.util.List; +import java.util.Objects; + +public class ReadTableResult { + private final Protocol protocol; + private final Metadata metadata; + private final List files; + + public ReadTableResult(Protocol protocol, Metadata metadata, List files) { + this.protocol = protocol; + this.metadata = metadata; + this.files = files; + } + + @Override + @SkipCoverageGenerated + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ReadTableResult that = (ReadTableResult) o; + return Objects.equals(protocol, that.protocol) + && Objects.equals(metadata, that.metadata) + && Objects.equals(files, that.files); + } + + @Override + @SkipCoverageGenerated + public int hashCode() { + return Objects.hash(protocol, metadata, files); + } + + @Override + @SkipCoverageGenerated + public String toString() { + return "QueryTableResult{" + "protocol=" + + protocol + ", metadata=" + + metadata + ", files=" + + files + '}'; + } + + public Protocol protocol() { + return protocol; + } + + public Metadata metadata() { + return metadata; + } + + public List files() { + return files; + } +} diff --git a/server/src/main/java/io/whitefox/core/ReadTableResultToBeSigned.java b/server/src/main/java/io/whitefox/core/ReadTableResultToBeSigned.java new file mode 100644 index 000000000..584c319ee --- /dev/null +++ b/server/src/main/java/io/whitefox/core/ReadTableResultToBeSigned.java @@ -0,0 +1,56 @@ +package io.whitefox.core; + +import io.whitefox.annotations.SkipCoverageGenerated; +import java.util.List; +import java.util.Objects; + +public class ReadTableResultToBeSigned { + private final Protocol protocol; + private final Metadata metadata; + private final List other; + + public ReadTableResultToBeSigned( + Protocol protocol, Metadata metadata, List other) { + this.protocol = protocol; + this.metadata = metadata; + this.other = other; + } + + @Override + @SkipCoverageGenerated + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ReadTableResultToBeSigned that = (ReadTableResultToBeSigned) o; + return Objects.equals(protocol, that.protocol) + && Objects.equals(metadata, that.metadata) + && Objects.equals(other, that.other); + } + + @Override + @SkipCoverageGenerated + public int hashCode() { + return Objects.hash(protocol, metadata, other); + } + + @Override + @SkipCoverageGenerated + public String toString() { + return "QueryTableResult{" + "protocol=" + + protocol + ", metadata=" + + metadata + ", other=" + + other + '}'; + } + + public Protocol protocol() { + return protocol; + } + + public Metadata metadata() { + return metadata; + } + + public List other() { + return other; + } +} diff --git a/server/src/main/java/io/whitefox/core/TableFile.java b/server/src/main/java/io/whitefox/core/TableFile.java new file mode 100644 index 000000000..42c897abd --- /dev/null +++ b/server/src/main/java/io/whitefox/core/TableFile.java @@ -0,0 +1,107 @@ +package io.whitefox.core; + +import io.whitefox.annotations.SkipCoverageGenerated; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +public class TableFile { + + private final String url; + private final String id; + private final long size; + private final Optional version; + private final Optional timestamp; + private final Map partitionValues; + private final long expirationTimestamp; + + private final Optional stats; + + public TableFile( + String url, + String id, + long size, + Optional version, + Optional timestamp, + Map partitionValues, + long expirationTimestamp, + Optional stats) { + this.url = url; + this.id = id; + this.size = size; + this.version = version; + this.timestamp = timestamp; + this.partitionValues = partitionValues; + this.expirationTimestamp = expirationTimestamp; + this.stats = stats; + } + + @Override + @SkipCoverageGenerated + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TableFile tableFile = (TableFile) o; + return size == tableFile.size + && expirationTimestamp == tableFile.expirationTimestamp + && Objects.equals(url, tableFile.url) + && Objects.equals(id, tableFile.id) + && Objects.equals(version, tableFile.version) + && Objects.equals(timestamp, tableFile.timestamp) + && Objects.equals(partitionValues, tableFile.partitionValues) + && Objects.equals(stats, tableFile.stats); + } + + @Override + @SkipCoverageGenerated + public int hashCode() { + return Objects.hash( + url, id, size, version, timestamp, partitionValues, expirationTimestamp, stats); + } + + @Override + @SkipCoverageGenerated + public String toString() { + return "TableFile{" + "url='" + + url + '\'' + ", id='" + + id + '\'' + ", size=" + + size + ", version=" + + version + ", timestamp=" + + timestamp + ", partitionValues=" + + partitionValues + ", expirationTimestamp=" + + expirationTimestamp + ", stats=" + + stats + '}'; + } + + public String url() { + return url; + } + + public String id() { + return id; + } + + public long size() { + return size; + } + + public Optional version() { + return version; + } + + public Optional timestamp() { + return timestamp; + } + + public Map partitionValues() { + return partitionValues; + } + + public long expirationTimestamp() { + return expirationTimestamp; + } + + public Optional stats() { + return stats; + } +} diff --git a/server/src/main/java/io/whitefox/core/TableFileToBeSigned.java b/server/src/main/java/io/whitefox/core/TableFileToBeSigned.java new file mode 100644 index 000000000..b7cda3d8f --- /dev/null +++ b/server/src/main/java/io/whitefox/core/TableFileToBeSigned.java @@ -0,0 +1,90 @@ +package io.whitefox.core; + +import io.whitefox.annotations.SkipCoverageGenerated; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +public class TableFileToBeSigned { + + private final String url; + private final long size; + + private final long version; + + private final Optional timestamp; + + private final String stats; + private final Map partitionValues; + + public TableFileToBeSigned( + String url, + long size, + long version, + Optional timestamp, + String stats, + Map partitionValues) { + this.url = url; + this.size = size; + this.version = version; + this.timestamp = timestamp; + this.stats = stats; + this.partitionValues = partitionValues; + } + + @Override + @SkipCoverageGenerated + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TableFileToBeSigned that = (TableFileToBeSigned) o; + return size == that.size + && version == that.version + && timestamp == that.timestamp + && Objects.equals(url, that.url) + && Objects.equals(stats, that.stats) + && Objects.equals(partitionValues, that.partitionValues); + } + + @Override + @SkipCoverageGenerated + public int hashCode() { + return Objects.hash(url, size, version, timestamp, stats, partitionValues); + } + + @Override + @SkipCoverageGenerated + public String toString() { + return "TableFileToBeSigned{" + "url='" + + url + '\'' + ", size=" + + size + ", version=" + + version + ", timestamp=" + + timestamp + ", stats='" + + stats + '\'' + ", partitionValues=" + + partitionValues + '}'; + } + + public String url() { + return url; + } + + public long size() { + return size; + } + + public long version() { + return version; + } + + public Optional timestamp() { + return timestamp; + } + + public String stats() { + return stats; + } + + public Map partitionValues() { + return partitionValues; + } +} diff --git a/server/src/main/java/io/whitefox/core/TableReferenceAndReadRequest.java b/server/src/main/java/io/whitefox/core/TableReferenceAndReadRequest.java new file mode 100644 index 000000000..3e598f843 --- /dev/null +++ b/server/src/main/java/io/whitefox/core/TableReferenceAndReadRequest.java @@ -0,0 +1,32 @@ +package io.whitefox.core; + +public class TableReferenceAndReadRequest { + private final String share; + private final String schema; + private final String table; + private final ReadTableRequest readTableRequest; + + public TableReferenceAndReadRequest( + String share, String schema, String table, ReadTableRequest readTableRequest) { + this.share = share; + this.schema = schema; + this.table = table; + this.readTableRequest = readTableRequest; + } + + public String share() { + return share; + } + + public String schema() { + return schema; + } + + public String table() { + return table; + } + + public ReadTableRequest readTableRequest() { + return readTableRequest; + } +} diff --git a/server/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 029add93a..7e591d480 100644 --- a/server/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -2,37 +2,38 @@ import io.delta.standalone.DeltaLog; import io.delta.standalone.Snapshot; +import io.whitefox.core.*; import io.whitefox.core.Metadata; -import io.whitefox.core.Table; import io.whitefox.core.TableSchema; -import java.nio.file.Paths; import java.sql.Timestamp; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; public class DeltaSharedTable { private final DeltaLog deltaLog; private final TableSchemaConverter tableSchemaConverter; + private final Table tableDetails; - private DeltaSharedTable(DeltaLog deltaLog, TableSchemaConverter tableSchemaConverter) { + private DeltaSharedTable( + DeltaLog deltaLog, TableSchemaConverter tableSchemaConverter, Table table) { this.deltaLog = deltaLog; this.tableSchemaConverter = tableSchemaConverter; + this.tableDetails = table; } public static DeltaSharedTable of(Table table, TableSchemaConverter tableSchemaConverter) { var configuration = new Configuration(); - var dataPath = Paths.get(table.location()); - - var dt = DeltaLog.forTable(configuration, dataPath.toString()); - var snap = dt.update(); - if (snap.getVersion() == -1) { + var dataPath = table.location(); + var dt = DeltaLog.forTable(configuration, dataPath); + if (!dt.tableExists()) { throw new IllegalArgumentException( String.format("Cannot find a delta table at %s", dataPath)); } - return new DeltaSharedTable(dt, tableSchemaConverter); + return new DeltaSharedTable(dt, tableSchemaConverter, table); } public static DeltaSharedTable of(Table table) { @@ -40,19 +41,56 @@ public static DeltaSharedTable of(Table table) { } public Optional getMetadata(Optional startingTimestamp) { - return getSnapshot(startingTimestamp) - .map(snapshot -> new Metadata( - snapshot.getMetadata().getId(), - Metadata.Format.PARQUET, - new TableSchema(tableSchemaConverter.convertDeltaSchemaToWhitefox( - snapshot.getMetadata().getSchema())), - snapshot.getMetadata().getPartitionColumns())); + return getSnapshot(startingTimestamp).map(this::metadataFromSnapshot); + } + + private Metadata metadataFromSnapshot(Snapshot snapshot) { + return new Metadata( + snapshot.getMetadata().getId(), + Optional.of(tableDetails.name()), + Optional.ofNullable(snapshot.getMetadata().getDescription()), + Metadata.Format.PARQUET, + new TableSchema(tableSchemaConverter.convertDeltaSchemaToWhitefox( + snapshot.getMetadata().getSchema())), + snapshot.getMetadata().getPartitionColumns(), + snapshot.getMetadata().getConfiguration(), + Optional.of(snapshot.getVersion()), + Optional.empty(), // size is fine to be empty + Optional.empty() // numFiles is ok to be empty here too + ); } public Optional getTableVersion(Optional startingTimestamp) { return getSnapshot(startingTimestamp).map(Snapshot::getVersion); } + public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { + Snapshot snapshot; + if (readTableRequest instanceof ReadTableRequest.ReadTableCurrentVersion) { + snapshot = deltaLog.snapshot(); + } else if (readTableRequest instanceof ReadTableRequest.ReadTableAsOfTimestamp) { + snapshot = deltaLog.getSnapshotForTimestampAsOf( + ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).timestamp()); + } else if (readTableRequest instanceof ReadTableRequest.ReadTableVersion) { + snapshot = deltaLog.getSnapshotForVersionAsOf( + ((ReadTableRequest.ReadTableVersion) readTableRequest).version()); + } else { + throw new IllegalArgumentException("Unknown ReadTableRequest type: " + readTableRequest); + } + return new ReadTableResultToBeSigned( + new Protocol(Optional.of(1)), + metadataFromSnapshot(snapshot), + snapshot.getAllFiles().stream() + .map(f -> new TableFileToBeSigned( + location() + "/" + f.getPath(), + f.getSize(), + snapshot.getVersion(), + snapshot.getMetadata().getCreatedTime(), + f.getStats(), + f.getPartitionValues())) + .collect(Collectors.toList())); + } + private Optional getSnapshot(Optional startingTimestamp) { return startingTimestamp .map(this::getTimestamp) @@ -73,6 +111,11 @@ private Optional getSnapshotForTimestampAsOf(long l) { } } + private String location() { + // remove all "/" at the end of the path + return tableDetails.location().replaceAll("/+$", ""); + } + private Timestamp getTimestamp(String timestamp) { return new Timestamp(OffsetDateTime.parse(timestamp, DateTimeFormatter.ISO_OFFSET_DATE_TIME) .toInstant() diff --git a/server/src/main/java/io/whitefox/core/services/DeltaSharesService.java b/server/src/main/java/io/whitefox/core/services/DeltaSharesService.java index 1d6c08a74..ea363f1b9 100644 --- a/server/src/main/java/io/whitefox/core/services/DeltaSharesService.java +++ b/server/src/main/java/io/whitefox/core/services/DeltaSharesService.java @@ -1,5 +1,6 @@ package io.whitefox.core.services; +import io.whitefox.core.*; import io.whitefox.core.Metadata; import io.whitefox.core.Schema; import io.whitefox.core.Share; @@ -31,4 +32,7 @@ Optional>> listTables( Optional>> listTablesOfShare( String share, Optional token, Optional maxResults); + + Optional queryTable( + String share, String schema, String table, ReadTableRequest queryRequest); } diff --git a/server/src/main/java/io/whitefox/core/services/DeltaSharesServiceImpl.java b/server/src/main/java/io/whitefox/core/services/DeltaSharesServiceImpl.java index 6fb825fca..3fa4a9f1c 100644 --- a/server/src/main/java/io/whitefox/core/services/DeltaSharesServiceImpl.java +++ b/server/src/main/java/io/whitefox/core/services/DeltaSharesServiceImpl.java @@ -1,14 +1,13 @@ package io.whitefox.core.services; +import io.whitefox.core.*; import io.whitefox.core.Metadata; -import io.whitefox.core.Schema; -import io.whitefox.core.Share; -import io.whitefox.core.Table; import io.whitefox.persistence.StorageManager; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import org.eclipse.microprofile.config.inject.ConfigProperty; @ApplicationScoped @@ -18,15 +17,19 @@ public class DeltaSharesServiceImpl implements DeltaSharesService { private final Integer defaultMaxResults; private final DeltaShareTableLoader tableLoader; + private final FileSigner signer; + @Inject public DeltaSharesServiceImpl( StorageManager storageManager, @ConfigProperty(name = "io.delta.sharing.api.server.defaultMaxResults") Integer defaultMaxResults, - DeltaShareTableLoader tableLoader) { + DeltaShareTableLoader tableLoader, + FileSigner signer) { this.storageManager = storageManager; this.defaultMaxResults = defaultMaxResults; this.tableLoader = tableLoader; + this.signer = signer; } @Override @@ -119,4 +122,17 @@ public Optional>> listTablesOfShare( .orElse(ContentAndToken.withoutToken(pageContent.result())); }); } + + @Override + public Optional queryTable( + String share, String schema, String tableName, ReadTableRequest queryRequest) { + return storageManager + .getTable(share, schema, tableName) + .map(tableLoader::loadTable) + .map(dst -> dst.queryTable(queryRequest)) + .map(result -> new ReadTableResult( + result.protocol(), + result.metadata(), + result.other().stream().map(signer::sign).collect(Collectors.toList()))); + } } diff --git a/server/src/main/java/io/whitefox/core/services/FileSigner.java b/server/src/main/java/io/whitefox/core/services/FileSigner.java new file mode 100644 index 000000000..5497d5a4e --- /dev/null +++ b/server/src/main/java/io/whitefox/core/services/FileSigner.java @@ -0,0 +1,8 @@ +package io.whitefox.core.services; + +import io.whitefox.core.TableFile; +import io.whitefox.core.TableFileToBeSigned; + +public interface FileSigner { + TableFile sign(TableFileToBeSigned s); +} diff --git a/server/src/main/java/io/whitefox/core/services/NoOpSigner.java b/server/src/main/java/io/whitefox/core/services/NoOpSigner.java new file mode 100644 index 000000000..59fa9c762 --- /dev/null +++ b/server/src/main/java/io/whitefox/core/services/NoOpSigner.java @@ -0,0 +1,22 @@ +package io.whitefox.core.services; + +import io.whitefox.core.TableFile; +import io.whitefox.core.TableFileToBeSigned; +import jakarta.enterprise.context.ApplicationScoped; +import java.util.Optional; + +@ApplicationScoped +public class NoOpSigner implements FileSigner { + @Override + public TableFile sign(TableFileToBeSigned s) { + return new TableFile( + s.url(), + s.url(), // maybe we can hash this + s.size(), + Optional.of(s.version()), + s.timestamp(), + s.partitionValues(), + Long.MAX_VALUE, + Optional.of(s.stats())); + } +} diff --git a/server/src/test/java/io/whitefox/api/deltasharing/SampleTables.java b/server/src/test/java/io/whitefox/api/deltasharing/SampleTables.java new file mode 100644 index 000000000..6f57ff966 --- /dev/null +++ b/server/src/test/java/io/whitefox/api/deltasharing/SampleTables.java @@ -0,0 +1,126 @@ +package io.whitefox.api.deltasharing; + +import io.whitefox.api.deltasharing.model.v1.generated.*; +import io.whitefox.core.Table; +import io.whitefox.persistence.StorageManager; +import io.whitefox.persistence.memory.InMemoryStorageManager; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class SampleTables { + public static final String currentPath = + Paths.get(".").toAbsolutePath().normalize().toString(); + + public static final String deltaTable1Path = + currentPath + "/src/test/resources/delta/samples/delta-table/"; + public static final String deltaTableWithHistory1Path = + currentPath + "/src/test/resources/delta/samples/delta-table-with-history/"; + public static final StorageManager storageManager = + new InMemoryStorageManager(List.of(new io.whitefox.core.Share( + "name", + "key", + Map.of( + "default", + new io.whitefox.core.Schema( + "default", + List.of( + new io.whitefox.core.Table( + "table1", "file://" + deltaTable1Path, "default", "name"), + new Table( + "table-with-history", + "file://" + deltaTableWithHistory1Path, + "default", + "name")), + "name"))))); + + public static final MetadataObject deltaTable1Metadata = new MetadataObject() + .metadata(new MetadataObjectMetadata() + .id("56d48189-cdbc-44f2-9b0e-2bded4c79ed7") + .name("table1") + .format(new FormatObject().provider("parquet")) + .schemaString( + "{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}") + .partitionColumns(List.of()) + .version(0L) + ._configuration(Map.of())); + public static final MetadataObject deltaTableWithHistory1Metadata = new MetadataObject() + .metadata(new MetadataObjectMetadata() + .id("56d48189-cdbc-44f2-9b0e-2bded4c79ed7") + .name("table-with-history") + .format(new FormatObject().provider("parquet")) + .schemaString( + "{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}") + .partitionColumns(List.of()) + .version(0L) + ._configuration(Map.of())); + public static final ProtocolObject deltaTable1Protocol = + new ProtocolObject().protocol(new ProtocolObjectProtocol().minReaderVersion(1)); + public static final Set deltaTable1Files = Set.of( + new FileObject() + ._file(new FileObjectFile() + .url("file://" + deltaTable1Path + + "part-00003-049d1c60-7ad6-45a3-af3f-65ffcabcc974-c000.snappy.parquet") + .id("file://" + deltaTable1Path + + "part-00003-049d1c60-7ad6-45a3-af3f-65ffcabcc974-c000.snappy.parquet") + .partitionValues(Map.of()) + .size(478L) + .stats( + "{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}") + .version(0L) + .timestamp(1695976443161L) + .expirationTimestamp(9223372036854775807L)), + new FileObject() + ._file(new FileObjectFile() + .url("file://" + deltaTable1Path + + "part-00001-a67388a6-e20e-426e-a872-351c390779a5-c000.snappy.parquet") + .id("file://" + deltaTable1Path + + "part-00001-a67388a6-e20e-426e-a872-351c390779a5-c000.snappy.parquet") + .partitionValues(Map.of()) + .size(478L) + .stats( + "{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0}}") + .version(0L) + .timestamp(1695976443161L) + .expirationTimestamp(9223372036854775807L)), + new FileObject() + ._file(new FileObjectFile() + .url("file://" + deltaTable1Path + + "part-00007-3e861bbf-fe8b-44d0-bac7-712b8cf4608c-c000.snappy.parquet") + .id("file://" + deltaTable1Path + + "part-00007-3e861bbf-fe8b-44d0-bac7-712b8cf4608c-c000.snappy.parquet") + .partitionValues(Map.of()) + .size(478L) + .stats( + "{\"numRecords\":1,\"minValues\":{\"id\":3},\"maxValues\":{\"id\":3},\"nullCount\":{\"id\":0}}") + .version(0L) + .timestamp(1695976443161L) + .expirationTimestamp(9223372036854775807L)), + new FileObject() + ._file(new FileObjectFile() + .url("file://" + deltaTable1Path + + "part-00005-e7b9aad4-adf6-42ad-a17c-fbc93689b721-c000.snappy.parquet") + .id("file://" + deltaTable1Path + + "part-00005-e7b9aad4-adf6-42ad-a17c-fbc93689b721-c000.snappy.parquet") + .partitionValues(Map.of()) + .size(478L) + .stats( + "{\"numRecords\":1,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":2},\"nullCount\":{\"id\":0}}") + .version(0L) + .timestamp(1695976443161L) + .expirationTimestamp(9223372036854775807L)), + new FileObject() + ._file(new FileObjectFile() + .url("file://" + deltaTable1Path + + "part-00009-90280af8-7b24-4519-9e49-82db78a1651b-c000.snappy.parquet") + .id("file://" + deltaTable1Path + + "part-00009-90280af8-7b24-4519-9e49-82db78a1651b-c000.snappy.parquet") + .partitionValues(Map.of()) + .size(478L) + .stats( + "{\"numRecords\":1,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}") + .version(0L) + .timestamp(1695976443161L) + .expirationTimestamp(9223372036854775807L))); +} diff --git a/server/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index e9b5ac596..43e8699ae 100644 --- a/server/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -1,52 +1,48 @@ package io.whitefox.api.deltasharing.server; import static io.restassured.RestAssured.given; +import static io.whitefox.api.deltasharing.SampleTables.*; import static org.hamcrest.Matchers.*; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import com.fasterxml.jackson.databind.ObjectMapper; import io.quarkus.test.junit.QuarkusMock; import io.quarkus.test.junit.QuarkusTest; +import io.restassured.http.Header; import io.whitefox.OpenApiValidationFilter; import io.whitefox.api.deltasharing.encoders.DeltaPageTokenEncoder; -import io.whitefox.core.Schema; -import io.whitefox.core.Share; -import io.whitefox.core.Table; +import io.whitefox.api.deltasharing.model.v1.generated.*; import io.whitefox.core.services.ContentAndToken; import io.whitefox.persistence.StorageManager; -import io.whitefox.persistence.memory.InMemoryStorageManager; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; +import java.io.IOException; import java.nio.file.Paths; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; @QuarkusTest public class DeltaSharesApiImplTest { - @BeforeAll public static void setup() { - var storageManager = new InMemoryStorageManager(List.of(new Share( - "name", - "key", - Map.of( - "default", - new Schema( - "default", - List.of(new Table( - "table1", "src/test/resources/delta/samples/delta-table", "default", "name")), - "name"))))); QuarkusMock.installMockForType(storageManager, StorageManager.class); } private final DeltaPageTokenEncoder encoder; + private final ObjectMapper objectMapper; @Inject - public DeltaSharesApiImplTest(DeltaPageTokenEncoder encoder) { + public DeltaSharesApiImplTest(DeltaPageTokenEncoder encoder, ObjectMapper objectMapper) { this.encoder = encoder; + this.objectMapper = objectMapper; } private static final String specLocation = Paths.get(".") @@ -156,7 +152,7 @@ public void listTables() { .get("delta-api/v1/shares/{share}/schemas/{schema}/tables", "name", "default") .then() .statusCode(200) - .body("items", hasSize(1)) + .body("items", hasSize(2)) .body("items[0].name", is("table1")) .body("items[0].schema", is("default")) .body("items[0].share", is("name")) @@ -179,22 +175,36 @@ public void tableMetadataNotFound() { @Test @DisabledOnOs(OS.WINDOWS) - public void tableMetadata() { + public void tableMetadata() throws IOException { + var responseBodyLines = given() + .when() + .filter(filter) + .get( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/metadata", + "name", + "default", + "table1") + .then() + .statusCode(200) + .extract() + .asString() + .split("\n"); + assertEquals(2, responseBodyLines.length); + assertEquals( + new ProtocolObject().protocol(new ProtocolObjectProtocol().minReaderVersion(1)), + objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class)); assertEquals( - "{\"protocol\":{\"minReaderVersion\":1}}\n" - + "{\"metadata\":{\"id\":\"56d48189-cdbc-44f2-9b0e-2bded4c79ed7\",\"format\":{\"provider\":\"parquet\"},\"schemaString\":\"{\\\"type\\\":\\\"struct\\\",\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"long\\\",\\\"nullable\\\":true,\\\"metadata\\\":{}}]}\",\"partitionColumns\":[]}}", - given() - .when() - .filter(filter) - .get( - "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/metadata", - "name", - "default", - "table1") - .then() - .statusCode(200) - .extract() - .asPrettyString()); + new MetadataObject() + .metadata(new MetadataObjectMetadata() + .id("56d48189-cdbc-44f2-9b0e-2bded4c79ed7") + .name("table1") + .format(new FormatObject().provider("parquet")) + .schemaString( + "{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}") + .partitionColumns(List.of()) + .version(0L) + ._configuration(Map.of())), + objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class)); } @Test @@ -205,7 +215,7 @@ public void listAllTables() { .get("delta-api/v1/shares/{share}/all-tables", "name") .then() .statusCode(200) - .body("items", hasSize(1)) + .body("items", hasSize(2)) .body("items[0].name", is("table1")) .body("items[0].schema", is("default")) .body("items[0].share", is("name")) @@ -284,4 +294,123 @@ public void getTableVersionBadTimestamp() { .then() .statusCode(502); } + + @DisabledOnOs(OS.WINDOWS) + @Test + public void queryTableCurrentVersion() throws IOException { + var responseBodyLines = given() + .when() + .filter(filter) + .body("{}") + .header(new Header("Content-Type", "application/json")) + .post( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query", + "name", + "default", + "table1") + .then() + .statusCode(200) + .extract() + .body() + .asString() + .split("\n"); + + assertEquals( + deltaTable1Protocol, + objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class)); + assertEquals( + deltaTable1Metadata, + objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class)); + var files = Arrays.stream(responseBodyLines) + .skip(2) + .map(line -> { + try { + return objectMapper.reader().readValue(line, FileObject.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toSet()); + assertEquals(files, deltaTable1Files); + assertEquals(7, responseBodyLines.length); + } + + @DisabledOnOs(OS.WINDOWS) + @Test + public void queryTableByVersion() throws IOException { + var responseBodyLines = given() + .when() + .filter(filter) + .body("{\"version\": 0}") + .header(new Header("Content-Type", "application/json")) + .post( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query", + "name", + "default", + "table1") + .then() + .statusCode(200) + .extract() + .body() + .asString() + .split("\n"); + + assertEquals( + deltaTable1Protocol, + objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class)); + assertEquals( + deltaTable1Metadata, + objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class)); + var files = Arrays.stream(responseBodyLines) + .skip(2) + .map(line -> { + try { + return objectMapper.reader().readValue(line, FileObject.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toSet()); + assertEquals(deltaTable1Files, files); + assertEquals(7, responseBodyLines.length); + } + + @Test + @Disabled + public void queryTableByTs() throws IOException { + var responseBodyLines = given() + .when() + .filter(filter) + .body("{\"timestamp\": \"2023-10-19T17:16:00Z\"}") + .header(new Header("Content-Type", "application/json")) + .post( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query", + "name", + "default", + "table-with-history") + .then() + .statusCode(200) + .extract() + .body() + .asString() + .split("\n"); + + assertEquals( + deltaTable1Protocol, + objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class)); + assertEquals( + deltaTableWithHistory1Metadata, + objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class)); + assertDoesNotThrow(() -> Arrays.stream(responseBodyLines) + .skip(2) + .map(line -> { + try { + return objectMapper.reader().readValue(line, FileObject.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toSet())); + assertEquals(7, responseBodyLines.length); + } } diff --git a/server/src/test/java/io/whitefox/services/DeltaShareServiceTest.java b/server/src/test/java/io/whitefox/services/DeltaShareServiceTest.java index 527ab8761..9406e6545 100644 --- a/server/src/test/java/io/whitefox/services/DeltaShareServiceTest.java +++ b/server/src/test/java/io/whitefox/services/DeltaShareServiceTest.java @@ -9,6 +9,7 @@ import io.whitefox.core.services.DeltaShareTableLoader; import io.whitefox.core.services.DeltaSharesService; import io.whitefox.core.services.DeltaSharesServiceImpl; +import io.whitefox.core.services.NoOpSigner; import io.whitefox.persistence.StorageManager; import io.whitefox.persistence.memory.InMemoryStorageManager; import java.util.Collections; @@ -24,11 +25,12 @@ public class DeltaShareServiceTest { DeltaShareTableLoader loader = new DeltaShareTableLoader(); Integer defaultMaxResults = 10; + NoOpSigner signer = new NoOpSigner(); @Test public void getUnknownShare() throws ExecutionException, InterruptedException { DeltaSharesService deltaSharesService = - new DeltaSharesServiceImpl(new InMemoryStorageManager(), defaultMaxResults, loader); + new DeltaSharesServiceImpl(new InMemoryStorageManager(), defaultMaxResults, loader, signer); assertEquals(Optional.empty(), deltaSharesService.getShare("unknown")); } @@ -37,7 +39,7 @@ public void getShare() throws ExecutionException, InterruptedException { var shares = List.of(new Share("name", "key", Collections.emptyMap())); StorageManager storageManager = new InMemoryStorageManager(shares); DeltaSharesService deltaSharesService = - new DeltaSharesServiceImpl(storageManager, defaultMaxResults, loader); + new DeltaSharesServiceImpl(storageManager, defaultMaxResults, loader, signer); var share = deltaSharesService.getShare("name"); assertTrue(share.isPresent()); assertEquals("name", share.get().name()); @@ -49,7 +51,7 @@ public void listShares() throws ExecutionException, InterruptedException { var shares = List.of(new Share("name", "key", Collections.emptyMap())); StorageManager storageManager = new InMemoryStorageManager(shares); DeltaSharesService deltaSharesService = - new DeltaSharesServiceImpl(storageManager, defaultMaxResults, loader); + new DeltaSharesServiceImpl(storageManager, defaultMaxResults, loader, signer); var sharesWithNextToken = deltaSharesService.listShares(Optional.empty(), Optional.of(30)); assertEquals(1, sharesWithNextToken.getContent().size()); assertTrue(sharesWithNextToken.getToken().isEmpty()); @@ -60,7 +62,7 @@ public void listSharesWithToken() throws ExecutionException, InterruptedExceptio var shares = List.of(new Share("name", "key", Collections.emptyMap())); StorageManager storageManager = new InMemoryStorageManager(shares); DeltaSharesService deltaSharesService = - new DeltaSharesServiceImpl(storageManager, defaultMaxResults, loader); + new DeltaSharesServiceImpl(storageManager, defaultMaxResults, loader, signer); var sharesWithNextToken = deltaSharesService.listShares(Optional.empty(), Optional.of(30)); assertEquals(1, sharesWithNextToken.getContent().size()); assertTrue(sharesWithNextToken.getToken().isEmpty()); @@ -70,7 +72,8 @@ public void listSharesWithToken() throws ExecutionException, InterruptedExceptio public void listSchemasOfEmptyShare() throws ExecutionException, InterruptedException { var shares = List.of(new Share("name", "key", Collections.emptyMap())); StorageManager storageManager = new InMemoryStorageManager(shares); - DeltaSharesService deltaSharesService = new DeltaSharesServiceImpl(storageManager, 100, loader); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, signer); var resultSchemas = deltaSharesService.listSchemas("name", Optional.empty(), Optional.empty()); assertTrue(resultSchemas.isPresent()); assertTrue(resultSchemas.get().getContent().isEmpty()); @@ -82,7 +85,8 @@ public void listSchemas() throws ExecutionException, InterruptedException { var shares = List.of(new Share( "name", "key", Map.of("default", new Schema("default", Collections.emptyList(), "name")))); StorageManager storageManager = new InMemoryStorageManager(shares); - DeltaSharesService deltaSharesService = new DeltaSharesServiceImpl(storageManager, 100, loader); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, signer); var resultSchemas = deltaSharesService.listSchemas("name", Optional.empty(), Optional.empty()); assertTrue(resultSchemas.isPresent()); assertEquals(1, resultSchemas.get().getContent().size()); @@ -97,7 +101,8 @@ public void listSchemasOfUnknownShare() throws ExecutionException, InterruptedEx var shares = List.of(new Share( "name", "key", Map.of("default", new Schema("default", Collections.emptyList(), "name")))); StorageManager storageManager = new InMemoryStorageManager(shares); - DeltaSharesService deltaSharesService = new DeltaSharesServiceImpl(storageManager, 100, loader); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, signer); var resultSchemas = deltaSharesService.listSchemas("notKey", Optional.empty(), Optional.empty()); assertTrue(resultSchemas.isEmpty()); @@ -113,7 +118,8 @@ public void listTables() throws ExecutionException, InterruptedException { new Schema( "default", List.of(new Table("table1", "location1", "default", "name")), "name")))); StorageManager storageManager = new InMemoryStorageManager(shares); - DeltaSharesService deltaSharesService = new DeltaSharesServiceImpl(storageManager, 100, loader); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, signer); var resultSchemas = deltaSharesService.listTables("name", "default", Optional.empty(), Optional.empty()); assertTrue(resultSchemas.isPresent()); @@ -137,7 +143,8 @@ public void listAllTables() throws ExecutionException, InterruptedException { new Schema( "other", List.of(new Table("table2", "location2", "default", "name")), "name")))); StorageManager storageManager = new InMemoryStorageManager(shares); - DeltaSharesService deltaSharesService = new DeltaSharesServiceImpl(storageManager, 100, loader); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, signer); var resultSchemas = deltaSharesService.listTablesOfShare("name", Optional.empty(), Optional.empty()); assertTrue(resultSchemas.isPresent()); @@ -173,7 +180,8 @@ public void listAllTablesEmpty() throws ExecutionException, InterruptedException "name"))), new Share("name2", "key2", Map.of())); StorageManager storageManager = new InMemoryStorageManager(shares); - DeltaSharesService deltaSharesService = new DeltaSharesServiceImpl(storageManager, 100, loader); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, signer); var resultSchemas = deltaSharesService.listTablesOfShare("name2", Optional.empty(), Optional.empty()); assertTrue(resultSchemas.isPresent()); @@ -184,7 +192,8 @@ public void listAllTablesEmpty() throws ExecutionException, InterruptedException @Test public void listAllTablesNoShare() throws ExecutionException, InterruptedException { StorageManager storageManager = new InMemoryStorageManager(); - DeltaSharesService deltaSharesService = new DeltaSharesServiceImpl(storageManager, 100, loader); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, signer); var resultSchemas = deltaSharesService.listTablesOfShare("name2", Optional.empty(), Optional.empty()); assertTrue(resultSchemas.isEmpty()); @@ -203,7 +212,8 @@ public void getTableMetadata() { List.of(new Table("table1", tablePath("delta-table"), "default", "name")), "name")))); StorageManager storageManager = new InMemoryStorageManager(shares); - DeltaSharesService deltaSharesService = new DeltaSharesServiceImpl(storageManager, 100, loader); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, signer); var tableMetadata = deltaSharesService.getTableMetadata("name", "default", "table1", null); assertTrue(tableMetadata.isPresent()); assertEquals("56d48189-cdbc-44f2-9b0e-2bded4c79ed7", tableMetadata.get().id()); @@ -219,7 +229,8 @@ public void tableMetadataNotFound() { new Schema( "default", List.of(new Table("table1", "location1", "default", "name")), "name")))); StorageManager storageManager = new InMemoryStorageManager(shares); - DeltaSharesService deltaSharesService = new DeltaSharesServiceImpl(storageManager, 100, loader); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, signer); var resultTable = deltaSharesService.getTableMetadata("name", "default", "tableNotFound", null); assertTrue(resultTable.isEmpty()); } diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00000-38ea829c-0e1b-49ba-b71f-590b79b55a4a-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00000-38ea829c-0e1b-49ba-b71f-590b79b55a4a-c000.snappy.parquet.crc new file mode 100644 index 000000000..e083ec87d Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00000-38ea829c-0e1b-49ba-b71f-590b79b55a4a-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00000-f0ef34d7-c7a2-4da8-9f52-f5e9cfa345d3-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00000-f0ef34d7-c7a2-4da8-9f52-f5e9cfa345d3-c000.snappy.parquet.crc new file mode 100644 index 000000000..04082d7e7 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00000-f0ef34d7-c7a2-4da8-9f52-f5e9cfa345d3-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00001-a67388a6-e20e-426e-a872-351c390779a5-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00001-a67388a6-e20e-426e-a872-351c390779a5-c000.snappy.parquet.crc new file mode 100644 index 000000000..fe3a6258e Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00001-a67388a6-e20e-426e-a872-351c390779a5-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00001-addd981e-cbd7-4f0f-bacf-6d52f3e7d0a7-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00001-addd981e-cbd7-4f0f-bacf-6d52f3e7d0a7-c000.snappy.parquet.crc new file mode 100644 index 000000000..cb1f6e1b2 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00001-addd981e-cbd7-4f0f-bacf-6d52f3e7d0a7-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00002-776d1d42-ee7e-404e-88dd-adbabdeb0240-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00002-776d1d42-ee7e-404e-88dd-adbabdeb0240-c000.snappy.parquet.crc new file mode 100644 index 000000000..623601017 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00002-776d1d42-ee7e-404e-88dd-adbabdeb0240-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00003-049d1c60-7ad6-45a3-af3f-65ffcabcc974-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00003-049d1c60-7ad6-45a3-af3f-65ffcabcc974-c000.snappy.parquet.crc new file mode 100644 index 000000000..d5e19e43a Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00003-049d1c60-7ad6-45a3-af3f-65ffcabcc974-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00003-154b0fe9-58c5-455a-a2fa-1d86b51c8aaa-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00003-154b0fe9-58c5-455a-a2fa-1d86b51c8aaa-c000.snappy.parquet.crc new file mode 100644 index 000000000..b674c58d6 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00003-154b0fe9-58c5-455a-a2fa-1d86b51c8aaa-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00004-1eca673f-7846-425e-aff1-775eb6b95ef5-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00004-1eca673f-7846-425e-aff1-775eb6b95ef5-c000.snappy.parquet.crc new file mode 100644 index 000000000..e51f7f61f Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00004-1eca673f-7846-425e-aff1-775eb6b95ef5-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00005-4fc30438-c1fa-451d-bdc5-e161e2745de3-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00005-4fc30438-c1fa-451d-bdc5-e161e2745de3-c000.snappy.parquet.crc new file mode 100644 index 000000000..be613b6ac Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00005-4fc30438-c1fa-451d-bdc5-e161e2745de3-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00005-e7b9aad4-adf6-42ad-a17c-fbc93689b721-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00005-e7b9aad4-adf6-42ad-a17c-fbc93689b721-c000.snappy.parquet.crc new file mode 100644 index 000000000..40544516f Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00005-e7b9aad4-adf6-42ad-a17c-fbc93689b721-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00006-08194aae-84c6-42f9-a628-cae765c91679-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00006-08194aae-84c6-42f9-a628-cae765c91679-c000.snappy.parquet.crc new file mode 100644 index 000000000..a14dc1f25 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00006-08194aae-84c6-42f9-a628-cae765c91679-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00007-3e861bbf-fe8b-44d0-bac7-712b8cf4608c-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00007-3e861bbf-fe8b-44d0-bac7-712b8cf4608c-c000.snappy.parquet.crc new file mode 100644 index 000000000..caa0eea6e Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00007-3e861bbf-fe8b-44d0-bac7-712b8cf4608c-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00007-c23fbfb7-73ce-45e8-8e3c-1ddba521e173-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00007-c23fbfb7-73ce-45e8-8e3c-1ddba521e173-c000.snappy.parquet.crc new file mode 100644 index 000000000..95e7bb20c Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00007-c23fbfb7-73ce-45e8-8e3c-1ddba521e173-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00008-315ab056-d9dc-4bc2-8c50-78064b679333-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00008-315ab056-d9dc-4bc2-8c50-78064b679333-c000.snappy.parquet.crc new file mode 100644 index 000000000..a4a181a13 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00008-315ab056-d9dc-4bc2-8c50-78064b679333-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00009-7ef5c591-7290-4493-9c3e-73f17d960190-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00009-7ef5c591-7290-4493-9c3e-73f17d960190-c000.snappy.parquet.crc new file mode 100644 index 000000000..0e3e25664 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00009-7ef5c591-7290-4493-9c3e-73f17d960190-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/.part-00009-90280af8-7b24-4519-9e49-82db78a1651b-c000.snappy.parquet.crc b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00009-90280af8-7b24-4519-9e49-82db78a1651b-c000.snappy.parquet.crc new file mode 100644 index 000000000..1f3c5de55 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/.part-00009-90280af8-7b24-4519-9e49-82db78a1651b-c000.snappy.parquet.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/.00000000000000000000.json.crc b/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 000000000..88640447f Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/.00000000000000000000.json.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/.00000000000000000001.json.crc b/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/.00000000000000000001.json.crc new file mode 100644 index 000000000..6ad8953d5 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/.00000000000000000001.json.crc differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/00000000000000000000.json b/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/00000000000000000000.json new file mode 100644 index 000000000..54fa98a93 --- /dev/null +++ b/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/00000000000000000000.json @@ -0,0 +1,8 @@ +{"commitInfo":{"timestamp":1695976443439,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"2686"},"engineInfo":"Apache-Spark/3.3.0 Delta-Lake/2.3.0","txnId":"6448a489-e48d-45a9-b7ea-5ec4107939d2"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"56d48189-cdbc-44f2-9b0e-2bded4c79ed7","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1695976443161}} +{"add":{"path":"part-00001-a67388a6-e20e-426e-a872-351c390779a5-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1695976443000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00003-049d1c60-7ad6-45a3-af3f-65ffcabcc974-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1695976443000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00005-e7b9aad4-adf6-42ad-a17c-fbc93689b721-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1695976443000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":2},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00007-3e861bbf-fe8b-44d0-bac7-712b8cf4608c-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1695976443000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3},\"maxValues\":{\"id\":3},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00009-90280af8-7b24-4519-9e49-82db78a1651b-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1695976443000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/00000000000000000001.json b/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/00000000000000000001.json new file mode 100644 index 000000000..0d11ed5b5 --- /dev/null +++ b/server/src/test/resources/delta/samples/delta-table-with-history/_delta_log/00000000000000000001.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1697735872041,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4779"},"engineInfo":"Apache-Spark/3.3.0 Delta-Lake/2.3.0","txnId":"8ec31521-31a9-44dd-82db-59826235aa53"}} +{"add":{"path":"part-00000-f0ef34d7-c7a2-4da8-9f52-f5e9cfa345d3-c000.snappy.parquet","partitionValues":{},"size":477,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":10},\"maxValues\":{\"id\":10},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00001-addd981e-cbd7-4f0f-bacf-6d52f3e7d0a7-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":11},\"maxValues\":{\"id\":11},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00002-776d1d42-ee7e-404e-88dd-adbabdeb0240-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":12},\"maxValues\":{\"id\":12},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00003-154b0fe9-58c5-455a-a2fa-1d86b51c8aaa-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":13},\"maxValues\":{\"id\":13},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00004-1eca673f-7846-425e-aff1-775eb6b95ef5-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":14},\"maxValues\":{\"id\":14},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00005-4fc30438-c1fa-451d-bdc5-e161e2745de3-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":15},\"maxValues\":{\"id\":15},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00006-08194aae-84c6-42f9-a628-cae765c91679-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":16},\"maxValues\":{\"id\":16},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00007-c23fbfb7-73ce-45e8-8e3c-1ddba521e173-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":17},\"maxValues\":{\"id\":17},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00008-315ab056-d9dc-4bc2-8c50-78064b679333-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":18},\"maxValues\":{\"id\":18},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00009-7ef5c591-7290-4493-9c3e-73f17d960190-c000.snappy.parquet","partitionValues":{},"size":478,"modificationTime":1697735871000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":19},\"maxValues\":{\"id\":19},\"nullCount\":{\"id\":0}}"}} diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00000-38ea829c-0e1b-49ba-b71f-590b79b55a4a-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00000-38ea829c-0e1b-49ba-b71f-590b79b55a4a-c000.snappy.parquet new file mode 100644 index 000000000..26ff20549 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00000-38ea829c-0e1b-49ba-b71f-590b79b55a4a-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00000-f0ef34d7-c7a2-4da8-9f52-f5e9cfa345d3-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00000-f0ef34d7-c7a2-4da8-9f52-f5e9cfa345d3-c000.snappy.parquet new file mode 100644 index 000000000..f22911aea Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00000-f0ef34d7-c7a2-4da8-9f52-f5e9cfa345d3-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00001-a67388a6-e20e-426e-a872-351c390779a5-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00001-a67388a6-e20e-426e-a872-351c390779a5-c000.snappy.parquet new file mode 100644 index 000000000..d6cec93b8 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00001-a67388a6-e20e-426e-a872-351c390779a5-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00001-addd981e-cbd7-4f0f-bacf-6d52f3e7d0a7-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00001-addd981e-cbd7-4f0f-bacf-6d52f3e7d0a7-c000.snappy.parquet new file mode 100644 index 000000000..a6be14094 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00001-addd981e-cbd7-4f0f-bacf-6d52f3e7d0a7-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00002-776d1d42-ee7e-404e-88dd-adbabdeb0240-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00002-776d1d42-ee7e-404e-88dd-adbabdeb0240-c000.snappy.parquet new file mode 100644 index 000000000..7be94e5df Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00002-776d1d42-ee7e-404e-88dd-adbabdeb0240-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00003-049d1c60-7ad6-45a3-af3f-65ffcabcc974-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00003-049d1c60-7ad6-45a3-af3f-65ffcabcc974-c000.snappy.parquet new file mode 100644 index 000000000..114c2e107 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00003-049d1c60-7ad6-45a3-af3f-65ffcabcc974-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00003-154b0fe9-58c5-455a-a2fa-1d86b51c8aaa-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00003-154b0fe9-58c5-455a-a2fa-1d86b51c8aaa-c000.snappy.parquet new file mode 100644 index 000000000..9ba709a0a Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00003-154b0fe9-58c5-455a-a2fa-1d86b51c8aaa-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00004-1eca673f-7846-425e-aff1-775eb6b95ef5-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00004-1eca673f-7846-425e-aff1-775eb6b95ef5-c000.snappy.parquet new file mode 100644 index 000000000..ddaa29080 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00004-1eca673f-7846-425e-aff1-775eb6b95ef5-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00005-4fc30438-c1fa-451d-bdc5-e161e2745de3-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00005-4fc30438-c1fa-451d-bdc5-e161e2745de3-c000.snappy.parquet new file mode 100644 index 000000000..ab6fa60bf Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00005-4fc30438-c1fa-451d-bdc5-e161e2745de3-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00005-e7b9aad4-adf6-42ad-a17c-fbc93689b721-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00005-e7b9aad4-adf6-42ad-a17c-fbc93689b721-c000.snappy.parquet new file mode 100644 index 000000000..c655515d2 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00005-e7b9aad4-adf6-42ad-a17c-fbc93689b721-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00006-08194aae-84c6-42f9-a628-cae765c91679-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00006-08194aae-84c6-42f9-a628-cae765c91679-c000.snappy.parquet new file mode 100644 index 000000000..57e39f60c Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00006-08194aae-84c6-42f9-a628-cae765c91679-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00007-3e861bbf-fe8b-44d0-bac7-712b8cf4608c-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00007-3e861bbf-fe8b-44d0-bac7-712b8cf4608c-c000.snappy.parquet new file mode 100644 index 000000000..32faa5bbc Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00007-3e861bbf-fe8b-44d0-bac7-712b8cf4608c-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00007-c23fbfb7-73ce-45e8-8e3c-1ddba521e173-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00007-c23fbfb7-73ce-45e8-8e3c-1ddba521e173-c000.snappy.parquet new file mode 100644 index 000000000..ef3c6aee8 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00007-c23fbfb7-73ce-45e8-8e3c-1ddba521e173-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00008-315ab056-d9dc-4bc2-8c50-78064b679333-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00008-315ab056-d9dc-4bc2-8c50-78064b679333-c000.snappy.parquet new file mode 100644 index 000000000..b579eb750 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00008-315ab056-d9dc-4bc2-8c50-78064b679333-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00009-7ef5c591-7290-4493-9c3e-73f17d960190-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00009-7ef5c591-7290-4493-9c3e-73f17d960190-c000.snappy.parquet new file mode 100644 index 000000000..2c8d9cce1 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00009-7ef5c591-7290-4493-9c3e-73f17d960190-c000.snappy.parquet differ diff --git a/server/src/test/resources/delta/samples/delta-table-with-history/part-00009-90280af8-7b24-4519-9e49-82db78a1651b-c000.snappy.parquet b/server/src/test/resources/delta/samples/delta-table-with-history/part-00009-90280af8-7b24-4519-9e49-82db78a1651b-c000.snappy.parquet new file mode 100644 index 000000000..80b30c4b8 Binary files /dev/null and b/server/src/test/resources/delta/samples/delta-table-with-history/part-00009-90280af8-7b24-4519-9e49-82db78a1651b-c000.snappy.parquet differ