Update delta protocol written spec

Update openapi spec

wip

wip

wip

Add nix stuff

wippp

protocol/delta-sharing-protocol-api.yml

@@ -527,6 +527,12 @@ paths:
           description: 'If set to true, return the historical metadata if seen in the delta log. This is for the streaming client to check if the table schema is still read compatible.'
           schema:
             type: boolean
+        - in: header
+          name: delta-sharing-capabilities
+          required: false
+          description: 'Delta Sharing Capabilities'
+          schema:
+            type: string
       responses:
         '400':
           $ref: "#/components/responses/400"
@@ -581,11 +587,14 @@ components:
         not 	            Unary 	Represents a logical not check. This op should have once child.
         The supported value types:
           ValueType 	Description
-          "bool" 	Represents an Boolean type.
-          "int" 	Represents an Integer type.
-          "long" 	Represents a Long type.
-          "string" 	Represents a String type.
-          "date" 	Represents a Date type in "yyyy-mm-dd" format.
+          "bool" 	  Represents an Boolean type.
+          "int" 	  Represents an Integer type.
+          "long" 	  Represents a Long type.
+          "string" 	  Represents a String type.
+          "date" 	  Represents a Date type in "yyyy-mm-dd" format.
+          "float"     Represents a Float type.
+          "double"    Represents a Double type.
+          "timestamp" Represents a timestamp in ISO8601 format, in the UTC timezone.
 
     ListShareResponse:
       type: object
@@ -771,27 +780,31 @@ components:
       properties:
         protocol:
           # it refers to ./delta-sharing-protocol.md#protocol
-          $ref: '#/components/schemas/ProtocolObject'
+          $ref: '#/components/schemas/ParquetProtocolObject'
         metadata:
           # it refers to ./delta-sharing-protocol.md#metadata
-          $ref: '#/components/schemas/MetadataObject'
+          $ref: '#/components/schemas/ParquetMetadataObject'
 
     # This is not used for the spec but comes handy for autogeneration
     TableQueryResponseObject:
+      oneOf:
+        - $ref: '#/components/schemas/ParquetTableQueryResponseObject'
+        - $ref: '#/components/schemas/DeltaTableQueryResponseObject'
+    ParquetTableQueryResponseObject:
       type: object
       properties:
         protocol:
           # it refers to ./delta-sharing-protocol.md#protocol
-          $ref: '#/components/schemas/ProtocolObject'
+          $ref: '#/components/schemas/ParquetProtocolObject'
         metadata:
           # it refers to ./delta-sharing-protocol.md#metadata
-          $ref: '#/components/schemas/MetadataObject'
+          $ref: '#/components/schemas/ParquetMetadataObject'
         files:
           type: array
           items:
             # it refers to ./delta-sharing-protocol.md#file
-            $ref: '#/components/schemas/FileObject'
-    FileObject:
+            $ref: '#/components/schemas/ParquetFileObject'
+    ParquetFileObject:
       type: object
       properties:
         file:
@@ -825,7 +838,7 @@ components:
             - id
             - partitionValues
             - size
-    ProtocolObject:
+    ParquetProtocolObject:
       type: object
       properties:
         protocol:
@@ -834,15 +847,14 @@ components:
             minReaderVersion:
               type: integer
               format: int32
-    FormatObject:
+    ParquetFormatObject:
       type: object
       properties:
         provider:
           type: string
       required:
         - provider
-
-    MetadataObject:
+    ParquetMetadataObject:
       type: object
       properties:
         metaData:
@@ -855,7 +867,7 @@ components:
             description:
               type: string
             format:
-              $ref: '#/components/schemas/FormatObject'
+              $ref: '#/components/schemas/ParquetFormatObject'
             schemaString:
               type: string
             partitionColumns:
@@ -881,7 +893,240 @@ components:
             - format
             - schemaString
             - partitionColumns
-
+    DeltaTableQueryResponseObject:
+      type: object
+      properties:
+        protocol:
+          # it refers to ./delta-sharing-protocol.md#protocol
+          $ref: '#/components/schemas/DeltaProtocolObject'
+        metadata:
+          # it refers to ./delta-sharing-protocol.md#metadata
+          $ref: '#/components/schemas/DeltaMetadataObject'
+        files:
+          type: array
+          items:
+            # it refers to ./delta-sharing-protocol.md#file
+            $ref: '#/components/schemas/DeltaFileObject'
+    DeltaProtocolObject:
+      type: object
+      properties:
+        protocol:
+          type: object
+          properties:
+            deltaProtocol:
+              type: object
+              properties:
+                minReaderVersion:
+                  type: integer
+                  format: int32
+                minWriterVersion:
+                  type: integer
+                  format: int32
+    DeltaFormatObject:
+      type: object
+      properties:
+        provider:
+          type: string
+        options:
+          type: object
+          additionalProperties:
+            type: string
+      required:
+        - provider
+    DeltaMetadata:
+      type: object
+      description: see https://github.com/delta-io/delta/blob/master/PROTOCOL.md#change-metadata
+      required:
+        - id
+        - format
+        - schemaString
+        - partitionColumns
+        - configuration
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+        description:
+          type: string
+        format:
+          $ref: '#/components/schemas/DeltaFormatObject'
+        schemaString:
+          type: string
+        partitionColumns:
+          type: array
+          items:
+            type: string
+        createdTime:
+          type: integer
+          format: int64
+        configuration:
+          type: object
+          additionalProperties:
+            type: string
+    DeltaMetadataObject:
+      type: object
+      properties:
+        metaData:
+          type: object
+          properties:
+            version:
+              type: integer
+              format: int64
+            size:
+              type: integer
+              format: int64
+            numFiles:
+              type: integer
+              format: int64
+            deltaMetadata:
+              $ref: '#/components/schemas/DeltaMetadata'
+          required: [ deltaMetadata ]
+    DeltaFileObject:
+      required:
+        - id
+        - deltaSingleAction
+      properties:
+        id:
+          type: string
+        deletionVectorFileId:
+          type: string
+        version:
+          type: integer
+          format: int64
+        timestamp:
+          type: integer
+          format: int64
+        expirationTimestamp:
+          type: integer
+          format: int64
+        deltaSingleAction:
+          $ref: '#/components/schemas/DeltaSingleAction'
+    DeltaSingleAction:
+      type: object
+      description: only one field can be not null, container of delta actions such as file, add, cdf or remove see https://github.com/delta-io/delta/tree/master/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions
+      properties:
+        add:
+          $ref: '#/components/schemas/DeltaAddFileForCDFAction'
+        cdf:
+          $ref: '#/components/schemas/DeltaAddCDCFileAction'
+        file:
+          $ref: '#/components/schemas/DeltaAddFileAction'
+        remove:
+          $ref: '#/components/schemas/DeltaRemoveFileAction'
+    DeltaAddFileForCDFAction:
+      type: object
+      description: see io.delta.sharing.server.model.AddFileForCDF
+      properties:
+        url:
+          type: string
+        id:
+          type: string
+        partitionValues:
+          type: object
+          additionalProperties:
+            type: string
+        size:
+          type: integer
+          format: int64
+        expirationTimestamp:
+          type: integer
+          format: int64
+        version:
+          type: integer
+          format: int64
+        timestamp:
+          type: integer
+          format: int64
+        stats:
+          type: string
+    DeltaAddCDCFileAction:
+      type: object
+      description: see io.delta.sharing.server.model.AddCDCFile
+      properties:
+        url:
+          type: string
+        id:
+          type: string
+        partitionValues:
+          type: object
+          additionalProperties:
+            type: string
+        size:
+          type: integer
+          format: int64
+        expirationTimestamp:
+          type: integer
+          format: int64
+        timestamp:
+          type: integer
+          format: int64
+        version:
+          type: integer
+          format: int64
+    DeltaAddFileAction:
+      type: object
+      description: see io.delta.sharing.server.model.AddFile
+      properties:
+        url:
+          type: string
+        id:
+          type: string
+        partitionValues:
+          type: object
+          additionalProperties:
+            type: string
+        size:
+          type: integer
+          format: int64
+        stats:
+          type: string
+        expirationTimestamp:
+          type: integer
+          format: int64
+        timestamp:
+          type: integer
+          format: int64
+        version:
+          type: integer
+          format: int64
+    DeltaRemoveFileAction:
+      type: object
+      properties:
+        url:
+          type: string
+        id:
+          type: string
+        partitionValues:
+          type: object
+          additionalProperties:
+            type: string
+        size:
+          type: integer
+          format: int64
+        expirationTimestamp:
+          type: integer
+          format: int64
+        timestamp:
+          type: integer
+          format: int64
+        version:
+          type: integer
+          format: int64
+    DeltaEndStreamAction:
+      description: An action that is returned as the last line of the streaming response. It allows the server to include additional data that might be dynamically generated while the streaming message is sent
+      type: object
+      properties:
+        refreshToken:
+          type: string
+          description: a token used to refresh pre-signed urls for a long running query
+        nextPageToken:
+          type: string
+          description: a token used to retrieve the subsequent page of a query
+        minUrlExpirationTimestamp:
+          description: the minimum url expiration timestamp of the urls returned in current response
+          type: integer
+          format: int64
   responses:
     "400":
       description: The request is malformed

sbin/buildNativeContainer.sh

@@ -1,2 +1,5 @@
 #!/bin/bash
-./gradlew server:imageBuild -Dquarkus.package.type=native -Dquarkus.native.container-build=true -Dquarkus.container-image.name=server-native
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+pushd $SCRIPT_DIR/..
+nix-shell --run './gradlew server:app:imageBuild -Dquarkus.package.type=native -Dquarkus.native.container-build=true -Dquarkus.container-image.name=server-native'
+popd

sbin/startDocusaurus.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+pushd $SCRIPT_DIR/..
+nix-shell --run './gradlew docsite:npm_run_start'
+popd

server/app/build.gradle.kts

@@ -49,6 +49,7 @@ val serverGeneratorProperties = mapOf(
     "dateLibrary" to "java8",
     "disallowAdditionalPropertiesIfNotPresent" to "false",
     "generateBuilders" to "false",
+    "legacyDiscriminatorBehavior" to "false",
     "generatePom" to "false",
     "interfaceOnly" to "true",
     "library" to "quarkus",