Adopt ObjectStore (#761)

* feat: update datafusion and object store

* feat: adopt ObjectStore

* chore: add integration test feature

* refactor: object store as ground truth for table uri

* refactor: make creating builder infallible

* fix: fix python build

* fix: datafusion tests

* chore: flatten directories

* fix: allow configuring non https connections

* feat: draft integration set scaffolding

* refactor: clean up storage errors

* refactor: remove StorageError

* test: port tests from object_store crate

* test: include local StorageIntegration

* ci: update build scripts

* chore: add exempts for ruby build

* chore: cleanup / consolidate tests

* fix: repair s3 clock client integration

* test: migrate s3 repair tests

* fix: s3 common importa

* cleanup

* test: datafusion integration tests

* fix: checkpoint integration tests

* fix: environemtn during integration tests

* fix: write non empty test fiules to work with azure listing

* fix: use correct storage backend in azure comit tests

* fix: py3.7 tests

* fix: parquet2 merge errors

* fix: add missing rustls feature gates

* fix: add missing dev dependency

* chore: pr comments

* Apply suggestions from code review

Co-authored-by: Will Jones <[email protected]>

* fix: doctest

Co-authored-by: Will Jones <[email protected]>

.github/workflows/build.yml

@@ -14,7 +14,7 @@ jobs:
   format:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install minimal stable with clippy and rustfmt
         uses: actions-rs/toolchain@v1
         with:
@@ -30,11 +30,12 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
+          - macos-11
           - macOS-10.15
           - windows-latest
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install minimal stable with clippy and rustfmt
         uses: actions-rs/toolchain@v1
         with:
@@ -55,11 +56,12 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
+          - macos-11
           - macOS-10.15
           - windows-latest
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install minimal stable with clippy and rustfmt
         uses: actions-rs/toolchain@v1
         with:
@@ -70,22 +72,76 @@ jobs:
       - name: Run tests
         run: cargo test --verbose --features datafusion-ext,azure
 
-  s3_test:
+  integration_test:
     runs-on: ubuntu-latest
+    services:
+      # fake-gcs:
+      #   image: fsouza/fake-gcs-server
+      #   ports:
+      #     - 4443:4443
+      localstack:
+        image: localstack/localstack:0.14.4
+        ports:
+          - 4566:4566
+      azurite:
+        image: mcr.microsoft.com/azure-storage/azurite
+        ports:
+          - 10000:10000
+
+    container:
+      image: amd64/rust
+      env:
+        # Disable full debug symbol generation to speed up CI build and keep memory down
+        # "1" means line tables only, which is useful for panic tracebacks.
+        RUSTFLAGS: "-C debuginfo=1"
+        # https://github.com/rust-lang/cargo/issues/10280
+        CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+        RUST_BACKTRACE: "1"
+        # Run integration tests
+        TEST_INTEGRATION: 1
+        AWS_DEFAULT_REGION: "us-east-1"
+        AWS_ACCESS_KEY_ID: test
+        AWS_SECRET_ACCESS_KEY: test
+        AWS_ENDPOINT_URL: http://localstack:4566
+        EC2_METADATA_ENDPOINT: http://ec2-metadata:1338
+        GOOGLE_ENDPOINT_URL: https://fake-gcs:4443/storage/v1/b
+        GOOGLE_USE_EMULATOR: "1"
+        AZURE_USE_EMULATOR: "1"
+        AZURITE_BLOB_STORAGE_URL: "http://azurite:10000"
+        OBJECT_STORE_BUCKET: test-bucket
+        AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;"
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+
+      # - name: Configure Fake GCS Server (GCP emulation)
+      #   run: |
+      #     curl --insecure -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "https://fake-gcs:4443/storage/v1/b"
+      #     echo '{"gcs_base_url": "https://fake-gcs:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT"
+      - name: Setup LocalStack (AWS emulation)
+        run: |
+          cd /tmp
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          ./aws/install
+      - name: Configure Azurite (Azure emulation)
+        # the magical connection string is from
+        # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
       - name: Install minimal stable with clippy and rustfmt
         uses: actions-rs/toolchain@v1
         with:
           profile: default
           toolchain: stable
           override: true
+
       - uses: Swatinem/rust-cache@v1
-      - name: Setup localstack
-        run: docker-compose up setup
+
       - name: Run tests
         run: |
-          cargo test --features s3,datafusion-ext
+          cargo test --features integration_test,azure,s3,datafusion-ext
 
   paruqet2_test:
     runs-on: ubuntu-latest

.gitignore

@@ -15,3 +15,5 @@ tlaplus/*.toolbox/*/[0-9]*-[0-9]*-[0-9]*-[0-9]*-[0-9]*-[0-9]*/
 **/.python-version
 .coverage
 *.env
+__azurite*
+__blobstorage__