Decouple FileFormat from datafusion_data_access

.asf.yaml

@@ -21,7 +21,7 @@ notifications:
   pullrequests: [email protected]
   jira_options: link label worklog
 github:
-  description: "Apache Arrow DataFusion and Ballista query engines"
+  description: "Apache Arrow DataFusion SQL Query Engine"
   homepage: https://arrow.apache.org/datafusion
   labels:
     - arrow

.github/pull_request_template.md

@@ -25,3 +25,23 @@ If there are user-facing changes then we may require documentation to be updated
 <!--
 If there are any breaking changes to public APIs, please add the `api change` label.
 -->
+
+# Does this PR break compatibility with Ballista?
+
+<!--
+The CI checks will attempt to build [arrow-ballista](https://github.com/apache/arrow-ballista) against this PR. If 
+this check fails then it indicates that this PR makes a breaking change to the DataFusion API.
+
+If possible, try to make the change in a way that is not a breaking API change. For example, if code has moved 
+ around, try adding `pub use` from the original location to preserve the current API.
+
+If it is not possible to avoid a breaking change (such as when adding enum variants) then follow this process:
+
+- Make a corresponding PR against `arrow-ballista` with the changes required there
+- Update `dev/build-arrow-ballista.sh` to clone the appropriate `arrow-ballista` repo & branch
+- Merge this PR when CI passes
+- Merge the Ballista PR
+- Create a new PR here to reset `dev/build-arrow-ballista.sh` to point to `arrow-ballista` master again
+
+_If you would like to help improve this process, please see https://github.com/apache/arrow-datafusion/issues/2583_
+-->

.github/workflows/dev.yml

@@ -66,9 +66,8 @@ jobs:
           #
           # ignore subproject CHANGELOG.md because they are machine generated
           npx [email protected] --write \
-            '{ballista,datafusion,datafusion-examples,docs,python}/**/*.md' \
-            '!{ballista,datafusion,python}/CHANGELOG.md' \
+            '{datafusion,datafusion-examples,docs,python}/**/*.md' \
+            '!{datafusion,python}/CHANGELOG.md' \
             README.md \
-            CONTRIBUTING.md \
-            'ballista/**/*.{ts,tsx}'
+            CONTRIBUTING.md
           git diff --exit-code

.github/workflows/dev_pr/labeler.yml

@@ -20,10 +20,6 @@ datafusion:
   - datafusion-cli/**/*
   - datafusion-examples/**/*
 
-ballista:
-  - ballista/**/*
-  - ballista-examples/**/*
-
 python:
   - python/**/*
 
@@ -41,5 +37,4 @@ documentation:
   - README.md
   - ./**/README.md
   - DEVELOPERS.md
-  - ballista/docs/**.*
   - datafusion/docs/**.*

.github/workflows/rust.yml

@@ -121,8 +121,6 @@ jobs:
           cargo run --example csv_sql
           cargo run --example parquet_sql
           cargo run --example avro_sql --features=datafusion/avro
-          cd ../ballista-examples
-          cargo run --example test_sql --features=ballista/standalone
         env:
           CARGO_HOME: "/github/home/.cargo"
           CARGO_TARGET_DIR: "/github/home/target"
@@ -146,6 +144,9 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.x'
       - name: Cache Cargo
         uses: actions/cache@v2
         with:
@@ -162,16 +163,9 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: ${{ matrix.rust }}
-      # Ballista is currently not part of the main workspace so requires a separate test step
-      - name: Run Ballista tests
+      - name: Run tests
         run: |
-          export ARROW_TEST_DATA=$(pwd)/testing/data
-          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
-          cd ballista/rust
-          # snmalloc requires cmake so build without default features
-          cargo test --no-default-features --features sled
-          # Ensure also compiles in standalone mode
-          cargo test --no-default-features --features standalone
+          ./dev/build-arrow-ballista.sh
         env:
           CARGO_HOME: "/github/home/.cargo"
           CARGO_TARGET_DIR: "/github/home/target"
@@ -237,8 +231,6 @@ jobs:
           POSTGRES_PASSWORD: postgres
       - name: Build datafusion-cli
         run: (cd datafusion-cli && cargo build)
-      - name: Build ballista-cli
-        run: (cd ballista-cli && cargo build)
       - name: Test Psql Parity
         run: python -m pytest -v integration-tests/test_psql_parity.py
         env:

.gitignore

@@ -96,3 +96,6 @@ venv/*
 
 # apache release artifacts
 dev/dist
+
+# CI
+arrow-ballista

CONTRIBUTING.md

@@ -35,9 +35,6 @@ list to help you get started.
 
 This section describes how you can get started at developing DataFusion.
 
-For information on developing with Ballista, see the
-[Ballista developer documentation](ballista/docs/README.md).
-
 ### Bootstrap environment
 
 DataFusion is written in Rust and it uses a standard rust toolkit:
@@ -168,7 +165,7 @@ The benchmark will automatically remove any generated parquet file on exit, howe
 
 ### Upstream Benchmark Suites
 
-Instructions and tooling for running upstream benchmark suites against DataFusion and/or Ballista can be found in [benchmarks](./benchmarks).
+Instructions and tooling for running upstream benchmark suites against DataFusion can be found in [benchmarks](./benchmarks).
 
 These are valuable for comparative evaluation against alternative Arrow implementations and query engines.
 
@@ -263,5 +260,5 @@ $ prettier --version
 After you've confirmed your prettier version, you can format all the `.md` files:
 
 ```bash
-prettier -w {ballista,datafusion,datafusion-cli,datafusion-examples,dev,docs}/**/*.md
+prettier -w {datafusion,datafusion-cli,datafusion-examples,dev,docs}/**/*.md
 ```

Cargo.toml

@@ -25,15 +25,11 @@ members = [
     "datafusion/physical-expr",
     "datafusion/proto",
     "datafusion/row",
+    "datafusion/sql",
     "datafusion-examples",
     "benchmarks",
-    "ballista/rust/client",
-    "ballista/rust/core",
-    "ballista/rust/executor",
-    "ballista/rust/scheduler",
-    "ballista-examples",
 ]
-exclude = ["ballista-cli", "datafusion-cli"]
+exclude = ["datafusion-cli"]
 
 [profile.release]
 codegen-units = 1

README.md

@@ -60,7 +60,7 @@ Projects that adapt to or serve as plugins to DataFusion:
 
 Here are some of the projects known to use DataFusion:
 
-- [Ballista](ballista) Distributed Compute Platform
+- [Ballista](https://github.com/apache/arrow-ballista) Distributed Compute Platform
 - [Cloudfuse Buzz](https://github.com/cloudfuse-io/buzz-rust)
 - [Cube Store](https://github.com/cube-js/cube.js/tree/master/rust)
 - [delta-rs](https://github.com/delta-io/delta-rs)