From 21170cb87a9524ec3f828d989452218ff7195596 Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Sun, 18 Apr 2021 18:30:39 +0000 Subject: [PATCH 1/8] Fixed CI. --- .github/workflows/cancel.yml | 77 ----------- .github/workflows/dev.yml | 59 +-------- .github/workflows/integration.yml | 83 ------------ .github/workflows/rust.yml | 204 +++--------------------------- 4 files changed, 18 insertions(+), 405 deletions(-) delete mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index de980eb6d05b..e1c6ed98ae8b 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -30,20 +30,6 @@ jobs: steps: # Unfortunately, we need to define a separate cancellation step for # each workflow where we want to cancel stale runs. - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale C++ runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: cpp.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale C# runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: csharp.yml - skipEventTypes: '["push", "schedule"]' - uses: potiuk/cancel-workflow-runs@master name: "Cancel stale Dev runs" with: @@ -51,69 +37,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} workflowFileName: dev.yml skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Go runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: go.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Integration runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: integration.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Java JNI runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: java_jni.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Java runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: java.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale JS runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: js.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Julia runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: julia.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Python runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: python.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale R runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: r.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Ruby runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: ruby.yml - skipEventTypes: '["push", "schedule"]' - uses: potiuk/cancel-workflow-runs@master name: "Cancel stale Rust runs" with: diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index d1b018480047..9d8146a2f1d6 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -31,66 +31,13 @@ jobs: lint: name: Lint C++, Python, R, Rust, Docker, RAT runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh + - uses: actions/checkout@v2 - name: Setup Python uses: actions/setup-python@v1 with: python-version: 3.8 - name: Setup Archery run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ubuntu-lint - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ubuntu-lint - - release: - name: Source Release and Merge Script - runs-on: ubuntu-20.04 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - env: - GIT_AUTHOR_NAME: Github Actions - GIT_AUTHOR_EMAIL: github@actions - GIT_COMMITTER_NAME: Github Actions - GIT_COMMITTER_EMAIL: github@actions - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Python - uses: actions/setup-python@v1 - with: - python-version: '3.6' - - name: Install Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: '2.6' - - name: Install Dependencies - shell: bash - run: | - pip install cython setuptools pytest jira - - name: Run Release Test - shell: bash - run: | - ci/scripts/release_test.sh $(pwd) - - name: Run Merge Script Test - shell: bash - run: | - pytest -v dev/test_merge_arrow_pr.py + - name: Lint + run: archery lint --rat diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 20112553ea25..000000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,83 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Integration - -on: - push: - paths: - - '.github/workflows/integration.yml' - - 'ci/**' - - 'dev/archery/**' - - 'go/**' - - 'integration/**' - - 'js/**' - - 'cpp/**' - - 'java/**' - - 'format/**' - - 'rust/**' - pull_request: - paths: - - '.github/workflows/integration.yml' - - 'ci/**' - - 'dev/archery/**' - - 'go/**' - - 'integration/**' - - 'js/**' - - 'cpp/**' - - 'java/**' - - 'format/**' - - 'rust/**' - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: AMD64 Conda Integration Test - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: conda-${{ hashFiles('cpp/**') }} - restore-keys: conda- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run conda-integration - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push conda-integration diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4bb17a2ecafa..3f1031c44486 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -123,7 +123,6 @@ jobs: run: | export CARGO_HOME="/github/home/.cargo" export CARGO_TARGET_DIR="/github/home/target" - cd rust # run tests on all workspace members with default feature list cargo test # test datafusion examples @@ -131,14 +130,6 @@ jobs: cargo test --no-default-features cargo run --example csv_sql cargo run --example parquet_sql - cd .. - cd arrow - # re-run tests on arrow workspace with additional features - cargo test --features=prettyprint - cargo run --example builders - cargo run --example dynamic_types - cargo run --example read_csv - cargo run --example read_csv_infer_schema # Ballista is currently not part of the main workspace so requires a separate test step - name: Run Ballista tests run: | @@ -148,49 +139,6 @@ jobs: # snmalloc requires cmake so build without default features cargo test --no-default-features - # test the --features "simd" of the arrow crate. This requires nightly. - linux-test-simd: - name: Test SIMD on AMD64 Rust ${{ matrix.rust }} - runs-on: ubuntu-latest - strategy: - matrix: - arch: [amd64] - rust: [nightly-2021-03-24] - container: - image: ${{ matrix.arch }}/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - ARROW_TEST_DATA: /__w/arrow/arrow/testing/data - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /github/home/.cargo - # this key equals the ones on `linux-build-lib` for re-use - key: cargo-cache- - - name: Cache Rust dependencies - uses: actions/cache@v2 - with: - path: /github/home/target - # this key equals the ones on `linux-build-lib` for re-use - key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }} - - name: Setup Rust toolchain - run: | - rustup toolchain install ${{ matrix.rust }} - rustup default ${{ matrix.rust }} - rustup component add rustfmt - - name: Run tests - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cd rust/arrow - cargo test --features "simd" - windows-and-macos: name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }} runs-on: ${{ matrix.os }} @@ -216,9 +164,23 @@ jobs: export PARQUET_TEST_DATA=$(pwd)/cpp/submodules/parquet-testing/data # do not produce debug symbols to keep memory usage down export RUSTFLAGS="-C debuginfo=0" - cd rust cargo test + lint: + name: Lint + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v2 + - name: Setup toolchain + run: | + rustup toolchain install stable + rustup default stable + rustup component add rustfmt + - name: Run + run: cargo fmt --all -- --check + clippy: name: Clippy needs: [linux-build-lib] @@ -258,7 +220,6 @@ jobs: run: | export CARGO_HOME="/github/home/.cargo" export CARGO_TARGET_DIR="/github/home/target" - cd rust cargo clippy --all-targets --workspace -- -D warnings -A clippy::redundant_field_names miri-checks: @@ -290,7 +251,6 @@ jobs: RUST_LOG: 'trace' run: | export MIRIFLAGS="-Zmiri-disable-isolation" - cd rust cargo miri setup cargo clean # Ignore MIRI errors until we can get a clean run @@ -330,141 +290,7 @@ jobs: # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0 # see https://github.com/xd009642/tarpaulin/issues/618 cargo install --version 0.16.0 cargo-tarpaulin - cd rust cargo tarpaulin --out Xml - name: Report coverage continue-on-error: true run: bash <(curl -s https://codecov.io/bash) - - # test FFI against the C-Data interface exposed by pyarrow - pyarrow-integration-test: - name: Test Pyarrow C Data Interface - runs-on: ubuntu-latest - strategy: - matrix: - rust: [stable] - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Setup Rust toolchain - run: | - rustup toolchain install ${{ matrix.rust }} - rustup default ${{ matrix.rust }} - rustup component add rustfmt clippy - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /home/runner/.cargo - key: cargo-maturin-cache- - - name: Cache Rust dependencies - uses: actions/cache@v2 - with: - path: /home/runner/target - # this key is not equal because maturin uses different compilation flags. - key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{ matrix.rust }}- - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Run tests - run: | - export CARGO_HOME="/home/runner/.cargo" - export CARGO_TARGET_DIR="/home/runner/target" - - cd rust/arrow-pyarrow-integration-testing - - python -m venv venv - source venv/bin/activate - - pip install maturin==0.8.2 toml==0.10.1 pyarrow==1.0.0 - maturin develop - python -m unittest discover tests - - # test the arrow crate builds against wasm32 in stable rust - wasm32-build: - name: Build wasm32 on AMD64 Rust ${{ matrix.rust }} - runs-on: ubuntu-latest - strategy: - matrix: - arch: [amd64] - rust: [nightly-2021-03-24] - container: - image: ${{ matrix.arch }}/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - ARROW_TEST_DATA: /__w/arrow/arrow/testing/data - PARQUET_TEST_DATA: /__w/arrow/arrow/cpp/submodules/parquet-testing/data - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /github/home/.cargo - # this key equals the ones on `linux-build-lib` for re-use - key: cargo-cache- - - name: Cache Rust dependencies - uses: actions/cache@v2 - with: - path: /github/home/target - key: ${{ runner.os }}-${{ matrix.arch }}-target-wasm32-cache-${{ matrix.rust }} - - name: Setup Rust toolchain - run: | - rustup toolchain install ${{ matrix.rust }} - rustup override set ${{ matrix.rust }} - rustup component add rustfmt - rustup target add wasm32-unknown-unknown - - name: Build arrow crate - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cd rust/arrow - cargo build --target wasm32-unknown-unknown - - # test the projects can build without default features - default-build: - name: Check No Defaults on AMD64 Rust ${{ matrix.rust }} - runs-on: ubuntu-latest - strategy: - matrix: - arch: [amd64] - rust: [stable] - container: - image: ${{ matrix.arch }}/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - ARROW_TEST_DATA: /__w/arrow/arrow/testing/data - PARQUET_TEST_DATA: /__w/arrow/arrow/cpp/submodules/parquet-testing/data - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /github/home/.cargo - # this key equals the ones on `linux-build-lib` for re-use - key: cargo-cache- - - name: Cache Rust dependencies - uses: actions/cache@v2 - with: - path: /github/home/target - key: ${{ runner.os }}-${{ matrix.arch }}-target-wasm32-cache-${{ matrix.rust }} - - name: Setup Rust toolchain - run: | - rustup toolchain install ${{ matrix.rust }} - rustup override set ${{ matrix.rust }} - rustup component add rustfmt - - name: Build arrow crate - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cd rust/arrow - cargo check --all-targets --no-default-features From e0908bcab760c099ff242dc034cdba73d45b26ff Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Sun, 18 Apr 2021 18:39:26 +0000 Subject: [PATCH 2/8] More cleanup. --- .github/workflows/archery.yml | 64 ----- .github/workflows/cpp.yml | 395 --------------------------- .github/workflows/cpp_cron.yml | 149 ---------- .github/workflows/csharp.yml | 121 -------- .github/workflows/dev.yml | 4 - .github/workflows/dev_pr/labeler.yml | 52 ---- .github/workflows/go.yml | 125 --------- .github/workflows/java.yml | 112 -------- .github/workflows/java_jni.yml | 83 ------ .github/workflows/js.yml | 122 --------- .github/workflows/julia.yml | 53 ---- .github/workflows/python.yml | 154 ----------- .github/workflows/python_cron.yml | 141 ---------- .github/workflows/r.yml | 255 ----------------- .github/workflows/ruby.yml | 290 -------------------- 15 files changed, 2120 deletions(-) delete mode 100644 .github/workflows/archery.yml delete mode 100644 .github/workflows/cpp.yml delete mode 100644 .github/workflows/cpp_cron.yml delete mode 100644 .github/workflows/csharp.yml delete mode 100644 .github/workflows/go.yml delete mode 100644 .github/workflows/java.yml delete mode 100644 .github/workflows/java_jni.yml delete mode 100644 .github/workflows/js.yml delete mode 100644 .github/workflows/julia.yml delete mode 100644 .github/workflows/python.yml delete mode 100644 .github/workflows/python_cron.yml delete mode 100644 .github/workflows/r.yml delete mode 100644 .github/workflows/ruby.yml diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml deleted file mode 100644 index 761e04595436..000000000000 --- a/.github/workflows/archery.yml +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Archery & Crossbow - -on: - push: - paths: - - '.github/workflows/archery.yml' - - 'dev/archery/**' - - 'dev/tasks/**' - - 'docker-compose.yml' - pull_request: - paths: - - '.github/workflows/archery.yml' - - 'dev/archery/**' - - 'dev/tasks/**' - - 'docker-compose.yml' - -jobs: - - test: - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - name: Archery Unittests and Crossbow Check Config - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Git Fixup - if: ${{ github.event_name == 'pull_request' }} - shell: bash - run: git branch master origin/master - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: '3.6' - - name: Install Archery, Crossbow- and Test Dependencies - run: pip install pytest responses -e dev/archery[all] - - name: Archery Unittests - working-directory: dev/archery - run: pytest -v archery - - name: Archery Docker Validation - run: archery docker - - name: Crossbow Check Config - working-directory: dev/tasks - run: archery crossbow check-config diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml deleted file mode 100644 index 0bcf3460ad4b..000000000000 --- a/.github/workflows/cpp.yml +++ /dev/null @@ -1,395 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C++ - -on: - push: - paths: - - '.github/workflows/cpp.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/util_*' - - 'cpp/**' - - 'format/Flight.proto' - pull_request: - paths: - - '.github/workflows/cpp.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/util_*' - - 'cpp/**' - - 'format/Flight.proto' - -env: - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: ${{ matrix.title }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - image: - - conda-cpp - - ubuntu-cpp-sanitizer - include: - - image: conda-cpp - title: AMD64 Conda C++ - - image: ubuntu-cpp-sanitizer - title: AMD64 Ubuntu 20.04 C++ ASAN UBSAN - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.image }}- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ${{ matrix.image }} - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - docker-arm: - # NOTE: this job is specific for self-hosted runners - # CACHING: don't use the cache plugin because of various permission - # issues and keep the cached docker volumes permanently on the - # host - # PYTHON: no distributions are built for arm machines by the github - # actions team, so python>3.6 must be preinstalled on the self - # hosted machines - name: ${{ matrix.title }} - runs-on: ${{ matrix.runner }} - # TODO(kszucs): re-enable once the self-hosted workers are properly - # registered to github - if: false && github.event_name == 'push' - defaults: - # To use certain environment variables set by .bashrc, an interactive - # bash shell must be used - run: - shell: bash -i {0} - strategy: - fail-fast: false - matrix: - name: - - arm32v7-debian-10-cpp - - arm64v8-ubuntu-20.04-cpp - include: - - name: arm32v7-debian-10-cpp - debian: 10 - title: ARM32v7 Debian 10 C++ - image: | - -e CPP_MAKE_PARALLELISM=2 \ - -e CXXFLAGS=-Wno-psabi \ - -e ARROW_PARQUET=OFF \ - -e ARROW_FLIGHT=OFF \ - -e ARROW_GANDIVA=OFF \ - -e ARROW_ORC=OFF \ - -e CMAKE_ARGS=-DARROW_CPU_FLAG=armv7 \ - debian-cpp - arch: 'arm32v7' - runner: [self-hosted, linux, ARM] - - name: arm64v8-ubuntu-20.04-cpp - ubuntu: 20.04 - title: ARM64v8 Ubuntu 20.04 C++ - image: | - -e CPP_MAKE_PARALLELISM=1 \ - -e ARROW_PARQUET=OFF \ - ubuntu-cpp - arch: 'arm64v8' - runner: [self-hosted, linux, ARM64] - env: - # the defaults here should correspond to the values in .env - ARCH: ${{ matrix.arch || 'arm64v8' }} - DEBIAN: ${{ matrix.debian || 10 }} - FEDORA: ${{ matrix.fedora || 32 }} - UBUNTU: ${{ matrix.ubuntu || 18.04 }} - LLVM: 8 - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Setup Archery - run: pip install -U -e dev/archery[docker] - - name: Execute Docker Build - # parallelism is reduced because the ARM builders are low on memory - run: | - ulimit -c unlimited - archery docker run ${{ matrix.image }} - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - build-example: - name: C++ Minimal Build Example - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Run - run: | - cd cpp/examples/minimal_build - docker-compose run --rm minimal - - macos: - name: AMD64 MacOS 10.15 C++ - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - env: - ARROW_BUILD_TESTS: ON - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_HDFS: ON - ARROW_HOME: /usr/local - ARROW_JEMALLOC: ON - # TODO(kszucs): link error in the tests - ARROW_ORC: OFF - ARROW_PARQUET: ON - ARROW_PLASMA: ON - ARROW_S3: ON - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Dependencies - shell: bash - run: | - rm -f /usr/local/bin/2to3 - brew update --preinstall - brew unlink gcc@8 gcc@9 - brew bundle --file=cpp/Brewfile - - name: Build - shell: bash - run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: | - sudo sysctl -w kern.coredump=1 - sudo sysctl -w kern.corefile=core.%N.%P - ulimit -c unlimited # must enable within the same shell - ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 ${{ matrix.name }} C++ - runs-on: ${{ matrix.os }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - os: - - windows-latest - include: - - os: windows-latest - name: Windows 2019 - generator: Visual Studio 16 2019 - env: - ARROW_BOOST_USE_SHARED: OFF - ARROW_BUILD_BENCHMARKS: ON - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_DATASET: ON - ARROW_FLIGHT: OFF - ARROW_HDFS: ON - ARROW_HOME: /usr - ARROW_JEMALLOC: OFF - ARROW_MIMALLOC: ON - ARROW_PARQUET: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: OFF - ARROW_WITH_BZ2: OFF - ARROW_WITH_LZ4: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - BOOST_SOURCE: BUNDLED - CMAKE_ARGS: '-A x64 -DOPENSSL_ROOT_DIR=C:\Program Files\OpenSSL-Win64' - CMAKE_GENERATOR: ${{ matrix.generator }} - CMAKE_INSTALL_LIBDIR: bin - CMAKE_INSTALL_PREFIX: /usr - CMAKE_UNITY_BUILD: ON - NPROC: 2 - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Installed Packages - run: choco list -l - - name: Install Dependencies - run: choco install -y --no-progress openssl - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: | - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} C++ - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - mingw-n-bits: - - 32 - - 64 - env: - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_BUILD_TYPE: release - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_HDFS: OFF - ARROW_HOME: /mingw${{ matrix.mingw-n-bits }} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_PYTHON: ON - ARROW_S3: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - # Don't use preinstalled Boost by empty BOOST_ROOT and - # -DBoost_NO_BOOST_CMAKE=ON - BOOST_ROOT: "" - CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} - -DBoost_NO_BOOST_CMAKE=ON - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW${{ matrix.mingw-n-bits }} - update: true - - name: Setup MSYS2 - shell: msys2 {0} - run: | - ci/scripts/msys2_setup.sh cpp - - name: Cache ccache - uses: actions/cache@v2 - with: - path: ccache - key: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} - restore-keys: cpp-ccache-mingw${{ matrix.mingw-n-bits }}- - - name: Build - shell: msys2 {0} - run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS - ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" - - name: Download MinIO - shell: msys2 {0} - run: | - mkdir -p /usr/local/bin - wget \ - --output-document /usr/local/bin/minio.exe \ - https://dl.min.io/server/minio/release/windows-amd64/minio.exe - chmod +x /usr/local/bin/minio.exe - - name: Test - shell: msys2 {0} - run: | - python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[0:2])))") - export PYTHONHOME="$(cygpath --windows ${MINGW_PREFIX})\lib\python${python_version}" - PYTHONPATH="${PYTHONHOME}" - PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\lib-dynload" - PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\site-packages" - export PYTHONPATH - ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build" diff --git a/.github/workflows/cpp_cron.yml b/.github/workflows/cpp_cron.yml deleted file mode 100644 index 9e4f3cf388b3..000000000000 --- a/.github/workflows/cpp_cron.yml +++ /dev/null @@ -1,149 +0,0 @@ - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C++ Cron - -on: - push: - paths: - - '.github/workflows/cpp_cron.yml' - pull_request: - paths: - - '.github/workflows/cpp_cron.yml' - schedule: - - cron: | - 0 */12 * * * - -env: - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - docker: - name: ${{ matrix.title }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }} - strategy: - fail-fast: false - matrix: - name: - - amd64-debian-10-cpp - - amd64-fedora-33-cpp - - amd64-ubuntu-18.04-cpp - - amd64-ubuntu-20.04-cpp - include: - - name: amd64-debian-10-cpp - image: debian-cpp - title: AMD64 Debian 10 C++ - debian: 10 - - name: amd64-fedora-33-cpp - image: fedora-cpp - title: AMD64 Fedora 33 C++ - fedora: 33 - - name: amd64-ubuntu-18.04-cpp - image: ubuntu-cpp - title: AMD64 Ubuntu 18.04 C++ - ubuntu: 18.04 - - name: amd64-ubuntu-20.04-cpp - image: ubuntu-cpp - title: AMD64 Ubuntu 20.04 C++ - ubuntu: 20.04 - env: - # the defaults here should correspond to the values in .env - ARCH: 'amd64' - DEBIAN: ${{ matrix.debian || 10 }} - FEDORA: ${{ matrix.fedora || 33 }} - UBUNTU: ${{ matrix.ubuntu || 18.04 }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ${{ matrix.name }}-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.name }}- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ${{ matrix.image }} - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - oss-fuzz: - name: OSS-Fuzz build check - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }} - strategy: - fail-fast: false - matrix: - ubuntu: [18.04] - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh - - name: Checkout OSS-Fuzz - uses: actions/checkout@v1 - with: - path: oss-fuzz - repository: google/oss-fuzz - ref: master - - name: Install dependencies - working-directory: ../oss-fuzz - run: | - python3 -m pip install setuptools - python3 -m pip install -r infra/ci/requirements.txt - - name: Build image - shell: bash - working-directory: ../oss-fuzz - run: | - python3 infra/helper.py build_image --pull arrow - - name: Build fuzzers - shell: bash - working-directory: ../oss-fuzz - run: | - python3 infra/helper.py build_fuzzers arrow `pwd`/../arrow - - name: Check build - shell: bash - working-directory: ../oss-fuzz - run: | - python3 infra/helper.py check_build arrow diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml deleted file mode 100644 index 03a297bb9142..000000000000 --- a/.github/workflows/csharp.yml +++ /dev/null @@ -1,121 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C# - -on: - push: - paths: - - '.github/workflows/csharp.yml' - - 'ci/scripts/csharp_*' - - 'csharp/**' - pull_request: - paths: - - '.github/workflows/csharp.yml' - - 'ci/scripts/csharp_*' - - 'csharp/**' - -jobs: - - ubuntu: - name: AMD64 Ubuntu 18.04 C# ${{ matrix.dotnet }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - dotnet: ['3.1.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v1 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Source Link - shell: bash - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) - - windows: - name: AMD64 Windows 2019 18.04 C# ${{ matrix.dotnet }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - dotnet: ['3.1.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v1 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Source Link - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) - - macos: - name: AMD64 MacOS 10.15 C# ${{ matrix.dotnet }} - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - dotnet: ['3.1.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v1 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Install Source Link - shell: bash - run: dotnet tool install --global sourcelink - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 9d8146a2f1d6..6c6dd830e161 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -22,10 +22,6 @@ on: push: pull_request: -env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - jobs: lint: diff --git a/.github/workflows/dev_pr/labeler.yml b/.github/workflows/dev_pr/labeler.yml index 098e1bad7f47..5eb722da4186 100644 --- a/.github/workflows/dev_pr/labeler.yml +++ b/.github/workflows/dev_pr/labeler.yml @@ -15,60 +15,8 @@ # specific language governing permissions and limitations # under the License. -"lang-c++": - - cpp/**/* - -lang-c-glib: - - c_glib/**/* - -lang-csharp: - - csharp/**/* - -lang-go: - - go/**/* - -lang-java: - - java/**/* - -lang-js: - - js/**/* - -lang-julia: - - julia/**/* - -lang-python: - - python/**/* - -lang-R: - - r/**/* - -lang-ruby: - - ruby/**/* - -lang-rust: - - rust/**/* - datafusion: - rust/datafusion/**/* ballista: - rust/ballista/**/* - -flight: - - cpp/src/arrow/flight/**/* - - r/R/flight.* - - rust/arrow-flight/**/* - - python/pyarrow/*flight.* - -gandiva: - - c_glib/gandiva-glib/**/* - - cpp/src/gandiva/**/* - - ruby/red-gandiva/**/* - - python/pyarrow/gandiva.* - -parquet: - - c_glib/parquet-glib/**/* - - cpp/src/parquet/**/* - - r/R/parquet.* - - ruby/red-parquet/**/* - - rust/parquet*/**/* diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 574795f5e9b1..000000000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,125 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Go - -on: - push: - paths: - - '.github/workflows/go.yml' - - 'ci/docker/*_go.dockerfile' - - 'ci/scripts/go_*' - - 'go/**' - pull_request: - paths: - - '.github/workflows/go.yml' - - 'ci/docker/*_go.dockerfile' - - 'ci/docker/**' - - 'ci/scripts/go_*' - - 'go/**' - -env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: AMD64 Debian 10 Go ${{ matrix.go }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - go: [1.15] - env: - GO: ${{ matrix.go }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run debian-go - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-go - - windows: - name: AMD64 Windows 2019 Go ${{ matrix.go }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - go: [1.15] - steps: - - name: Install go - uses: actions/setup-go@v1 - with: - go-version: ${{ matrix.go }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: ci/scripts/go_build.sh . - - name: Test - shell: bash - run: ci/scripts/go_test.sh . - - macos: - name: AMD64 MacOS 10.15 Go ${{ matrix.go }} - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - go: [1.15] - steps: - - name: Install go - uses: actions/setup-go@v1 - with: - go-version: ${{ matrix.go }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: ci/scripts/go_build.sh . - - name: Test - shell: bash - run: ci/scripts/go_test.sh . diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml deleted file mode 100644 index 7f6f29f0f444..000000000000 --- a/.github/workflows/java.yml +++ /dev/null @@ -1,112 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java - -on: - push: - paths: - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'format/Flight.proto' - - 'java/**' - pull_request: - paths: - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'format/Flight.proto' - - 'java/**' - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - debian: - name: AMD64 Debian 9 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - jdk: [11] - maven: [3.6.2] - env: - JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run debian-java - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-java - - macos: - name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }} - runs-on: macos-latest - if: github.event_name == 'push' - strategy: - fail-fast: false - matrix: - jdk: [11] - steps: - - name: Set up Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.jdk }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml deleted file mode 100644 index 5f25e8c053d8..000000000000 --- a/.github/workflows/java_jni.yml +++ /dev/null @@ -1,83 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java JNI - -on: - push: - paths: - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'java/**' - pull_request: - paths: - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'java/**' - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset) - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - jdk: [8] - maven: [3.5.2] - env: - JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run debian-java-jni - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-java-jni diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml deleted file mode 100644 index 354c45c60d30..000000000000 --- a/.github/workflows/js.yml +++ /dev/null @@ -1,122 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: NodeJS - -on: - push: - paths: - - '.github/workflows/js.yml' - - 'ci/docker/*js.dockerfile' - - 'ci/scripts/js_*' - - 'js/**' - pull_request: - paths: - - '.github/workflows/js.yml' - - 'ci/docker/*js.dockerfile' - - 'ci/scripts/js_*' - - 'js/**' - -env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: AMD64 Debian 10 NodeJS 14 - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run debian-js - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-js - - macos: - name: AMD64 MacOS 10.15 NodeJS ${{ matrix.node }} - runs-on: macos-latest - if: github.event_name == 'push' - strategy: - fail-fast: false - matrix: - node: [14] - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install NodeJS - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node }} - - name: Build - shell: bash - run: ci/scripts/js_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/js_test.sh $(pwd) - - # TODO(kszucs): the windows build fails with platform specific npm error - # windows: - # name: AMD64 Windows 2019 NodeJS ${{ matrix.node }} - # runs-on: windows-latest - # if: github.event_name == 'push' - # strategy: - # fail-fast: false - # matrix: - # node: [14] - # steps: - # - name: Checkout Arrow - # uses: actions/checkout@v1 - # with: - # submodules: true - # - name: Install NodeJS - # uses: actions/setup-node@v1 - # with: - # node-version: ${{ matrix.node }} - # - name: Install Platform Dependencies - # shell: bash - # run: yarn add -g cross-env - # - name: Build - # shell: bash - # run: ci/scripts/js_build.sh $(pwd) - # - name: Test - # shell: bash - # run: ci/scripts/js_test.sh $(pwd) diff --git a/.github/workflows/julia.yml b/.github/workflows/julia.yml deleted file mode 100644 index 64ea6c947a15..000000000000 --- a/.github/workflows/julia.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Julia -on: - push: - paths: - - '.github/workflows/julia.yml' - - 'julia/**' - pull_request: - paths: - - '.github/workflows/julia.yml' - - 'julia/**' - -jobs: - test: - name: AMD64 ${{ matrix.os }} Julia ${{ matrix.version }} - env: - JULIA_NUM_THREADS: 2 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - version: - - '1.3' - - '1' # automatically expands to the latest stable 1.x release of Julia - - 'nightly' - os: - - ubuntu-latest - - windows-latest - steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.version }} - arch: x64 - - uses: julia-actions/julia-runtest@v1 - with: - project: julia/Arrow diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index 9062e93e6651..000000000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,154 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Python - -on: - push: - paths: - - '.github/workflows/python.yml' - - 'ci/**' - - 'cpp/**' - - 'python/**' - pull_request: - paths: - - '.github/workflows/python.yml' - - 'ci/**' - - 'cpp/**' - - 'python/**' - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: ${{ matrix.title }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - name: - - conda-python-3.8-nopandas - - conda-python-3.6-pandas-0.23 - - conda-python-3.7-pandas-latest - include: - - name: conda-python-3.8-nopandas - cache: conda-python-3.8 - image: conda-python - title: AMD64 Conda Python 3.8 Without Pandas - python: 3.8 - - name: conda-python-3.6-pandas-0.23 - cache: conda-python-3.6 - image: conda-python-pandas - title: AMD64 Conda Python 3.6 Pandas 0.23 - python: 3.6 - pandas: 0.23 - - name: conda-python-3.7-pandas-latest - cache: conda-python-3.7 - image: conda-python-pandas - title: AMD64 Conda Python 3.7 Pandas latest - python: 3.7 - pandas: latest - env: - PYTHON: ${{ matrix.python || 3.7 }} - UBUNTU: ${{ matrix.ubuntu || 18.04 }} - PANDAS: ${{ matrix.pandas || 'latest' }} - NUMPY: ${{ matrix.numpy || 'latest' }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.cache }}- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ${{ matrix.image }} - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - macos: - name: AMD64 MacOS 10.15 Python 3 - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - env: - ARROW_HOME: /usr/local - ARROW_DATASET: ON - ARROW_JEMALLOC: ON - ARROW_ORC: ON - ARROW_PYTHON: ON - ARROW_PLASMA: ON - ARROW_GANDIVA: ON - ARROW_PARQUET: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_ZSTD: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_BROTLI: ON - ARROW_BUILD_TESTS: OFF - CMAKE_ARGS: "-DPython3_EXECUTABLE=/usr/local/bin/python3" - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Dependencies - shell: bash - run: | - rm -f /usr/local/bin/2to3 - brew update --preinstall - brew unlink gcc@8 gcc@9 - brew bundle --file=cpp/Brewfile - brew install coreutils - python3 -mpip install \ - -r python/requirements-build.txt \ - -r python/requirements-test.txt - - name: Build - shell: bash - run: | - export PYTHON=python3 - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - ci/scripts/python_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/python_test.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/python_cron.yml b/.github/workflows/python_cron.yml deleted file mode 100644 index 7a4401af1c3b..000000000000 --- a/.github/workflows/python_cron.yml +++ /dev/null @@ -1,141 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Python Cron - -on: - push: - paths: - - '.github/workflows/python_cron.yml' - pull_request: - paths: - - '.github/workflows/python_cron.yml' - schedule: - - cron: | - 0 */12 * * * - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: ${{ matrix.title }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }} - strategy: - fail-fast: false - matrix: - name: - - debian-10-python-3 - - fedora-33-python-3 - - ubuntu-18.04-python-3 - - conda-python-3.7-dask-latest - - conda-python-3.7-turbodbc-latest - - conda-python-3.7-kartothek-latest - - conda-python-3.7-pandas-0.24 - - conda-python-3.7-pandas-master - - conda-python-3.7-hdfs-2.9.2 - include: - - name: debian-10-python-3 - cache: debian-10-python-3 - image: debian-python - title: AMD64 Debian 10 Python 3 - debian: 10 - - name: fedora-33-python-3 - cache: fedora-33-python-3 - image: fedora-python - title: AMD64 Fedora 33 Python 3 - fedora: 33 - - name: ubuntu-18.04-python-3 - cache: ubuntu-18.04-python-3 - image: ubuntu-python - title: AMD64 Ubuntu 18.04 Python 3 - ubuntu: 18.04 - - name: conda-python-3.7-dask-latest - cache: conda-python-3.7 - image: conda-python-dask - title: AMD64 Conda Python 3.7 Dask latest - dask: latest - - name: conda-python-3.7-turbodbc-latest - cache: conda-python-3.7 - image: conda-python-turbodbc - title: AMD64 Conda Python 3.7 Turbodbc latest - turbodbc: latest - - name: conda-python-3.7-kartothek-latest - cache: conda-python-3.7 - image: conda-python-kartothek - title: AMD64 Conda Python 3.7 Kartothek latest - kartothek: latest - - name: conda-python-3.7-pandas-0.24 - cache: conda-python-3.7 - image: conda-python-pandas - title: AMD64 Conda Python 3.7 Pandas 0.24 - pandas: 0.24 - - name: conda-python-3.7-pandas-master - cache: conda-python-3.7 - image: --no-leaf-cache conda-python-pandas - title: AMD64 Conda Python 3.7 Pandas master - pandas: master - - name: conda-python-3.7-hdfs-2.9.2 - cache: conda-python-3.7 - image: conda-python-hdfs - title: AMD64 Conda Python 3.7 HDFS 2.9.2 - hdfs: 2.9.2 - env: - # the defaults here should correspond to the values in .env - DEBIAN: ${{ matrix.debian || 10 }} - FEDORA: ${{ matrix.fedora || 33 }} - UBUNTU: ${{ matrix.ubuntu || 18.04 }} - PYTHON: ${{ matrix.python || 3.7 }} - HDFS: ${{ matrix.hdfs || '2.9.2' }} - DASK: ${{ matrix.dask || 'latest' }} - TURBODBC: ${{ matrix.turbodbc || 'latest' }} - PANDAS: ${{ matrix.pandas || 'latest' }} - KARTOTHEK: ${{ matrix.kartothek || 'latest' }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.cache }}- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ${{ matrix.image }} - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ${{ matrix.image }} diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml deleted file mode 100644 index 7851b6b1915e..000000000000 --- a/.github/workflows/r.yml +++ /dev/null @@ -1,255 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: R - -on: - push: - paths: - - ".github/workflows/r.yml" - - "ci/scripts/r_*.sh" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/PKGBUILD" - - "ci/etc/rprofile" - - "ci/docker/**" - - "cpp/**" - - "r/**" - pull_request: - paths: - - ".github/workflows/r.yml" - - "ci/scripts/r_*.sh" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/PKGBUILD" - - "ci/etc/rprofile" - - "ci/docker/**" - - "cpp/**" - - "r/**" - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - r: ["3.6"] - ubuntu: [18.04] - env: - R: ${{ matrix.r }} - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/**') }} - restore-keys: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}- - - name: Check pkgdown reference sections - run: ci/scripts/r_pkgdown_check.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ubuntu-r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ubuntu-r - - bundled: - name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{ matrix.config.tag }}" - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - config: - - { org: "rstudio", image: "r-base", tag: "4.0-centos7" } - - { org: "rhub", image: "debian-gcc-devel", tag: "latest" } - env: - R_ORG: ${{ matrix.config.org }} - R_IMAGE: ${{ matrix.config.image }} - R_TAG: ${{ matrix.config.tag }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ${{ matrix.config.image }}-r-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.config.image }}-r- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push r - - windows: - name: AMD64 Windows RTools ${{ matrix.rtools }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - rtools: [35, 40] - env: - TEST_R_WITH_ARROW: "TRUE" - ARROW_R_CXXFLAGS: "-Werror" - _R_CHECK_TESTS_NLINES_: 0 - steps: - - run: git config --global core.autocrlf false - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Make R tests verbose - # If you get a segfault/mysterious test Execution halted, - # make this `true` to see where it dies. - if: false - shell: cmd - run: | - cd r/tests - sed -i.bak -E -e 's/"arrow"/"arrow", reporter = "location"/' testthat.R - rm -f testthat.R.bak - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV - # We must enable actions/cache before r-lib/actions/setup-r to ensure - # using system tar instead of tar provided by Rtools. - # We can use tar provided by Rtools when we drop support for Rtools 3.5. - # Because Rtools 4.0 or later has zstd. actions/cache requires zstd - # when tar is GNU tar. - - name: Cache ccache - uses: actions/cache@v2 - with: - path: ccache - key: r-${{ matrix.rtools }}-ccache-mingw-${{ hashFiles('cpp/**') }} - restore-keys: r-${{ matrix.rtools }}-ccache-mingw- - # We use the makepkg-mingw setup that is included in rtools40 even when - # we use the rtools35 compilers, so we always install R 4.0/Rtools40 - - uses: r-lib/actions/setup-r@master - with: - rtools-version: 40 - r-version: "4.0" - Ncpus: 2 - - uses: r-lib/actions/setup-r@master - if: ${{ matrix.rtools == 35 }} - with: - rtools-version: 35 - r-version: "3.6" - Ncpus: 2 - - name: Build Arrow C++ - shell: bash - env: - RTOOLS_VERSION: ${{ matrix.rtools }} - run: ci/scripts/r_windows_build.sh - - uses: actions/upload-artifact@v1 - with: - name: Rtools ${{ matrix.rtools }} Arrow C++ - path: libarrow.zip - - name: Install R package dependencies - shell: Rscript {0} - run: | - options(pkgType="win.binary") - install.packages(c("remotes", "rcmdcheck")) - remotes::install_deps("r", dependencies = TRUE) - - name: Check - shell: Rscript {0} - run: | - Sys.setenv( - RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "libarrow.zip"), - MAKEFLAGS = paste0("-j", parallel::detectCores()) - ) - rcmdcheck::rcmdcheck("r", - build_args = '--no-build-vignettes', - args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'), - error_on = 'warning', - check_dir = 'check', - timeout = 3600 - ) - - name: Dump install logs - shell: cmd - run: cat check/arrow.Rcheck/00install.out - if: always() - # We can remove this when we drop support for Rtools 3.5. - - name: Ensure using system tar in actions/cache - run: | - Write-Output "${Env:windir}\System32" | ` - Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml deleted file mode 100644 index 2b99cddf8da1..000000000000 --- a/.github/workflows/ruby.yml +++ /dev/null @@ -1,290 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C GLib & Ruby - -on: - push: - paths: - - '.github/workflows/ruby.yml' - - 'ci/docker/**' - - 'ci/scripts/c_glib_*' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/ruby_*' - - 'ci/scripts/util_*' - - 'c_glib/**' - - 'cpp/**' - - 'ruby/**' - pull_request: - paths: - - '.github/workflows/ruby.yml' - - 'ci/docker/**' - - 'ci/scripts/c_glib_*' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/ruby_*' - - 'ci/scripts/util_*' - - 'c_glib/**' - - 'cpp/**' - - 'ruby/**' - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - ubuntu: - - 18.04 - - 20.04 - env: - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ubuntu-${{ matrix.ubuntu }}-ruby-${{ hashFiles('cpp/**') }} - restore-keys: ubuntu-${{ matrix.ubuntu }}-ruby- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ubuntu-ruby - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - shell: bash - run: archery docker push ubuntu-ruby - - macos: - name: AMD64 MacOS 10.15 GLib & Ruby - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - env: - ARROW_BUILD_TESTS: OFF - ARROW_GANDIVA: ON - ARROW_GLIB_DEVELOPMENT_MODE: true - ARROW_GLIB_GTK_DOC: true - ARROW_HOME: /usr/local - ARROW_JEMALLOC: OFF - ARROW_ORC: OFF - ARROW_PARQUET: ON - ARROW_WITH_BROTLI: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - XML_CATALOG_FILES: /usr/local/etc/xml/catalog - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Homebrew Dependencies - shell: bash - run: | - rm -f /usr/local/bin/2to3 - brew update --preinstall - brew unlink gcc@8 gcc@9 - brew bundle --file=cpp/Brewfile - brew bundle --file=c_glib/Brewfile - - name: Install Ruby Dependencies - run: | - export MAKEFLAGS="-j$(sysctl -n hw.ncpu)" - bundle install --gemfile c_glib/Gemfile - bundle install --gemfile ruby/Gemfile - for ruby_package_gemfile in ruby/*/Gemfile; do \ - bundle install --gemfile ${ruby_package_gemfile} - done - - name: Setup ccache - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - run: | - echo "::set-output name=cache-dir::$(ccache --get-config cache_dir)" - - name: Cache ccache - uses: actions/cache@v2 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: ruby-ccache-macos-${{ hashFiles('cpp/**') }} - restore-keys: ruby-ccache-macos- - - name: Build C++ - run: | - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Build GLib - run: | - ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build - - name: Test GLib - shell: bash - run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/build - - name: Test Ruby - shell: bash - run: ci/scripts/ruby_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} GLib & Ruby - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - strategy: - fail-fast: false - matrix: - mingw-n-bits: - - 64 - ruby-version: - - 2.6 - env: - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_BUILD_TYPE: release - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_HDFS: OFF - ARROW_HOME: /mingw${{ matrix.mingw-n-bits }} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_PYTHON: OFF - ARROW_S3: ON - ARROW_USE_GLOG: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - # Don't use preinstalled Boost by empty BOOST_ROOT and - # -DBoost_NO_BOOST_CMAKE=ON - BOOST_ROOT: "" - CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} - -DBoost_NO_BOOST_CMAKE=ON - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby-version }} - - name: Upgrade MSYS2 - run: | - ridk exec bash ci\scripts\msys2_system_upgrade.sh - taskkill /F /FI "MODULES eq msys-2.0.dll" - - name: Clean MSYS2 - run: | - ridk exec bash ci\scripts\msys2_system_clean.sh - - name: Setup MSYS2 - run: | - ridk exec bash ci\scripts\msys2_setup.sh ruby - - name: Cache ccache - uses: actions/cache@v2 - with: - path: ccache - key: ruby-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} - restore-keys: ruby-ccache-mingw${{ matrix.mingw-n-bits }}- - - name: Build C++ - run: | - $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\cpp_build.sh "${source_dir}" "${build_dir}" - - name: Build GLib - run: | - $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_build.sh "${source_dir}" "${build_dir}" - - name: RubyGems info - id: rubygems-info - run: | - Write-Output "::set-output name=gem-dir::$(ridk exec gem env gemdir)" - - name: Cache RubyGems - uses: actions/cache@v2 - with: - path: ${{ steps.rubygems-info.outputs.gem-dir }} - key: ruby-rubygems-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('**/Gemfile', 'ruby/*/*.gemspec') }} - restore-keys: ruby-rubygems-mingw${{ matrix.mingw-n-bits }}- - - name: Install test dependencies - run: | - bundle install --gemfile c_glib\Gemfile - bundle install --gemfile ruby\Gemfile - Get-ChildItem ruby\*\Gemfile | ` - ForEach-Object {bundle install --gemfile $_} - - name: Test GLib - run: | - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_test.sh "${source_dir}" "${build_dir}" - - name: Test Ruby - run: | - $Env:PKG_CONFIG_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/pkgconfig")" - $Env:GI_TYPELIB_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/girepository-1.0")" - $Env:RUBYOPTS = "-rdevkit" - $Env:MAKE = "ridk exec make" - $ErrorActionPreference = "Continue" - rake -f ruby\Rakefile From eed1b82b1c2ff817129f75533170f8e6148dbf07 Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Sun, 18 Apr 2021 18:41:41 +0000 Subject: [PATCH 3/8] More cleanups. --- .gitignore | 6 ++ dev/.gitignore | 1 + dev/archery/archery/cli.py | 5 +- dev/archery/archery/utils/source.py | 8 +- dev/release/rat_exclude_files.txt | 111 +--------------------------- 5 files changed, 15 insertions(+), 116 deletions(-) diff --git a/.gitignore b/.gitignore index 5397fe371dfd..31bdf49ce43e 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,9 @@ cpp/Brewfile.lock.json target Cargo.lock +rusty-tags.vi +.history +.flatbuffers/ + +.vscode +venv/* diff --git a/dev/.gitignore b/dev/.gitignore index b0792939f2ad..399c30926260 100644 --- a/dev/.gitignore +++ b/dev/.gitignore @@ -18,3 +18,4 @@ # Python virtual environments for dev tools .venv*/ +__pycache__ diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index bcaddf1c795d..4bbde75b74cf 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -289,8 +289,7 @@ def decorate_lint_command(cmd): @archery.command(short_help="Check Arrow source tree for errors") -@click.option("--src", metavar="", default=None, - callback=validate_arrow_sources, +@click.option("--src", metavar="", default=".", help="Specify Arrow source directory") @click.option("--fix", is_flag=True, type=BOOL, default=False, help="Toggle fixing the lint errors if the linter supports it.") @@ -301,6 +300,8 @@ def decorate_lint_command(cmd): @decorate_lint_command @click.pass_context def lint(ctx, src, fix, iwyu_all, **checks): + src = ArrowSources(src) + if checks.pop('all'): # "--all" is given => enable all non-selected checks for k, v in checks.items(): diff --git a/dev/archery/archery/utils/source.py b/dev/archery/archery/utils/source.py index d30b4f152e54..1ae0fe025049 100644 --- a/dev/archery/archery/utils/source.py +++ b/dev/archery/archery/utils/source.py @@ -45,13 +45,7 @@ def __init__(self, path): ---------- path : src """ - path = Path(path) - # validate by checking a specific path in the arrow source tree - if not (path / 'cpp' / 'CMakeLists.txt').exists(): - raise InvalidArrowSource( - "No Arrow C++ sources found in {}.".format(path) - ) - self.path = path + self.path = Path(path) @property def archery(self): diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 68f5668098e7..ead9c8db16f4 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -12,37 +12,6 @@ ci/etc/rprofile ci/etc/*.patch ci/vcpkg/*.patch CHANGELOG.md -cpp/CHANGELOG_PARQUET.md -cpp/src/arrow/io/mman.h -cpp/src/arrow/util/random.h -cpp/src/arrow/status.cc -cpp/src/arrow/status.h -cpp/src/arrow/vendored/* -cpp/build-support/asan_symbolize.py -cpp/build-support/cpplint.py -cpp/build-support/lint_exclusions.txt -cpp/build-support/iwyu/* -cpp/cmake_modules/FindPythonLibsNew.cmake -cpp/cmake_modules/SnappyCMakeLists.txt -cpp/cmake_modules/SnappyConfig.h -cpp/examples/parquet/parquet-arrow/cmake_modules/FindArrow.cmake -cpp/src/parquet/.parquetcppversion -cpp/src/generated/parquet_constants.cpp -cpp/src/generated/parquet_constants.h -cpp/src/generated/parquet_types.cpp -cpp/src/generated/parquet_types.h -cpp/src/plasma/thirdparty/ae/ae.c -cpp/src/plasma/thirdparty/ae/ae.h -cpp/src/plasma/thirdparty/ae/ae_epoll.c -cpp/src/plasma/thirdparty/ae/ae_evport.c -cpp/src/plasma/thirdparty/ae/ae_kqueue.c -cpp/src/plasma/thirdparty/ae/ae_select.c -cpp/src/plasma/thirdparty/ae/config.h -cpp/src/plasma/thirdparty/ae/zmalloc.h -cpp/src/plasma/thirdparty/dlmalloc.c -cpp/thirdparty/flatbuffers/include/flatbuffers/base.h -cpp/thirdparty/flatbuffers/include/flatbuffers/flatbuffers.h -cpp/thirdparty/flatbuffers/include/flatbuffers/stl_emulation.h dev/requirements*.txt dev/archery/MANIFEST.in dev/archery/requirements*.txt @@ -115,56 +84,11 @@ dev/tasks/linux-packages/apache-arrow/debian/source/format dev/tasks/linux-packages/apache-arrow/debian/watch dev/tasks/requirements*.txt dev/tasks/conda-recipes/* -docs/requirements.txt -go/arrow/flight/Flight_grpc.pb.go -go/arrow/go.sum -go/arrow/Gopkg.lock -go/arrow/flight/Flight.pb.go -go/arrow/flight/Flight_grpc.pb.go -go/arrow/internal/cpu/* -go/arrow/type_string.go -go/*.tmpldata -go/*.s -go/parquet/go.sum -go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go -go/parquet/internal/gen-go/parquet/parquet-consts.go -go/parquet/internal/gen-go/parquet/parquet.go -js/.npmignore -js/closure-compiler-scripts/* -js/src/fb/*.ts -js/yarn.lock -js/.eslintignore -python/cmake_modules -python/cmake_modules/FindPythonLibsNew.cmake -python/cmake_modules/SnappyCMakeLists.txt -python/cmake_modules/SnappyConfig.h -python/MANIFEST.in -python/manylinux1/.dockerignore -python/pyarrow/includes/__init__.pxd -python/pyarrow/tests/__init__.py -python/pyarrow/vendored/* -python/requirements*.txt pax_global_header MANIFEST.in __init__.pxd __init__.py requirements.txt -csharp/.gitattributes -csharp/dummy.git/* -csharp/src/Apache.Arrow/Flatbuf/* -csharp/Apache.Arrow.sln -csharp/examples/FluentBuilderExample/FluentBuilderExample.csproj -csharp/examples/Examples.sln -csharp/src/Apache.Arrow/Apache.Arrow.csproj -csharp/src/Apache.Arrow/Properties/Resources.Designer.cs -csharp/src/Apache.Arrow/Properties/Resources.resx -csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj -csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj -csharp/test/Apache.Arrow.Tests/app.config -csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj -csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj -csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj -csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj *.html *.sgml *.css @@ -173,35 +97,8 @@ csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj *.svg *.devhelp2 *.scss -r/R/arrowExports.R -r/src/arrowExports.cpp -r/DESCRIPTION -r/LICENSE.md -r/NAMESPACE -r/.Rbuildignore -r/arrow.Rproj -r/README.md -r/README.Rmd -r/man/*.Rd -r/cran-comments.md -r/vignettes/*.Rmd -r/tests/testthat/test-*.txt -r/inst/include/cpp11.hpp -r/inst/include/cpp11/*.hpp .gitattributes -ruby/red-arrow/.yardopts -rust/arrow/test/data/*.csv -rust/rust-toolchain -rust/arrow-flight/src/arrow.flight.protocol.rs -julia/Arrow/Project.toml -julia/Arrow/README.md -julia/Arrow/docs/Manifest.toml -julia/Arrow/docs/Project.toml -julia/Arrow/docs/make.jl -julia/Arrow/docs/mkdocs.yml -julia/Arrow/docs/src/index.md -julia/Arrow/docs/src/manual.md -julia/Arrow/docs/src/reference.md -rust/ballista/rust/benchmarks/tpch/queries/q*.sql -rust/ballista/rust/scheduler/testdata/* -rust/ballista/ui/scheduler/yarn.lock +rust-toolchain +ballista/rust/benchmarks/tpch/queries/q*.sql +ballista/rust/scheduler/testdata/* +ballista/ui/scheduler/yarn.lock From 72b2f9b356164bd63036a1b7db763685a2c7503f Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Sun, 18 Apr 2021 18:57:02 +0000 Subject: [PATCH 4/8] More cleanups. --- .github/workflows/rust.yml | 13 ++++++------- ballista/rust/benchmarks/tpch/README.md | 6 +++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3f1031c44486..e2e604ece9d4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -70,14 +70,13 @@ jobs: run: | export CARGO_HOME="/github/home/.cargo" export CARGO_TARGET_DIR="/github/home/target" - cd rust cargo build # Ballista is currently not part of the main workspace so requires a separate build step - name: Build Ballista run: | export CARGO_HOME="/github/home/.cargo" export CARGO_TARGET_DIR="/github/home/target" - cd rust/ballista/rust + cd ballista/rust # snmalloc requires cmake so build without default features cargo build --no-default-features @@ -96,8 +95,8 @@ jobs: # Disable full debug symbol generation to speed up CI build and keep memory down # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" - ARROW_TEST_DATA: /__w/arrow/arrow/testing/data - PARQUET_TEST_DATA: /__w/arrow/arrow/cpp/submodules/parquet-testing/data + ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data + PARQUET_TEST_DATA: /__w/arrow-rs/arrow-rs/parquet-testing/data steps: - uses: actions/checkout@v2 with: @@ -135,7 +134,7 @@ jobs: run: | export CARGO_HOME="/github/home/.cargo" export CARGO_TARGET_DIR="/github/home/target" - cd rust/ballista/rust + cd ballista/rust # snmalloc requires cmake so build without default features cargo test --no-default-features @@ -161,7 +160,7 @@ jobs: shell: bash run: | export ARROW_TEST_DATA=$(pwd)/testing/data - export PARQUET_TEST_DATA=$(pwd)/cpp/submodules/parquet-testing/data + export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data # do not produce debug symbols to keep memory usage down export RUSTFLAGS="-C debuginfo=0" cargo test @@ -285,7 +284,7 @@ jobs: export CARGO_TARGET_DIR="/home/runner/target" export ARROW_TEST_DATA=$(pwd)/testing/data - export PARQUET_TEST_DATA=$(pwd)/cpp/submodules/parquet-testing/data + export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0 # see https://github.com/xd009642/tarpaulin/issues/618 diff --git a/ballista/rust/benchmarks/tpch/README.md b/ballista/rust/benchmarks/tpch/README.md index 6d77694b91b6..20c4fc71de35 100644 --- a/ballista/rust/benchmarks/tpch/README.md +++ b/ballista/rust/benchmarks/tpch/README.md @@ -41,7 +41,7 @@ To run the benchmarks it is necessary to have at least one Ballista scheduler an To run the scheduler from source: ```bash -cd $ARROW_HOME/rust/ballista/rust/scheduler +cd $ARROW_HOME/ballista/rust/scheduler RUST_LOG=info cargo run --release ``` @@ -50,7 +50,7 @@ By default the scheduler will bind to `0.0.0.0` and listen on port 50050. To run the executor from source: ```bash -cd $ARROW_HOME/rust/ballista/rust/executor +cd $ARROW_HOME/ballista/rust/executor RUST_LOG=info cargo run --release ``` @@ -65,7 +65,7 @@ RUST_LOG=info RUSTFLAGS='-C target-cpu=native -C lto -C codegen-units=1 -C embed To run the benchmarks: ```bash -cd $ARROW_HOME/rust/ballista/rust/benchmarks/tpch +cd $ARROW_HOME/ballista/rust/benchmarks/tpch cargo run --release benchmark --host localhost --port 50050 --query 1 --path $(pwd)/data --format tbl ``` From 17a08020b2c902a68d9d7901c22780bc340806bd Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Sun, 18 Apr 2021 19:13:59 +0000 Subject: [PATCH 5/8] Bumped arrow to latest. --- ballista/rust/benchmarks/tpch/Cargo.toml | 4 ++-- ballista/rust/client/Cargo.toml | 2 +- ballista/rust/core/Cargo.toml | 4 ++-- ballista/rust/executor/Cargo.toml | 4 ++-- ballista/rust/scheduler/Cargo.toml | 2 +- benchmarks/Cargo.toml | 4 ++-- datafusion-examples/Cargo.toml | 4 ++-- datafusion/Cargo.toml | 4 ++-- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ballista/rust/benchmarks/tpch/Cargo.toml b/ballista/rust/benchmarks/tpch/Cargo.toml index 9a7d65199266..8d62e20e17e1 100644 --- a/ballista/rust/benchmarks/tpch/Cargo.toml +++ b/ballista/rust/benchmarks/tpch/Cargo.toml @@ -28,8 +28,8 @@ edition = "2018" ballista = { path="../../client" } datafusion = { path = "../../../../datafusion" } -arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } -parquet = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } +arrow = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } +parquet = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } env_logger = "0.8" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread"] } diff --git a/ballista/rust/client/Cargo.toml b/ballista/rust/client/Cargo.toml index bf40cdb499ac..6ac86875169b 100644 --- a/ballista/rust/client/Cargo.toml +++ b/ballista/rust/client/Cargo.toml @@ -31,5 +31,5 @@ futures = "0.3" log = "0.4" tokio = "1.0" -arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } +arrow = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } datafusion = { path = "../../../datafusion" } diff --git a/ballista/rust/core/Cargo.toml b/ballista/rust/core/Cargo.toml index 58e6d22734ac..e9d7682473f1 100644 --- a/ballista/rust/core/Cargo.toml +++ b/ballista/rust/core/Cargo.toml @@ -40,8 +40,8 @@ tokio = "1.0" tonic = "0.4" uuid = { version = "0.8", features = ["v4"] } -arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } -arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } +arrow = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } +arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } datafusion = { path = "../../../datafusion" } diff --git a/ballista/rust/executor/Cargo.toml b/ballista/rust/executor/Cargo.toml index ccf30cf16eb7..79ceabe2dd66 100644 --- a/ballista/rust/executor/Cargo.toml +++ b/ballista/rust/executor/Cargo.toml @@ -45,8 +45,8 @@ tokio-stream = "0.1" tonic = "0.4" uuid = { version = "0.8", features = ["v4"] } -arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } -arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } +arrow = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } +arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } datafusion = { path = "../../../datafusion" } diff --git a/ballista/rust/scheduler/Cargo.toml b/ballista/rust/scheduler/Cargo.toml index 197a2319154d..ce8ca09e15b2 100644 --- a/ballista/rust/scheduler/Cargo.toml +++ b/ballista/rust/scheduler/Cargo.toml @@ -52,7 +52,7 @@ tonic = "0.4" tower = { version = "0.4" } warp = "0.3" -arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } +arrow = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } datafusion = { path = "../../../datafusion" } [dev-dependencies] diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 7fd844471686..66a81be26b36 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -31,8 +31,8 @@ simd = ["datafusion/simd"] snmalloc = ["snmalloc-rs"] [dependencies] -arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } -parquet = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } +arrow = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } +parquet = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } datafusion = { path = "../datafusion" } structopt = { version = "0.3", default-features = false } tokio = { version = "^1.0", features = ["macros", "rt", "rt-multi-thread"] } diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index 1a060504df72..7f7c239d0f54 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -29,8 +29,8 @@ publish = false [dev-dependencies] -arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } -arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" } +arrow = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } +arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "08a662f" } datafusion = { path = "../datafusion" } prost = "0.7" tonic = "0.4" diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml index 88a50dd5a443..443bd7e02041 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/Cargo.toml @@ -50,8 +50,8 @@ unicode_expressions = ["unicode-segmentation"] [dependencies] ahash = "0.7" hashbrown = "0.11" -arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c", features = ["prettyprint"] } -parquet = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c", features = ["arrow"] } +arrow = { git = "https://github.com/apache/arrow-rs", rev = "08a662f", features = ["prettyprint"] } +parquet = { git = "https://github.com/apache/arrow-rs", rev = "08a662f", features = ["arrow"] } sqlparser = "0.9.0" clap = "2.33" rustyline = {version = "7.0", optional = true} From 61a739b61275c7427ed30772f609b59d9dbc66c4 Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Sun, 18 Apr 2021 19:23:28 +0000 Subject: [PATCH 6/8] Always test. --- .github/workflows/rust.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e2e604ece9d4..62460226b26b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -18,16 +18,9 @@ name: Rust on: + # always trigger push: - paths: - - '.github/workflows/rust.yml' - - 'rust/**' - - 'format/Flight.proto' pull_request: - paths: - - '.github/workflows/rust.yml' - - 'rust/**' - - 'format/Flight.proto' jobs: From 2da1e03e288ab705a1e0cbcc15619e56fa31d144 Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Sun, 18 Apr 2021 19:44:19 +0000 Subject: [PATCH 7/8] Removed unused files. --- .readthedocs.yml | 19 ------ .travis.yml | 165 ----------------------------------------------- 2 files changed, 184 deletions(-) delete mode 100644 .readthedocs.yml delete mode 100644 .travis.yml diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 11a7d70c2513..000000000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -conda: - file: python/doc/environment.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2cf70cca982f..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,165 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -dist: bionic - -language: minimal - -cache: - directories: - - $TRAVIS_BUILD_DIR/.docker - -addons: - apt: - packages: - - python3-pip - -services: - - docker - -# Note that the global "env" setting isn't inherited automatically by -# matrix entries with their own "env", so we have to insert it explicitly. -env: &global_env - ARROW_ENABLE_TIMING_TESTS: "OFF" - COMPOSE_DOCKER_CLI_BUILD: 1 - DOCKER_BUILDKIT: 0 - DOCKER_VOLUME_PREFIX: $TRAVIS_BUILD_DIR/.docker/ - -jobs: - include: - - name: "C++ on ARM" - os: linux - arch: arm64 - env: - <<: *global_env - ARCH: arm64v8 - ARROW_CI_MODULES: "CPP" - DOCKER_IMAGE_ID: ubuntu-cpp - # ARROW_USE_GLOG=OFF is needed to avoid build error caused by - # glog and CMAKE_UNITY_BUILD=ON. - # - # Disable ARROW_S3 because it often causes "No output has - # been received in the last 10m0s, this potentially indicates - # a stalled build or something wrong with the build itself." - # on Travis CI. - # - # Limiting CPP_MAKE_PARALLELISM is required to avoid random compiler - # crashes. - DOCKER_RUN_ARGS: >- - " - -e ARROW_BUILD_STATIC=OFF - -e ARROW_ORC=OFF - -e ARROW_S3=OFF - -e ARROW_USE_GLOG=OFF - -e CMAKE_UNITY_BUILD=ON - -e CPP_MAKE_PARALLELISM=4 - " - # The LLVM's APT repository provides only arm64 binaries. - # We should use LLVM provided by Ubuntu. - LLVM: "10" - UBUNTU: "20.04" - - - name: "C++ on s390x" - os: linux - arch: s390x - env: - <<: *global_env - ARCH: s390x - ARROW_CI_MODULES: "CPP" - DOCKER_IMAGE_ID: ubuntu-cpp - # Can't enable ARROW_MIMALLOC because of failures in memory pool tests. - # Can't enable ARROW_S3 because compiler is killed while compiling - # aws-sdk-cpp. - DOCKER_RUN_ARGS: >- - " - -e ARROW_BUILD_STATIC=OFF - -e ARROW_FLIGHT=ON - -e ARROW_MIMALLOC=OFF - -e ARROW_ORC=OFF - -e ARROW_PARQUET=OFF - -e ARROW_S3=OFF - -e CMAKE_UNITY_BUILD=ON - -e CPP_MAKE_PARALLELISM=4 - -e PARQUET_BUILD_EXAMPLES=OFF - -e PARQUET_BUILD_EXECUTABLES=OFF - -e Protobuf_SOURCE=BUNDLED - -e cares_SOURCE=BUNDLED - -e gRPC_SOURCE=BUNDLED - " - # The LLVM's APT repository provides only arm64 binaries. - # We should use LLVM provided by Ubuntu. - LLVM: "10" - UBUNTU: "20.04" - - - name: "Go on s390x" - os: linux - arch: s390x - env: - <<: *global_env - ARCH: s390x - ARROW_CI_MODULES: "GO" - DOCKER_IMAGE_ID: debian-go - - - name: "Java on s390x" - os: linux - arch: s390x - env: - <<: *global_env - ARCH: s390x - ARROW_CI_MODULES: "JAVA" - DOCKER_IMAGE_ID: debian-java - JDK: 11 - - allow_failures: - - name: "Go on s390x" - - name: "Java on s390x" - -before_install: - - eval "$(python ci/detect-changes.py)" - - | - arrow_ci_affected=no - for arrow_ci_module in ${ARROW_CI_MODULES}; do - arrow_ci_affected_variable=ARROW_CI_${arrow_ci_module}_AFFECTED - if [ "$(eval "echo \$${arrow_ci_affected_variable}")" = "1" ]; then - arrow_ci_affected=yes - fi - done - if [ "${arrow_ci_affected}" = "no" ]; then - travis_terminate 0 - fi - -install: - - pip3 install -e dev/archery[docker] - -script: - - sudo sysctl -w kernel.core_pattern="core.%e.%p" - # This isn't allowed on Travis CI: - # /home/travis/.travis/functions: line 109: ulimit: core file size: cannot modify limit: Operation not permitted - - | - ulimit -c unlimited || : - - | - archery docker run \ - ${DOCKER_RUN_ARGS} \ - --volume ${PWD}/build:/build \ - ${DOCKER_IMAGE_ID} - -after_success: - - | - if [ "${TRAVIS_EVENT_TYPE}" = "push" -a \ - "${TRAVIS_REPO_SLUG}" = "apache/arrow" ]; then - archery docker push ${DOCKER_IMAGE_ID} || : - fi From ec16c0f469903c37057d763e9745227d21861d7d Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Sun, 18 Apr 2021 19:51:19 +0000 Subject: [PATCH 8/8] Added submodule. --- .gitmodules | 3 +++ testing | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 testing diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000000..1f0e91227ff0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "testing"] + path = testing + url = https://github.com/apache/arrow-testing diff --git a/testing b/testing new file mode 160000 index 000000000000..b658b087767b --- /dev/null +++ b/testing @@ -0,0 +1 @@ +Subproject commit b658b087767b041b2081766814655b4dd5a9a439