Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Simplified CI #452

Merged
merged 1 commit into from
Sep 25, 2021
+66 −223
Merged

Simplified CI #452

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
289 changes: 66 additions & 223 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,167 +3,75 @@ name: Check and test
on: [push, pull_request]

jobs:
# build the library, a compilation step used by multiple steps below
linux-build-lib:
name: Build
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-cache-stable
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
- name: Build
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
cargo build
# test the crate
linux-test:
name: Test
needs: [linux-build-lib]
ubuntu-latest:
name: Test full
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-cache-stable
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
submodules: true # needed to test IPC, which are located in a submodule
- name: Install Rust
run: rustup update stable
- uses: Swatinem/rust-cache@v1
- name: Setup parquet files
run: |
run: |
apt update && apt install python3-pip python3-venv -y -q
python3 -m venv venv
venv/bin/pip install pip --upgrade
venv/bin/pip install pyarrow==3
venv/bin/python parquet_integration/write_parquet.py
- name: Run
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
cargo test --features full
run: cargo test --features full

windows-and-macos:
name: ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
with:
submodules: true
# TODO: this won't cache anything, which is expensive. Setup this action
# with a OS-dependent path.
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
submodules: true # needed to test IPC, which are located in a submodule
- name: Install Rust
run: rustup update stable
- uses: Swatinem/rust-cache@v1
- name: Run
shell: bash
# no need to run the whole thing. Rust guarantees interoperability
run: ARROW2_IGNORE_PARQUET= cargo test --features full

clippy:
name: Clippy
needs: [linux-build-lib]
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-cache-stable
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt clippy
- name: Run
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
cargo clippy --features full
- name: Install Rust
run: rustup update stable
- uses: Swatinem/rust-cache@v1
- name: Install clippy
run: rustup component add clippy
- name: "clippy --all"
run: cargo clippy --all --tests

miri-checks:
name: Miri
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
submodules: true
- uses: actions/cache@v2
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-miri-${{ hashFiles('**/Cargo.lock') }}
- name: Setup toolchain
toolchain: nightly-2021-07-09
override: true
- uses: Swatinem/rust-cache@v1
- name: Install Miri
run: |
rustup toolchain install nightly-2021-07-03
rustup default nightly-2021-07-03
rustup component add rustfmt miri
rustup component add miri
cargo miri setup
- name: Run
env:
RUST_BACKTRACE: full
RUST_LOG: 'trace'
# --skip io: miri can't handle opening of files, so we skip those
run: |
cargo miri setup
cargo clean
cargo miri test --features full -- --skip io::parquet --skip io::ipc
run: cargo miri test --features full -- --skip io::parquet --skip io::ipc

coverage:
name: Coverage
Expand All @@ -172,138 +80,73 @@ jobs:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /home/runner/.cargo
# this key is not equal because the user is different than on a container (runner vs github)
key: cargo-coverage-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /home/runner/target
# this key is not equal because coverage uses different compilation flags.
key: ${{ runner.os }}-amd64-target-coverage-cache-stable-
- name: Setup parquet files
run: |
run: |
apt update && apt install python3-pip python3-venv -y -q
python3 -m venv venv
venv/bin/pip install pip --upgrade
venv/bin/pip install pyarrow==3
venv/bin/python parquet_integration/write_parquet.py
- uses: Swatinem/rust-cache@v1
- name: Install tarpaulin
run: cargo install cargo-tarpaulin
- name: Run coverage
run: |
export CARGO_HOME="/home/runner/.cargo"
export CARGO_TARGET_DIR="/home/runner/target"
# 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
# see https://github.com/xd009642/tarpaulin/issues/618
cargo install --version 0.16.0 cargo-tarpaulin
cargo tarpaulin --features full --out Xml
run: cargo tarpaulin --features full --out Xml
- name: Report coverage
continue-on-error: true
run: bash <(curl -s https://codecov.io/bash)

feature-compilation:
name: Feature coverage
needs: [linux-build-lib]
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-cache-stable
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt clippy
cargo install cargo-all-features
- name: Install Rust
run: rustup update stable
- name: Setup all features
run: cargo install cargo-all-features
- uses: Swatinem/rust-cache@v1
- name: Run
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
cargo check-all-features
run: cargo check-all-features

# test the arrow crate builds against wasm32 in stable rust
wasm32-build:
name: wasm32 build
cross:
name: cross
runs-on: ubuntu-latest
container:
image: amd64/rust
strategy:
matrix:
target:
- wasm32-unknown-unknown
- i686-unknown-linux-gnu
- powerpc-unknown-linux-gnu
- powerpc64-unknown-linux-gnu
- mips-unknown-linux-gnu
- arm-linux-androideabi
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
- uses: actions-rs/toolchain@v1
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
toolchain: nightly
target: ${{ matrix.target }}
override: true
- uses: Swatinem/rust-cache@v1
- uses: actions-rs/cargo@v1
with:
path: /github/home/target
key: ${{ runner.os }}-amd64-target-wasm32-cache-nightly
- name: Setup toolchain
run: |
rustup toolchain install nightly
rustup override set nightly
rustup component add rustfmt
rustup target add wasm32-unknown-unknown
- name: Build arrow crate
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
# no need
cargo build --no-default-features --features=merge_sort,io_ipc,io_csv,io_print,io_json,io_parquet --target wasm32-unknown-unknown
use-cross: true
command: check
args: --features=merge_sort,io_ipc,io_csv,io_print,io_json,io_parquet --target ${{ matrix.target }}

linux-simd-test:
name: SIMD
needs: [linux-build-lib]
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-simd-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-simd-cache-
- name: Setup toolchain
run: |
rustup toolchain install nightly
rustup default nightly
toolchain: nightly
override: true
- uses: Swatinem/rust-cache@v1
- name: Run
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
# no need to run over all features: simd only affects the core
cargo test --no-default-features --features compute,simd --lib
# no need to run over all features: simd only affects the core
run: cargo test --tests --no-default-features --features compute,simd