Skip to content
Celeborn CI

[CELEBORN-1721][CIP-12] Support HARD_SPLIT in PushMergedData #12605

Sign in to view logs

[CELEBORN-1721][CIP-12] Support HARD_SPLIT in PushMergedData

[CELEBORN-1721][CIP-12] Support HARD_SPLIT in PushMergedData #12605

Workflow file for this run

.github/workflows/maven.yml at 4b9cf99
# This workflow will build a Java project with Maven
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
name: Celeborn CI
on:
push:
branches:
- main
- branch-*
pull_request:
branches:
- main
- branch-*
jobs:
service:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
java:
- 8
- 11
- 17
steps:
- uses: actions/checkout@v4
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: zulu
java-version: ${{ matrix.java }}
cache: maven
check-latest: false
- name: Test Service with Maven
run: build/mvn -Pgoogle-mirror test
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v4
with:
name: service-java-${{ matrix.java }}-unit-test-log
path: |
**/target/test-reports/**
**/target/unit-tests.log
- name: Upload coverage to Codecov
if: matrix.java == 8
uses: codecov/codecov-action@v3
spark2:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
java:
- 8
spark:
- '2.4'
steps:
- uses: actions/checkout@v4
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: zulu
java-version: ${{ matrix.java }}
cache: maven
check-latest: false
- name: Test with Maven
run: |
SPARK_BINARY_VERSION=${{ matrix.spark }}
SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}"
TEST_MODULES="client-spark/common,client-spark/spark-${SPARK_MAJOR_VERSION},client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it"
build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
build/mvn $PROFILES -pl $TEST_MODULES test
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v4
with:
name: spark-${{ matrix.spark }}-java-${{ matrix.java }}-unit-test-log
path: |
**/target/test-reports/**
**/target/unit-tests.log
spark3:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
java:
- 8
- 11
- 17
spark:
- '3.0'
- '3.1'
- '3.2'
- '3.3'
- '3.4'
- '3.5'
shuffle-plugin-class:
- 'org.apache.spark.shuffle.sort.io.LocalDiskShuffleDataIO'
- 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
exclude:
# SPARK-33772: Spark supports JDK 17 since 3.3.0
- java: 17
spark: '3.0'
- java: 17
spark: '3.1'
- java: 17
spark: '3.2'
# ShuffleDriverComponents#supportsReliableStorage was introduced in 3.5.0
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
spark: '3.0'
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
spark: '3.1'
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
spark: '3.2'
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
spark: '3.3'
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
spark: '3.4'
steps:
- uses: actions/checkout@v4
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: zulu
java-version: ${{ matrix.java }}
cache: maven
check-latest: false
- name: Test with Maven
run: |
SPARK_BINARY_VERSION=${{ matrix.spark }}
SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}"
TEST_MODULES="client-spark/common,client-spark/spark-${SPARK_MAJOR_VERSION},client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it"
build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
build/mvn $PROFILES -pl $TEST_MODULES -Dspark.shuffle.sort.io.plugin.class=${{ matrix.shuffle-plugin-class }} test
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v4
with:
name: spark-${{ matrix.spark }}-java-${{ matrix.java }}-unit-test-log
path: |
**/target/test-reports/**
**/target/unit-tests.log
flink:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
java:
- 8
- 11
flink:
- '1.14'
- '1.15'
- '1.16'
- '1.17'
- '1.18'
- '1.19'
- '1.20'
steps:
- uses: actions/checkout@v4
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: zulu
java-version: ${{ matrix.java }}
cache: maven
check-latest: false
- name: Test with Maven
run: |
PROFILES="-Pgoogle-mirror,flink-${{ matrix.flink }}"
TEST_MODULES="client-flink/common,client-flink/flink-${{ matrix.flink }},client-flink/flink-${{ matrix.flink }}-shaded,tests/flink-it"
build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
build/mvn $PROFILES -pl $TEST_MODULES test
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v4
with:
name: flink-${{ matrix.flink }}-java-${{ matrix.java }}-unit-test-log
path: |
**/target/test-reports/**
**/target/unit-tests.log
mr:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
java:
- 8
- 11
steps:
- uses: actions/checkout@v4
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: zulu
java-version: ${{ matrix.java }}
cache: maven
check-latest: false
- name: Test with Maven
run: |
PROFILES="-Pgoogle-mirror,mr"
TEST_MODULES="client-mr/mr,client-mr/mr-shaded,tests/mr-it"
build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
build/mvn $PROFILES -pl $TEST_MODULES test
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v4
with:
name: mr-java-${{ matrix.java }}-unit-test-log
path: |
**/target/test/
**/target/test-reports/**
**/target/unit-tests.log