From 85e6e13925dace5a77f2c7a688ba41538de2a555 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 09:50:46 -0600 Subject: [PATCH 01/15] Add workflow to publish Docker images --- .dockerignore | 11 ++++++ .github/workflows/docker-publish.yml | 51 ++++++++++++++++++++++++++++ kube/Dockerfile | 20 +++++++---- pom.xml | 1 + 4 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/docker-publish.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..85a7bfd7c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.github +.idea +bin +conf +dev +docs +metastore_db +spark-warehouse +target +venv diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..2877a5c32 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PR Build + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +on: + push: + tags: + - '*.*.*' + - '*.*.*-rc*' + +docker: + name: Docker + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v3 + - name: Build and push Docker image + run: | + docker build -t datafusion-comet:latest -f kube/Dockerfile . + export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1 --pretty='%h') || echo '')" + if [[ $DOCKER_TAG =~ ^[0-9\.]+-rc[0-9]+$ ]] + then + echo "publishing docker tag $DOCKER_TAG" + docker tag datafusion-comet:latest ghcr.io/apache/datafusion-comet:$DOCKER_TAG + docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS" + docker push ghcr.io/apache/datafusion-comet:$DOCKER_TAG + fi + env: + DOCKER_USER: ${{ github.actor }} + DOCKER_PASS: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/kube/Dockerfile b/kube/Dockerfile index d6244c113..f86959b69 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -21,7 +21,6 @@ USER root # Installing JDK11 as the image comes with JRE RUN apt update \ - && apt install -y git \ && apt install -y curl \ && apt install -y openjdk-11-jdk \ && apt clean @@ -32,14 +31,23 @@ ENV RUSTFLAGS="-C debuginfo=line-tables-only -C incremental=false" ENV SPARK_VERSION=3.4 ENV SCALA_VERSION=2.12 +# Add source files to Docker image +RUN mkdir /comet +ADD .mvn /comet +ADD common /comet +ADD fuzz-testing /comet +ADD native /comet +ADD spark /comet +ADD spark-integration /comet +ADD .scalafix.conf /comet +ADD Makefile /comet +ADD pom.xml /comet + # Pick the JDK instead of JRE to compile Comet -RUN cd /opt \ - && git clone https://github.com/apache/datafusion-comet.git \ - && cd datafusion-comet \ - && JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" +RUN JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" FROM apache/spark:3.4.2 ENV SPARK_VERSION=3.4 ENV SCALA_VERSION=2.12 USER root -COPY --from=builder /opt/datafusion-comet/spark/target/comet-spark-spark${SPARK_VERSION}_$SCALA_VERSION-0.1.0-SNAPSHOT.jar $SPARK_HOME/jars \ No newline at end of file +COPY --from=builder /opt/datafusion-comet/spark/target/comet-spark-spark${SPARK_VERSION}_$SCALA_VERSION-0.2.0-SNAPSHOT.jar $SPARK_HOME/jars \ No newline at end of file diff --git a/pom.xml b/pom.xml index 311437cc9..3752d94b1 100644 --- a/pom.xml +++ b/pom.xml @@ -938,6 +938,7 @@ under the License. **/build/** **/target/** **/apache-spark/** + .dockerignore .git/** .github/** .gitignore From 2c52741def01ee18a48536893bc3f202fcae4910 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 09:52:13 -0600 Subject: [PATCH 02/15] update workflow name --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2877a5c32..65a310596 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: PR Build +name: Publish Docker images concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} From 073d202fdc7fbe2b0e51db499da345187d1819e1 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 09:57:19 -0600 Subject: [PATCH 03/15] remove regex check --- .github/workflows/docker-publish.yml | 13 +++++-------- kube/Dockerfile | 4 +++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 65a310596..d68269300 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -37,15 +37,12 @@ docker: - uses: actions/checkout@v3 - name: Build and push Docker image run: | - docker build -t datafusion-comet:latest -f kube/Dockerfile . + docker build -t datafusion-comet:spark-3.4-latest -f kube/Dockerfile . export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1 --pretty='%h') || echo '')" - if [[ $DOCKER_TAG =~ ^[0-9\.]+-rc[0-9]+$ ]] - then - echo "publishing docker tag $DOCKER_TAG" - docker tag datafusion-comet:latest ghcr.io/apache/datafusion-comet:$DOCKER_TAG - docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS" - docker push ghcr.io/apache/datafusion-comet:$DOCKER_TAG - fi + echo "publishing docker tag $DOCKER_TAG" + docker tag datafusion-comet:spark-3.4-latest ghcr.io/apache/datafusion-comet:$DOCKER_TAG + docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS" + docker push ghcr.io/apache/datafusion-comet:$DOCKER_TAG env: DOCKER_USER: ${{ github.actor }} DOCKER_PASS: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/kube/Dockerfile b/kube/Dockerfile index f86959b69..6c9f50caf 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -33,6 +33,8 @@ ENV SCALA_VERSION=2.12 # Add source files to Docker image RUN mkdir /comet +WORKDIR /comet + ADD .mvn /comet ADD common /comet ADD fuzz-testing /comet @@ -44,7 +46,7 @@ ADD Makefile /comet ADD pom.xml /comet # Pick the JDK instead of JRE to compile Comet -RUN JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" +RUN JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" FROM apache/spark:3.4.2 ENV SPARK_VERSION=3.4 From 342ae8403850f95479ccaa5d8e6a61f38060c77f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 10:21:39 -0600 Subject: [PATCH 04/15] improve --- .github/workflows/docker-publish.yml | 16 ++++++++++++++-- kube/Dockerfile | 23 ++++++++++++----------- native/Cargo.toml | 18 +++++++++--------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d68269300..6e69ac98c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -35,12 +35,24 @@ docker: packages: write steps: - uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + - name: Extract Maven version + id: extract_version + run: | + COMET_VERSION=$(xmllint --xpath "string(//project/version)" pom.xml) + echo "COMET_VERSION=$COMET_VERSION" >> $GITHUB_ENV + - name: Echo Maven version + run: echo "The current Maven version is ${{ env.COMET_VERSION }}" - name: Build and push Docker image run: | - docker build -t datafusion-comet:spark-3.4-latest -f kube/Dockerfile . + docker build -t datafusion-comet:spark-3.4.2-scala-2.12-${{ env.COMET_VERSION }} -f kube/Dockerfile . export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1 --pretty='%h') || echo '')" echo "publishing docker tag $DOCKER_TAG" - docker tag datafusion-comet:spark-3.4-latest ghcr.io/apache/datafusion-comet:$DOCKER_TAG + docker tag datafusion-comet:spark-3.4.2-scala-2.12-${{ env.COMET_VERSION }} ghcr.io/apache/datafusion-comet:$DOCKER_TAG docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS" docker push ghcr.io/apache/datafusion-comet:$DOCKER_TAG env: diff --git a/kube/Dockerfile b/kube/Dockerfile index 6c9f50caf..18a79e423 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -31,22 +31,23 @@ ENV RUSTFLAGS="-C debuginfo=line-tables-only -C incremental=false" ENV SPARK_VERSION=3.4 ENV SCALA_VERSION=2.12 -# Add source files to Docker image +# copy source files to Docker image RUN mkdir /comet WORKDIR /comet -ADD .mvn /comet -ADD common /comet -ADD fuzz-testing /comet -ADD native /comet -ADD spark /comet -ADD spark-integration /comet -ADD .scalafix.conf /comet -ADD Makefile /comet -ADD pom.xml /comet +COPY .mvn /comet/.mvn +COPY common /comet/common +COPY fuzz-testing /comet/fuzz-testing +COPY native /comet/native +COPY spark /comet/spark +COPY spark-integration /comet/spark-integration +COPY .scalafix.conf /comet/.scalafix.conf +COPY Makefile /comet/Makefile +COPY pom.xml /comet/pom.xml # Pick the JDK instead of JRE to compile Comet -RUN JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" +RUN cd /comet \ + && JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" FROM apache/spark:3.4.2 ENV SPARK_VERSION=3.4 diff --git a/native/Cargo.toml b/native/Cargo.toml index 9977ceece..2e73c5445 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -39,15 +39,15 @@ arrow-buffer = { version = "52.2.0" } arrow-data = { version = "52.2.0" } arrow-schema = { version = "52.2.0" } parquet = { version = "52.2.0", default-features = false, features = ["experimental"] } -datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1" } -datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", features = ["unicode_expressions", "crypto_expressions"] } -datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", features = ["crypto_expressions"] } -datafusion-functions-nested = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } +datafusion-common = { version = "41.0.0" } +datafusion = { default-features = false, version = "41.0.0", features = ["unicode_expressions", "crypto_expressions"] } +datafusion-functions = { version = "41.0.0", features = ["crypto_expressions"] } +datafusion-functions-nested = { version = "41.0.0", default-features = false } +datafusion-expr = { version = "41.0.0", default-features = false } +datafusion-execution = { version = "41.0.0", default-features = false } +datafusion-physical-plan = { version = "41.0.0", default-features = false } +datafusion-physical-expr-common = { version = "41.0.0", default-features = false } +datafusion-physical-expr = { version = "41.0.0", default-features = false } datafusion-comet-spark-expr = { path = "spark-expr", version = "0.2.0" } datafusion-comet-proto = { path = "proto", version = "0.2.0" } chrono = { version = "0.4", default-features = false, features = ["clock"] } From e7f7c5d9c6e151ecf5be76e368a294c510b6995c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 10:41:36 -0600 Subject: [PATCH 05/15] fix --- kube/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/kube/Dockerfile b/kube/Dockerfile index 18a79e423..4000ef95f 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -36,6 +36,7 @@ RUN mkdir /comet WORKDIR /comet COPY .mvn /comet/.mvn +COPY mvnw /comet/mvnw COPY common /comet/common COPY fuzz-testing /comet/fuzz-testing COPY native /comet/native From 91203b4ee59c4f40964ed4f4db9567680360b6e5 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 10:45:00 -0600 Subject: [PATCH 06/15] use maven to get project version --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6e69ac98c..6f95c3af4 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -43,7 +43,7 @@ docker: - name: Extract Maven version id: extract_version run: | - COMET_VERSION=$(xmllint --xpath "string(//project/version)" pom.xml) + COMET_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) echo "COMET_VERSION=$COMET_VERSION" >> $GITHUB_ENV - name: Echo Maven version run: echo "The current Maven version is ${{ env.COMET_VERSION }}" From f2b0a76eb6ec59e756d28158f2ddffe07e3db3b7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 10:50:07 -0600 Subject: [PATCH 07/15] add scalastyle config --- kube/Dockerfile | 1 + pom.xml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/kube/Dockerfile b/kube/Dockerfile index 4000ef95f..f7f62671d 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -38,6 +38,7 @@ WORKDIR /comet COPY .mvn /comet/.mvn COPY mvnw /comet/mvnw COPY common /comet/common +COPY dev/scalastyle-config.xml /comet/dev/scalastyle-config.xml COPY fuzz-testing /comet/fuzz-testing COPY native /comet/native COPY spark /comet/spark diff --git a/pom.xml b/pom.xml index 3752d94b1..a65cfb28f 100644 --- a/pom.xml +++ b/pom.xml @@ -588,6 +588,10 @@ under the License. + + scala-2.12 + + scala-2.13 From d6a397947b71bd44657584c7555eee669bf044ea Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 10:51:07 -0600 Subject: [PATCH 08/15] fix --- .dockerignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 85a7bfd7c..cf152cd97 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,10 @@ .idea bin conf -dev +dev/changelog +dev/copyright +dev/diffs +dev/release docs metastore_db spark-warehouse From 68be56470132073a65b61d731b5cb9ca1f8eed38 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 10:55:21 -0600 Subject: [PATCH 09/15] remove java distro name --- .github/workflows/docker-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6f95c3af4..9ce12589d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,7 +38,6 @@ docker: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: 'temurin' java-version: '17' - name: Extract Maven version id: extract_version From ec6d0e56cf8c198cf4c272759fb6e970b58768e5 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 10:56:34 -0600 Subject: [PATCH 10/15] add dev folder --- .dockerignore | 4 ---- kube/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index cf152cd97..43896c4d5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,10 +3,6 @@ .idea bin conf -dev/changelog -dev/copyright -dev/diffs -dev/release docs metastore_db spark-warehouse diff --git a/kube/Dockerfile b/kube/Dockerfile index f7f62671d..f8c0f7700 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -38,7 +38,7 @@ WORKDIR /comet COPY .mvn /comet/.mvn COPY mvnw /comet/mvnw COPY common /comet/common -COPY dev/scalastyle-config.xml /comet/dev/scalastyle-config.xml +COPY dev /comet/dev COPY fuzz-testing /comet/fuzz-testing COPY native /comet/native COPY spark /comet/spark From 005f137b88ef04a6988dd1d36db1331e6a553d69 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 11:15:49 -0600 Subject: [PATCH 11/15] save progress --- .dockerignore | 12 ++++++++++-- docs/source/user-guide/installation.md | 6 +++++- kube/Dockerfile | 14 ++++++++++++-- pom.xml | 2 +- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index 43896c4d5..332acff44 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,8 +3,16 @@ .idea bin conf -docs +docs/build +docs/temp +docs/venv metastore_db -spark-warehouse target +common/target +spark-integration/target +fuzz-testing/target +spark/target +native/target +core/target +spark-warehouse venv diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md index 45f988e29..07fc689c5 100644 --- a/docs/source/user-guide/installation.md +++ b/docs/source/user-guide/installation.md @@ -32,7 +32,11 @@ Make sure the following requirements are met and software installed on your mach - JDK 8 and up - GLIBC 2.17 (Centos 7) and up -## Using a Published Binary Release +## Using a Published Docker Image + +Docker images are available at https://github.com/orgs/apache/packages?repo_name=datafusion-comet + +## Using a Published JAR File There are no published binary releases yet. diff --git a/kube/Dockerfile b/kube/Dockerfile index f8c0f7700..d9d11acc1 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -35,14 +35,22 @@ ENV SCALA_VERSION=2.12 RUN mkdir /comet WORKDIR /comet +# build native code first so that this layer can be re-used +# if only Scala code gets modified +COPY rust-toolchain.toml /comet/rust-toolchain.toml +COPY native /comet/native +RUN cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release + +# copy the reset of the project COPY .mvn /comet/.mvn COPY mvnw /comet/mvnw COPY common /comet/common COPY dev /comet/dev +COPY docs /comet/docs COPY fuzz-testing /comet/fuzz-testing -COPY native /comet/native COPY spark /comet/spark COPY spark-integration /comet/spark-integration +COPY scalafmt.conf /comet/scalafmt.conf COPY .scalafix.conf /comet/.scalafix.conf COPY Makefile /comet/Makefile COPY pom.xml /comet/pom.xml @@ -55,4 +63,6 @@ FROM apache/spark:3.4.2 ENV SPARK_VERSION=3.4 ENV SCALA_VERSION=2.12 USER root -COPY --from=builder /opt/datafusion-comet/spark/target/comet-spark-spark${SPARK_VERSION}_$SCALA_VERSION-0.2.0-SNAPSHOT.jar $SPARK_HOME/jars \ No newline at end of file + +# ntoe the use of a wildcard in the file name so that this works with both snapshot and final release versions +COPY --from=builder /opt/datafusion-comet/spark/target/comet-spark-spark${SPARK_VERSION}_$SCALA_VERSION-0.2.0*.jar $SPARK_HOME/jars \ No newline at end of file diff --git a/pom.xml b/pom.xml index a65cfb28f..d41a57dbc 100644 --- a/pom.xml +++ b/pom.xml @@ -968,7 +968,7 @@ under the License. docs/source/_static/images/** dev/release/rat_exclude_files.txt dev/release/requirements.txt - native/core/src/execution/generated/** + native/proto/src/generated/** From 19e2ba3d457688c587b624e2d69c28e2db57ee04 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 11:18:10 -0600 Subject: [PATCH 12/15] docker build works --- .github/workflows/docker-publish.yml | 4 ++-- kube/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9ce12589d..07d7f970b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -48,10 +48,10 @@ docker: run: echo "The current Maven version is ${{ env.COMET_VERSION }}" - name: Build and push Docker image run: | - docker build -t datafusion-comet:spark-3.4.2-scala-2.12-${{ env.COMET_VERSION }} -f kube/Dockerfile . + docker build -t datafusion-comet:spark-3.4-scala-2.12-${{ env.COMET_VERSION }} -f kube/Dockerfile . export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1 --pretty='%h') || echo '')" echo "publishing docker tag $DOCKER_TAG" - docker tag datafusion-comet:spark-3.4.2-scala-2.12-${{ env.COMET_VERSION }} ghcr.io/apache/datafusion-comet:$DOCKER_TAG + docker tag datafusion-comet:spark-3.4-scala-2.12t-${{ env.COMET_VERSION }} ghcr.io/apache/datafusion-comet:$DOCKER_TAG docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS" docker push ghcr.io/apache/datafusion-comet:$DOCKER_TAG env: diff --git a/kube/Dockerfile b/kube/Dockerfile index d9d11acc1..c770fab99 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -65,4 +65,4 @@ ENV SCALA_VERSION=2.12 USER root # ntoe the use of a wildcard in the file name so that this works with both snapshot and final release versions -COPY --from=builder /opt/datafusion-comet/spark/target/comet-spark-spark${SPARK_VERSION}_$SCALA_VERSION-0.2.0*.jar $SPARK_HOME/jars \ No newline at end of file +COPY --from=builder /comet/spark/target/comet-spark-spark${SPARK_VERSION}_$SCALA_VERSION-0.2.0*.jar $SPARK_HOME/jars \ No newline at end of file From 0565cd62110eff4c285b168604aae48e816e16d2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 12:53:23 -0600 Subject: [PATCH 13/15] Update kube/Dockerfile Co-authored-by: Oleks V --- kube/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/Dockerfile b/kube/Dockerfile index c770fab99..4e15794ab 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -41,7 +41,7 @@ COPY rust-toolchain.toml /comet/rust-toolchain.toml COPY native /comet/native RUN cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release -# copy the reset of the project +# copy the rest of the project COPY .mvn /comet/.mvn COPY mvnw /comet/mvnw COPY common /comet/common From 51f70a7ced5b40a0ee417c0a6245d190c793d3d8 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 13:56:21 -0600 Subject: [PATCH 14/15] Update .github/workflows/docker-publish.yml Co-authored-by: Edmondo Porcu --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 07d7f970b..3d11a3c46 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -29,7 +29,7 @@ on: docker: name: Docker - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: contents: read packages: write From 70dbd93ef7a6cb995c41c0c5ae9e1273fedda3f1 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Aug 2024 14:14:36 -0600 Subject: [PATCH 15/15] address feedback --- .github/workflows/docker-publish.yml | 40 +++++++++++++++------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 07d7f970b..5bae483bb 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -26,6 +26,7 @@ on: tags: - '*.*.*' - '*.*.*-rc*' + - 'test-docker-publish-*' docker: name: Docker @@ -34,26 +35,29 @@ docker: contents: read packages: write steps: - - uses: actions/checkout@v3 - name: Set up Java - uses: actions/setup-java@v3 - with: - java-version: '17' - - name: Extract Maven version + uses: actions/setup-java@v3 + with: + java-version: '17' + - name: Extract Comet version id: extract_version run: | COMET_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) echo "COMET_VERSION=$COMET_VERSION" >> $GITHUB_ENV - - name: Echo Maven version - run: echo "The current Maven version is ${{ env.COMET_VERSION }}" - - name: Build and push Docker image - run: | - docker build -t datafusion-comet:spark-3.4-scala-2.12-${{ env.COMET_VERSION }} -f kube/Dockerfile . - export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1 --pretty='%h') || echo '')" - echo "publishing docker tag $DOCKER_TAG" - docker tag datafusion-comet:spark-3.4-scala-2.12t-${{ env.COMET_VERSION }} ghcr.io/apache/datafusion-comet:$DOCKER_TAG - docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS" - docker push ghcr.io/apache/datafusion-comet:$DOCKER_TAG - env: - DOCKER_USER: ${{ github.actor }} - DOCKER_PASS: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + - name: Echo Comet version + run: echo "The current Comet version is ${{ env.COMET_VERSION }}" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64,linux/arm64 + push: true + tags: apache/datafusion-comet:spark-3.4-scala-2.12-${{ env.COMET_VERSION }} + file: kube/Dockerfile