diff --git a/.github/actions/setup-hadoop/action.yaml b/.github/actions/setup-hadoop/action.yaml new file mode 100644 index 000000000000..e1ee5cd9f63c --- /dev/null +++ b/.github/actions/setup-hadoop/action.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Setup Hadoop +description: 'Prepare Hadoop' + +inputs: + need-hadoop: + description: "This setup needs hadoop or not" + +runs: + using: "composite" + steps: + - name: Cache hadoop + id: cache-hadoop + uses: actions/cache@v4 + if: inputs.need-hadoop == 'true' + with: + path: /home/runner/hadoop-3.3.5 + key: cache-hadoop-3.3.5 + + - name: Build hadoop if not cached + if: steps.cache-hadoop.outputs.cache-hit != 'true' && inputs.need-hadoop == 'true' + shell: bash + run: | + set -e + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + + + + + diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index b19588f7d3aa..652b8a493bcd 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -31,8 +31,6 @@ inputs: github-token: description: "Github Token" default: "" - need-hadoop: - description: "This setup needs hadoop or not" runs: using: "composite" @@ -154,18 +152,3 @@ runs: rm foundationdb-clients_7.1.17-1_amd64.deb rm foundationdb-server_7.1.17-1_amd64.deb - - name: Cache hadoop - id: cache-hadoop - uses: actions/cache@v4 - if: inputs.need-hadoop == 'true' - with: - path: /home/runner/hadoop-3.3.5 - key: r0-hadoop-3.3.5 - - - name: Build hadoop if not cached - if: steps.cache-hadoop.outputs.cache-hit != 'true' && inputs.need-hadoop == 'true' - shell: bash - run: | - set -e - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - diff --git a/.github/services/hdfs/hdfs_cluster/action.yml b/.github/services/hdfs/hdfs_cluster/action.yml index 099bb86761b5..eae84b234725 100644 --- a/.github/services/hdfs/hdfs_cluster/action.yml +++ b/.github/services/hdfs/hdfs_cluster/action.yml @@ -30,6 +30,10 @@ runs: with: distribution: temurin java-version: "11" + - name: Setup hadoop + uses: ./.github/actions/setup-hadoop + with: + need-hadoop: true - name: Setup hadoop env shell: bash run: | diff --git a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml index 4742d86a5bdd..860b6137a147 100644 --- a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml @@ -33,6 +33,7 @@ runs: - name: Setup hadoop env shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner export HADOOP_HOME=/home/runner/hadoop-3.3.5 echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml index 0f855196434e..d4bdce0765d1 100644 --- a/.github/services/hdfs/hdfs_default/action.yml +++ b/.github/services/hdfs/hdfs_default/action.yml @@ -29,6 +29,8 @@ runs: - name: Setup shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) diff --git a/.github/services/hdfs/hdfs_default_gcs/action.yml b/.github/services/hdfs/hdfs_default_gcs/action.yml index 1e644e009490..69aabeba75d8 100644 --- a/.github/services/hdfs/hdfs_default_gcs/action.yml +++ b/.github/services/hdfs/hdfs_default_gcs/action.yml @@ -37,6 +37,8 @@ runs: - name: Setup shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/gcs-connector-hadoop3-2.2.19-shaded.jar https://github.com/GoogleCloudDataproc/hadoop-connectors/releases/download/v2.2.19/gcs-connector-hadoop3-2.2.19-shaded.jar diff --git a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml index 5e62dcb9de24..94a2763af763 100644 --- a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml +++ b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml @@ -43,6 +43,8 @@ runs: OPENDAL_AZBLOB_ACCOUNT_NAME=devstoreaccount1 OPENDAL_AZBLOB_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/hadoop-azure-3.3.5.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.3.5/hadoop-azure-3.3.5.jar diff --git a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml index 7853083dcdfe..e75b8e415627 100644 --- a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml +++ b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml @@ -40,6 +40,8 @@ runs: - name: Setup shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/hadoop-aws-3.3.5.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.5/hadoop-aws-3.3.5.jar diff --git a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml index 2f1190ecf9a9..b8de8671611a 100644 --- a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml @@ -29,6 +29,8 @@ runs: - name: Setup shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) diff --git a/.github/workflows/test_behavior_binding_nodejs.yml b/.github/workflows/test_behavior_binding_nodejs.yml index 191420cfd238..510fbe40401e 100644 --- a/.github/workflows/test_behavior_binding_nodejs.yml +++ b/.github/workflows/test_behavior_binding_nodejs.yml @@ -43,7 +43,6 @@ jobs: need-nextest: true need-protoc: true need-rocksdb: true - need-hadoop: true github-token: ${{ secrets.GITHUB_TOKEN }} # TODO: 1password is only supported on linux diff --git a/.github/workflows/test_behavior_binding_python.yml b/.github/workflows/test_behavior_binding_python.yml index bc4a5fc6794b..8f3d4bbaf48a 100644 --- a/.github/workflows/test_behavior_binding_python.yml +++ b/.github/workflows/test_behavior_binding_python.yml @@ -43,7 +43,6 @@ jobs: need-nextest: true need-protoc: true need-rocksdb: true - need-hadoop: true github-token: ${{ secrets.GITHUB_TOKEN }} # TODO: 1password is only supported on linux diff --git a/.github/workflows/test_behavior_core.yml b/.github/workflows/test_behavior_core.yml index 08212f134840..ef49723c5ed1 100644 --- a/.github/workflows/test_behavior_core.yml +++ b/.github/workflows/test_behavior_core.yml @@ -42,7 +42,6 @@ jobs: with: need-protoc: true need-rocksdb: true - need-hadoop: true github-token: ${{ secrets.GITHUB_TOKEN }} # TODO: 1password is only supported on linux