# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # name: Build and test on: push: branches: - trunk-* pull_request: branches: - trunk-* concurrency: group: test-${{ github.head_ref || github.run_id }} cancel-in-progress: true env: MAVEN_ARGS: --batch-mode --no-transfer-progress jobs: default: name: Unit test runs-on: ubuntu-20.04 strategy: fail-fast: false max-parallel: 6 matrix: java: - 17 modules: - -pl :hadoop-hdfs - -pl :hadoop-yarn-server-resourcemanager - -pl :hadoop-mapreduce-client-jobclient - -pl :hadoop-common,:hadoop-yarn-server-nodemanager,:hadoop-yarn-client,:hadoop-distcp - -pl :hadoop-hdfs-rbf,:hadoop-yarn-applications-distributedshell,:hadoop-yarn-services-core - -pl :hadoop-minikdc,:hadoop-auth,:hadoop-auth-examples,:hadoop-nfs,:hadoop-kms,:hadoop-registry -pl :hadoop-hdfs-client,:hadoop-hdfs-native-client,:hadoop-hdfs-httpfs,:hadoop-hdfs-nfs,:hadoop-yarn-api -pl :hadoop-yarn-common,:hadoop-yarn-server-common,:hadoop-yarn-server-applicationhistoryservice -pl :hadoop-yarn-server-timelineservice,:hadoop-yarn-server-web-proxy,:hadoop-yarn-server-tests -pl :hadoop-mapreduce-client-core,:hadoop-mapreduce-client-common,:hadoop-mapreduce-client-shuffle -pl :hadoop-mapreduce-client-app,:hadoop-mapreduce-client-hs,:hadoop-minicluster,:hadoop-federation-balance -pl :hadoop-yarn-server-sharedcachemanager,:hadoop-yarn-server-timeline-pluginstorage -pl :hadoop-yarn-server-timelineservice-hbase-common,:hadoop-yarn-server-timelineservice-hbase-client -pl :hadoop-yarn-server-timelineservice-hbase-server-2,:hadoop-yarn-server-timelineservice-hbase-tests -pl :hadoop-yarn-server-router,:hadoop-yarn-server-timelineservice-documentstore -pl :hadoop-yarn-server-globalpolicygenerator,:hadoop-yarn-applications-unmanaged-am-launcher -pl :hadoop-yarn-services-api,:hadoop-yarn-applications-catalog-webapp,:hadoop-yarn-registry -pl org.apache.hadoop.applications.mawo:hadoop-yarn-applications-mawo-core,:hadoop-yarn-csi -pl :hadoop-mapreduce-client-hs-plugins,:hadoop-mapreduce-client-nativetask -pl :hadoop-mapreduce-client-uploader,:hadoop-mapreduce-examples,:hadoop-streaming,:hadoop-client -pl :hadoop-dynamometer-workload,:hadoop-dynamometer-infra,:hadoop-dynamometer-blockgen -pl :hadoop-dynamometer-dist,:hadoop-archives,:hadoop-archive-logs,:hadoop-rumen,:hadoop-gridmix -pl :hadoop-datajoin,:hadoop-extras,:hadoop-aws,:hadoop-kafka,:hadoop-azure,:hadoop-aliyun,:hadoop-sls -pl :hadoop-resourceestimator,:hadoop-azure-datalake,:hadoop-fs2img,:hadoop-tools-dist,:hadoop-benchmark -pl :hadoop-compat-bench,:hadoop-client-api,:hadoop-client-runtime,:hadoop-client-minicluster -pl :hadoop-client-integration-tests,:hadoop-dist,:hadoop-cos,:hadoop-huaweicloud,:hadoop-cloud-storage steps: - name: Checkout code uses: actions/checkout@v4 - name: Setup JDK ${{ matrix.java }} uses: actions/setup-java@v4 with: distribution: zulu java-version: ${{ matrix.java }} cache: 'maven' check-latest: false - name: Prepare Environment run: | WORKING_DIR=$PWD cd dev-support/docker export DEBIAN_FRONTEND=noninteractive export DEBCONF_TERSE=true ###### # Platform package dependency resolver ###### chmod a+x pkg-resolver/*.sh pkg-resolver/*.py chmod a+r pkg-resolver/*.json ###### # Install packages from apt ###### sudo apt-get -q update sudo apt-get -q -vvv install -y --no-install-recommends \ clang libprotobuf-dev libcurl4-openssl-dev libsasl2-dev libsnappy-dev libzstd-dev yasm # bats doxygen gnupg-agent hugo libprotoc-dev sudo apt-get clean sudo rm -rf /var/lib/apt/lists/* locale-gen en_US.UTF-8 export LANG='en_US.UTF-8' export LANGUAGE='en_US:en' export LC_ALL='en_US.UTF-8' export PYTHONIOENCODING=utf-8 ####### # Set env vars for SpotBugs 4.2.2 ####### export SPOTBUGS_HOME=/opt/spotbugs ####### # Set env vars for Google Protobuf 3.21.12 ####### export PROTOBUF_HOME=/opt/protobuf export PATH="${PATH}:/opt/protobuf/bin" ####### # Skip gpg verification when downloading Yetus via yetus-wrapper ####### export HADOOP_SKIP_YETUS_VERIFICATION=true ####### # Install packages ####### pkg-resolver/install-common-pkgs.sh pkg-resolver/install-spotbugs.sh ubuntu:focal pkg-resolver/install-boost.sh ubuntu:focal pkg-resolver/install-protobuf.sh ubuntu:focal pkg-resolver/install-hadolint.sh ubuntu:focal pkg-resolver/install-intel-isa-l.sh ubuntu:focal cd $WORKING_DIR - name: Build run: mvn ${{ matrix.modules }} -Pnative clean install -am -DskipTests - name: Test run: mvn ${{ matrix.modules }} -Pnative test -Dsurefire.excludesFile=$PWD/dev-support/java-17/exclude-tests.txt - name: Upload test logs if: failure() uses: actions/upload-artifact@v3 with: name: logs-${{ matrix.modules }} path: | **/target/*.log