build_and_test.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Build and test

on:
  push:
    branches:
      - trunk-*
  pull_request:
    branches:
      - trunk-*

concurrency:
  group: test-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  MAVEN_ARGS: --batch-mode --no-transfer-progress

jobs:
  default:
    name: Unit test
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      max-parallel: 6
      matrix:
        java:
          - 17
        modules:
          - -pl :hadoop-hdfs
          - -pl :hadoop-yarn-server-resourcemanager
          - -pl :hadoop-mapreduce-client-jobclient
          - -pl :hadoop-common,:hadoop-yarn-server-nodemanager,:hadoop-yarn-client,:hadoop-distcp
          - -pl :hadoop-hdfs-rbf,:hadoop-yarn-applications-distributedshell,:hadoop-yarn-services-core
          - -pl :hadoop-minikdc,:hadoop-auth,:hadoop-auth-examples,:hadoop-nfs,:hadoop-kms,:hadoop-registry
            -pl :hadoop-hdfs-client,:hadoop-hdfs-native-client,:hadoop-hdfs-httpfs,:hadoop-hdfs-nfs,:hadoop-yarn-api
            -pl :hadoop-yarn-common,:hadoop-yarn-server-common,:hadoop-yarn-server-applicationhistoryservice
            -pl :hadoop-yarn-server-timelineservice,:hadoop-yarn-server-web-proxy,:hadoop-yarn-server-tests
            -pl :hadoop-mapreduce-client-core,:hadoop-mapreduce-client-common,:hadoop-mapreduce-client-shuffle
            -pl :hadoop-mapreduce-client-app,:hadoop-mapreduce-client-hs,:hadoop-minicluster,:hadoop-federation-balance
            -pl :hadoop-yarn-server-sharedcachemanager,:hadoop-yarn-server-timeline-pluginstorage
            -pl :hadoop-yarn-server-timelineservice-hbase-common,:hadoop-yarn-server-timelineservice-hbase-client
            -pl :hadoop-yarn-server-timelineservice-hbase-server-2,:hadoop-yarn-server-timelineservice-hbase-tests
            -pl :hadoop-yarn-server-router,:hadoop-yarn-server-timelineservice-documentstore
            -pl :hadoop-yarn-server-globalpolicygenerator,:hadoop-yarn-applications-unmanaged-am-launcher
            -pl :hadoop-yarn-services-api,:hadoop-yarn-applications-catalog-webapp,:hadoop-yarn-registry
            -pl org.apache.hadoop.applications.mawo:hadoop-yarn-applications-mawo-core,:hadoop-yarn-csi
            -pl :hadoop-mapreduce-client-hs-plugins,:hadoop-mapreduce-client-nativetask
            -pl :hadoop-mapreduce-client-uploader,:hadoop-mapreduce-examples,:hadoop-streaming,:hadoop-client
            -pl :hadoop-dynamometer-workload,:hadoop-dynamometer-infra,:hadoop-dynamometer-blockgen
            -pl :hadoop-dynamometer-dist,:hadoop-archives,:hadoop-archive-logs,:hadoop-rumen,:hadoop-gridmix
            -pl :hadoop-datajoin,:hadoop-extras,:hadoop-aws,:hadoop-kafka,:hadoop-azure,:hadoop-aliyun,:hadoop-sls
            -pl :hadoop-resourceestimator,:hadoop-azure-datalake,:hadoop-fs2img,:hadoop-tools-dist,:hadoop-benchmark
            -pl :hadoop-compat-bench,:hadoop-client-api,:hadoop-client-runtime,:hadoop-client-minicluster
            -pl :hadoop-client-integration-tests,:hadoop-dist,:hadoop-cos,:hadoop-huaweicloud,:hadoop-cloud-storage
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Setup JDK ${{ matrix.java }}
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: ${{ matrix.java }}
          cache: 'maven'
          check-latest: false
      - name: Prepare Environment
        run: |
          WORKING_DIR=$PWD
          cd dev-support/docker
          export DEBIAN_FRONTEND=noninteractive
          export DEBCONF_TERSE=true
          ######
          # Platform package dependency resolver
          ######
          chmod a+x pkg-resolver/*.sh pkg-resolver/*.py
          chmod a+r pkg-resolver/*.json
          ######
          # Install packages from apt
          ######
          sudo apt-get -q update
          sudo apt-get -q -vvv install -y --no-install-recommends \
            clang libprotobuf-dev libcurl4-openssl-dev libsasl2-dev libsnappy-dev libzstd-dev yasm # bats doxygen gnupg-agent hugo libprotoc-dev
          sudo apt-get clean
          sudo rm -rf /var/lib/apt/lists/*
          locale-gen en_US.UTF-8
          export LANG='en_US.UTF-8'
          export LANGUAGE='en_US:en'
          export LC_ALL='en_US.UTF-8'
          export PYTHONIOENCODING=utf-8
          #######
          # Set env vars for SpotBugs 4.2.2
          #######
          export SPOTBUGS_HOME=/opt/spotbugs
          #######
          # Set env vars for Google Protobuf 3.21.12
          #######
          export PROTOBUF_HOME=/opt/protobuf
          export PATH="${PATH}:/opt/protobuf/bin"
          #######
          # Skip gpg verification when downloading Yetus via yetus-wrapper
          #######
          export HADOOP_SKIP_YETUS_VERIFICATION=true
          #######
          # Install packages
          #######
          pkg-resolver/install-common-pkgs.sh
          pkg-resolver/install-spotbugs.sh ubuntu:focal
          pkg-resolver/install-boost.sh ubuntu:focal
          pkg-resolver/install-protobuf.sh ubuntu:focal
          pkg-resolver/install-hadolint.sh ubuntu:focal
          pkg-resolver/install-intel-isa-l.sh ubuntu:focal
          cd $WORKING_DIR
      - name: Build
        run: mvn ${{ matrix.modules }} -Pnative clean install -am -DskipTests
      - name: Test
        run: mvn ${{ matrix.modules }} -Pnative test -Dsurefire.excludesFile=$PWD/dev-support/java-17/exclude-tests.txt
      - name: Upload test logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: logs-${{ matrix.modules }}
          path: |
            **/target/*.log