build-test-deploy.yml

# build-executable-{arch} job builds the gprofiler executable by running build_{arch}_executable.sh scripts, then uploads it as a job artifact
# test-executable-{arch} job downloads gprofiler exe artifact and then runs test_executable.py tests from tests dir
# deploy-executable job runs only on tagged commits and it deploys gprofiler executables as release assets
# build-container-{arch} job downloads gprofiler exe artifact, produces gprofiler docker image from it, exports it into the file, and uploads image as a job artifact
# test-container-{arch} job downloads gprofiler image artifact and runs tests/test.sh script
# deploy-container job runs only on tagged commits and it deploys gprofiler images to dockerhub

name: Build, test and deploy

on:
  pull_request:
  push:
    tags:
      - '**'

jobs:
  build-executable-x64:
   runs-on: ubuntu-latest
   steps:
     - name: Checkout Code
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true

     # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11"
     - name: Build gProfiler executable
       # Using BuildKit although it has another cache mechanism which is not supported by satackey/action-docker-layer-caching@v0.0.11
       # We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching,
       # decided not to use buildkit here to get nice caches.
       run: |
        mkdir -p output
        ./scripts/build_x86_64_executable.sh
        mv build/x86_64/gprofiler output/gprofiler_x86_64
        cp output/gprofiler_x86_64 output/gprofiler  # for backwards compatibility, we upload both with arch suffix and without

     - name: Upload the executables as job artifacts
       uses: actions/upload-artifact@v4
       with:
         name: gprofiler_x86_64
         path: output/
         retention-days: 1

  test-executable-x64:
    if: ${{ !startsWith(github.ref, 'refs/tags/') }}
    runs-on: ubuntu-20.04  # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running
                           # on ubuntu-22.04/ubuntu-latest:
                           # stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError)
    needs: build-executable-x64
    strategy:
      fail-fast: false
      matrix:
        containers:
          - alpine
          - ubuntu:14.04
          - ubuntu:16.04
          - ubuntu:18.04
          - ubuntu:20.04
          - ubuntu:20.10
          - ubuntu:22.04
          - centos:6
          - centos:7
          - centos:8
          - debian:8
          - debian:9
          - debian:10
          - debian:11
    steps:
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: "3.8"

      - name: Install Java
        uses: actions/setup-java@v1
        with:
          java-version: '8.0.275'
          java-package: jdk
          architecture: x64

      - name: Install Node.JS
        uses: actions/setup-node@v2
        with:
          # same version as used in tests/containers/nodejs/Dockerfile
          node-version: 10.x

      - name: Checkout Code
        uses: actions/checkout@v3
        with:
          submodules: true

      - name: Download the executable from previous job
        uses: actions/download-artifact@v4
        with:
          name: gprofiler_x86_64
          path: dist/

      - name: Run gProfiler tests
        run: |
          mv dist/gprofiler_x86_64 dist/gprofiler
          chmod +x dist/gprofiler
          NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable

  build-executable-aarch64:
    if: startsWith(github.ref, 'refs/tags/')
    runs-on:
      - self-hosted
      - public
      - ARM64
      - gprofiler
      - EC2
      - linux
    steps:
     - name: Checkout Code
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true

     - name: Set up requirements
       run: scripts/setup_runner_requirements.sh

     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v2

     - name: Build gProfiler executable
       run: |
         mkdir -p output
         ./scripts/build_aarch64_executable.sh
         mv build/aarch64/gprofiler output/gprofiler_aarch64

     - name: Upload the executables as job artifacts
       uses: actions/upload-artifact@v4
       with:
         name: gprofiler_aarch64
         path: output/
         retention-days: 1

  deploy-executable:
    if: startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-20.04
    needs:
      - build-executable-x64
      - build-executable-aarch64

    steps:
      - name: Checkout Code
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 0

      # verification before we deploy
      - name: Verify release tag matches gProfiler version
        run: ./scripts/verify_tag.sh

      - name: Download x86_64 executable from a previous job
        uses: actions/download-artifact@v4
        with:
          name: gprofiler_x86_64
          path: output/

      - name: Download aarch64 executable from a previous job
        uses: actions/download-artifact@v4
        with:
          name: gprofiler_aarch64
          path: output/

      - name: Release
        uses: softprops/action-gh-release@v1
        with:
          files: |
            output/gprofiler_x86_64
            output/gprofiler_aarch64
            output/gprofiler
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  build-container-x64:
    runs-on: ubuntu-latest
    needs:
      - build-executable-x64

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3
      with:
        submodules: true
        fetch-depth: 0

    - name: Download executables from the previous job
      uses: actions/download-artifact@v4
      with:
        name: gprofiler_x86_64
        path: output/

    # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs
    # more nicely. the tests will then be able to use the built image.
    - name: Build gProfiler image
      # see https://github.com/docker/buildx/issues/1507 about the --provenance flag, I decided to go safe without the extra manifest.
      # --skip-exe-build needs to be first!
      run: ./scripts/build_x86_64_container.sh --skip-exe-build --provenance=false --build-arg EXE_PATH=output/gprofiler_x86_64 -t gprofiler_x86_64

    - name: Export gProfiler image
      run: mkdir -p output && docker image save gprofiler_x86_64 > output/gprofiler_x86_64.img

    - name: Upload the image artifact
      uses: actions/upload-artifact@v4
      with:
        name: gprofiler_x86_64.img
        path: output/
        retention-days: 1

  test-container-x64:
    if: ${{ !startsWith(github.ref, 'refs/tags/') }}
    needs: build-container-x64

    runs-on: ubuntu-20.04  # for 3.10.0 python tag

    strategy:
      fail-fast: false  # helps detecting flakiness / errors specific to one Python version
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10.0"  # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests.

    steps:
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install Java
      uses: actions/setup-java@v1
      with:
        java-version: '8.0.275'
        java-package: jdk
        architecture: x64

    - name: Install Node.JS
      uses: actions/setup-node@v2
      with:
        # same version as used in tests/containers/nodejs/Dockerfile
        node-version: 10.x

    - name: Checkout Code
      uses: actions/checkout@v3
      with:
        submodules: true

    - name: Download the executable from previous job
      uses: actions/download-artifact@v4
      with:
        name: gprofiler_x86_64
        path: output/

    - name: Download the image from previous job
      uses: actions/download-artifact@v4
      with:
        name: gprofiler_x86_64.img
        path: output/

    - name: Add +x to gprofiler
      run: chmod +x ./output/gprofiler_x86_64

    - name: Extract resources from gProfiler executable
      run: sudo ./output/gprofiler_x86_64 extract-resources --resources-dest=./gprofiler/resources

    # used in the tests
    - name: Import gProfiler image
      run: docker image load < output/gprofiler_x86_64.img

    # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11"
    - name: Run gProfiler tests
      run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py

  build-container-aarch64:
    if: startsWith(github.ref, 'refs/tags/')
    needs: build-executable-aarch64
    runs-on:
      - self-hosted
      - public
      - ARM64
      - gprofiler
      - EC2
      - linux
    steps:
      - name: Checkout Code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: true

      - name: Set up runner requirements
        run: scripts/setup_runner_requirements.sh

      # this gets GH_REPO and RELEASE_VERSION
      - name: Get and verify tag value
        run: ./scripts/verify_tag.sh

      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1

      - name: Download executables from the previous job
        uses: actions/download-artifact@v4
        with:
          name: gprofiler_aarch64
          path: output/

      # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11"
      - name: Build and push
        run: |
          set -x

          BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}"
          AARCH64_IMAGE="$BASE_IMAGE-aarch64"

          # build & push Aarch64
          # --skip-exe-build needs to be first!
          ./scripts/build_aarch64_container.sh --skip-exe-build --provenance=false --build-arg EXE_PATH=output/gprofiler_aarch64 -t "$AARCH64_IMAGE" --push

  deploy-container:
    if: startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    needs:
      - build-container-x64
      - build-container-aarch64
    steps:
      - name: Checkout Code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: true

      # this gets GH_REPO and RELEASE_VERSION
      - name: Verify release tag matches gProfiler version
        run: ./scripts/verify_tag.sh

      # build-container-x64 has uploaded the image as an artifact, we donwload it here.
      # build-container-aarch64 has pushed the image to DockerHub, so we'll pull it later when creating
      # the manifest.
      - name: Download the x86_64 image from previous job
        uses: actions/download-artifact@v4
        with:
          name: gprofiler_x86_64.img
          path: output/

      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1

      - name: Push manifest
        run: |
          set -x

          BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}"
          LATEST_IMAGE="${{ env.GH_REPO }}:latest"
          AARCH64_IMAGE="$BASE_IMAGE-aarch64"
          X86_64_IMAGE="$BASE_IMAGE-x86_64"

          docker pull --platform=linux/aarch64 $AARCH64_IMAGE

          docker image load < output/gprofiler_x86_64.img
          docker tag gprofiler_x86_64 $X86_64_IMAGE
          docker push $X86_64_IMAGE

          # create manifests for the tag + for 'latest'
          docker buildx imagetools create -t "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE"
          docker buildx imagetools create -t "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE"