docker-publish.yml

name: Build and Publish Mlflow Docker Image

on:
  push:
    branches: [main]
    # Publish semver tags as releases.
    tags:
      - "*.*.*"
      - "*.*.*.*"
    paths:
      - ".github/workflows/docker-publish.yml"
      - ".env"
      - "poetry.lock"
      - "poetry.toml"
      - "pyproject.toml"
      - "Dockerfile"
      - "mlflowstack/**"
      - "docker-compose.*.yaml"
      - "tests/**"
      - "test-containers/**"

  pull_request:
    branches: [main]

  workflow_dispatch:

jobs:
  integrationtest:
    runs-on: ubuntu-latest

    strategy:
      matrix:
        database-combo:
          - {
              postgres: "15",
              mysql: "8.0",
              minio: "RELEASE.2024-11-07T00-52-20Z",
              gcs: "1.50.2",
              azurite: "3.33.0",
              azure-storage-blob: "12.26.0",
            }
          - {
              postgres: "16",
              mysql: "8.4",
              minio: "RELEASE.2024-11-07T00-52-20Z",
              gcs: "1.50.2",
              azurite: "3.33.0",
              azure-storage-blob: "12.26.0",
            }
          - {
              postgres: "17",
              mysql: "9.1",
              minio: "RELEASE.2024-11-07T00-52-20Z",
              gcs: "1.50.2",
              azurite: "3.33.0",
              azure-storage-blob: "12.26.0",
            }

    name: Test PostgreSQL ${{ matrix.database-combo.postgres }} & MySQL ${{ matrix.database-combo.mysql }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: 3.12

      #----------------------------------------------
      #  -----  install & configure poetry  -----
      #----------------------------------------------
      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      #----------------------------------------------
      #       load cached venv if cache exists
      #----------------------------------------------
      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      #----------------------------------------------
      # install dependencies if cache does not exist
      #----------------------------------------------
      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      #----------------------------------------------
      # install your root project, if required
      #----------------------------------------------
      - name: Install library
        run: poetry install --no-interaction

      - name: Cache Docker images
        id: cache-docker-images
        uses: actions/cache@v4
        with:
          path: ${{ github.workspace }}/.docker-cache
          key: ${{ runner.os }}-docker-images-${{ matrix.database-combo.postgres }}-${{ matrix.database-combo.mysql }}-${{ matrix.database-combo.minio }}-${{ matrix.database-combo.gcs }}-${{ matrix.database-combo.azurite }}
          restore-keys: |
            ${{ runner.os }}-docker-images-

      - name: Load Cached Docker Images
        if: steps.cache-docker-images.outputs.cache-hit == 'true'
        run: |
          docker load --input ${{ github.workspace }}/.docker-cache/docker-images.tar

      - name: Pull and Save Docker Images if not Cached
        if: steps.cache-docker-images.outputs.cache-hit != 'true'
        run: |
          mkdir -p ${{ github.workspace }}/.docker-cache
          docker pull postgres:${{ matrix.database-combo.postgres }}
          docker pull mysql:${{ matrix.database-combo.mysql }}
          docker pull minio/minio:${{ matrix.database-combo.minio }}
          docker pull fsouza/fake-gcs-server:${{ matrix.database-combo.gcs }}
          docker pull mcr.microsoft.com/azure-storage/azurite:${{ matrix.database-combo.azurite }}

          docker save -o ${{ github.workspace }}/.docker-cache/docker-images.tar \
            postgres:${{ matrix.database-combo.postgres }} \
            mysql:${{ matrix.database-combo.mysql }} \
            minio/minio:${{ matrix.database-combo.minio }} \
            fsouza/fake-gcs-server:${{ matrix.database-combo.gcs }} \
            mcr.microsoft.com/azure-storage/azurite:${{ matrix.database-combo.azurite }}

      #----------------------------------------------
      # Overwrite .env file to test different version of databases
      #----------------------------------------------
      - name: Set up .env file
        run: |
          echo "POSTGRES_VERSION=${{ matrix.database-combo.postgres }}" > .env
          echo "MYSQL_VERSION=${{ matrix.database-combo.mysql }}" >> .env
          echo "MINIO_VERSION=${{ matrix.database-combo.minio }}" >> .env
          echo "FAKE_GCS_SERVER_VERSION=${{ matrix.database-combo.gcs }}" >> .env
          echo "AZURITE_VERSION=${{ matrix.database-combo.azurite }}" >> .env
          echo "AZURE_STORAGE_BLOB_VERSION=${{ matrix.database-combo.azure-storage-blob }}" >> .env

      #----------------------------------------------
      #              run test suite
      #----------------------------------------------
      - name: Run tests
        env:
          POSTGRES_VERSION: ${{ matrix.database-combo.postgres }}
          MYSQL_VERSION: ${{ matrix.database-combo.mysql }}
        run: |
          source .venv/bin/activate
          poetry run pytest

  buildtestpush:
    runs-on: ubuntu-latest

    needs: integrationtest

    env:
      IMAGE_NAME: ${{ github.repository }}

    permissions:
      contents: read
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write
      # To upload sarif files
      security-events: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Install ORAS
        id: install_oras
        uses: oras-project/setup-oras@main

      - name: Install Cosign
        id: install_cosign
        uses: sigstore/cosign-installer@v3.7.0

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3.3.0
        with:
          platforms: all

      # Workaround: https://github.com/docker/build-push-action/issues/461
      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v3

      # Login against a Docker Hub registry except on PR
      # https://github.com/docker/login-action
      - name: Login to Docker Hub
        id: docker_hub_login
        if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/')
        uses: docker/login-action@v3
        with:
          registry: docker.io
          username: ${{ github.actor }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}

      # Login against a Docker registry except on PR
      # https://github.com/docker/login-action
      - name: Login to GitHub Container Registry
        id: ghcr_login
        if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/')
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Login to GitHub Container Registry (ORAS)
        id: oras_ghcr_login
        if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/')
        run: |
          echo ${{ secrets.GITHUB_TOKEN }} | oras login ghcr.io --username ${{ github.actor }} --password-stdin

      - name: Publish Artifact Hub Manifest
        id: publish_ah_manifest
        if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/')
        run: |
          oras push \
            ghcr.io/${{ env.IMAGE_NAME }}:artifacthub.io \
            --config /dev/null:application/vnd.cncf.artifacthub.config.v1+yaml \
            artifacthub-repo.yml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml

      # Extract metadata (tags, labels) for Docker
      # https://github.com/docker/metadata-action
      - name: Extract Docker metadata
        if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/')
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            docker.io/${{ env.IMAGE_NAME }}
            ghcr.io/${{ env.IMAGE_NAME }}
          flavor: |
            latest=true
          tags: |
            type=ref,event=tag
          labels: |
            io.artifacthub.package.readme-url=https://raw.githubusercontent.com/burakince/mlflow/main/README.md
            io.artifacthub.package.maintainers=[{"name":"burakince","email":"burak.ince@linux.org.tr"}]
            io.artifacthub.package.logo-url=https://raw.githubusercontent.com/mlflow/mlflow/master/assets/logo.svg
            io.artifacthub.package.keywords=machine-learning,ai,ml,model-management,mlflow,mlflow-tracking-server,mlflow-docker,mlflow-tracking,mlflow-kube
            io.artifacthub.package.license=MIT
            io.artifacthub.package.alternative-locations=docker.io/${{ env.IMAGE_NAME }}

      # Build Docker image with Buildx and don't push it
      - name: Build Docker image
        id: build-docker-image
        uses: docker/build-push-action@v6
        with:
          context: .
          platforms: linux/arm64/v8,linux/amd64
          push: false
          # load: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Run Snyk to check Docker image for vulnerabilities
        id: docker-image-scan
        continue-on-error: true
        uses: snyk/actions/docker@0.4.0
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          image: docker.io/${{ env.IMAGE_NAME }}:latest
          args: --file=Dockerfile --severity-threshold=medium --sarif-file-output=snyk.sarif

      - name: Upload result to GitHub Code Scanning
        if: hashFiles('snyk.sarif') != ''
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: snyk.sarif

      # Break the pipeline with failure status if security scan failed
      # - name: Check docker image scan status
      #   if: ${{ steps.docker-image-scan.outcome == 'failure' }}
      #   run: exit 1

      - name: Install Dive
        run: |
          DIVE_VERSION=$(curl -sL "https://api.github.com/repos/wagoodman/dive/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/')
          curl -OL https://github.com/wagoodman/dive/releases/download/v${DIVE_VERSION}/dive_${DIVE_VERSION}_linux_amd64.deb
          sudo apt install ./dive_${DIVE_VERSION}_linux_amd64.deb

      - name: Analyse Efficiency and Wastes on Image
        run: dive docker.io/${{ env.IMAGE_NAME }}:latest

      - name: Push All Docker Image Tags
        if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/')
        run: |
          docker push --all-tags docker.io/${{ env.IMAGE_NAME }}
          docker push --all-tags ghcr.io/${{ env.IMAGE_NAME }}

      # Sign the resulting Docker image digest except on PRs.
      # This will only write to the public Rekor transparency log when the Docker
      # repository is public to avoid leaking data.  If you would like to publish
      # transparency data even for private images, pass --force to cosign below.
      # https://github.com/sigstore/cosign
      - name: Sign the published Docker images
        if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/')
        run: |
          echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign -y {}@${{ steps.build-docker-image.outputs.digest }}