Merge pull request #64 from populationgenomics/alpha

First Main Release

.dockerignore

@@ -0,0 +1 @@
+.venv

.github/renovate-config.json

@@ -1,44 +1,40 @@
 {
-    "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-    "onboarding": true,
-    "requireConfig": "ignored",
-    "platform": "github",
-    "repositories": ["populationgenomics/cpg-flow"],
-    "prConcurrentLimit": 10,
-    "prHourlyLimit": 5,
-    "semanticCommits": "enabled",
-    "enabledManagers": ["pip-compile"],
-    "constraints": {
-      "python": "==3.10"
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "onboarding": true,
+  "requireConfig": "ignored",
+  "platform": "github",
+  "repositories": ["populationgenomics/cpg-flow"],
+  "prConcurrentLimit": 10,
+  "prHourlyLimit": 5,
+  "semanticCommits": "enabled",
+  "enabledManagers": ["pep621"],
+  "constraints": {
+    "python": "==3.10"
+  },
+  "pep621": {
+    "lockFileMaintenance": {
+      "enabled": true
+    }
+  },
+  "packageRules": [
+    {
+      "groupName": "all non-major dependencies",
+      "groupSlug": "all-minor-patch",
+      "matchUpdateTypes": [
+        "minor",
+        "patch"
+      ],
+      "matchPackagePatterns": ["*"],
+      "minimumReleaseAge": "48 hours"
     },
-    "platformCommit": "enabled",
-    "pip-compile": {
-      "fileMatch": ["^requirements/main\\.txt$", "^requirements/dev\\.txt$", "^requirements/test\\.txt$"],
-      "lockFileMaintenance": {
-        "enabled": true,
-        "branchTopic": "pip-compile-refresh",
-        "commitMessageAction": "Refresh pip-compile outputs"
-      }
-    },
-    "packageRules": [
-      {
-        "groupName": "all non-major dependencies",
-        "groupSlug": "all-minor-patch",
-        "matchUpdateTypes": [
-          "minor",
-          "patch"
-        ],
-        "matchPackagePatterns": ["*"],
-        "minimumReleaseAge": "48 hours"
-      },
-      {
-        "groupName": "all major dependencies",
-        "groupSlug": "all-major",
-        "matchUpdateTypes": [
-          "major"
-        ],
-        "matchPackagePatterns": ["*"],
-        "minimumReleaseAge": "48 hours"
-      }
-    ]
-  }
+    {
+      "groupName": "all major dependencies",
+      "groupSlug": "all-major",
+      "matchUpdateTypes": [
+        "major"
+      ],
+      "matchPackagePatterns": ["*"],
+      "minimumReleaseAge": "48 hours"
+    }
+  ]
+}

.github/workflows/docker.yaml

@@ -0,0 +1,92 @@
+name: Docker
+on:
+  pull_request:
+    branches:
+      - main
+      - alpha
+
+  push:
+    branches:
+      - main
+      - alpha
+
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag to use (defaults to "test")'
+        default: 'test'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  VERSION: 0.1.0-alpha.18
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    environment: production
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      DOCKER_BUILDKIT: 1
+      BUILDKIT_PROGRESS: plain
+      CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+      IMAGE_NAME: cpg_flow
+      DOCKER_DEV: australia-southeast1-docker.pkg.dev/cpg-common/images-dev
+      DOCKER_MAIN: australia-southeast1-docker.pkg.dev/cpg-common/images
+      DOCKER_TMP: australia-southeast1-docker.pkg.dev/cpg-common/images-tmp
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log the PR commit sha
+        if: ${{ github.event_name == 'pull_request' }}
+        run: "echo 'PR commit sha: ${{ github.event.pull_request.head.sha }}'"
+
+      - id: 'google-cloud-auth'
+        name: 'Authenticate to Google Cloud'
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: 'projects/1051897107465/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+          service_account: '[email protected]'
+
+      - name: set up gcloud sdk
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: cpg-common
+
+      - name: gcloud docker auth
+        run: |
+          gcloud auth configure-docker australia-southeast1-docker.pkg.dev
+
+      - name: build
+        run: |
+          docker build . -f Dockerfile --tag $IMAGE_NAME:${{ github.sha }}
+
+      - name: push commit sha tag
+        if: ${{ github.ref_name != 'main' && github.ref_name != 'alpha' }}
+        run: |
+          docker tag $IMAGE_NAME:${{ github.sha }} $DOCKER_TMP/$IMAGE_NAME:${{ github.event.pull_request.head.sha }}
+          docker push $DOCKER_TMP/$IMAGE_NAME:${{ github.event.pull_request.head.sha }}
+          echo "DOCKER_IMAGE=$DOCKER_TMP/$IMAGE_NAME:${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+
+      - name: push production
+        if: ${{ github.ref_name == 'main' || github.ref_name == 'alpha' }}
+        run: |
+          docker tag $IMAGE_NAME:${{ github.sha }} $DOCKER_MAIN/$IMAGE_NAME:$VERSION
+          docker push $DOCKER_MAIN/$IMAGE_NAME:$VERSION
+          echo "DOCKER_IMAGE=$DOCKER_MAIN/$IMAGE_NAME:$VERSION" >> $GITHUB_ENV
+
+      - name: manually triggered build
+        if: ${{ github.event_name == 'workflow_dispatch' && github.ref_name != 'main' }}
+        run: |
+          docker tag $IMAGE_NAME:${{ github.sha }} $DOCKER_DEV/$IMAGE_NAME:${{ github.event.inputs.tag }}
+          docker push $DOCKER_DEV/$IMAGE_NAME:${{ github.event.inputs.tag }}
+          echo "DOCKER_IMAGE=$DOCKER_DEV/$IMAGE_NAME:${{ github.event.inputs.tag }}" >> $GITHUB_ENV
+
+      - name: print docker tag
+        run: echo "Pushed Docker Tag ${{ env.DOCKER_IMAGE }}"

.github/workflows/lint.yaml

@@ -1,25 +1,31 @@
 name: Lint
-on: [push]
+on:
+  push:
 
 jobs:
   lint:
+    name: Lint
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
       - uses: actions/checkout@v4
-
-      - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          submodules: 'true'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install Python
+        run: uv python install
 
-      - name: Install packages
-        run: pip install -r requirements/dev.txt
+      - name: Install pre-commit from dev dependencies
+        run: uv sync --only-dev
 
       - name: Install pre-commit hooks
-        run: pre-commit install --install-hooks
+        run: uv run pre-commit install --install-hooks
 
       - name: Run pre-commit
-        run: pre-commit run --all-files
+        run: uv run pre-commit run --all-files

.github/workflows/package.yaml

@@ -0,0 +1,51 @@
+name: Package
+on:
+  push:
+    branches:
+      - main
+      - alpha
+
+permissions:
+  id-token: write
+  contents: write
+
+jobs:
+  package:
+    name: Package
+    environment: production
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Setup | Force correct release branch on workflow sha
+        run: |
+          git clone https://github.com/${{ github.repository }} .
+          git checkout -B ${{ github.ref_name }} ${{ github.sha }}
+          git config user.name "cpg-software-ci-bot"
+          git config user.email "[email protected]"
+          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
+
+      - name: Action | Semantic Version Release
+        id: release
+        uses: python-semantic-release/[email protected]
+        with:
+          root_options: "-v"
+          git_committer_email: "[email protected]"
+          git_committer_name: "cpg-software-ci-bot"
+          github_token: ${{ secrets.BOT_ACCESS_TOKEN }}
+
+      - name: Publish | Upload package to PyPI
+        uses: pypa/[email protected]
+        if: steps.release.outputs.released == 'true'
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+
+      - name: Publish | Upload to GitHub Release Assets
+        uses: python-semantic-release/[email protected]
+        if: steps.release.outputs.released == 'true'
+        with:
+          github_token: ${{ secrets.BOT_ACCESS_TOKEN }}
+          tag: ${{ steps.release.outputs.tag }}

.github/workflows/renovate.yaml

@@ -2,15 +2,7 @@ name: Renovate
 on:
   schedule:
     - cron: '0 0 * * Sun'
-  push:
-    branches:
-      - dev
   workflow_dispatch:
-    inputs:
-      branch:
-        description: 'Branch to run on'
-        default: 'dev'
-        required: true
 
 permissions:
   contents: write
@@ -28,8 +20,6 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.branch }}
 
       - name: Self-hosted Renovate
         uses: renovatebot/[email protected]

.github/workflows/security.yaml

@@ -0,0 +1,25 @@
+name: Security Checks
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  pip-audit:
+    runs-on: ubuntu-latest
+    name: Pip Audit
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install project dependencies
+        run: |
+          uv sync
+          source .venv/bin/activate
+
+      - uses: pypa/[email protected]
+        with:
+          virtual-environment: .venv
+          summary: true

.github/workflows/test.yaml

@@ -1,5 +1,7 @@
 name: Test
 on:
+  # Building on pull-requests, manual dispatch, and pushes to main; but restricting
+  # publishing only to main pushes and manual dispatch with `if`s in specific steps.
   push:
 
 jobs:
@@ -15,10 +17,7 @@ jobs:
           submodules: 'true'
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: '0.5.6'
-          enable-cache: true
+        uses: astral-sh/setup-uv@v5
 
       - name: Install project dev dependencies
         run: uv sync --only-dev
@@ -60,6 +59,8 @@ jobs:
           echo "COVERAGE=$COVERAGE" >> badge_data.env
           echo "COVERAGE_COLOR=$COVERAGE_COLOR" >> badge_data.env
 
+          cat badge_data.env
+
       - name: 'Save badge data as an Artifact'
         uses: actions/upload-artifact@v4
         with:
@@ -71,14 +72,14 @@ jobs:
         uses: actions/github-script@v6
         with:
           script: |
-            core.setFailed('Unit tests failed with rc = ${{ steps.runtests.outputs.rc }}')
+            core.setFailed('Unittests failed with rc = ${{ steps.runtests.outputs.rc }}')
 
   sonarqube:
     name: SonarQube scan
     runs-on: ubuntu-latest
     needs: test
     environment: production
-    if: github.ref == 'refs/heads/main'
+    if: github.ref_name == 'main'
     steps:
       - uses: actions/checkout@v4
         with: