Merge change from upstream

ACCESS-NRI · Feb 7, 2024 · b7b7ba3 · b7b7ba3
1 parent 8ed9ad1
commit b7b7ba3
Show file tree

Hide file tree

Showing 27 changed files with 561 additions and 231 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,25 @@
+name: Build conda env
+on: 
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+jobs:
+  build:
+    name: Build environment
+    runs-on: ubuntu-latest
+    steps:
+      - name: Build environment
+        uses: appleboy/[email protected]
+        with:
+          host: gadi.nci.org.au
+          username: ${{secrets.GADI_USER}}
+          key: ${{secrets.DEPLOY_KEY}}
+          command_timeout: 120m
+          script: |
+            export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts
+            eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh )
+            eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh )
+            cd $JOB_LOG_DIR
+            qsub -N build_${{ inputs.environment }} -lncpus=1,mem=20GB,walltime=2:00:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR,CONDA_ENVIRONMENT=${{ inputs.environment }} -P kr06 -q copyq -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/build.sh
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -1,30 +1,24 @@
 name: Deploy conda env
 on:
-  push:
-    branches: main
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
 jobs:
   deploy:
+    name: Deploy environment
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        ### Latest at time of writing
-        uses: actions/[email protected]
-      - name: Sync repository to Gadi
-        ### Latest at time of writing
-        uses: up9cloud/[email protected]
-        env:
-          HOST: gadi.nci.org.au
-          TARGET: ${{secrets.GADI_REPO_PATH}}
-          KEY: ${{secrets.DEPLOY_KEY}}
-          USER: ${{secrets.GADI_USER}}
-      - name: Deploy conda environment
+      - name: Deploy environment
         uses: appleboy/[email protected]
         with:
           host: gadi.nci.org.au
           username: ${{secrets.GADI_USER}}
           key: ${{secrets.DEPLOY_KEY}}
           script: |
             export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts
+            export CONDA_ENVIRONMENT=${{ inputs.environment }}
             eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh )
             eval $( grep BUILD_STAGE_DIR $SCRIPT_DIR/install_config.sh )
             "${SCRIPT_DIR}"/deploy.sh
diff --git a/.github/workflows/get_changed_env.yml b/.github/workflows/get_changed_env.yml
@@ -0,0 +1,41 @@
+name: Get changed conda envs
+on:
+  workflow_call:
+    outputs:
+      matrix: 
+        value: ${{ jobs.generate_matrix.outputs.matrix }}
+jobs:
+  generate_matrix:
+    name: Determine changed environments
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.diff.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check changed environments
+        id: diff
+        run: |
+          # See https://github.community/t/check-pushed-file-changes-with-git-diff-tree-in-github-actions/17220/10
+          ### https://stackoverflow.com/questions/59977364/github-actions-how-use-strategy-matrix-with-script
+          if [ $GITHUB_BASE_REF ]; then
+            # Pull Request
+            git fetch origin $GITHUB_BASE_REF --depth=1
+            export DIFF=$( git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA )
+            echo "Diff between origin/$GITHUB_BASE_REF and $GITHUB_SHA"
+          else
+            # Push
+            git fetch origin ${{ github.event.before }} --depth=1
+            export DIFF=$( git diff --name-only ${{ github.event.before }} $GITHUB_SHA )
+            echo "Diff between ${{ github.event.before }} and $GITHUB_SHA"
+          fi
+          json="{\"include\":["
+          for line in $DIFF; do
+            if [[ $line =~ environments ]]; then
+                env_name=$( basename ${line%/*} )
+                if ! [[ $json =~ $env_name ]]; then
+                    json="$json{\"environment\":\"$env_name\"},"
+                fi
+            fi
+          done
+          ### https://github.com/actions/runner/issues/2947
+          echo "matrix=$( echo "${json%,}]}" | jq -r 'tostring' )" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/manual_trigger.yml b/.github/workflows/manual_trigger.yml
@@ -0,0 +1,54 @@
+name: Force environment update
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Environment to update"
+        required: true
+        type: string
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Sync repository to Gadi
+        uses: up9cloud/[email protected]
+        env:
+          HOST: gadi.nci.org.au
+          TARGET: ${{secrets.GADI_REPO_PATH}}
+          KEY: ${{secrets.DEPLOY_KEY}}
+          USER: ${{secrets.GADI_USER}}
+      - name: Create Admin dirs on Gadi
+        uses: appleboy/[email protected]
+        with:
+          host: gadi.nci.org.au
+          username: ${{secrets.GADI_USER}}
+          key: ${{secrets.DEPLOY_KEY}}
+          script: |
+            source ${{secrets.GADI_REPO_PATH}}/scripts/install_config.sh
+            source ${{secrets.GADI_REPO_PATH}}/scripts/functions.sh
+            mkdir -p $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR
+            set_admin_perms $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR
+
+  build:
+    needs: setup
+    uses: ./.github/workflows/build.yml
+    with:
+      environment: ${{ inputs.environment }}
+    secrets: inherit
+
+  test:
+    needs: build
+    uses: ./.github/workflows/test.yml
+    with:
+      environment: ${{ inputs.environment }}
+    secrets: inherit
+
+  deploy:
+    needs: test
+    uses: ./.github/workflows/deploy.yml
+    with:
+      environment: ${{ inputs.environment }}
+    secrets: inherit
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -0,0 +1,71 @@
+name: Build and test conda env
+on: pull_request
+jobs:
+  generate_matrix:
+    uses: ./.github/workflows/get_changed_env.yml
+
+  setup:
+    runs-on: ubuntu-latest
+    needs: generate_matrix
+    if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }}
+    container:
+      image: quay.io/singularity/singularity:v3.11.4
+      options: --privileged
+    steps:
+      - name: Checkout repository
+        ### Latest at time of writing
+        uses: actions/checkout@v4
+      - name: Check if container definition has changed
+        id: changed-container-def
+        uses: tj-actions/changed-files@v41
+        with:
+          files_yaml: |
+            containerdef:
+              - container/container.def
+      - name: Build container if definition has changed
+        if: steps.changed-container-def.outputs.containerdef_any_changed == 'true'
+        run: |
+            echo ${{ needs.generate_matrix.outputs.matrix }}
+            sudo -E singularity build container/base.sif container/container.def
+      - name: Sync repository to Gadi
+        ### Latest at time of writing
+        uses: up9cloud/[email protected]
+        env:
+          HOST: gadi.nci.org.au
+          TARGET: ${{secrets.GADI_REPO_PATH}}
+          KEY: ${{secrets.DEPLOY_KEY}}
+          USER: ${{secrets.GADI_USER}}
+      - name: Create Admin dirs on Gadi
+        uses: appleboy/[email protected]
+        with:
+          host: gadi.nci.org.au
+          username: ${{secrets.GADI_USER}}
+          key: ${{secrets.DEPLOY_KEY}}
+          script: |
+            source ${{secrets.GADI_REPO_PATH}}/scripts/install_config.sh
+            source ${{secrets.GADI_REPO_PATH}}/scripts/functions.sh
+            mkdir -p $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR
+            set_admin_perms $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR
+
+  build:
+    needs: [ generate_matrix, setup ]
+    uses: ./.github/workflows/build.yml
+    with:
+      environment: ${{ matrix.environment }}
+    secrets: inherit
+    if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }}
+    strategy:
+      matrix: ${{fromJson(needs.generate_matrix.outputs.matrix)}}
+      max-parallel: 1
+      fail-fast: false
+  test:
+    needs: [ generate_matrix, build ]
+    uses: ./.github/workflows/test.yml
+    with:
+      environment: ${{ matrix.environment }}
+    secrets: inherit
+    if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }}
+    strategy:
+      matrix: ${{fromJson(needs.generate_matrix.outputs.matrix)}}
+      max-parallel: 1
+      fail-fast: false
diff --git a/.github/workflows/push_to_main.yml b/.github/workflows/push_to_main.yml
@@ -0,0 +1,35 @@
+name: Deploy conda env
+on:
+  push:
+    branches: main
+jobs:
+  generate_matrix:
+    uses: ./.github/workflows/get_changed_env.yml
+
+  setup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        ### Latest at time of writing
+        uses: actions/checkout@v4
+      - name: Sync repository to Gadi
+        ### Latest at time of writing
+        uses: up9cloud/[email protected]
+        env:
+          HOST: gadi.nci.org.au
+          TARGET: ${{secrets.GADI_REPO_PATH}}
+          KEY: ${{secrets.DEPLOY_KEY}}
+          USER: ${{secrets.GADI_USER}}
+
+  deploy:
+    needs: [ generate_matrix, setup ]
+    uses: ./.github/workflows/deploy.yml
+    with:
+      environment: ${{ matrix.environment }}
+    secrets: inherit
+    if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }}
+    strategy: 
+      matrix: ${{fromJson(needs.generate_matrix.outputs.matrix)}}
+      max-parallel: 1
+      fail-fast: false
+
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,24 @@
+name: Test conda env
+on: 
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+jobs:
+  test:
+    name: Test environment
+    runs-on: ubuntu-latest
+    steps:
+      - name: Test environment
+        uses: appleboy/[email protected]
+        with:
+          host: gadi.nci.org.au
+          username: ${{secrets.GADI_USER}}
+          key: ${{secrets.DEPLOY_KEY}}
+          script: |
+            export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts
+            eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh )
+            eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh )
+            cd $JOB_LOG_DIR
+            qsub -N test_${{ inputs.environment }} -lncpus=4,mem=20GB,walltime=0:20:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR,CONDA_ENVIRONMENT=${{ inputs.environment }} -P kr06 -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/test.sh
diff --git a/container/container.def b/container/container.def
@@ -31,6 +31,8 @@ Bootstrap: scratch
         mkdir -p ${SINGULARITY_ROOTFS}/run
         mkdir -p ${SINGULARITY_ROOTFS}/system
         mkdir -p ${SINGULARITY_ROOTFS}/usr
+        mkdir -p ${SINGULARITY_ROOTFS}/sys/fs/cgroup
+        mkdir -p ${SINGULARITY_ROOTFS}/iointensive
 
         ### Mount points for service sockets
         mkdir -p ${SINGULARITY_ROOTFS}/var/lib/sss
@@ -40,6 +42,9 @@ Bootstrap: scratch
         mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-22.10
         mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.01
         mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.04
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.07
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.10
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-24.01
 
 %runscript
         /usr/bin/bash -l