NO MERGE: WIP dev branch for non-openhpc slurm update

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @stackhpc/batch

.github/workflows/fatimage.yml

@@ -0,0 +1,62 @@
+
+name: Build fat image
+on:
+  workflow_dispatch:
+jobs:
+  openstack:
+    name: openstack-imagebuild
+    concurrency: ${{ github.ref }} # to branch/PR
+    runs-on: ubuntu-20.04
+    env:
+      ANSIBLE_FORCE_COLOR: True
+      OS_CLOUD: openstack
+      CI_CLOUD: ${{ vars.CI_CLOUD }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup ssh
+        run: |
+          set -x
+          mkdir ~/.ssh
+          echo "${{ secrets[format('{0}_SSH_KEY', vars.CI_CLOUD)] }}" > ~/.ssh/id_rsa
+          chmod 0600 ~/.ssh/id_rsa
+        shell: bash
+
+      - name: Add bastion's ssh key to known_hosts
+        run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
+        shell: bash
+
+      - name: Install ansible etc
+        run: dev/setup-env.sh
+
+      - name: Write clouds.yaml
+        run: |
+          mkdir -p ~/.config/openstack/
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', vars.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
+        shell: bash
+
+      - name: Setup environment
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+
+      - name: Build fat image with packer
+        id: packer_build
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          cd packer/
+          packer init .
+          PACKER_LOG=1 packer build -only openstack.openhpc -on-error=${{ vars.PACKER_ON_ERROR }} -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
+
+      - name: Get created image name from manifest
+        id: manifest
+        run: |
+          . venv/bin/activate
+          IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
+          while ! openstack image show -f value -c name $IMAGE_ID; do
+            sleep 30
+          done
+          IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+          echo "::set-output name=IMAGE_ID::$IMAGE_ID"
+          echo "::set-output name=IMAGE_NAME::$IMAGE_NAME"

.github/workflows/stackhpc.yml

@@ -1,5 +1,5 @@
 
-name: Test deployment and image build on OpenStack
+name: Test deployment and reimage on OpenStack
 on:
   workflow_dispatch:
   push:
@@ -8,103 +8,100 @@ on:
   pull_request:
 jobs:
   openstack:
-    name: openstack-ci-${{ matrix.cloud }}
-    strategy:
-      matrix:
-        cloud:
-          - "arcus"   # Arcus OpenStack in rcp-cloud-portal-demo project, with RoCE
-      fail-fast: false # as want clouds to continue independently
-    concurrency: ${{ matrix.cloud }}
+    name: openstack-ci
+    concurrency: ${{ github.ref }} # to branch/PR
     runs-on: ubuntu-20.04
+    env:
+      ANSIBLE_FORCE_COLOR: True
+      OS_CLOUD: openstack
+      TF_VAR_cluster_name: ci${{ github.run_id }}
+      CI_CLOUD: ${{ vars.CI_CLOUD }}
     steps:
       - uses: actions/checkout@v2
 
+      - name: Record which cloud CI is running on
+        run: |
+          echo CI_CLOUD: ${{ vars.CI_CLOUD }}
+
       - name: Setup ssh
         run: |
           set -x
           mkdir ~/.ssh
-          echo "${${{ matrix.cloud }}_SSH_KEY}" > ~/.ssh/id_rsa
+          echo "${{ secrets[format('{0}_SSH_KEY', vars.CI_CLOUD)] }}" > ~/.ssh/id_rsa
           chmod 0600 ~/.ssh/id_rsa
-        env:
-          smslabs_SSH_KEY: ${{ secrets.SSH_KEY }}
-          arcus_SSH_KEY: ${{ secrets.ARCUS_SSH_KEY }}
-
+        shell: bash
+
       - name: Add bastion's ssh key to known_hosts
-        run: cat environments/${{ matrix.cloud }}/bastion_fingerprint >> ~/.ssh/known_hosts
+        run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
         shell: bash
 
       - name: Install ansible etc
         run: dev/setup-env.sh
 
       - name: Install terraform
         uses: hashicorp/setup-terraform@v1
+        with:
+          terraform: v1.5.5
 
       - name: Initialise terraform
         run: terraform init
-        working-directory: ${{ github.workspace }}/environments/${{ matrix.cloud }}/terraform
+        working-directory: ${{ github.workspace }}/environments/.stackhpc/terraform
 
       - name: Write clouds.yaml
         run: |
           mkdir -p ~/.config/openstack/
-          echo "${${{ matrix.cloud }}_CLOUDS_YAML}" > ~/.config/openstack/clouds.yaml
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', vars.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
         shell: bash
-        env:
-          smslabs_CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
-          arcus_CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
-
-      - name: Provision infrastructure
-        id: provision
+
+      - name: Setup environment-specific inventory/terraform inputs
         run: |
           . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform apply -auto-approve
+          . environments/.stackhpc/activate
+          ansible-playbook ansible/adhoc/generate-passwords.yml
+          echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
         env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
-
-      - name: Get server provisioning failure messages
-        id: provision_failure
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+
+      - name: Provision nodes using fat image
+        id: provision_servers
         run: |
           . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
+          . environments/.stackhpc/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          TF_FAIL_MSGS="$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
-          echo $TF_FAIL_MSGS
-          echo "::set-output name=messages::${TF_FAIL_MSGS}"
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
-        if: always() && steps.provision.outcome == 'failure'
-
-      - name: Delete infrastructure if failed due to lack of hosts
+          terraform apply -auto-approve -var-file="${{ vars.CI_CLOUD }}.tfvars"
+
+      - name: Delete infrastructure if provisioning failed
         run: |
           . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
+          . environments/.stackhpc/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform destroy -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
-        if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
+          terraform destroy -auto-approve -var-file="${{ vars.CI_CLOUD }}.tfvars"
+        if: failure() && steps.provision_servers.outcome == 'failure'
 
-      - name: Directly configure cluster
+      - name: Configure cluster
         run: |
           . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
+          . environments/.stackhpc/activate
           ansible all -m wait_for_connection
-          ansible-playbook ansible/adhoc/generate-passwords.yml
-          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
-          ansible-playbook -vv ansible/site.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
-
+          ansible-playbook -v ansible/site.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
+
+      - name: Run MPI-based tests
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/adhoc/hpctests.yml
+
+      # - name: Run EESSI tests
+      #   run: |
+      #     . venv/bin/activate
+      #     . environments/.stackhpc/activate
+      #     ansible-playbook -vv ansible/ci/check_eessi.yml
+
       - name: Confirm Open Ondemand is up (via SOCKS proxy)
         run: |
           . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
+          . environments/.stackhpc/activate
 
           # load ansible variables into shell:
           ansible-playbook ansible/ci/output_vars.yml \
@@ -126,68 +123,61 @@ jobs:
             --server-response \
             --no-check-certificate \
             --http-user=testuser \
-            --http-password=${TEST_USER_PASSWORD} https://${openondemand_servername} \
+            --http-password=${TESTUSER_PASSWORD} https://${openondemand_servername} \
             2>&1)
           (echo $statuscode | grep "200 OK") || (echo $statuscode  && exit 1)
         env:
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Build packer images
-        run: |
-          . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
-          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
-          cd packer/
-          PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+      # - name: Build environment-specific compute image
+      #   id: packer_build
+      #   run: |
+      #     . venv/bin/activate
+      #     . environments/.stackhpc/activate
+      #     cd packer/
+      #     packer init
+      #     PACKER_LOG=1 packer build -except openstack.fatimage -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+      #     ../dev/output_manifest.py packer-manifest.json # Sets NEW_COMPUTE_IMAGE_ID outputs
 
-      - name: Test reimage of nodes
+      # - name: Test reimage of compute nodes to new environment-specific image (via slurm)
+      #   run: |
+      #     . venv/bin/activate
+      #     . environments/.stackhpc/activate
+      #     ansible login -v -a "sudo scontrol reboot ASAP nextstate=RESUME reason='rebuild image:${{ steps.packer_build.outputs.NEW_COMPUTE_IMAGE_ID }}' ${TF_VAR_cluster_name}-compute-[0-3]"
+      #     ansible compute -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
+      #     ansible-playbook -v ansible/ci/check_slurm.yml
+
+      - name: Test reimage of login and control nodes (via rebuild adhoc)
         run: |
           . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
-          ansible all -m wait_for_connection
-          ansible-playbook -vv ansible/ci/test_reimage.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
+          . environments/.stackhpc/activate
+          ansible-playbook -v --limit control,login ansible/adhoc/rebuild.yml
+          ansible all -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
+          ansible-playbook -v ansible/site.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
 
-      - name: Run MPI-based tests
+      - name: Check sacct state survived reimage
         run: |
           . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
-          ansible-playbook -vv ansible/adhoc/hpctests.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          OS_CLOUD: openstack
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/ci/check_sacct_hpctests.yml
 
       - name: Check MPI-based tests are shown in Grafana
         run: |
           . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
+          . environments/.stackhpc/activate
           ansible-playbook -vv ansible/ci/check_grafana.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          OS_CLOUD: openstack
 
       - name: Delete infrastructure
         run: |
           . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
+          . environments/.stackhpc/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform destroy -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
+          terraform destroy -auto-approve -var-file="${{ vars.CI_CLOUD }}.tfvars"
         if: ${{ success() || cancelled() }}
 
-      - name: Delete images
-        run: |
-          . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
-          ansible-playbook -vv ansible/ci/delete_images.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
+      # - name: Delete images
+      #   run: |
+      #     . venv/bin/activate
+      #     . environments/.stackhpc/activate
+      #     ansible-playbook -vv ansible/ci/delete_images.yml

ansible/.gitignore

@@ -4,8 +4,8 @@ roles/*
 # Whitelist roles that are checked into this repository.
 !roles/filebeat/
 !roles/filebeat/**
-!roles/opendistro/
-!roles/opendistro/**
+!roles/opensearch/
+!roles/opensearch/**
 !roles/podman/
 !roles/podman/**
 !roles/grafana-dashboards/
@@ -26,3 +26,21 @@ roles/*
 !roles/slurm_exporter/**
 !roles/firewalld/
 !roles/firewalld/**
+!roles/etc_hosts/
+!roles/etc_hosts/**
+!roles/cloud_init/
+!roles/cloud_init/**
+!roles/mysql/
+!roles/mysql/**
+!roles/systemd/
+!roles/systemd/**
+!roles/cuda/
+!roles/cuda/**
+!roles/freeipa/
+!roles/freeipa/**
+!roles/proxy/
+!roles/proxy/**
+!roles/resolv_conf/
+!roles/resolv_conf/**
+!roles/cve-2023-41914
+!roles/cve-2023-41914/**

ansible/adhoc/backup-keytabs.yml

@@ -0,0 +1,11 @@
+# Use ONE of the following tags on this playbook:
+#   - retrieve: copies keytabs out of the state volume to the environment
+#   - deploy: copies keytabs from the environment to the state volume
+
+- hosts: freeipa_client
+  become: yes
+  gather_facts: no
+  tasks:
+    - import_role:
+        name: freeipa
+        tasks_from: backup-keytabs.yml

ansible/adhoc/cudatests.yml

@@ -0,0 +1,8 @@
+- hosts: cuda
+  become: yes
+  gather_facts: no
+  tags: cuda_samples
+  tasks:
+    - import_role:
+        name: cuda
+        tasks_from: samples.yml

ansible/adhoc/cve-2023-41914.yml

@@ -0,0 +1,6 @@
+- hosts: openhpc
+  gather_facts: no
+  become: yes
+  tasks:
+    - import_role:
+        name: cve-2023-41914