Dev kmr

.github/actions/install-crucible/install-crucible.sh

@@ -72,13 +72,30 @@ done
 validate_ci_run_environment
 validate_ci_endpoint
 
+ci_auth_file="/root/crucible-ci-engines-token.json"
+production_auth_file="/root/crucible-production-engines-token.json"
+if [ -e "${ci_auth_file}" -a -s "${ci_auth_file}" -a -e "${production_auth_file}" -a -s "${production_auth_file}" ]; then
+    echo "ERROR: It does not make sense for both the ci (${ci_auth_file}) and production (${production_auth_file}) client-server registry auth token files to exist"
+    exit 1
+fi
+
 AUTH_TOKEN_FILE_FOUND=0
-auth_file="/root/crucible-ci-engines-token.json"
-if [ -e "${auth_file}" -a -s "${auth_file}" ]; then
-    echo "Found client-server registry auth token file: ${auth_file}"
+if [ -e "${ci_auth_file}" -a -s "${ci_auth_file}" ]; then
+    echo "Found ci client-server registry auth token file: ${ci_auth_file}"
+    auth_file=${ci_auth_file}
     AUTH_TOKEN_FILE_FOUND=1
+    AUTH_TOKEN_TYPE="CI"
 else
-    echo "No client-server registry auth token file found: ${auth_file}"
+    echo "No ci client-server registry auth token file found: ${ci_auth_file}"
+
+    if [ -e "${production_auth_file}" -a -s "${production_auth_file}" ]; then
+        echo "Found production client-server registry auth token file: ${production_auth_file}"
+        auth_file=${production_auth_file}
+        AUTH_TOKEN_FILE_FOUND=1
+        AUTH_TOKEN_TYPE="PRODUCTION"
+    else
+        echo "No production client-server registry auth token file found: ${production_auth_file}"
+    fi
 fi
 
 # ensure endpoint availability
@@ -118,8 +135,16 @@ if pushd ~/ > /dev/null; then
     fi
     if [ ${AUTH_TOKEN_FILE_FOUND} == 1 ]; then
         INSTALLER_ARGS+=" --client-server-auth-file ${auth_file}"
-        CONTAINER_REGISTRY="quay.io/crucible/crucible-ci-engines"
         REGISTRY_TLS_VERIFY="true"
+
+        case "${AUTH_TOKEN_TYPE}" in
+            "CI")
+                CONTAINER_REGISTRY="quay.io/crucible/crucible-ci-engines"
+                ;;
+            "PRODUCTION")
+                CONTAINER_REGISTRY="quay.io/crucible/client-server"
+                ;;
+        esac
     fi
     CONTROLLER_REGISTRY_ARGS=""
     if [ "${CI_CONTROLLER}" == "yes" ]; then

.github/actions/integration-tests/run-ci-stage1

@@ -618,6 +618,8 @@ run_cmd "crucible ls --type run-id"
 
 run_cmd "crucible ls --type tags"
 
+run_cmd "crucible opensearch repair"
+
 run_cmd "crucible opensearch init"
 
 run_cmd "crucible opensearch rebuild"

.github/workflows/benchmark-crucible-ci.yaml

@@ -21,6 +21,10 @@ on:
     secrets:
       registry_auth:
         required: false
+      ci_registry_auth:
+        required: false
+      production_registry_auth:
+        required: false
 
 concurrency:
   group: ${{ inputs.ci_target }}/${{ github.ref }}/benchmark-crucible-ci
@@ -131,11 +135,18 @@ jobs:
         repository: perftool-incubator/bench-${{ inputs.ci_target }}
         ref: ${{ inputs.ci_target_branch }}
         path: "bench-${{ inputs.ci_target }}"
-    - name: import secret
+
+    - name: import ci secret
       env:
-        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth }}
-      if: env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != ''
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth != '' && secrets.registry_auth || secrets.ci_registry_auth }}
+      if: ${{ env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
       run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-ci-engines-token.json"
+    - name: import production secret
+      env:
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.production_registry_auth }}
+      if: ${{ env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
+      run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-production-engines-token.json"
+
     - name: run crucible-ci->integration-tests
       if: ${{ matrix.scenario.enabled }}
       uses: ./crucible-ci/.github/actions/integration-tests
@@ -190,11 +201,18 @@ jobs:
         repository: perftool-incubator/bench-${{ inputs.ci_target }}
         ref: ${{ inputs.ci_target_branch }}
         path: "bench-${{ inputs.ci_target }}"
-    - name: import secret
+
+    - name: import ci secret
       env:
-        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth }}
-      if: env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != ''
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth != '' && secrets.registry_auth || secrets.ci_registry_auth }}
+      if: ${{ needs.gen-params.outputs.build_controller == 'yes' && env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
       run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-ci-engines-token.json"
+    - name: import production secret
+      env:
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.production_registry_auth }}
+      if: ${{ needs.gen-params.outputs.build_controller == 'yes' && env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
+      run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-production-engines-token.json"
+
     - name: run crucible-ci->integration-tests
       if: ${{ matrix.scenario.enabled }}
       uses: ./crucible-ci/.github/actions/integration-tests

.github/workflows/core-crucible-ci.yaml

@@ -26,6 +26,10 @@ on:
     secrets:
       registry_auth:
         required: false
+      ci_registry_auth:
+        required: false
+      production_registry_auth:
+        required: false
 
 concurrency:
   group: ${{ inputs.crucible_ci_test }}/${{ inputs.ci_target }}/${{ github.ref }}/core-crucible-ci
@@ -199,11 +203,17 @@ jobs:
         ref: ${{ inputs.ci_target_branch }}
         path: ${{ needs.gen-params.outputs.ci_target_dir }}
 
-    - name: import secret
+    - name: import ci secret
       env:
-        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth }}
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth != '' && secrets.registry_auth || secrets.ci_registry_auth }}
       if: ${{ needs.gen-params.outputs.build_controller == 'yes' && env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
       run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-ci-engines-token.json"
+    - name: import production secret
+      env:
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.production_registry_auth }}
+      if: ${{ needs.gen-params.outputs.build_controller == 'yes' && env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
+      run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-production-engines-token.json"
+
     - name: run crucible-ci->install-crucible
       if: ${{ needs.gen-params.outputs.build_controller == 'yes' }}
       uses: ./crucible-ci/.github/actions/install-crucible
@@ -269,11 +279,17 @@ jobs:
         ref: ${{ inputs.crucible_ci_test_branch }}
         path: crucible-ci
 
-    - name: import secret
+    - name: import ci secret
       env:
-        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth }}
-      if: env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != ''
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth != '' && secrets.registry_auth || secrets.ci_registry_auth }}
+      if: ${{ env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
       run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-ci-engines-token.json"
+    - name: import production secret
+      env:
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.production_registry_auth }}
+      if: ${{ env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
+      run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-production-engines-token.json"
+
     - name: run crucible-ci->integration-tests
       if: ${{ matrix.scenario.enabled }}
       uses: ./crucible-ci/.github/actions/integration-tests
@@ -334,11 +350,17 @@ jobs:
         ref: ${{ inputs.crucible_ci_test_branch }}
         path: crucible-ci
 
-    - name: import secret
+    - name: import ci secret
       env:
-        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth }}
-      if: env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != ''
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth != '' && secrets.registry_auth || secrets.ci_registry_auth }}
+      if: ${{ needs.gen-params.outputs.build_controller == 'yes' && env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
       run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-ci-engines-token.json"
+    - name: import production secret
+      env:
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.production_registry_auth }}
+      if: ${{ needs.gen-params.outputs.build_controller == 'yes' && env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
+      run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-production-engines-token.json"
+
     - name: run crucible-ci->integration-tests
       if: ${{ matrix.scenario.enabled }}
       uses: ./crucible-ci/.github/actions/integration-tests

.github/workflows/run-core-crucible-ci.yaml

@@ -21,4 +21,4 @@ jobs:
       ci_target_branch: "${{ github.ref }}"
       github_workspace: "$GITHUB_WORKSPACE"
     secrets:
-      registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}
+      ci_registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}

.github/workflows/test-benchmark-crucible-ci.yaml

@@ -33,7 +33,7 @@ jobs:
       crucible_ci_test_branch: "${{ github.ref }}"
       github_workspace: "$GITHUB_WORKSPACE"
     secrets:
-      registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}
+      ci_registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}
 
   test-benchmark-crucible-ci-complete:
     runs-on: ubuntu-latest

.github/workflows/test-core-crucible-ci.yaml

@@ -34,7 +34,7 @@ jobs:
       crucible_ci_test_branch: "${{ github.ref }}"
       github_workspace: "$GITHUB_WORKSPACE"
     secrets:
-      registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}
+      ci_registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}
 
   test-core-crucible-ci-complete:
     runs-on: ubuntu-latest

.github/workflows/test-tool-crucible-ci.yaml

@@ -31,7 +31,7 @@ jobs:
       crucible_ci_test_branch: "${{ github.ref }}"
       github_workspace: "$GITHUB_WORKSPACE"
     secrets:
-      registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}
+      ci_registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}
 
   test-tool-crucible-ci-complete:
     runs-on: ubuntu-latest

.github/workflows/tool-crucible-ci.yaml

@@ -21,6 +21,10 @@ on:
     secrets:
       registry_auth:
         required: false
+      ci_registry_auth:
+        required: false
+      production_registry_auth:
+        required: false
 
 concurrency:
   group: ${{ inputs.ci_target }}/${{ github.ref }}/tool-crucible-ci
@@ -123,11 +127,18 @@ jobs:
         repository: perftool-incubator/tool-${{ inputs.ci_target }}
         ref: ${{ inputs.ci_target_branch }}
         path: "tool-${{ inputs.ci_target }}"
-    - name: import secret
+
+    - name: import ci secret
       env:
-        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth }}
-      if: env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != ''
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth != '' && secrets.registry_auth || secrets.ci_registry_auth }}
+      if: ${{ env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
       run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-ci-engines-token.json"
+    - name: import production secret
+      env:
+        CLIENT_SERVER_REGISTRY_AUTH_SECRET: ${{ secrets.production_registry_auth }}
+      if: ${{ env.CLIENT_SERVER_REGISTRY_AUTH_SECRET != '' }}
+      run: sudo bash -c "echo \"$CLIENT_SERVER_REGISTRY_AUTH_SECRET\" > /root/crucible-production-engines-token.json"
+
     - name: run crucible-ci->integration-tests
       if: ${{ matrix.scenario.enabled }}
       uses: ./crucible-ci/.github/actions/integration-tests