WIP: create nomad-whisper state

jenkins/groovy/provision-nomad-whisper-pool/Jenkinsfile

@@ -0,0 +1,51 @@
+def utils
+pipeline {
+    agent any
+    options {
+        ansiColor('xterm')
+        timestamps()
+        buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10'))
+    }
+    stages {
+        stage ("setup") {
+            steps {
+                script {
+                    def rootDir = pwd()
+                    utils = load "${rootDir}/jenkins/groovy/Utils.groovy"
+                    utils.SetupRepos(env.VIDEO_INFRA_BRANCH)
+                    utils.SetupOCI()
+                    sh 'mkdir -p test-results'
+                }
+            }
+        }
+        stage ("Whisper Instance Configuration and Autoscaling Group Provisioning") {
+            steps {
+                script {
+                    echo 'Start provisioning whisper instance configuration'
+                    dir('infra-provisioning') {
+                        withCredentials([
+                            string(credentialsId: 'oci-jenkins-terraform-aws-secret', variable: 'AWS_SECRET_ACCESS_KEY'),
+                            string(credentialsId: 'oci-jenkins-terraform-aws-id', variable: 'AWS_ACCESS_KEY_ID'),
+                            sshUserPrivateKey(credentialsId: 'ssh-ubuntu', keyFileVariable: 'USER_PRIVATE_KEY_PATH', usernameVariable: 'SSH_USERNAME'),
+                            string(credentialsId: 'asap-jwt-kid-dev', variable: 'ASAP_JWT_KID_DEV'),
+                            string(credentialsId: 'asap-jwt-kid-prod', variable: 'ASAP_JWT_KID_PROD'),
+                            file(credentialsId: 'asap-jwt-key-dev', variable: 'ASAP_JWT_KEY_DEV'),
+                            file(credentialsId: 'asap-jwt-key-prod', variable: 'ASAP_JWT_KEY_PROD')
+                        ]) {
+                            utils.SetupAnsible()
+                            utils.SetupSSH()
+                            sshagent (credentials: ['ssh-ubuntu']) {
+                                sh """#!/bin/bash
+                                export USER_PUBLIC_KEY_PATH=~/.ssh/ssh_key.pub
+                                export ORACLE_GIT_BRANCH=\"$RELEASE_BRANCH\"
+                                export ENVIRONMENT=\"$ENVIRONMENT\"
+
+                                scripts/create-or-rotate-whisper.sh $SSH_USERNAME"""
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

jenkins/jobs/provision-nomad-whisper-pool.yaml

@@ -0,0 +1,56 @@
+- job:
+    name: provision-nomad-whisper-pool
+    display-name: provision nomad whisper autoscaling group
+    description: 'Provision a nomad GPU instance autoscaling group for a new environment/region.'
+    concurrent: true
+    parameters:
+      - string:
+          name: VIDEO_INFRA_BRANCH
+          default: main
+          description: "Controls checkout branch for infra repos, defaults to 'main'."
+          trim: true
+      - string:
+          name: ENVIRONMENT
+          default: lonely
+          description: "Environment to build in, defaults to 'lonely'."
+          trim: true
+      - string:
+          name: ORACLE_REGION
+          description: "Oracle Region to build in"
+          trim: true
+      - string:
+          name: CLOUD_PROVIDER
+          default: oracle
+          description: "Cloud Provider, defaults to 'oracle'"
+          trim: true
+      - string:
+          name: RELEASE_BRANCH
+          default: main
+          description: "Name of an existing infra branch to use when creating the release, defaults to 'main'."
+          trim: true
+      - string:
+          name: INFRA_CONFIGURATION_REPO
+          default: [email protected]:jitsi/infra-configuration.git
+          description: "Repo for configuration code (ansible etc), defaults to '[email protected]:jitsi/infra-configuration.git'."
+          trim: true
+      - string:
+          name: INFRA_CUSTOMIZATIONS_REPO
+          default: [email protected]:jitsi/infra-customizations.git
+          description: "Repo with customized configurations, defaults to '[email protected]:jitsi/infra-customizations.git'."
+          trim: true
+
+    project-type: pipeline
+    sandbox: true
+    pipeline-scm:
+      scm:
+        - git:
+            url: [email protected]:jitsi/infra-provisioning.git
+            credentials-id: "video-infra"
+            branches:
+              - "origin/${{VIDEO_INFRA_BRANCH}}"
+            browser: githubweb
+            browser-url: https://github.com/jitsi/infra-provisioning
+            submodule:
+              recursive: true
+      script-path: jenkins/groovy/provision-nomad-whisper-pool/Jenkinsfile
+      lightweight-checkout: true

scripts/create-or-rotate-whisper.sh

@@ -0,0 +1,163 @@
+#!/bin/bash
+
+set -x
+
+#IF THE CURRENT DIRECTORY HAS stack-env.sh THEN INCLUDE IT
+[ -e ./stack-env.sh ] && . ./stack-env.sh
+
+if [ -z "$ENVIRONMENT" ]; then
+  echo "No ENVIRONMENT found. Exiting..."
+  exit 203
+fi
+
+[ -e ./sites/$ENVIRONMENT/stack-env.sh ] && . ./sites/$ENVIRONMENT/stack-env.sh
+
+LOCAL_PATH=$(dirname "${BASH_SOURCE[0]}")
+
+source $LOCAL_PATH/../clouds/all.sh
+
+
+[ -z "$CLOUD_NAME" ] && CLOUD_NAME="oracle"
+
+source $LOCAL_PATH/../clouds/"$CLOUD_NAME".sh
+
+#pull in cloud-specific variables, e.g. tenancy
+[ -e "$LOCAL_PATH/../clouds/oracle.sh" ] && . "$LOCAL_PATH/../clouds/oracle.sh"
+
+[ -z "$ORACLE_GIT_BRANCH" ] && ORACLE_GIT_BRANCH="$RELEASE_BRANCH"
+
+
+[ -z "$CONFIG_VARS_FILE" ] && CONFIG_VARS_FILE="$LOCAL_PATH/../config/vars.yml"
+[ -z "$ENVIRONMENT_VARS_FILE" ] && ENVIRONMENT_VARS_FILE="$LOCAL_PATH/../sites/$ENVIRONMENT/vars.yml"
+
+
+export AUTOSCALER_BACKEND="${ENVIRONMENT}-${ORACLE_REGION}"
+export AUTOSCALER_URL="https://${ENVIRONMENT}-${ORACLE_REGION}-autoscaler.$TOP_LEVEL_DNS_ZONE_NAME"
+
+[ -z $ENABLE_AUTO_SCALE ] && ENABLE_AUTO_SCALE="true"
+[ -z $ENABLE_LAUNCH ] && ENABLE_LAUNCH="true"
+
+# run as user
+if [ -z "$1" ]; then
+  SSH_USER=$(whoami)
+  echo "Ansible SSH user is not defined. We use current user: $SSH_USER"
+else
+  SSH_USER=$1
+  echo "Run ansible as $SSH_USER"
+fi
+
+[ -z "$TAG_NAMESPACE" ] && TAG_NAMESPACE="jitsi"
+
+export ORACLE_REGION=$ORACLE_REGION
+export ORACLE_GIT_BRANCH=${ORACLE_GIT_BRANCH}
+export TAG_NAMESPACE=${TAG_NAMESPACE}
+
+ORACLE_CLOUD_NAME="$ORACLE_REGION-$ENVIRONMENT-oracle"
+[ -e "$LOCAL_PATH/../clouds/${ORACLE_CLOUD_NAME}.sh" ] && . $LOCAL_PATH/../clouds/${ORACLE_CLOUD_NAME}.sh
+
+
+echo "Creating whisper configuration"
+$LOCAL_PATH/../terraform/nomad-whisper/create-nomad-whisper-configuration.sh $SSH_USER
+if [ $? == 0 ]; then
+    echo "Whisper instance configuration created successfully"
+else
+    echo "Failed to create whisper instance configuration"
+    exit 214
+fi
+
+export INSTANCE_CONFIG_NAME="$ENVIRONMENT-$ORACLE_REGION-whisper-InstanceConfig"
+INSTANCE_CONFIGURATION_ID=$(oci compute-management instance-configuration list --region "$ORACLE_REGION" -c "$COMPARTMENT_OCID" --sort-by TIMECREATED --sort-order DESC --all --query "data[?\"display-name\" == '$INSTANCE_CONFIG_NAME'].id" | jq -r  '.[0]')
+
+if [ -z "$INSTANCE_CONFIGURATION_ID" ]; then
+    echo "No Instance configuration was found. Exiting ..."
+    exit 201
+fi
+
+echo "Instance configuration id is: $INSTANCE_CONFIGURATION_ID"
+
+SCALE_INCREASE_RATE=1
+SCALE_DECREASE_RATE=1
+GRACE_PERIOD_TTL_SEC=600
+PROTECTED_TTL_SEC=600
+MIN_COUNT=1
+MAX_COUNT=1
+SCALE_UP_THRESHOLD=0.6
+SCALE_DOWN_THRESHOLD=0.1
+SCALE_PERIOD=60
+SCALE_UP_PERIODS_COUNT=2
+SCALE_DOWN_PERIODS_COUNT=10
+[ -z $DESIRED_COUNT ] && DESIRED_COUNT=$MIN_COUNT
+
+export TYPE="whisper"
+export INSTANCE_CONFIGURATION_ID=$INSTANCE_CONFIGURATION_ID
+export GROUP_NAME="${ENVIRONMENT}-${ORACLE_REGION}-whisper"
+export ENABLE_AUTO_SCALE=${ENABLE_AUTO_SCALE}
+export ENABLE_LAUNCH=${ENABLE_LAUNCH}
+export ENABLE_SCHEDULER="false"
+export ENABLE_RECONFIGURATION="false"
+export GRACE_PERIOD_TTL_SEC=${GRACE_PERIOD_TTL_SEC}
+export PROTECTED_TTL_SEC=${PROTECTED_TTL_SEC}
+export MAX_COUNT=${MAX_COUNT}
+export MIN_COUNT=${MIN_COUNT}
+export DESIRED_COUNT=${DESIRED_COUNT}
+export SCALE_UP_THRESHOLD=${SCALE_UP_THRESHOLD}
+export SCALE_DOWN_THRESHOLD=${SCALE_DOWN_THRESHOLD}
+export SCALING_INCREASE_RATE=${SCALE_INCREASE_RATE}
+export SCALING_DECREASE_RATE=${SCALE_DECREASE_RATE}
+export SCALE_PERIOD=${SCALE_PERIOD}
+export SCALE_UP_PERIODS_COUNT=${SCALE_UP_PERIODS_COUNT}
+export SCALE_DOWN_PERIODS_COUNT=${SCALE_DOWN_PERIODS_COUNT}
+
+[ -z "$WAIT_FOR_POSTINSTALL" ] && WAIT_FOR_POSTINSTALL="TRUE"
+[ -z "$SLEEP_SECONDS_BEFORE_POSTINSTALL_CHECKS" ] && SLEEP_SECONDS_BEFORE_POSTINSTALL_CHECKS=480
+
+####
+
+echo "Creating Whisper autoscaling group"
+$LOCAL_PATH/custom-autoscaler-create-group.sh
+CREATE_GROUP_RESULT="$?"
+
+if [ $CREATE_GROUP_RESULT -gt 0 ]; then
+  echo "Failed to create the custom autoscaler group $GROUP_NAME. Exiting."
+  exit 213
+fi
+
+if [ -z "$AUTOSCALER_URL" ]; then
+  echo "No AUTOSCALER_URL provided or found. Exiting.. "
+  exit 212
+fi
+
+if [ -z "$JWT_ENV_FILE" ]; then 
+  if [ -z "$SIDECAR_ENV_VARIABLES" ]; then
+    echo "No SIDECAR_ENV_VARIABLES provided or found. Exiting.. "
+    exit 211
+  fi
+
+  JWT_ENV_FILE="/etc/jitsi/autoscaler-sidecar/$SIDECAR_ENV_VARIABLES"
+fi
+
+[ -z "$TOKEN" ] && TOKEN=$(JWT_ENV_FILE=$JWT_ENV_FILE /opt/jitsi/jitsi-autoscaler-sidecar/scripts/jwt.sh)
+
+if [ $WAIT_FOR_POSTINSTALL == "TRUE" ]; then
+  echo "Wait for Whisper instances to launch"
+
+  if [ $SLEEP_SECONDS_BEFORE_POSTINSTALL_CHECKS -gt 0 ]; then
+    echo "Sleeping for $SLEEP_SECONDS_BEFORE_POSTINSTALL_CHECKS seconds before checking postinstall result"
+    sleep $SLEEP_SECONDS_BEFORE_POSTINSTALL_CHECKS
+  fi
+
+  echo "Checking postinstall result and waiting for it to complete..."
+  export GROUP_NAME
+  export SIDECAR_ENV_VARIABLES
+  export AUTOSCALER_URL
+  export TOKEN
+  export EXPECTED_COUNT="$DESIRED_COUNT"
+  export CHECK_SCALE_UP="true"
+  $LOCAL_PATH/check-jvb-count-custom-autoscaler-oracle.sh
+  POSTINSTALL_RESULT=$?
+
+  if [ $POSTINSTALL_RESULT -gt 0 ]; then
+    echo "Posinstall did not succeeded for $GROUP_NAME. Exiting."
+    exit 214
+  fi
+fi