feat: adding benchmark scripts for load testing

benchmark-load-testing/bin/initialize-benchmark-entities.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+set -e
+set -o pipefail
+# set -x
+
+#Functions
+function usage {
+  echo "Setup of test clients for the benchmark test with a pre-defined secret"
+  echo "Usage: $(basename $0) -k KEYCLOAK_HOME -r REALM_NAME -c CLIENT_ID [-d]" 2>&1
+  echo "-k keycloak home path ex: /home/opt/keycloak-18.0.0 : default value KEYCLOAK_HOME env variable"
+  echo "-d delete the client and realm before re-creating them: default value is false"
+  echo "-r realm : default value test-realm"
+  echo "-c service account enabled client name : default value gatling"
+  echo "-d delete the client and realm : default value is false"
+  echo "-u user : default username user-0. The user's password will always be the username plus the suffix -password, ex: for user-0 this would be user-0-password"
+  exit 1
+}
+
+function set_kcb_in_path {
+  echo -e "Setting up kcadm.sh in PATH\n"
+  export PATH=$PATH:$KEYCLOAK_HOME/bin
+}
+
+function create_realm {
+  if kcadm.sh get realms/${REALM_NAME} | grep -q ${REALM_NAME}; then
+    echo -e "INFO: Skipping Realm creation as realm exists\n"
+  else
+    echo -e "INFO: Creating Realm with realm id: ${REALM_NAME}"
+    kcadm.sh create realms -s realm=$REALM_NAME -s enabled=true -o > /dev/null
+  fi
+}
+
+function create_service_enabled_client_assign_roles {
+  CID=$(kcadm.sh create clients -r $REALM_NAME -i -f - <<EOF
+{
+  "clientId": "$CLIENT_ID",
+  "enabled": true,
+  "clientAuthenticatorType": "client-secret",
+  "secret": "setup-for-benchmark",
+  "redirectUris": [
+    "*"
+  ],
+  "serviceAccountsEnabled": true,
+  "publicClient": false,
+  "protocol": "openid-connect",
+  "attributes": {
+    "post.logout.redirect.uris": "+"
+  }
+}
+EOF
+)
+  echo "INFO: Created New ${CLIENT_ID} Client with Client ID ${CID}"
+  #Assign the necessary Service Account based roles to the client
+  kcadm.sh add-roles -r $REALM_NAME --uusername service-account-$CLIENT_ID --cclientid realm-management --rolename manage-clients --rolename view-users --rolename manage-realm --rolename manage-users
+}
+
+function create_oidc_client {
+  #Create the client application with OIDC for Auth Code scenarios with confidential client secret
+  CID=$(kcadm.sh create clients -r $REALM_NAME -i -f - <<EOF
+{
+  "clientId": "client-0",
+  "enabled": true,
+  "clientAuthenticatorType": "client-secret",
+  "secret": "client-0-secret",
+  "redirectUris": [
+    "*"
+  ],
+  "serviceAccountsEnabled": true,
+  "publicClient": false,
+  "protocol": "openid-connect",
+  "attributes": {
+    "post.logout.redirect.uris": "+"
+  }
+}
+EOF
+)
+  echo "INFO: Created New client-0 Client"
+}
+
+function create_user {
+ kcadm.sh create users -s username=$USER_NAME -s enabled=true -s firstName=Firstname -s lastName=Lastname -s email=$USER_NAME@keycloak.org -r $REALM_NAME
+ kcadm.sh set-password -r $REALM_NAME --username $USER_NAME --new-password $USER_NAME-password
+ echo "INFO: Created New user ${USER_NAME} in ${REALM_NAME}"
+}
+
+function delete_entities {
+  if kcadm.sh get realms/${REALM_NAME} | grep -q ${REALM_NAME}; then
+    kcadm.sh delete realms/${REALM_NAME} > /dev/null
+    echo -e "Successfully Deleted the Client and Realm\n"
+  fi
+}
+
+#main()
+#setting default values
+OPTIND=1
+REALM_NAME=test-realm
+CLIENT_ID=gatling
+DELETE_ENTITIES='false'
+USER_NAME=user-0
+
+while getopts "k:r:u:c:d" arg; do
+  case $arg in
+    k) KEYCLOAK_HOME="$OPTARG";
+        ;;
+    r) REALM_NAME="$OPTARG";
+        ;;
+    c) CLIENT_ID="$OPTARG";
+        ;;
+    d) DELETE_ENTITIES=true;
+        ;;
+    u) USER_NAME="$OPTARG";
+        ;;
+    \?) echo "ERROR: Invalid option: -${OPTARG}" >&2
+        usage
+        ;;
+  esac
+done
+
+#Handle missing opt args
+if (( ${OPTIND} == 1 ))
+then
+  echo -e "ERROR: No Options Specified\n"
+  usage
+fi
+shift $(( OPTIND -1 ))
+
+#Setup the KCB client tool in the system PATH
+set_kcb_in_path
+
+#Create or Re-Create Client and Realm
+if ${DELETE_ENTITIES}; then
+ echo "INFO: deleting the Client and Realm"
+ delete_entities
+else
+ echo "WARN: not deleting the Client and Realm"
+fi
+
+create_realm
+create_service_enabled_client_assign_roles
+create_oidc_client
+create_user

benchmark-load-testing/bin/kc-chaos.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Use this for simulating failures of pods when testing Keycloak's capabilities to recover.
+set -e
+
+: ${INITIAL_DELAY_SECS:=30}
+: ${CHAOS_DELAY_SECS:=10}
+: ${PROJECT:="runner-keycloak"}
+
+LOGS_DIR=$1
+
+echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Entering Chaos mode, with an initial delay of $INITIAL_DELAY_SECS seconds\033[0m"
+sleep $INITIAL_DELAY_SECS
+echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Running Chaos scenario - Delete random Keycloak pod\033[0m"
+
+ATTEMPT=0
+while true; do
+  ATTEMPT=$[ATTEMPT + 1]
+  RANDOM_KC_POD=$(kubectl \
+    -n "${PROJECT}" \
+    -o 'jsonpath={.items[*].metadata.name}' \
+    get pods -l app=keycloak | \
+      tr " " "\n" | \
+      shuf | \
+      head -n 1)
+
+  kubectl get pods -n "${PROJECT}" -l app=keycloak -o wide
+  kubectl logs -f -n "${PROJECT}" "${RANDOM_KC_POD}" > "$LOGS_DIR/${ATTEMPT}-${RANDOM_KC_POD}.log" 2>&1 &
+  kubectl describe -n "${PROJECT}" pod "${RANDOM_KC_POD}" > "$LOGS_DIR/${ATTEMPT}-${RANDOM_KC_POD}-complete-resource.log" 2>&1
+  kubectl top -n "${PROJECT}" pod -l app=keycloak --sum=true > "$LOGS_DIR/${ATTEMPT}-top.log" 2>&1
+  echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Killing Pod '${RANDOM_KC_POD}' and waiting for ${CHAOS_DELAY_SECS} seconds\033[0m"
+  kubectl delete pod -n "${PROJECT}" "${RANDOM_KC_POD}" --grace-period=1
+
+  START=$(date +%s)
+
+  kubectl wait --for=condition=Available --timeout=600s deployments.apps/keycloak-operator -n "${PROJECT}" || true
+  kubectl wait --for=condition=Ready --timeout=600s keycloaks.k8s.keycloak.org/keycloak -n "${PROJECT}" || true
+
+  END=$(date +%s)
+  DIFF=$(( END - START ))
+
+  echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Keycloak pod ${RANDOM_KC_POD} took ${DIFF} seconds to recover\033[0m"
+  sleep "${CHAOS_DELAY_SECS}"
+done

benchmark-load-testing/bin/kc-failover.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# Script simulating different xsite failover scenarios
+set -e
+
+if [[ "$RUNNER_DEBUG" == "1" ]]; then
+  set -x
+fi
+
+function activeClusterDown() {
+  DOMAIN=$1
+  CLIENT_IPS=$(dig +short client.${DOMAIN} | sort)
+  PRIMARY_IPS=$(dig +short primary.${DOMAIN} | sort)
+  BACKUP_IPS=$(dig +short backup.${DOMAIN} | sort)
+
+  [[ "${CLIENT_IPS}" == "${BACKUP_IPS}" && "${CLIENT_IPS}" != "${PRIMARY_IPS}" ]]
+  return
+}
+
+function scaleDownResource() {
+  kubectl -n $1 scale --replicas=0 $2
+  kubectl -n $1 rollout status --watch --timeout=600s $2
+}
+
+# Removes the Keycloak aws-health-route so that Route53 will eventually failover
+function killHealthRoute() {
+  kubectl -n ${PROJECT} delete route aws-health-route || true
+}
+
+# Remove all Keycloak routes so that Route53 will failover from the active to the passive cluster and the old DNS ips will fail
+function killKeycloakRoutes() {
+  scaleDownResource ${PROJECT} deployment/keycloak-operator
+  kubectl -n ${PROJECT} delete ingress keycloak-ingress || true
+  killHealthRoute
+}
+
+# Delete the Keycloak + Infinispan pods to simulate cluster crash
+function killKeycloakCluster() {
+  scaleDownResource openshift-operators deployment/infinispan-operator-controller-manager
+  scaleDownResource ${PROJECT} deployment/keycloak-operator
+  kubectl -n ${PROJECT} delete pods --all --force --grace-period=0
+  kubectl -n ${PROJECT} delete statefulset --all
+}
+
+# Delete the Infinispan GossipRouter
+function killGossipRouter() {
+  scaleDownResource openshift-operators deployment/infinispan-operator-controller-manager
+  kubectl -n ${PROJECT} delete pods -l app=infinispan-router-pod --force --grace-period=0
+  kubectl -n ${PROJECT} delete deployment/infinispan-router || true
+}
+
+# Scale Infinispan and Keycloak Operators so that the original cluster is recreated
+function reviveKeycloakCluster() {
+  echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Running Recovery scenario - ${RECOVERY_MODE}\033[0m"
+  cat << EOF | kubectl -n ${PROJECT} apply -f -
+  apiVersion: route.openshift.io/v1
+  kind: Route
+  metadata:
+    name: aws-health-route
+  spec:
+    host: "$1.${DOMAIN}"
+    port:
+      targetPort: https
+    tls:
+      insecureEdgeTerminationPolicy: Redirect
+      termination: passthrough
+    to:
+      kind: Service
+      name: keycloak-service
+EOF
+  kubectl -n openshift-operators scale --replicas=1 deployment/infinispan-operator-controller-manager
+  kubectl -n ${PROJECT} scale --replicas=1 deployment/keycloak-operator
+  kubectl -n ${PROJECT} rollout status --watch --timeout=600s statefulset/infinispan
+  kubectl -n ${PROJECT} rollout status --watch --timeout=600s statefulset/keycloak
+  exit
+}
+
+function waitForFailover() {
+  START=$(date +%s)
+  until activeClusterDown ${DOMAIN}
+  do
+    sleep 0.1
+  done
+  END=$(date +%s)
+  DIFF=$(( END - START ))
+
+  echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Route53 took ${DIFF} seconds to failover\033[0m"
+}
+
+function clusterFailover() {
+  killKeycloakCluster
+}
+
+: ${PROJECT:="runner-keycloak"}
+: ${FAILOVER_DELAY:=60}
+
+PROJECT=${PROJECT:-"runner-keycloak"}
+
+if [ -z "${RECOVERY_MODE}" ] && [ -z "${FAILOVER_MODE}" ]; then
+  echo "RECOVERY_MODE or FAILOVER_MODE env must be defined"
+  exit 1
+fi
+
+if [ -z "${DOMAIN}" ]; then
+  echo "DOMAIN env must be defined"
+  exit 1
+fi
+
+if [ -n "${RECOVERY_MODE}" ]; then
+  if [ "${RECOVERY_MODE^^}" == "ACTIVE" ]; then
+    reviveKeycloakCluster primary
+  elif [ "${RECOVERY_MODE^^}" == "PASSIVE" ]; then
+    reviveKeycloakCluster backup
+  else
+    echo "Unknown RECOVERY_MODE=${RECOVERY_MODE}"
+    exit 1
+  fi
+fi
+
+echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Entering Failover mode, with an initial delay of ${FAILOVER_DELAY} seconds\033[0m"
+sleep ${FAILOVER_DELAY}
+echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Running Failover scenario - ${FAILOVER_MODE}\033[0m"
+
+CLIENT_IPS=$(dig +short client.${DOMAIN} | sort)
+PRIMARY_IPS=$(dig +short primary.${DOMAIN} | sort)
+BACKUP_IPS=$(dig +short backup.${DOMAIN} | sort)
+
+if [ "${FAILOVER_MODE^^}" == "HEALTH_PROBE" ]; then
+  killHealthRoute
+elif [ "${FAILOVER_MODE^^}" == "KEYCLOAK_ROUTES" ]; then
+  killKeycloakRoutes
+elif [ "${FAILOVER_MODE^^}" == "CLUSTER_FAIL" ]; then
+  killKeycloakCluster
+elif [ "${FAILOVER_MODE^^}" == "GOSSIP_ROUTER_FAIL" ]; then
+  killGossipRouter
+  exit
+fi
+
+waitForFailover

benchmark-load-testing/bin/kc-rolling-restart.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Use this for deleting all Keycloak or Infinispan pods in sequence.
+set -e
+
+if [[ "$RUNNER_DEBUG" == "1" ]]; then
+  set -x
+fi
+
+: ${PROJECT:="runner-keycloak"}
+
+# Ensure POD_LABEL is set
+if [[ -z "${POD_LABEL}" ]]; then
+  echo "POD_LABEL is not set. Please export POD_LABEL before running the script."
+  exit 1
+fi
+
+echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Deleting all pods with label '${POD_LABEL}' in sequence\033[0m"
+
+ALL_PODS=$(kubectl -n "${PROJECT}" -o 'jsonpath={.items[*].metadata.name}' get pods -l app="${POD_LABEL}" | tr " " "\n")
+
+ATTEMPT=0
+for POD in $ALL_PODS; do
+  ATTEMPT=$((ATTEMPT + 1))
+  kubectl get pods -n "${PROJECT}" -l app="${POD_LABEL}"
+  echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') Killing Pod '${POD}'\033[0m"
+  kubectl delete pod -n "${PROJECT}" "${POD}"
+
+  START=$(date +%s)
+
+  if [[ "$POD_LABEL" == "keycloak" ]]; then
+    kubectl wait --for=condition=Available --timeout=120s deployments.apps/keycloak-operator -n "${PROJECT}" || true
+    kubectl wait --for=condition=Ready --timeout=120s keycloaks.k8s.keycloak.org/keycloak -n "${PROJECT}" || true
+  elif [[ "$POD_LABEL" == "infinispan-pod" ]]; then
+    kubectl wait --for condition=WellFormed --timeout=120s infinispans.infinispan.org -n "${PROJECT}" infinispan || true
+  fi
+
+  END=$(date +%s)
+  DIFF=$(( END - START ))
+
+  echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') ${POD} pod took ${DIFF} seconds to recover\033[0m"
+done
+
+kubectl get pods -n "${PROJECT}" -l app="${POD_LABEL}"
+echo -e "\033[0;31mINFO:$(date '+%F-%T-%Z') All ${POD_LABEL} pods have been restarted.\033[0m"