Detect SA token rotation

cni-plugin/deployment/scripts/install-cni.sh

@@ -56,11 +56,22 @@ HOST_CNI_NET="${CONTAINER_MOUNT_PREFIX}${DEST_CNI_NET_DIR}"
 # Location of legacy "interface mode" file, to be automatically deleted
 DEFAULT_CNI_CONF_PATH="${HOST_CNI_NET}/01-linkerd-cni.conf"
 KUBECONFIG_FILE_NAME=${KUBECONFIG_FILE_NAME:-ZZZ-linkerd-cni-kubeconfig}
+SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
+# This file contains the full path to the detected cni config file
+LINKERD_CONF_PATH="${HOST_CNI_NET}/linkerd.json"
 
 ############################
 ### Function definitions ###
 ############################
 
+# Removes the "linkerd-cni" entry from the cni config file
+remove_linkerd_plugin_conf() {
+  local file=$1
+  log "Removing linkerd-cni config from $file"
+  cni_data=$(jq 'del( .plugins[]? | select( .type == "linkerd-cni" ))' "$file")
+  echo "$cni_data" > "$file"
+}
+
 # Cleanup will remove any installed configuration from the host If there are any
 # *conflist files, then linkerd-cni configuration parameters will be removed
 # from them.
@@ -79,11 +90,7 @@ cleanup() {
   local cni_data=''
   find "${HOST_CNI_NET}" -maxdepth 1 -type f \( -iname '*conflist' \) -print0 |
     while read -r -d $'\0' file; do
-      log "Removing linkerd-cni config from $file"
-      cni_data=$(jq 'del( .plugins[]? | select( .type == "linkerd-cni" ))' "$file")
-      # TODO (matei): we should write this out to a temp file and then do a `mv`
-      # to be atomic. 
-      echo "$cni_data" > "$file"
+      remove_linkerd_plugin_conf "$file"
     done
 
   # Remove binary and kubeconfig file
@@ -137,8 +144,6 @@ create_cni_conf() {
 ${CNI_NETWORK_CONFIG}
 EOF
   fi
-
-  SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
   KUBE_CA_FILE=${KUBE_CA_FILE:-${SERVICE_ACCOUNT_PATH}/ca.crt}
   SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}
   # Pull out service account token.
@@ -237,6 +242,9 @@ install_cni_conf() {
   mv "${TMP_CONF}" "${cni_conf_path}" || exit_with_error 'Failed to mv files.'
   [ -n "$old_file_path" ] && rm -f "${old_file_path}" && log "Removing unwanted .conf file"
 
+  # store full path of cni config file into "linkerd.json"
+  jq -n --arg val "$cni_conf_path" '{"cni_conf_path": $val}' > "${LINKERD_CONF_PATH}"
+
   log "Created CNI config ${cni_conf_path}"
 }
 
@@ -257,14 +265,7 @@ sync() {
 
   local config_file_count
   local new_sha
-  if [ "$ev" = 'DELETE' ]; then
-    # When the event type is 'DELETE', we check to see if there are any `*conf` or `*conflist`
-    # files on the host's filesystem.
-    config_file_count=$(find "${HOST_CNI_NET}" -maxdepth 1 -type f \( -iname '*conflist' -o -iname '*conf' \) | sort | wc -l)
-    if [ "$config_file_count" -eq 0 ]; then
-      log "No active CNI configuration file found after $ev event"
-    fi
-  elif [ "$ev" = 'CREATE' ] || [ "$ev" = 'MOVED_TO' ] || [ "$ev" = 'MODIFY' ]; then
+  if [ "$ev" = 'CREATE' ] || [ "$ev" = 'MOVED_TO' ] || [ "$ev" = 'MODIFY' ]; then
     # When the event type is 'CREATE', 'MOVED_TO' or 'MODIFY', we check the
     # previously observed SHA (updated with each file watch) and compare it
     # against the new file's SHA. If they differ, it means something has
@@ -273,7 +274,7 @@ sync() {
     if [ "$new_sha" != "$prev_sha" ]; then
       # Create but don't rm old one since we don't know if this will be configured
       # to run as _the_ cni plugin.
-      log "New file [$filename] detected; re-installing"
+      log "New/changed file [$filename] detected; re-installing"
       install_cni_conf "$filepath"
     else
       # If the SHA hasn't changed or we get an unrecognised event, ignore it.
@@ -285,22 +286,53 @@ sync() {
   fi
 }
 
-# Monitor will start a watch on host's CNI config directory
-monitor() {
-  inotifywait -m "${HOST_CNI_NET}" -e create,delete,moved_to,modify |
+# monitor_cni_config starts a watch on the host's CNI config directory
+monitor_cni_config() {
+  inotifywait -m "${HOST_CNI_NET}" -e create,moved_to,modify |
     while read -r directory action filename; do
       if [[ "$filename" =~ .*.(conflist|conf)$ ]]; then 
         log "Detected change in $directory: $action $filename"
         sync "$filename" "$action" "$cni_conf_sha"
-        # When file exists (i.e we didn't deal with a DELETE ev)
-        # then calculate its sha to be used the next turn.
-        if [[ -e "$directory/$filename" && "$action" != 'DELETE' ]]; then
+        # calculate file SHA to use in the next iteration
+        if [[ -e "$directory/$filename" ]]; then
           cni_conf_sha="$(sha256sum "$directory/$filename" | while read -r s _; do echo "$s"; done)"
         fi
       fi
     done
 }
 
+# Kubernetes usually rolls out service account tokens by creating new
+# directories containing a new token file and re-creating the
+# /var/run/secrets/kubernetes.io/serviceaccount/token symlink pointing to it.
+# This function listens to creation events under the serviceaccount directory,
+# only reacting to direct creation of a "token" file, or creation of
+# directories containing a "token" file. We only proceed as long as there's
+# already a "linkerd.json" file in place, which contains the full path of the
+# cni config file that needs to be rebuilt.
+monitor_service_account_token() {
+    inotifywait -m "${SERVICE_ACCOUNT_PATH}" -e create |
+      while read -r directory _ filename; do
+        log "Detected creation of file in $directory: $filename"
+
+        if [[ ! -e "${LINKERD_CONF_PATH}" ]]; then
+          log "Ignoring event: ${LINKERD_CONF_PATH} not found"
+          continue
+        fi
+        cni_conf_path=$(jq -r .cni_conf_path "${LINKERD_CONF_PATH}")
+        if [[ -z "$cni_conf_path" ]]; then
+          log "Ignoring event: cni_conf_path unknown"
+          continue
+        fi
+
+        target=$(realpath "$directory/$filename")
+        # if a new token file was detected, remove the linkerd plugin config to
+        # trigger the cni config file rebuild
+        if [[ (-f "$target" && "${target##*/}" == "token") || (-d "$target" && -e "$target/token") ]]; then
+          remove_linkerd_plugin_conf "$cni_conf_path"
+        fi
+      done
+}
+
 log() {
   printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$1"
 }
@@ -349,5 +381,7 @@ fi
 # builtin, the reception of a signal for which a trap has been set will cause
 # the wait builtin to return immediately with an exit status greater than 128,
 # immediately after which the trap is executed."
-monitor &
-wait $!
+monitor_cni_config &
+monitor_service_account_token &
+# uses -n so that we exit when the first background job exits (when there's an error)
+wait -n