USAGOV-2174 - Deploy to Stage (#2175)

.circleci/config.yml

@@ -313,7 +313,7 @@ jobs:
 
   lint:
     docker:
-      - image: cimg/php:8.1.27-node
+      - image: cimg/php:8.3-node
     steps:
       - checkout
       - run:

.docker/src-cms/50_xdebug.ini

@@ -4,4 +4,4 @@ xdebug.client_host=host.docker.internal
 xdebug.idekey=VSCODE
 xdebug.max_nesting_level=1500
 xdebug.mode=debug
-# xdebug.start_with_request=1
+# xdebug.start_with_request=1

.docker/src-cron/etc/periodic/1min/cfevents.sh

@@ -1,6 +1,15 @@
 #!/usr/bin/env bash
 
 SPACE=$1
+if [ x$SPACE = x ]; then
+   SPACE=$(echo $VCAP_APPLICATION | jq -r '.space_name')
+else
+   shift
+fi
+
+if [ x$CFEVENT_RUN = x ]; then
+    exit 0;
+fi
 
 source ~/.profile $SPACE event &> /dev/null
 

.docker/src-cron/root/.profile

@@ -41,25 +41,34 @@ if [ ! -f ~/.certs-updated ]; then
    touch ~/.certs-updated
 fi
 
-CF_USERNAME=$(echo "$VCAP_SERVICES" | jq -r '.["cloud-gov-service-account"][]? | select(.name == "cron-service-account") | .credentials.username';)
-CF_PASSWORD=$(echo "$VCAP_SERVICES" | jq -r '.["cloud-gov-service-account"][]? | select(.name == "cron-service-account") | .credentials.password')
-
 case $TASK in
    event)
+      CF_USERNAME=$(echo "$VCAP_SERVICES" | jq -r '.["cloud-gov-service-account"][]? | select(.name == "cfevent-service-account") | .credentials.username';)
+      CF_PASSWORD=$(echo "$VCAP_SERVICES" | jq -r '.["cloud-gov-service-account"][]? | select(.name == "cfevent-service-account") | .credentials.password')
+      SERVICE_ACCOUNT="cfevent-service-account"
+      KEY_NAME="cfevent-service-key"
       export S3_BUCKET=$(echo "$VCAP_SERVICES" | jq -r '.["s3"][]? | select(.name == "cron-event-storage") | .credentials.bucket')
       export S3_ENDPOINT=$(echo "$VCAP_SERVICES" | jq -r '.["s3"][]? | select(.name == "cron-event-storage") | .credentials.fips_endpoint')
       export AWS_ACCESS_KEY_ID=$(echo "$VCAP_SERVICES" | jq -r '.["s3"][]? | select(.name == "cron-event-storage")    | .credentials.access_key_id')
       export AWS_SECRET_ACCESS_KEY=$(echo "$VCAP_SERVICES" | jq -r '.["s3"][]? | select(.name == "cron-event-storage")    | .credentials.secret_access_key')
       export AWS_DEFAULT_REGION=$(echo "$VCAP_SERVICES" | jq -r '.["s3"][]? | select(.name == "cron-event-storage")    | .credentials.region')
       ;;
    callwait)
+      CF_USERNAME=$(echo "$VCAP_SERVICES" | jq -r '.["cloud-gov-service-account"][]? | select(.name == "callcenter-service-account") | .credentials.username';)
+      CF_PASSWORD=$(echo "$VCAP_SERVICES" | jq -r '.["cloud-gov-service-account"][]? | select(.name == "callcenter-service-account") | .credentials.password')
+      SERVICE_ACCOUNT="callcenter-service-account"
+      KEY_NAME="callcenter-service-key"
       export S3_BUCKET=$(echo "$VCAP_SERVICES"             | jq -r '.["s3"][]? | select(.name == "cron-callwait-storage") | .credentials.bucket')
       export S3_ENDPOINT=$(echo "$VCAP_SERVICES"           | jq -r '.["s3"][]? | select(.name == "cron-callwait-storage") | .credentials.fips_endpoint')
       export AWS_ACCESS_KEY_ID=$(echo "$VCAP_SERVICES"     | jq -r '.["s3"][]? | select(.name == "cron-callwait-storage") | .credentials.access_key_id')
       export AWS_SECRET_ACCESS_KEY=$(echo "$VCAP_SERVICES" | jq -r '.["s3"][]? | select(.name == "cron-callwait-storage") | .credentials.secret_access_key')
       export AWS_DEFAULT_REGION=$(echo "$VCAP_SERVICES"    | jq -r '.["s3"][]? | select(.name == "cron-callwait-storage") | .credentials.region')
       ;;
-   *)
+      *)
+      CF_USERNAME=$(echo "$VCAP_SERVICES" | jq -r '.["cloud-gov-service-account"][]? | select(.name == "cron-service-account") | .credentials.username';)
+      CF_PASSWORD=$(echo "$VCAP_SERVICES" | jq -r '.["cloud-gov-service-account"][]? | select(.name == "cron-service-account") | .credentials.password')
+      SERVICE_ACCOUNT="cron-service-account"
+      KEY_NAME="cron-service-key"
       export S3_BUCKET=$(echo "$VCAP_SERVICES"             | jq -r '.["s3"][]? | select(.name == "cron-state-storage") | .credentials.bucket')
       export S3_ENDPOINT=$(echo "$VCAP_SERVICES"           | jq -r '.["s3"][]? | select(.name == "cron-state-storage") | .credentials.fips_endpoint')
       export AWS_ACCESS_KEY_ID=$(echo "$VCAP_SERVICES"     | jq -r '.["s3"][]? | select(.name == "cron-state-storage") | .credentials.access_key_id')
@@ -68,12 +77,6 @@ case $TASK in
       ;;
 esac
 
-#echo S3_BUCKET: $S3_BUCKET
-#echo S3_ENDPOINT: $S3_ENDPOINT
-#echo AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
-#echo AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
-#echo AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
-
 CF_API="https://api.fr.cloud.gov"
 CF_ORG=gsa-tts-usagov
 
@@ -84,13 +87,28 @@ TARGET_RESULT=0
 cf api "$CF_API" &> /dev/null
 API_RESULT=$?
 
-cf auth "$CF_USERNAME" "$CF_PASSWORD" &> /dev/null
+### First auth - use service account creds
+export CF_PASSWORD ### pass via env, not cmd line!
+cf auth "$CF_USERNAME" &> /dev/null
 AUTH_RESULT=$?
 
-cf target -o "$CF_ORG" -s "$CF_SPACE" &> /dev/null
+cf target -o $CF_ORG &> /dev/null
+ORG_TARGET_RESULT=$?
+
+echo cf service-key $SERVICE_ACCOUNT $KEY_NAME
+SERVICE_KEY=$(cf service-key $SERVICE_ACCOUNT $KEY_NAME | tail -n +3)
+KEY_USERNAME=$( echo ${SERVICE_KEY} | jq -r '.credentials.username')
+KEY_PASSWORD=$( echo ${SERVICE_KEY} | jq -r '.credentials.password')
+
+### Only after auth as service account can we auth with the service key, which holds the correct roles for audit log access
+export CF_PASSWORD="$KEY_PASSWORD" ### pass via env, not cmd line!
+cf auth "$KEY_USERNAME" &>/dev/null
+
+echo cf target -o "$CF_ORG" -s "$CF_SPACE"
+cf target -o "$CF_ORG" -s "$CF_SPACE" &>/dev/null
 TARGET_RESULT=$?
 
-if [ 0 -ne $API_RESULT -o 0 -ne $AUTH_RESULT -o 0 -ne $TARGET_RESULT ]; then
+if [ 0 -ne $API_RESULT -o 0 -ne $AUTH_RESULT -o 0 -ne $TARGET_RESULT -o 0 -ne $ORG_TARGET_RESULT ]; then
    echo "ERROR: Cloud Foundry Initialization Failed"
 fi
 
@@ -143,5 +161,3 @@ TASKLOCK_SCRIPT_ROOT=/opt/cron
 
 ### Maybe we should be using /var/run/tasks/ on the container?
 TASKLOCK_RUN_ROOT=/tmp/tasks/run
-
-#echo "Cron App Setup Complete"

.gitignore

@@ -110,6 +110,9 @@ web/themes/custom/usagov/css/styles.css.map
 web/themes/custom/usagov/npm-debug.log
 web/themes/custom/usagov/img
 
+# # Ignore published-pages.csv file
+web/modules/custom/usagov_ssg_postprocessing/files/published-pages.csv
+
 .vscode
 .vscode/settings.json
 workspace.code-workspace

README.md

@@ -243,15 +243,17 @@ PHPCodesniffer and the parallel linting tools should be installed automatically
 The following composer scripts are aliases for running these tools.
 
 * Check for code style errors across all project files. Must have zero errors:
-  `./bin/composer phpcs-errors`:
-* Check for code style errors and warnings across all project files.
-  `./bin/composer phpcs-strict`
+`./bin/composer phpcs-errors`
+* Check for code style errors and warnings across all project file
+`./bin/composer phpcs-strict`
 * Check for code style errors in current branch. Must have zero errors:
   `./bin/composer phpcs-changes`
 * Check for code style errors and warnings in current branch.
   `./bin/composer phpcs-changes-strict`
 * Check for PHP lint errors
   `./bin/composer php-lint`
+* Check for PHP 8.3 compatibility
+  `./bin/composer php-compatibility`
 
 ## Checking code with PHPStan
 

bin/cloudgov/create-service-account

@@ -62,15 +62,37 @@ echo $SERVICE_KEY
 SERVICE_USER=$( echo $SERVICE_KEY | jq -r '.credentials.username')
 
 # you must already be logged in as an OrgManager to have enough permissions
+if [ $keyname = 'cfevent' ]; then
+  if existsCFService ${keyname}-service-account; then
+    SERVICE_KEY=$(cf service-key ${keyname}-service-account ${keyname}-service-key | tail -n +3)
+    SERVICE_USER=$( echo ${SERVICE_KEY} | jq -r '.credentials.username')
+    $echo cf set-org-role ${SERVICE_USER} $startorg OrgAuditor
+    ### target our roles a bit:
+    if [ $SPACE = "prod" ]; then
+      for cfspace in dr dev stage prod tools shared-egress; do
+        $echo cf set-space-role ${SERVICE_USER} $startorg $cfspace SpaceAuditor
+      done
+    else
+      $echo cf set-space-role ${SERVICE_USER} $startorg $SPACE SpaceAuditor
+    fi
+    echo cf unset-space-role ${SERVICE_USER} $startorg $SPACE SpaceDeveloper
+  else
+    echo could not create Service Account for ${keyname} in ${SPACE}
+    exit 1
+  fi
+fi
+
 # to assign these cross-space developer roles to the service account's user
 # Note: for service users created in the prod space, grant access to lower spaces as well.  This is for CircleCI access
-cf set-space-role $SERVICE_USER $startorg $SPACE SpaceDeveloper
-if [ $SPACE = "prod" ]; then
-  cf set-space-role $SERVICE_USER $startorg dev SpaceDeveloper
-  cf set-space-role $SERVICE_USER $startorg stage SpaceDeveloper
-  cf set-space-role $SERVICE_USER $startorg prod SpaceDeveloper
-  cf set-space-role $SERVICE_USER $startorg tools SpaceDeveloper
-  cf set-space-role $SERVICE_USER $startorg shared-egress SpaceDeveloper
-  cf set-space-role $SERVICE_USER $startorg dr SpaceDeveloper
-  #cf set-space-role $SERVICE_USER $startorg shared-egress-dr SpaceDeveloper
+if [ $keyname = 'cci' ]; then
+  $echo cf set-org-role ${SERVICE_USER} $startorg OrgAuditor
+  cf set-space-role $SERVICE_USER $startorg $SPACE SpaceDeveloper
+  if [ $SPACE = "prod" ]; then
+    cf set-space-role $SERVICE_USER $startorg dev SpaceDeveloper
+    cf set-space-role $SERVICE_USER $startorg stage SpaceDeveloper
+    cf set-space-role $SERVICE_USER $startorg prod SpaceDeveloper
+    cf set-space-role $SERVICE_USER $startorg tools SpaceDeveloper
+    cf set-space-role $SERVICE_USER $startorg shared-egress SpaceDeveloper
+    cf set-space-role $SERVICE_USER $startorg dr SpaceDeveloper
+  fi
 fi

bin/cloudgov/deploy-cron

@@ -60,19 +60,18 @@ else
   $SCRIPT_DIR/create-service-account $SPACE $APPNAME
 fi
 
-if existsCFService ${APPNAME}-service-account; then
-  SERVICE_KEY=$(cf service-key ${APPNAME}-service-account ${APPNAME}-service-key | tail -n +3)
-  SERVICE_USER=$( echo ${SERVICE_KEY} | jq -r '.credentials.username')
-  ### target our roles a bit:
-  $echo cf set-space-role ${SERVICE_USER} $ORG $SPACE SpaceAuditor
-  $echo cf set-org-role ${SERVICE_USER} $ORG OrgAuditor
-  echo cf unset-space-role ${SERVICE_USER} $ORG $SPACE SpaceDeveloper
+if existsCFService cfevent-service-account; then
+  echo "User service cfevent-service-account already exists - good!"
+else
+  $SCRIPT_DIR/create-service-account $SPACE cfevent
+fi
+
+if existsCFService callcenter-service-account; then
+  echo "User service callcenter-service-account already exists - good!"
 else
-  echo could not create Service Account for ${APPNAME} in ${SPACE}
-  exit 1
+  $SCRIPT_DIR/create-service-account $SPACE callcenter
 fi
 
-# TODO: I want to come back to this and make CALLWAIT_STORAGE_SERVICE a shared service -- one s3 bucket for all three environs -akf
 STATE_STORAGE_SERVICE=${APPNAME}-state-storage
 EVENT_STORAGE_SERVICE=${APPNAME}-event-storage
 CALLWAIT_STORAGE_SERVICE=${APPNAME}-callwait-storage
@@ -109,10 +108,19 @@ function cf_push() {
 
 if existsCFApp ${APPNAME} ; then
   $SCRIPT_DIR/setup-egress-for-apps ${APPNAME}
+  retval=$?
+  if [ $retval != 0 ]; then
+    echo "ERROR: setup-egress-for-apps has failed.  Check output stream!"
+    exit 1
+  fi
   cf_push $APPSPEC $APPINSTANCES
 else
   cf_push $APPSPEC $APPINSTANCES
   $SCRIPT_DIR/setup-egress-for-apps ${APPNAME}
+  if [ $retval != 0 ]; then
+    echo "ERROR: setup-egress-for-apps has failed.  Check output stream!"
+    exit 1
+  fi
 fi
 
 ### These are set for some reason - unset them at deployment
@@ -129,9 +137,10 @@ cf set-env $APPNAME CALL_CENTER_ENVIRONMENT $CALL_CENTER_ENVIRONMENT
 cf set-env $APPNAME CALL_CENTER_EN_QUEUE_ID $CALL_CENTER_EN_QUEUE_ID
 cf set-env $APPNAME CALL_CENTER_SP_QUEUE_ID $CALL_CENTER_SP_QUEUE_ID
 
-# Enable call center script on prod only. (We can manually update this in dev, etc. for testing)
+# Enable call center / cfevents  scripts on prod only. (We can manually update this in dev, etc. for testing)
 if [ $SPACE = "prod" ]; then
     cf set-env $APPNAME CALL_CENTER_RUN "true"
+    cf set-env $APPNAME CFEVENT_RUN "true"
 fi
 
 cf restage $APPNAME

bin/cloudgov/events/cf-guids

@@ -1,5 +1,5 @@
 #!/bin/sh
-for SPACE in dev-dr dev stage prod shared-egress shared-egress-dr tools; do
+for SPACE in dr dev stage prod shared-egress tools; do
     if ! cf space $SPACE &> /dev/null; then
       echo "ERROR: CF space '$SPACE' does not exist!"
       exit 4

bin/cloudgov/events/get-events

@@ -65,14 +65,21 @@ SPACE_GUID=$(cf space $SPACE --guid)
 cf target -s $SPACE &>/dev/null
 
 declare -A SPACE_GUIDS=()
-# Keep the code needed to query multiple spaces for events. Not working now, but may be possible.
-#  for SPACE in dev stage prod shared-egress shared-egress-dr tools; do
-for SPC in $SPACE; do
+
+# Attempt to fetch uuids for all usagov cf spaces:
+SPACE_QUERY=''
+for SPC in dr dev stage prod shared-egress tools; do
+#for SPC in $SPACE; do
   if [ ! cf space $SPC &> /dev/null ]; then
     echo "ERROR: CF space '$SPC' does not exist!"
   else
     SPACE_GUID=$(cf space $SPC --guid)
     SPACE_GUIDS[$SPACE_GUID]=$SPC
+    if [ -z $SPACE_QUERY ]; then
+      SPACE_QUERY=$SPACE_GUID
+    else
+      SPACE_QUERY="$SPACE_QUERY,$SPACE_GUID"
+    fi
   fi
 done
 
@@ -105,7 +112,7 @@ for resource_type in ${!AUDIT_TYPES[@]}; do
     fields=${AUDIT_FIELDS[$resource_type]}
 
     # Manually create the query to fetch the first page of results
-    NEXT="/v3/audit_events?types=${types}&organization_guids=${ORG_GUID}&space_guids=${SPACE_GUID}&created_ats[gt]=${LAST_EVENT_TIME}&order_by=created_at"
+    NEXT="/v3/audit_events?types=${types}&organization_guids=${ORG_GUID}&space_guids=${SPACE_QUERY}&created_ats[gt]=${LAST_EVENT_TIME}&order_by=created_at"
     while [ -n "$NEXT" -a ! "$NEXT" == 'null' ]; do
 
         EVENTS=$(cf curl $NEXT)

bin/cloudgov/setup-egress-for-apps

@@ -15,7 +15,12 @@ fi
 if [ -f ./env.local ]; then
   . ./env.local
 fi
-
+if [ -f $SCRIPT_DIR/../deploy/includes ]; then
+  . $SCRIPT_DIR/../deploy/includes
+else
+   echo Cannot find $SCRIPT_DIR/../deploy/includes
+   exit 1
+fi
 
 usage="
 $0: (Re-)deploy egress proxy for an environment
@@ -160,6 +165,19 @@ for app in $applist; do
       fi
     fi
 
+    if [ "$app" = cron ]; then
+      for svc in callcenter-service-account cfevent-service-account; do
+        if existsCFService $svc; then
+          echo "Binding $app with $svc"
+          $output cf bind-service $app $svc
+          retval=$?
+        else
+          echo "Cannot bind $app with non-existant service $svc"
+          retval=1
+        fi
+      done
+    fi
+
     if [ $restartRequired -eq 1 ]; then
       $output cf restart "$app" --no-wait --strategy rolling
     fi

composer.json

@@ -15,6 +15,7 @@
         }
     ],
     "require": {
+        "php": ">=8.3",
         "composer/installers": "^2.3",
         "cweagans/composer-patches": "^1.7",
         "drupal/address": "^1.9",
@@ -49,7 +50,6 @@
         "drupal/log_stdout": "^1.3",
         "drupal/menu_block": "^1.7",
         "drupal/menu_breadcrumb": "^2.0@alpha",
-        "drupal/menu_entity_index": "^1.6",
         "drupal/menu_item_fields": "^1.10",
         "drupal/new_relic_rpm": "^2.1",
         "drupal/node_menus": "^3.0@alpha",
@@ -142,7 +142,7 @@
         },
         "patches": {
             "drupal/tome": {
-                "CSS on Amazon S3 via Flysystem not processing properly": "https://www.drupal.org/files/issues/2020-08-06/3161384-4.patch",
+                "Make tome work with js and css aggregation": "./patches/drupal/tomeAggregationFix.patch",
                 "De-duplicate invoke paths in StaticCommand's exportPaths": "./patches/drupal/deduplicateTomeInvokePaths.patch",
                 "Make tome work with drush 12": "https://www.drupal.org/files/issues/2023-08-02/tome_drush12-support.patch",
                 "Make tome work with path count > 1 without re-exporting pages multiple times" : "./patches/drupal/tomePathCountFixes.patch"
@@ -171,8 +171,10 @@
         "drupal/coder": "^8.3",
         "mglaman/phpstan-drupal": "*",
         "php-parallel-lint/php-parallel-lint": "^1.4",
+        "phpcompatibility/php-compatibility": "^9.3",
         "phpstan/phpstan": "*",
-        "phpstan/phpstan-deprecation-rules": "*"
+        "phpstan/phpstan-deprecation-rules": "*",
+        "rector/rector": "^2.0"
     },
     "scripts": {
         "changed-files": [
@@ -195,6 +197,9 @@
         ],
         "phpstan": [
             "vendor/bin/phpstan --memory-limit=1G"
+        ],
+        "php-compatibility": [
+            "vendor/bin/phpcs -pn --runtime-set testVersion 8.3- web/modules/custom web/themes/custom -d memory_limit=2G --standard=.phpcs.xml.dist"
         ]
     }
 }