Update test runner for Hello-broken (#2102)

Fixes #2086

run/hello-broken/package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "start": "node index.js",
     "test": "echo \"Error: no test specified\" && exit 0",
-    "system-test": "NAME=Cloud test/runner.sh mocha test/system.test.js --timeout=30000",
+    "system-test": "NAME=Cloud mocha test/system.test.js --timeout=180000",
     "lint": "eslint '**/*.js'",
     "fix": "eslint --fix '**/*.js'"
   },
@@ -20,6 +20,7 @@
     "express": "^4.17.1"
   },
   "devDependencies": {
+    "google-auth-library": "^6.1.3",
     "got": "^11.0.0",
     "mocha": "^8.0.0"
   }

run/hello-broken/test/e2e_test_cleanup.yaml

@@ -0,0 +1,19 @@
+steps:
+
+- id: 'Delete image and service'
+  name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: '/bin/bash'
+  args:
+    - '-c'
+    - |
+      ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \
+        "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet"
+
+      ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \
+        "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet"
+
+substitutions:
+  _SERVICE: hello-broken
+  _VERSION: manual
+  _REGION: us-central1
+  _PLATFORM: managed

run/hello-broken/test/e2e_test_setup.yaml

@@ -0,0 +1,41 @@
+steps:
+
+- id: 'Build Container Image'
+  name: 'gcr.io/cloud-builders/docker'
+  entrypoint: '/bin/bash'
+  args:
+    - '-c'
+    - |
+      ./test/retry.sh "docker build -t gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} ."
+
+- id: 'Push Container Image'
+  name: 'gcr.io/cloud-builders/docker'
+  entrypoint: '/bin/bash'
+  args:
+    - '-c'
+    - |
+      ./test/retry.sh "docker push gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}"
+
+- id: 'Deploy to Cloud Run'
+  name: 'gcr.io/cloud-builders/gcloud:latest'
+  entrypoint: /bin/bash
+  args:
+  - '-c'
+  - |
+    ./test/retry.sh "gcloud run deploy ${_SERVICE} \
+      --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \
+      --no-allow-unauthenticated \
+      --region ${_REGION} \
+      --platform ${_PLATFORM} \
+      --set-env-vars NAME=${_NAME}"
+
+
+images:
+- gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}
+
+substitutions:
+  _SERVICE: hello-broken
+  _VERSION: manual
+  _REGION: us-central1
+  _PLATFORM: managed
+  _NAME: ''

run/hello-broken/test/retry.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+# retry.sh
+# Provides utility function commonly needed across Cloud Build pipelines to
+# retry commands on failure.
+#
+# Usage:
+# 1. Retry single command:
+#
+# ./retry.sh "CMD"
+#
+# 2. Retry with check:
+#
+# ./retry.sh "gcloud RESOURCE EXISTS?" "gcloud ACTION"
+#
+##
+
+# Usage: try "cmd1" "cmd2"
+# If first cmd executes successfully then execute second cmd
+runIfSuccessful() {
+  echo "running: $1"
+  $($1 > /dev/null)
+  if [ $? -eq 0 ]; then
+    echo "running: $2"
+    $($2 > /dev/null)
+  fi
+}
+
+# Define max retries
+max_attempts=3;
+attempt_num=1;
+
+arg1="$1"
+arg2="$2"
+
+if [ $# -eq 1 ]
+then
+  cmd="$arg1"
+else
+  cmd="runIfSuccessful \"$arg1\" \"$arg2\""
+fi
+
+until eval $cmd
+do
+    if ((attempt_num==max_attempts))
+    then
+        echo "Attempt $attempt_num / $max_attempts failed! No more retries left!"
+        exit 1
+    else
+        echo "Attempt $attempt_num / $max_attempts failed!"
+        sleep $((attempt_num++))
+    fi
+done