Move nightly evergreen CI tasks out of commit to master builds (#6952)

* Early exit hang analyzer if git clone fails; move nightly tasks out of commit builds
* Update from review

evergreen/config.yml

@@ -3,8 +3,8 @@
 # * Pull Requests:
 #   * add the "for_pull_requests" tag to the task to run for pull requests
 # * Nightly Builds:
-#   * Add the '"for_nightly_tests"'' tag to the task for the "nightly_tests" alias to work
-#   * Add the 'activate: false' setting to the task under the build variant for each task to run during the nightly test runs
+#   * Add the `"for_nightly_tests"` tag to the task for the "nightly_tests" alias to work
+#   * Add the `allowed_requesters: [ "ad_hoc", "patch" ]` entry to the nightly task to only run for periodic tests
 # * Commit Builds:
 #   * All other tasks will be included in the set run when a PR is merged to master
 
@@ -290,7 +290,7 @@ functions:
         # Copy the baas_server log from the remote baas host if it exists
         if [[ ! -f baas_host.yml || ! -f .baas_ssh_key ]]; then
           echo "No remote baas host or remote baas host definitions not found"
-          exit
+          exit 0
         fi
 
         BAAS_HOST_NAME=$(tr -d '"[]{}' < baas_host.yml | cut -d , -f 1 | awk -F : '{print $2}')
@@ -394,14 +394,14 @@ functions:
 
         if [[ ! -f baas_host.yml || ! -f .baas_ssh_key ]]; then
           echo "No remote baas host or remote baas host definitions not found"
-          exit
+          exit 0
         fi
 
         BAAS_HOST_NAME=$(tr -d '"[]{}' < baas_host.yml | cut -d , -f 1 | awk -F : '{print $2}')
         export BAAS_HOST_NAME
 
         ssh_user="$(printf "ubuntu@%s" "$BAAS_HOST_NAME")"
-        ssh_options="-o ForwardAgent=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=No -o ConnectTimeout=10 -i .baas_ssh_key"
+        ssh_options="-o ForwardAgent=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o ConnectTimeout=60 -i .baas_ssh_key"
 
         # Kill the remote baas server process, if the definition files exist
         echo "Terminating baas server on remote host..."
@@ -414,12 +414,15 @@ functions:
         set -o errexit
         set -o verbose
 
-        if [[ ! -d realm-core ]]; then
+        TOP_DIR=$(pwd)/realm-core
+        HANG_ANALYZER_PATH=$TOP_DIR/evergreen/hang_analyzer
+        REQUIREMENTS_PATH=$HANG_ANALYZER_PATH/requirements.txt
+
+        if [[ ! -d $TOP_DIR || ! -d $REQUIREMENTS_PATH ]]; then
           echo "No source directory exists. Not running hang analyzer"
           exit 1
         fi
 
-        TOP_DIR=$(pwd)/realm-core
         mkdir realm-core/hang_analyzer_workdir; cd realm-core/hang_analyzer_workdir
         ${python3|python3} -m venv venv
 
@@ -456,7 +459,6 @@ functions:
 
         python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" || exit 1
 
-        REQUIREMENTS_PATH=$TOP_DIR/evergreen/hang_analyzer/requirements.txt
         if [ "Windows_NT" = "$OS" ]; then
           REQUIREMENTS_PATH=$(cygpath -w $REQUIREMENTS_PATH)
         fi
@@ -465,7 +467,6 @@ functions:
 
         echo "Going to run hang analyzer"
 
-        HANG_ANALYZER_PATH=$TOP_DIR/evergreen/hang_analyzer
         if [ "Windows_NT" = "$OS" ]; then
           HANG_ANALYZER_PATH=$(cygpath -w $HANG_ANALYZER_PATH)
         fi
@@ -686,6 +687,7 @@ tasks:
 - name: valgrind
   exec_timeout_secs: 14400
   tags: [ "for_nightly_tests" ]
+  allowed_requesters: [ "ad_hoc", "patch" ]
   commands:
   - func: "fetch source"
   - func: "fetch binaries"
@@ -805,6 +807,7 @@ tasks:
 
 - name: long-running-core-tests
   tags: [ "for_nightly_tests" ]
+  allowed_requesters: [ "ad_hoc", "patch" ]
   commands:
   - func: "run tests"
     # The long-running tests can take a really long time on Windows, so we give the test up to 4
@@ -1035,6 +1038,7 @@ tasks:
 
 - name: fuzzer
   tags: [ "for_nightly_tests" ]
+  allowed_requesters: [ "ad_hoc", "patch" ]
   commands:
   - command: shell.exec
     params:
@@ -1250,7 +1254,6 @@ buildvariants:
     distros:
     - ubuntu2004-large
   - name: long-running-tests
-    activate: false
   - name: bloaty
 
 - name: ubuntu2004-asan
@@ -1317,7 +1320,6 @@ buildvariants:
     enable_fuzzer: On
   tasks:
   - name: fuzzer-tests
-    activate: false
 
 - name: rhel70
   display_name: "RHEL 7 x86_64"
@@ -1345,7 +1347,6 @@ buildvariants:
     cmake_build_type: RelWithDebInfo
   tasks:
   - name: valgrind
-    activate: false
 
 - name: ubuntu2004-arm64
   display_name: "Ubuntu 20.04 ARM64"