adds mechanism to skip notebook directories for different run types

ci/gpu/build.sh

@@ -188,6 +188,8 @@ else
         TEST_MODE_FLAG=""
     fi
 
+    NOTEBOOK_TEST_MODE="ci"
+
     gpuci_logger "Running cuGraph test.sh..."
     if [[ $run_cpp_tests == "true" ]]; then
         ${WORKSPACE}/ci/test.sh ${TEST_MODE_FLAG} --run-cpp-tests --run-python-tests | tee testoutput.txt
@@ -200,7 +202,7 @@ else
 
     if [[ $run_nb_tests == "true" ]]; then
         gpuci_logger "Running cuGraph notebook test script..."
-        ${WORKSPACE}/ci/gpu/test-notebooks.sh 2>&1 | tee nbtest.log
+        ${WORKSPACE}/ci/gpu/test-notebooks.sh ${NOTEBOOK_TEST_MODE} 2>&1 | tee nbtest.log
         gpuci_logger "Ran cuGraph notebook test script : return code was: $?, gpu/build.sh exit code is now: $EXITCODE"
         python ${WORKSPACE}/ci/utils/nbtestlog2junitxml.py nbtest.log
     fi

ci/gpu/notebook_list.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,11 +12,28 @@
 # limitations under the License.
 
 import re
+import argparse
 import sys
 import glob
+import os
+from pathlib import Path
 
 from numba import cuda
 
+# for adding another run type and skip file name add to this dictionary
+runtype_dict = {"all" : "", "ci" : "SKIP_CI_TESTING", 
+                "nightly" : "SKIP_NIGHTLY", "weekly" : "SKIP_WEEKLY"}
+
+
+def skip_book_dir(runtype):
+    # Add all run types here, currently only CI supported
+
+    if runtype in runtype_dict.keys():
+        if Path(runtype_dict.get(runtype)).is_file():
+            return True
+    return False
+
+
 cuda_version_string = ".".join([str(n) for n in cuda.runtime.get_version()])
 #
 # Not strictly true... however what we mean is
@@ -25,6 +42,19 @@
 pascal = False
 ampere = False
 device = cuda.get_current_device()
+
+parser = argparse.ArgumentParser(description='Condition for running the notebook tests')
+parser.add_argument('runtype', type=str)
+
+args = parser.parse_args()
+
+runtype = args.runtype
+
+if runtype not in runtype_dict.keys():
+    print(f'Unknown Run Type  = {runtype}', file=sys.stderr)
+    exit()
+
+
 # check for the attribute using both pre and post numba 0.53 names
 cc = getattr(device, 'COMPUTE_CAPABILITY', None) or \
      getattr(device, 'compute_capability')
@@ -33,31 +63,35 @@
 if (cc[0] >= 8):
     ampere = True
 
+skip = False
 for filename in glob.iglob('**/*.ipynb', recursive=True):
     skip = False
-    for line in open(filename, 'r'):
-        if re.search('# Skip notebook test', line):
-            skip = True
-            print(f'SKIPPING {filename} (marked as skip)', file=sys.stderr)
-            break;
-        elif re.search('dask', line):
-            print(f'SKIPPING {filename} (suspected Dask usage, not currently automatable)', file=sys.stderr)
-            skip = True
-            break;
-        elif pascal and re.search('# Does not run on Pascal', line):
-            print(f'SKIPPING {filename} (does not run on Pascal)', file=sys.stderr)
-            skip = True
-            break;
-        elif ampere and re.search('# Does not run on Ampere', line):
-            print(f'SKIPPING {filename} (does not run on Ampere)', file=sys.stderr)
-            skip = True
-            break;
-        elif re.search('# Does not run on CUDA ', line) and \
+    if (skip_book_dir(runtype) == True):
+        print(f'SKIPPING {filename} (Notebook skipped for run type {runtype}) due to skip file in folder.', file=sys.stderr)
+        skip = True
+    else:
+        for line in open(filename, 'r'):
+            if re.search('# Skip notebook test', line):
+                skip = True
+                print(f'SKIPPING {filename} (marked as skip)', file=sys.stderr)
+                break;
+            elif re.search('dask', line):
+                print(f'SKIPPING {filename} (suspected Dask usage, not currently automatable)', file=sys.stderr)
+                skip = True
+                break;
+            elif pascal and re.search('# Does not run on Pascal', line):
+                print(f'SKIPPING {filename} (does not run on Pascal)', file=sys.stderr)
+                skip = True
+                break;
+            elif ampere and re.search('# Does not run on Ampere', line):
+                print(f'SKIPPING {filename} (does not run on Ampere)', file=sys.stderr)
+                skip = True
+                break;
+            elif re.search('# Does not run on CUDA ', line) and \
              (cuda_version_string in line):
-            print(f'SKIPPING {filename} (does not run on CUDA {cuda_version_string})',
+                print(f'SKIPPING {filename} (does not run on CUDA {cuda_version_string})',
                   file=sys.stderr)
-            skip = True
-            break;
-
+                skip = True
+                break;
     if not skip:
         print(filename)

ci/gpu/test-notebooks.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -22,6 +22,7 @@ NOTEBOOKS_DIR=${WORKSPACE}/notebooks
 NBTEST=${WORKSPACE}/ci/utils/nbtest.sh
 LIBCUDF_KERNEL_CACHE_PATH=${WORKSPACE}/.jitcache
 EXITCODE=0
+RUN_TYPE=$1
 
 cd ${NOTEBOOKS_DIR}
 TOPLEVEL_NB_FOLDERS=$(find . -name *.ipynb |cut -d'/' -f2|sort -u)
@@ -41,7 +42,13 @@ for folder in ${TOPLEVEL_NB_FOLDERS}; do
     echo "FOLDER: ${folder}"
     echo "========================================"
     cd ${NOTEBOOKS_DIR}/${folder}
-    NBLIST=$(python ${WORKSPACE}/ci/gpu/notebook_list.py)
+    if [ -z "$1" ]
+        then
+            runtype="all"
+        else
+            runtype=${1}
+    fi
+    NBLIST=$(python ${WORKSPACE}/ci/gpu/notebook_list.py ${runtype})
     for nb in ${NBLIST}; do
         nbBasename=$(basename ${nb})
         cd $(dirname ${nb})