Split tests into groups for parallel execution in CI/CD runners (#417)

Split tests into groups for parallel execution in CI/CD runners [ committed by @billschereriii ] [ reviewed by @al-rigazzi @ankona ]
CrayLabs · Nov 14, 2023 · 15ba145 · 15ba145
1 parent 96765d9
commit 15ba145
Show file tree

Hide file tree

Showing 44 changed files with 181 additions and 45 deletions.
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
@@ -44,11 +44,12 @@ env:
 
 jobs:
   run_tests:
-    name: Run tests with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }}
+    name: Run tests ${{ matrix.subset }} with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
+        subset: [backends, slow_tests, group_a, group_b]
         os: [macos-12, ubuntu-20.04] # Operating systems
         compiler: [8] # GNU compiler version
         rai: [1.2.7] # Redis AI versions
@@ -121,10 +122,19 @@ jobs:
       - name: Run Pylint
         run: make check-lint
 
+      # Run pytest (backends subdirectory)
       - name: Run Pytest
+        if: (matrix.subset == 'backends')
         run: |
           echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
-          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ ./tests/
+          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ ./tests/backends
+
+      # Run pytest (test subsets)
+      - name: Run Pytest
+        if: "!contains(matrix.subset, 'backends')" # if not running backend tests
+        run: |
+          echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
+          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests
 
       # Upload artifacts on failure, ignoring binary files
       - name: Upload Artifact
@@ -143,5 +153,5 @@ jobs:
       - name: Upload Pytest coverage to Codecov
         uses: codecov/codecov-action@v2
         with:
-          fail_ci_if_error: true
+          fail_ci_if_error: false
           files: ./coverage.xml
diff --git a/doc/changelog.rst b/doc/changelog.rst
@@ -19,6 +19,7 @@ To be released at some future point in time
 
 Description
 
+- Split tests into groups for parallel execution in CI/CD pipeline
 - Change signature of `Experiment.summary()`
 - Expose first_device parameter for scripts, functions, models
 - Added support for MINBATCHTIMEOUT in model execution
@@ -27,6 +28,7 @@ Description
 
 Detailed Notes
 
+- Split tests into `group_a`, `group_b`, `slow_tests` for parallel execution in CI/CD pipeline (PR417_)
 - Change `format` argument to `style` in `Experiment.summary()`, this is
   an API break (PR391_)
 - Added support for first_device parameter for scripts, functions,
@@ -43,6 +45,7 @@ Detailed Notes
 - Add support for creation of multiple databases with unique identifiers. (PR342_)
 
 
+  .. _PR417: https://github.com/CrayLabs/SmartSim/pull/417
   .. _PR391: https://github.com/CrayLabs/SmartSim/pull/391
   .. _PR342: https://github.com/CrayLabs/SmartSim/pull/342
   .. _PR394: https://github.com/CrayLabs/SmartSim/pull/394

diff --git a/tests/test_alps_settings.py b/tests/test_alps_settings.py
@@ -30,6 +30,9 @@
 from smartsim.error import SSUnsupportedError
 from smartsim.settings import AprunSettings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_aprun_settings():
     settings = AprunSettings("python")

diff --git a/tests/test_batch_settings.py b/tests/test_batch_settings.py
@@ -30,6 +30,9 @@
 from smartsim.settings import BsubBatchSettings, QsubBatchSettings, SbatchSettings
 from smartsim.settings.settings import create_batch_settings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_create_pbs_batch():
     pbs_batch = create_batch_settings(

diff --git a/tests/test_buildenv.py b/tests/test_buildenv.py
@@ -30,6 +30,9 @@
 from smartsim._core._install.buildenv import Version_
 from pkg_resources import packaging  # type: ignore
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_version_hash_eq():
     """Ensure hashes are equal if data is equal"""

diff --git a/tests/test_cli.py b/tests/test_cli.py
@@ -45,6 +45,10 @@
     configure_parser as validate_parser,
 )
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
+
 def mock_execute_custom(msg: str = None, good: bool = True) -> int:
     retval = 0 if good else 1
     print(msg)

diff --git a/tests/test_cobalt_parser.py b/tests/test_cobalt_parser.py
@@ -24,9 +24,13 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import pytest
 
 from smartsim._core.launcher.cobalt import cobaltParser
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_parse_step_id():
     output = "JobName      JobId \n" "=====================\n" "smartsim     507975 \n"

diff --git a/tests/test_colo_model_local.py b/tests/test_colo_model_local.py
@@ -32,6 +32,10 @@
 from smartsim.error import SSUnsupportedError
 from smartsim.entity import Model
 
+# The tests in this file belong to the slow_tests group
+pytestmark = pytest.mark.slow_tests
+
+
 if sys.platform == "darwin":
     supported_dbs = ["tcp", "deprecated"]
 else:

diff --git a/tests/test_colo_model_lsf.py b/tests/test_colo_model_lsf.py
@@ -33,6 +33,9 @@
 import smartsim.settings.base
 from smartsim.settings.lsfSettings import JsrunSettings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 is_mac = sys.platform == "darwin"
 

diff --git a/tests/test_config.py b/tests/test_config.py
@@ -28,14 +28,15 @@
 from pathlib import Path
 
 import os
-from re import A, L
 import pytest
-import shutil
 import typing as t
 
 from smartsim._core.config.config import Config
 from smartsim.error import SSConfigError
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_all_config_defaults():
     config = Config()

diff --git a/tests/test_containers.py b/tests/test_containers.py
@@ -32,11 +32,14 @@
 import pytest
 
 from smartsim import Experiment, status
-from smartsim._core.utils import installed_redisai_backends
 from smartsim.database import Orchestrator
-from smartsim.entity import Ensemble, Model
+from smartsim.entity import Ensemble
 from smartsim.settings.containers import Singularity
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
+
 # Check if singularity is available as command line tool
 singularity_exists = which("singularity") is not None
 containerURI = "docker://alrigazzi/smartsim-testing:latest"

diff --git a/tests/test_controller_errors.py b/tests/test_controller_errors.py
@@ -34,6 +34,9 @@
 from smartsim.error.errors import SSUnsupportedError
 from smartsim.settings import RunSettings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_finished_entity_orc_error():
     """Orchestrators are never 'finished', either run forever or stopped by user"""

diff --git a/tests/test_dbnode.py b/tests/test_dbnode.py
@@ -37,6 +37,9 @@
 from smartsim.entity.dbnode import DBNode, LaunchedShardData
 from smartsim.error.errors import SmartSimError
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_parse_db_host_error():
     orc = Orchestrator()

diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py
@@ -34,6 +34,10 @@
 from smartsim.error import EntityExistsError, SSUnsupportedError, UserStrategyError
 from smartsim.settings import RunSettings
 
+# The tests in this file belong to the slow_tests group
+pytestmark = pytest.mark.slow_tests
+
+
 """
 Test ensemble creation
 

diff --git a/tests/test_entitylist.py b/tests/test_entitylist.py
@@ -33,6 +33,9 @@
 from smartsim.entity import EntityList
 from smartsim.settings import RunSettings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_entity_list_init():
     with pytest.raises(NotImplementedError):

diff --git a/tests/test_experiment.py b/tests/test_experiment.py
@@ -24,15 +24,16 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import contextlib
-
 import pytest
 
 from smartsim import Experiment
 from smartsim.entity import Model
 from smartsim.error import SmartSimError
 from smartsim.settings import RunSettings
 
+# The tests in this file belong to the slow_tests group
+pytestmark = pytest.mark.slow_tests
+
 
 def test_model_prefix(fileutils):
     exp_name = "test_prefix"

diff --git a/tests/test_generator.py b/tests/test_generator.py
@@ -26,7 +26,6 @@
 
 import filecmp
 from os import path as osp
-
 import pytest
 
 from smartsim import Experiment
@@ -35,6 +34,10 @@
 from smartsim.settings import RunSettings
 from tabulate import tabulate
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
+
 rs = RunSettings("python", exe_args="sleep.py")
 
 

diff --git a/tests/test_helpers.py b/tests/test_helpers.py
@@ -24,9 +24,13 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import pytest
 
 from smartsim._core.utils.helpers import cat_arg_and_value
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_double_dash_concat():
     result = cat_arg_and_value("--foo", "FOO")

diff --git a/tests/test_init.py b/tests/test_init.py
@@ -25,6 +25,12 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+import pytest
+
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
+
 __author__ = "Sam Partee"
 
 

diff --git a/tests/test_interrupt.py b/tests/test_interrupt.py
@@ -24,7 +24,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-
+import pytest
 import os
 import signal
 import time
@@ -33,6 +33,9 @@
 from smartsim import Experiment
 from smartsim.settings import RunSettings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def keyboard_interrupt(pid):
     """Interrupt main thread"""

diff --git a/tests/test_launch_errors.py b/tests/test_launch_errors.py
@@ -32,6 +32,9 @@
 from smartsim.error import SSUnsupportedError
 from smartsim.settings import JsrunSettings, RunSettings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_unsupported_run_settings():
     exp_name = "test-unsupported-run-settings"

diff --git a/tests/test_local_launch.py b/tests/test_local_launch.py
@@ -24,9 +24,13 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-
+import pytest
 from smartsim import Experiment, status
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
+
 """
 Test the launch of simple entity types with local launcher
 """

diff --git a/tests/test_local_multi_run.py b/tests/test_local_multi_run.py
@@ -24,9 +24,13 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-
+import pytest
 from smartsim import Experiment, status
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
+
 """
 Test the launch of simple entity types with local launcher
 """

diff --git a/tests/test_local_restart.py b/tests/test_local_restart.py
@@ -24,9 +24,13 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-
+import pytest
 from smartsim import Experiment, status
 
+# The tests in this file belong to the group_b group
+pytestmark = pytest.mark.group_b
+
+
 """
 Test restarting ensembles and models.
 """

diff --git a/tests/test_lsf_parser.py b/tests/test_lsf_parser.py
@@ -24,9 +24,13 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-
+import pytest
 from smartsim._core.launcher.lsf import lsfParser
 
+# The tests in this file belong to the group_b group
+pytestmark = pytest.mark.group_b
+
+
 # -- bsub ---------------------------------------------------------
 
 

diff --git a/tests/test_lsf_settings.py b/tests/test_lsf_settings.py
@@ -25,13 +25,14 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-from pprint import pformat
-
 import pytest
 
 from smartsim.error import SSUnsupportedError
 from smartsim.settings import BsubBatchSettings, JsrunSettings
 
+# The tests in this file belong to the group_b group
+pytestmark = pytest.mark.group_b
+
 # ------ Jsrun ------------------------------------------------