[not for merge] DESC release branch

.github/workflows/ci.yaml

@@ -94,6 +94,13 @@ jobs:
         pip install .[monitoring,visualization]
         parsl/tests/test-viz.sh
 
+    # config_local_test comes after viz so that the large monitoring.db
+    # created by `make test` is still around
+    - name: make config_local_test
+      run: |
+        source .venv/bin/activate
+        make config_local_test
+
     - name: make coverage
       run: |
         source .venv/bin/activate
@@ -109,3 +116,88 @@ jobs:
           pytest-parsl/
           ci_job_info.txt
         compression-level: 9
+
+
+  gen3:
+        runs-on: ${{ matrix.os }}
+
+        strategy:
+            matrix:
+                os: [ ubuntu-latest ]
+                py: [ "3.11" ]
+                CC: [ gcc ]
+                CXX: [ g++ ]
+
+        defaults:
+            run:
+                # cf. https://github.com/conda-incubator/setup-miniconda#important
+                shell: bash -l {0}
+
+        steps:
+            - uses: actions/checkout@v2
+              with:
+                  repository: LSSTDESC/gen3_workflow
+
+            - uses: actions/checkout@master
+              with:
+                  path: ./parsl-repo
+
+            - name: Setup conda
+              uses: conda-incubator/setup-miniconda@v2
+              with:
+                  activate-environment: stack
+                  python-version: "3.11"
+                  condarc-file: etc/.condarc
+
+            - name: Install conda deps
+              run: |
+                conda info
+                conda list
+                conda install -y mamba
+                mamba install -y --file conda_requirements.txt
+                conda info
+                conda list
+
+
+            - name: Install workflow packages
+              run: |
+                # pip install -U --no-deps 'parsl[monitoring,workqueue] @ git+https://github.com/parsl/parsl@desc'
+                pip install -U --no-deps './parsl-repo[monitoring,workqueue]'
+                pip install typeguard tblib paramiko dill pyzmq globus-sdk sqlalchemy_utils
+                conda install -c conda-forge ndcctools=7.6.1=py311h689c632_0 --no-deps
+
+            - name: Clone gen3_workflow checkout the branch
+              shell: bash -l {0}
+              run: |
+                echo pwd
+                pwd 
+
+                echo ls
+                ls
+
+                echo Cloning gen3_workflow repo
+                git clone https://github.com/LSSTDESC/gen3_workflow
+
+                echo declaring gen3_workflow to eups
+                eups declare gen3_workflow -r ${PWD}/gen3_workflow -t current
+
+                cd gen3_workflow
+                git fetch origin master:TESTING
+                git checkout TESTING
+
+            - name: Run the test pipelines
+              run: |
+                setup gen3_workflow
+                (eups list lsst_distrib) 2>&1 | grep -v "Unknown tag"
+                (eups list gen3_workflow) 2>&1 | grep -v "Unknown tag"
+                cd tests
+                pytest test_query_workflow.py test_bps_restart.py
+
+
+            - name: Archive test logs
+              if: ${{ always() }}
+              uses: actions/upload-artifact@v2
+              with:
+                name: testinfo-${{ matrix.python-version }}
+                path: gen3_workflow/
+

Makefile

@@ -82,10 +82,16 @@ radical_local_test:
 	mkdir -p ~/.radical/pilot/configs && echo '{"localhost": {"virtenv_mode": "local"}}' > ~/.radical/pilot/configs/resource_local.json
 	pytest parsl/tests/ -k "not cleannet and not issue3328 and not executor_supports_std_stream_tuples" --config parsl/tests/configs/local_radical.py --random-order --durations 10
 
+.PHONY: workqueue_mon_test
+workqueue_mon_test: $(WORKQUEUE_INSTALL)  ## run all tests with workqueue_ex config
+	pip3 install ".[monitoring]"
+	PYTHONPATH=.:/tmp/cctools/lib/python3.8/site-packages  pytest parsl/tests/ -k "not cleannet and not issue363" --config parsl/tests/configs/workqueue_monitoring_config.py --cov=parsl --cov-append --cov-report= --random-order
+
+
 .PHONY: config_local_test
 config_local_test: $(CCTOOLS_INSTALL)
 	pip3 install ".[monitoring,visualization,proxystore,kubernetes]"
-	PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages pytest parsl/tests/ -k "not cleannet" --config local --random-order --durations 10
+	PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages pytest parsl/tests/ -k "not cleannet and not site" --config local --random-order --durations 10
 
 .PHONY: site_test
 site_test:
@@ -97,7 +103,7 @@ perf_test:
 	parsl-perf --time 5 --config parsl/tests/configs/local_threads.py
 
 .PHONY: test ## run all tests with all config types
-test: clean_coverage isort lint flake8 mypy local_thread_test htex_local_test htex_local_alternate_test wqex_local_test vineex_local_test radical_local_test config_local_test perf_test ## run all tests
+test: clean_coverage isort lint flake8 mypy local_thread_test htex_local_test htex_local_alternate_test wqex_local_test workqueue_mon_test vineex_local_test radical_local_test perf_test ## run all tests
 
 .PHONY: tag
 tag: ## create a tag in git. to run, do a 'make VERSION="version string" tag

docs/userguide/workflow.rst

@@ -241,3 +241,24 @@ Caching expensive initialisation between tasks
 Many tasks in workflows require a expensive "initialization" steps that, once performed, can be used across successive invocations for that task. For example, you may want to reuse a machine learning model for multiple interface tasks and avoid loading it onto GPUs more than once.
 
 `This ExaWorks tutorial <https://github.com/ExaWorks/warmable-function-calls>`_ gives examples of how to do this.
+
+Other useful patterns
+=====================
+
+This section is intended to list some patterns which are not to do with
+parallelism and concurrency, but still might be useful in parsl workflows
+
+Environment wrappers for bash_apps
+----------------------------------
+
+This usecase comes from DESC DRP v2.
+
+Sometimes a bash app command must be run in a particular environment, for
+example, inside a container started by shifter or singularity. [TODO URLs for
+those two]
+
+So although the app commandline might look like:
+
+.. code-block::
+
+    myscience input.txt 5 10

parsl/app/app.py

@@ -8,7 +8,6 @@
 from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
 import typeguard
-from typing_extensions import Literal
 
 from parsl.dataflow.dflow import DataFlowKernel
 from parsl.dataflow.futures import AppFuture
@@ -27,7 +26,7 @@ class AppBase(metaclass=ABCMeta):
     @typeguard.typechecked
     def __init__(self, func: Callable,
                  data_flow_kernel: Optional[DataFlowKernel] = None,
-                 executors: Union[List[str], Literal['all']] = 'all',
+                 executors: Union[List[str], str] = 'all',
                  cache: bool = False,
                  ignore_for_cache: Optional[Sequence[str]] = None) -> None:
         """Construct the App object.
@@ -80,7 +79,7 @@ def __call__(self, *args: Any, **kwargs: Any) -> AppFuture:
 def python_app(function: Optional[Callable] = None,
                data_flow_kernel: Optional[DataFlowKernel] = None,
                cache: bool = False,
-               executors: Union[List[str], Literal['all']] = 'all',
+               executors: Union[List[str], str] = 'all',
                ignore_for_cache: Optional[Sequence[str]] = None) -> Callable:
     """Decorator function for making python apps.
 
@@ -146,7 +145,7 @@ def wrapper(f: Callable) -> PythonApp:
             return PythonApp(f,
                              data_flow_kernel=data_flow_kernel,
                              cache=cache,
-                             executors=["_parsl_internal"],
+                             executors="_parsl_internal",
                              ignore_for_cache=ignore_for_cache,
                              join=True)
         return wrapper(func)
@@ -159,7 +158,7 @@ def wrapper(f: Callable) -> PythonApp:
 def bash_app(function: Optional[Callable] = None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
-             executors: Union[List[str], Literal['all']] = 'all',
+             executors: Union[List[str], str] = 'all',
              ignore_for_cache: Optional[Sequence[str]] = None) -> Callable:
     """Decorator function for making bash apps.
 

parsl/config.py

@@ -177,6 +177,8 @@ def _validate_executors(self) -> None:
         if len(duplicates) > 0:
             raise ConfigurationError('Executors must have unique labels ({})'.format(
                 ', '.join(['label={}'.format(repr(d)) for d in duplicates])))
+        if 'all' in labels:
+            raise ConfigurationError('Executor cannot be labelled "all"')
 
     def validate_usage_tracking(self, level: int) -> None:
         if not USAGE_TRACKING_DISABLED <= level <= USAGE_TRACKING_LEVEL_3: