Merge pull request #16 from tovrstra/script-redirect

Add support for standard output and error redirection to script driver

.pre-commit-config.yaml

@@ -25,12 +25,12 @@ repos:
   hooks:
     - id: remove-crlf
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.5
+  rev: v0.5.1
   hooks:
     - id: ruff-format
     - id: ruff
       args: ["--fix", "--show-fixes"]
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.28.4
+  rev: 0.29.0
   hooks:
     - id: check-github-workflows

docs/advanced_topics/amending_static_inputs/stdout.txt

@@ -7,6 +7,5 @@
 This is a config file.
 ────────────────────────────────────────────────────────────────────────────────
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/amending_steps/stdout.txt

@@ -19,6 +19,5 @@ Contents of input.txt:
 You better read this.
 ────────────────────────────────────────────────────────────────────────────────
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/blocked_steps/stdout.txt

@@ -11,6 +11,5 @@ step:cp -aT a.txt b.txt
 ────────────────────────────────────────────────────────────────────────────────
    WARNING │ Skipping cleanup due to incomplete build.
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/cyclic_dependencies/stdout.txt

@@ -15,19 +15,19 @@ Traceback (most recent call last):
   File "/home/toon/univ/reprep/stepup-core/stepup/core/api.py", line 303, in step
     return RPC_CLIENT.call.step(
            ^^^^^^^^^^^^^^^^^^^^^
-  File "/home/toon/univ/reprep/stepup-core/stepup/core/rpc.py", line 436, in __call__
+  File "/home/toon/univ/reprep/stepup-core/stepup/core/rpc.py", line 448, in __call__
     _handle_error(body, name, args, kwargs)
-  File "/home/toon/univ/reprep/stepup-core/stepup/core/rpc.py", line 68, in _handle_error
+  File "/home/toon/univ/reprep/stepup-core/stepup/core/rpc.py", line 69, in _handle_error
     raise RPCError(f"An exception was raised in the server during the call {fmt_call}: \n\n{body}")
 stepup.core.exceptions.RPCError: An exception was raised in the server during the call step('step:./plan.py', 'cp -aT b.txt a.txt', [Path('b.txt')], [], [Path('a.txt')], [], Path('./'), False, None, False):
 Traceback (most recent call last):
-  File "/home/toon/univ/reprep/stepup-core/stepup/core/rpc.py", line 193, in _handle_request
+  File "/home/toon/univ/reprep/stepup-core/stepup/core/rpc.py", line 194, in _handle_request
     result = call(*bound.args, **bound.kwargs)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/home/toon/univ/reprep/stepup-core/stepup/core/director.py", line 359, in step
+  File "/home/toon/univ/reprep/stepup-core/stepup/core/director.py", line 376, in step
     return self._workflow.define_step(
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/home/toon/univ/reprep/stepup-core/stepup/core/workflow.py", line 623, in define_step
+  File "/home/toon/univ/reprep/stepup-core/stepup/core/workflow.py", line 622, in define_step
     self.create_file(step_key, out_path, FileState.PENDING)
   File "/home/toon/univ/reprep/stepup-core/stepup/core/workflow.py", line 375, in create_file
     self.supply(creator_key, file_key)
@@ -51,6 +51,5 @@ cycle:
    WARNING │ Skipping cleanup due to incomplete build.
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
    WARNING │ Dissolving the workflow due to an exceptions while the graph was being changed.
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/environment_variables/stdout.txt

@@ -9,6 +9,5 @@
 foo
 ────────────────────────────────────────────────────────────────────────────────
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/here_and_root/stdout.txt

@@ -12,6 +12,5 @@
      START │ cp -aT example.txt ../../public/sub/example.txt  # wd=sub/
    SUCCESS │ cp -aT example.txt ../../public/sub/example.txt  # wd=sub/
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/optional_steps/stdout.txt

@@ -12,6 +12,5 @@
      START │ ./plot.py run
    SUCCESS │ ./plot.py run
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/pools/stdout.txt

@@ -22,6 +22,5 @@ A
 C
 ────────────────────────────────────────────────────────────────────────────────
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/static_deferred_glob/stdout.txt

@@ -9,6 +9,5 @@
 This is foo.
 ────────────────────────────────────────────────────────────────────────────────
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/static_named_glob/stdout.txt

@@ -28,6 +28,5 @@
      START │ cat ch4/sec4_1_summary.md > public/ch4.md
    SUCCESS │ cat ch4/sec4_1_summary.md > public/ch4.md
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/variable_substitution/stdout.txt

@@ -6,6 +6,5 @@
      START │ ./step.py < src_foo.txt > dst_foo.txt
    SUCCESS │ ./step.py < src_foo.txt > dst_foo.txt
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/advanced_topics/volatile_outputs/stdout.txt

@@ -6,6 +6,5 @@
      START │ date > date.txt
    SUCCESS │ date > date.txt
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/changelog.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add support for standard output and error redirection in the script driver.
+  The dictionary returned by the `info` or `case_info` functions
+  can include `"stdout"` and/or `"stderr"` items.
+  The values of these two fields are paths to which the standard output and/or error
+  of the run part of the script are redirected.
+
+
 ## [1.2.8] - 2024-06-28 {: #v1.2.8 }
 
 ### Fixed

docs/development.md

@@ -26,7 +26,7 @@ The documentation is created using [MkDocs](https://www.mkdocs.org/).
 Edit the documentation Markdown files with a live preview by running:
 
 ```bash
-mkdocs serve --watch stepup/core/
+mkdocs serve
 ```
 
 (Keep this running.)

docs/getting_started/copy_mkdir/stdout.txt

@@ -8,9 +8,8 @@
      START │ mkdir -p sub/
    SUCCESS │ mkdir -p sub/
      START │ cp -aT hello.txt sub/hello.txt
-   SUCCESS │ ./plan.py
    SUCCESS │ cp -aT hello.txt sub/hello.txt
+   SUCCESS │ ./plan.py
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/dependencies/stdout.txt

@@ -3,15 +3,14 @@
   DIRECTOR │ Launched worker 1
      PHASE │ run
      START │ ./plan.py
+   SUCCESS │ ./plan.py
      START │ echo First line. > story.txt; echo Second line. >> story.txt
    SUCCESS │ echo First line. > story.txt; echo Second line. >> story.txt
      START │ grep First story.txt
-   SUCCESS │ ./plan.py
    SUCCESS │ grep First story.txt
 ─────────────────────────────── Standard output ────────────────────────────────
 First line.
 ────────────────────────────────────────────────────────────────────────────────
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/distributed_plans/stdout.txt

@@ -16,6 +16,5 @@ This is part 2.
 This is part 1.
 ────────────────────────────────────────────────────────────────────────────────
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/first_step.md

@@ -81,8 +81,12 @@ StepUp will not know anymore that it already executed some steps and runs all of
   process when `plan.py` is executed by `stepup`.
   You should get the following screen output.
 
+{% macro incl() %}
+{% include "getting_started/first_step/stdout3.txt" %}
+{% endmacro %}
+
     ```
-    {% include 'getting_started/first_step/stdout3.txt' | indent(width=4) %}
+    {{ incl() | indent(width=4) }}
     ```
 
     This output contains internal details of StepUp,

docs/getting_started/first_step/stdout1.txt

@@ -5,10 +5,9 @@
    SUCCESS │ ./plan.py
      START │ echo Hello World
    SUCCESS │ echo Hello World
-───────────────────────────────────────── Standard output ──────────────────────────────────────────
+─────────────────────────────── Standard output ────────────────────────────────
 Hello World
-────────────────────────────────────────────────────────────────────────────────────────────────────
+────────────────────────────────────────────────────────────────────────────────
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/first_step/stdout2.txt

@@ -5,6 +5,5 @@
       SKIP │ ./plan.py
       SKIP │ echo Hello World
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/first_step/stdout3.txt

@@ -1,2 +0,0 @@
-filter_deferred([])
-step('dummy:', 'echo Hello World', [], [], [], [], Path('./'), False, None, False)

docs/getting_started/no_rules/stdout.txt

@@ -8,6 +8,5 @@
      START │ tr '[:lower:]' '[:upper:]' < lower2.txt > upper2.txt
    SUCCESS │ tr '[:lower:]' '[:upper:]' < lower2.txt > upper2.txt
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/script_multiple.md

@@ -23,6 +23,9 @@ def case_info(case: int):
     return {
         "inp": ..., # a single input path or a list of input paths
         "out": ..., # a single output path or a list of input paths
+        "static": ..., # declare a static file or a list of static files
+        "stdout": ..., # redirect the standard output to a file (StepUp 1.3.0)
+        "stderr": ..., # redirect the standard error to a file (StepUp 1.3.0)
         "just_any": "argument that you want to add",
     }
 
@@ -56,6 +59,8 @@ The script has the following elements:
 
 - The function `case_info()` is used to translate `args` and `kwargs` into a more detailed
   planning of the run steps.
+  The returned dictionary is handled in the same way as that of the `info()` function in
+  the previous tutorial [Script (Single Case)](script_single.md#single-case-script-driver).
 
 - The function `run()` works in the same way as for the single case script driver.
 
@@ -118,3 +123,21 @@ This produces the following figures:
         from settings import airports
         yield from airports
     ```
+
+- For debugging purposes, it is sometimes useful to run just a single case of a script.
+  To facilitate this type of debugging, the script can be called with the `cases` argument.
+  When you run the following command:
+
+    ```bash
+    ./plot.py cases
+    ```
+
+    you will get a list of different ways to execute the script's `run()` function:
+
+{% macro incl() %}
+{% include "getting_started/script_multiple/cases_out.txt" %}
+{% endmacro %}
+
+    ```bash
+    {{ incl() | indent(width=4) }}
+    ```

docs/getting_started/script_multiple/cases_out.txt

@@ -0,0 +1,2 @@
+./plot.py run -- 'plot_ebbr'
+./plot.py run -- 'plot_ebos'

docs/getting_started/script_multiple/main.sh

@@ -2,6 +2,7 @@
 git clean -qdfX .
 unset STEPUP_ROOT
 stepup -n -w 1 | sed -f ../../clean_stdout.sed > stdout.txt
+./plot.py cases > cases_out.txt
 
 # INP: plan.py
 # INP: plot.py

docs/getting_started/script_multiple/stdout.txt

@@ -10,6 +10,5 @@
      START │ ./plot.py run -- 'plot_ebos'
    SUCCESS │ ./plot.py run -- 'plot_ebos'
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/script_single.md

@@ -19,9 +19,9 @@ is roughly equivalent to:
 step("./executable plan", inp="sub/executable", workdir="sub/")
 ```
 
-where the subdirectory is optional.
-The step `./executable plan` is expected to define additional steps to actually run something useful with the executable.
-A common scenario is that it plans a single step `./executable run` with appropriate inputs and outputs.
+Note that the use of a subdirectory is not required.
+The `./executable plan` step is expected to define additional steps to actually run something useful with the executable.
+A common scenario is to plan a single `./executable run` step with appropriate inputs and outputs.
 
 When the `optional=True` keyword argument is given to the `script()` function,
 it executes `./executable plan --optional`.
@@ -55,6 +55,9 @@ def info():
     return {
         "inp": ..., # a single input path or a list of input paths
         "out": ..., # a single output path or a list of input paths
+        "static": ..., # declare a static file or a list of static files
+        "stdout": ..., # redirect the standard output to a file (StepUp 1.3.0)
+        "stderr": ..., # redirect the standard error to a file (StepUp 1.3.0)
         "just_any": "argument that you want to add",
     }
 
@@ -65,16 +68,24 @@ if __name__ == "__main__":
     driver()
 ```
 
-- The `info` function provides the necessary data to implement
-  the planning of the execution of the script.
+- The `info` function provides the data necessary to plan the execution of the script.
   It is executed when calling the script as `./script.py plan`.
 
+    !!! note
+
+        All dictionary items are optional.
+        The `info` function can even return an empty dictionary.
+        The keys `inp`, `out`, `static`, `stdout` and `stderr` affect the planning the run part,
+        as explained in the comments above.
+
 - The `run` function is called to perform the useful work and
-  is executed when running the script with `./script.py run`.
+  is executed when the script is executed as `./script.py run`.
+
+    !!! note
 
-Note that the `run` function can have any argument defined in the dictionary return by `info`,
-but it does not have to specify all of them.
-The argument list of `run` may also contain fewer arguments (or even none at all).
+        The `run` function can have any argument defined in the dictionary returned by `info`,
+        but it does not have to specify all of them.
+        The argument list of `run` can contain fewer arguments (or even none at all).
 
 
 ## Example

docs/getting_started/script_single/stdout.txt

@@ -8,6 +8,5 @@
      START │ ./generate.py run
    SUCCESS │ ./generate.py run
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/static_files/stdout.txt

@@ -6,6 +6,5 @@
      START │ nl limerick.txt > numbered.txt
    SUCCESS │ nl limerick.txt > numbered.txt
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/static_glob/stdout.txt

@@ -10,6 +10,5 @@
      START │ cp -aT src/foo.txt dst/foo.txt
    SUCCESS │ cp -aT src/foo.txt dst/foo.txt
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/static_glob_conditional/stdout1.txt

@@ -11,6 +11,5 @@
 2.580000
 ────────────────────────────────────────────────────────────────────────────────
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

docs/getting_started/static_glob_conditional/stdout2.txt

@@ -6,6 +6,5 @@
    SUCCESS │ ./plan.py
       SKIP │ cat average.txt
   WORKFLOW │ Dumped to .stepup/workflow.mpk.xz
-     PHASE │ watch
   DIRECTOR │ Stopping workers.
   DIRECTOR │ See you!

mkdocs.yml

@@ -103,7 +103,9 @@ plugins:
 - search:
     lang:
       - en
-- macros
+- macros:
+    on_error_fail: true
+    on_undefined: strict
 - mkdocstrings:
     default_handler: python
     handlers: