Add pixi run ribasim-models to run all test models (#1224)

This runs them all in a single Julia session to avoid latency. I didn't
add this to CI for now since we already run them all on TeamCity using
the CLI. But this is especially helpful during development to see which
models fail and why.

If I intentionally break two models you see this output at the end:

```
[ Info: Ran 32 models, 30 passed, 2 failed.
Failed models:
bucket
rating_curve
```

And in between the normal output:
```
[ Info: Running model bucket
ERROR: expect key: starttime
Stacktrace:
  [1] error(s::String)
    @ Base .\error.jl:35
  [2] macro expansion
    @ C:\Users\visser_mn\.julia\packages\Configurations\Yxczn\src\from_dict.jl:0 [inlined]
  [3] from_dict_specialize(::Type{Ribasim.config.Toml}, d::Dict{String, Any})
    @ Ribasim.config C:\Users\visser_mn\.julia\packages\Configurations\Yxczn\src\codegen.jl:362
  [4] #from_dict#4
    @ C:\Users\visser_mn\.julia\packages\Configurations\Yxczn\src\from_dict.jl:46 [inlined]
  [5] from_dict
    @ C:\Users\visser_mn\.julia\packages\Configurations\Yxczn\src\from_dict.jl:33 [inlined]
  [6] from_toml(::Type{Ribasim.config.Toml}, filename::String; kw::@kwargs{})
    @ Configurations C:\Users\visser_mn\.julia\packages\Configurations\Yxczn\src\from_toml.jl:18
  [7] from_toml
    @ C:\Users\visser_mn\.julia\packages\Configurations\Yxczn\src\from_toml.jl:8 [inlined]
  [8] #Config#62
    @ D:\Ribasim\core\src\config.jl:147 [inlined]
  [9] Config
    @ D:\Ribasim\core\src\config.jl:146 [inlined]
┌ Error: Simulation failed
│   modelname = "bucket"
└ @ Main D:\Ribasim\utils\testmodelrun.jl:15
```

---------

Co-authored-by: Hofer-Julian <[email protected]>

.github/workflows/core_testmodels.yml

@@ -0,0 +1,43 @@
+name: Julia Run Testmodels
+on:
+  push:
+    branches: [main, update/pixi-lock, update/julia-manifest]
+    paths-ignore: [".teamcity/**"]
+    tags: ["*"]
+  pull_request:
+  merge_group:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
+jobs:
+  test:
+    name: Julia ${{ matrix.os }} - ${{ matrix.arch }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          # https://github.com/Deltares/Ribasim/issues/825
+          # - macOS-latest
+          - windows-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/cache@v1
+        with:
+          cache-compiled: "true"
+          cache-registries: "true"
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: "v0.15.2"
+      - name: Prepare pixi
+        run: pixi run install-ci
+      - name: Run testmodels with Ribasim Core
+        run: |
+          pixi run ribasim-core-testmodels

Project.toml

@@ -22,6 +22,7 @@ DocumenterMarkdown = "997ab1e6-3595-5248-9280-8efb232c3433"
 EnumX = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b"
 IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a"

pixi.toml

@@ -29,7 +29,7 @@ install-ci = { depends_on = [
 install = { depends_on = [
     "install-ci",
     "install-qgis-plugins",
-    "install-pre-commit"
+    "install-pre-commit",
 ] }
 # Instantiate
 update-registry-julia = "julia --eval='using Pkg; Registry.update()'"
@@ -45,11 +45,13 @@ build-julia-docs = { cmd = "julia --project docs/make.jl", depends_on = [
 ] }
 quartodoc-build = { cmd = "quartodoc build && rm objects.json", cwd = "docs" }
 quarto-preview = { cmd = "quarto preview docs", depends_on = [
-    "quartodoc-build", "generate-testmodels"
+    "quartodoc-build",
+    "generate-testmodels",
 ] }
 quarto-check = { cmd = "quarto check all", depends_on = ["quartodoc-build"] }
 quarto-render = { cmd = "julia --project --eval 'using Pkg; Pkg.build(\"IJulia\")' && quarto render docs --to html --execute", depends_on = [
-    "quartodoc-build", "generate-testmodels"
+    "quartodoc-build",
+    "generate-testmodels",
 ] }
 docs = { depends_on = ["build-julia-docs", "quarto-preview"] }
 # Lint
@@ -92,7 +94,7 @@ test-ribasim-core-cov = { cmd = "julia --project=core --eval 'using Pkg; Pkg.tes
 generate-testmodels = "python utils/generate-testmodels.py"
 tests = { depends_on = ["lint", "test-ribasim-python", "test-ribasim-core"] }
 # Codegen
-generate-python = { cmd = "julia --project utils/gen_python.jl && ruff format python/ribasim/ribasim/schemas.py"}
+generate-python = { cmd = "julia --project utils/gen_python.jl && ruff format python/ribasim/ribasim/schemas.py" }
 codegen = { depends_on = ["initialize-julia", "generate-python"] }
 # Publish
 build-ribasim-python-wheel = { cmd = "rm --recursive --force dist && python -m build && twine check dist/*", cwd = "python/ribasim" }
@@ -124,13 +126,19 @@ test-ribasim-qgis-ui = { cmd = "python ribasim_qgis/scripts/run_qgis_ui_tests.py
 ] }
 test-ribasim-qgis = { cmd = "pytest --numprocesses=auto ribasim_qgis/tests", depends_on = [
     "install-ribasim-qgis",
-]}
+] }
 test-ribasim-qgis-cov = { cmd = "pytest --numprocesses=auto --cov=ribasim_qgis --cov-report=xml --cov-config=ribasim_qgis/.coveragerc ribasim_qgis/tests", depends_on = [
     "install-ribasim-qgis",
-]}
+] }
 mypy-ribasim-qgis = "mypy ribasim_qgis"
 # Run
-ribasim-model = "julia --project=core -e 'using Ribasim; Ribasim.main(ARGS)'"
+ribasim-core = { cmd = "julia --project=core -e 'using Ribasim; Ribasim.main(ARGS)'", depends_on = [
+    "initialize-julia",
+] }
+ribasim-core-testmodels = { cmd = "julia --project utils/testmodelrun.jl", depends_on = [
+    "generate-testmodels",
+    "initialize-julia",
+] }
 # Release
 github-release = "python utils/github-release.py"
 
@@ -145,7 +153,7 @@ libgdal-arrow-parquet = "*"
 matplotlib = "*"
 mypy = "*"
 netCDF4 = "*"
-pandas = "==2.1.4"  # Avoid excessive deprecation warnings from pandera (#984)
+pandas = "==2.1.4"          # Avoid excessive deprecation warnings from pandera (#984)
 pandas-stubs = "*"
 pandera = "*"
 pip = "*"

utils/runstats.jl

@@ -16,6 +16,8 @@ using DataStructures: OrderedDict
 using Dates
 using LibGit2
 
+include("utils.jl")
+
 "Add key config settings like solver settings to a dictionary"
 function add_config!(dict, config::Ribasim.Config)
     confdict = to_dict(getfield(config, :toml))
@@ -100,23 +102,6 @@ function run_dict(toml_path, config, timed)
     return dict
 end
 
-"Retrieve the names of the test models from a Python module"
-function get_testmodels()::Vector{String}
-    _, dirs, _ = first(walkdir("generated_testmodels"))
-
-    toml_paths = String[]
-    for dir in dirs
-        if !startswith(dir, "invalid_")
-            toml_path = normpath("generated_testmodels", dir, "ribasim.toml")
-            @assert isfile(toml_path)
-            push!(toml_paths, toml_path)
-        end
-    end
-
-    @assert length(toml_paths) > 10
-    return toml_paths
-end
-
 toml_paths = get_testmodels()
 runs = OrderedDict{String, Any}[]
 for toml_path in toml_paths

utils/testmodelrun.jl

@@ -0,0 +1,31 @@
+using Ribasim
+
+include("utils.jl")
+
+function main(ARGS)
+    toml_paths = get_testmodels()
+    n_model = length(toml_paths)
+    n_pass = 0
+    n_fail = 0
+    failed = String[]
+
+    for toml_path in toml_paths
+        modelname = basename(dirname(toml_path))
+        @info "Running model $modelname"
+        if Ribasim.main(toml_path) != 0
+            @error "Simulation failed" modelname
+            push!(failed, modelname)
+            n_fail += 1
+        else
+            n_pass += 1
+        end
+    end
+
+    @info "Ran $n_model models, $n_pass passed, $n_fail failed."
+    if n_fail > 0
+        println("Failed models:")
+        foreach(println, failed)
+    end
+end
+
+main(ARGS)

utils/utils.jl

@@ -0,0 +1,10 @@
+# Shared utility functions that are not part of the Ribasim core
+
+using Glob: glob
+
+"Retrieve the names of the valid test models"
+function get_testmodels()::Vector{String}
+    models_dir = normpath(@__DIR__, "..", "generated_testmodels")
+    toml_paths = sort(glob("**/ribasim.toml", models_dir))
+    filter(x -> !startswith(basename(dirname(x)), "invalid_"), toml_paths)
+end