ci(release): fix nightly build

.build_rtd_docs/conf.py

@@ -81,6 +81,18 @@
 dst = os.path.join(dstdir, fpth)
 shutil.copy(src, dst)
 
+# -- build the deprecations table --------------------------------------------
+print("Build the deprecations markdown table")
+pth = os.path.join("..", "doc", "mf6io", "mf6ivar")
+args = (sys.executable, "deprecations.py")
+# run the command
+proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=pth)
+stdout, stderr = proc.communicate()
+if stdout:
+    print(stdout.decode("utf-8"))
+if stderr:
+    print("Errors:\n{}".format(stderr.decode("utf-8")))
+
 # -- copy deprecations markdown ---------------------------------------------
 print("Copy the deprecations table")
 dstdir = "_mf6run"

.github/workflows/release.yml

@@ -73,7 +73,7 @@ jobs:
             version: ${{ inputs.compiler_version }}
             optimization: 2
             parallel: false
-          - os: macos-13
+          - os: macos-12
             compiler: ${{ inputs.compiler_toolchain }}
             version: ${{ inputs.compiler_version }}
             optimization: 2
@@ -459,6 +459,10 @@ jobs:
           name: deprecations
           path: modflow6/doc/mf6io/mf6ivar/md/deprecations.md
 
+      - name: Build MF6IO files from DFNs
+        working-directory: modflow6/doc/mf6io/mf6ivar
+        run: python mf6ivar.py
+
       - name: Build documentation
         env:
           # this step is lazy about building the mf6 examples PDF document, first

.github/workflows/release_dispatch.yml

@@ -146,7 +146,7 @@ jobs:
           echo "models=$models" >> $GITHUB_OUTPUT
   make_dist:
     name: Make distribution
-    uses: MODFLOW-USGS/modflow6/.github/workflows/release.yml@develop
+    uses: wpbonelli/modflow6/.github/workflows/release.yml@fix-nightly-build
     needs: set_options
     with:
       # If the workflow is manually triggered, the maintainer must manually set approve=true to approve a release.

distribution/benchmark.py

@@ -428,11 +428,10 @@ def test_run_benchmarks(tmp_path):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        prog="Benchmark MODFLOW 6 versions on example models",
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=textwrap.dedent(
             """\
-            Benchmarks the current version of MODFLOW 6 against the latest official release.
+            Benchmarks the current version of MODFLOW 6 against the latest official release,
             with the example models stored in the MODFLOW-USGS/modflow6-examples repository.
             """
         ),

distribution/build_dist.py

@@ -334,7 +334,7 @@ def build_distribution(
         bin_path=output_path / "bin",
         full=full,
         output_path=output_path / "doc",
-        overwrite=overwrite,
+        force=overwrite,
     )
 
 
@@ -378,16 +378,20 @@ def test_build_distribution(tmp_path, full):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        prog="Create a Modflow 6 distribution directory for release",
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=textwrap.dedent(
             """\
-            Create a distribution folder. If no output path is provided
-            distribution files are written to the distribution/ folder.
+            Create a MODFLOW 6 distribution. If output path is provided
+            distribution files are written to the selected path, if not
+            they are written to the distribution/ project subdirectory.
             By default a minimal distribution containing only binaries,
             mf6io documentation, release notes and metadata (code.json)
             is created. To create a full distribution including sources
-            and examples, use the --full flag.
+            and examples, use the --full flag. Models to be included in
+            the examples and documentation can be selected with --model
+            (or -m), which may be used multiple times. Use --force (-f)
+            to overwrite preexisting distribution artifacts; by default
+            the script is lazy and will only create what it can't find.
             """
         ),
     )

distribution/build_docs.py

@@ -7,7 +7,7 @@
 from datetime import datetime
 from os import PathLike, environ
 from pathlib import Path
-from pprint import pprint
+from pprint import pformat, pprint
 from tempfile import TemporaryDirectory
 from typing import List, Optional
 from urllib.error import HTTPError
@@ -71,6 +71,8 @@
 
 
 def clean_tex_files():
+    """Remove LaTeX files before a clean rebuild."""
+
     print("Cleaning latex files")
     exts = ["pdf", "aux", "bbl", "idx", "lof", "out", "toc"]
     pth = PROJ_ROOT_PATH / "doc" / "mf6io"
@@ -116,6 +118,8 @@ def download_benchmarks(
     verbose: bool = False,
     repo_owner: str = "MODFLOW-USGS",
 ) -> Optional[Path]:
+    """Try to download MF6 benchmarks from GitHub Actions."""
+
     output_path = Path(output_path).expanduser().absolute()
     name = "run-time-comparison"  # todo make configurable
     repo = (
@@ -169,9 +173,11 @@ def test_download_benchmarks(tmp_path, github_user):
 
 def build_benchmark_tex(
     output_path: PathLike,
-    overwrite: bool = False,
+    force: bool = False,
     repo_owner: str = "MODFLOW-USGS",
 ):
+    """Build LaTeX files for MF6 performance benchmarks to go into the release notes."""
+
     BENCHMARKS_PATH.mkdir(parents=True, exist_ok=True)
     benchmarks_path = BENCHMARKS_PATH / "run-time-comparison.md"
 
@@ -182,7 +188,7 @@ def build_benchmark_tex(
         )
 
     # run benchmarks again if no benchmarks found on GitHub or overwrite requested
-    if overwrite or not benchmarks_path.is_file():
+    if force or not benchmarks_path.is_file():
         run_benchmarks(
             build_path=PROJ_ROOT_PATH / "builddir",
             current_bin_path=PROJ_ROOT_PATH / "bin",
@@ -222,6 +228,8 @@ def test_build_benchmark_tex(tmp_path):
 
 
 def build_deprecations_tex():
+    """Build LaTeX files for the deprecations table to go into the release notes."""
+
     mf6ivar_path = MF6IO_PATH / "mf6ivar"
     md_path = mf6ivar_path / "md"
     md_path.mkdir(exist_ok=True)
@@ -260,70 +268,80 @@ def test_build_deprecations_tex():
     build_deprecations_tex()
 
 
-def build_mf6io_tex_from_dfn(
-    overwrite: bool = False, models: Optional[List[str]] = None
-):
-    if overwrite:
+def build_mf6io_tex(models: Optional[List[str]] = None, force: bool = False):
+    """Build LaTeX files for the MF6IO guide from DFN files."""
+    if force:
         clean_tex_files()
 
-    def files_match(tex_path, dfn_path, ignored):
-        dfn_names = [
-            f.stem
-            for f in dfn_path.glob("*")
+    def match(tex_names, dfn_names):
+        tex = set(tex_names)
+        dfn = set(dfn_names)
+        diff = tex ^ dfn
+        return not any(diff)
+
+    def assert_match(tex_names, dfn_names):
+        tex = set(tex_names)
+        dfn = set(dfn_names)
+        diff = tex ^ dfn
+        assert not any(diff), (
+            f"=> symmetric difference:\n{pformat(diff)}\n"
+            f"=> tex - dfn:\n{pformat(tex - dfn)}\n"
+            f"=> dfn - tex:\n{pformat(dfn - tex)}\n"
+        )
+
+    with set_dir(PROJ_ROOT_PATH / "doc" / "mf6io" / "mf6ivar"):
+        ignored = ["appendix", "common"] + list(
+            set(DEFAULT_MODELS) - set(models)
+        )
+        included = models + ["sim", "utl", "exg", "sln"]
+        tex_files = [
+            f
+            for f in Path("tex").glob("*.tex")
             if f.is_file()
-            and "dfn" in f.suffix
+            and any(pattern in f.name for pattern in included)
             and not any(pattern in f.name for pattern in ignored)
         ]
-        tex_names = [
-            f.stem.replace("-desc", "")
-            for f in tex_path.glob("*")
+        dfn_files = [
+            f
+            for f in Path("dfn").glob("*.dfn")
             if f.is_file()
-            and "tex" in f.suffix
+            and any(pattern in f.name for pattern in included)
             and not any(pattern in f.name for pattern in ignored)
         ]
-
-        return set(tex_names) == set(dfn_names)
-
-    with set_dir(PROJ_ROOT_PATH / "doc" / "mf6io" / "mf6ivar"):
-        ignored = ["appendix", "common"]
-        tex_pth = Path("tex")
-        dfn_pth = Path("dfn")
-        tex_files = [f for f in tex_pth.glob("*") if f.is_file()]
-        dfn_files = [f for f in dfn_pth.glob("*") if f.is_file()]
-
-        if (
-            not overwrite
-            and any(tex_files)
-            and any(dfn_files)
-            and files_match(tex_pth, dfn_pth, ignored)
-        ):
+        dfn_names = [f.stem for f in dfn_files]
+        tex_names = [f.stem.replace("-desc", "") for f in tex_files]
+        if match(tex_names, dfn_names) and not force:
             print("DFN files already exist:")
             pprint(dfn_files)
         else:
             for f in tex_files:
                 f.unlink()
 
-            # run python script
+            # run mf6ivar script and make sure a tex
+            # file was generated for each dfn
             args = [sys.executable, "mf6ivar.py"]
             if models is not None and any(models):
                 for model in models:
                     args += ["--model", model]
-            out, err, ret = run_cmd(*args)
+            out, err, ret = run_cmd(*args, verbose=True)
             assert not ret, out + err
-
-            # check that dfn and tex files match
-            assert files_match(tex_pth, dfn_pth, ignored)
+            assert_match(tex_names, dfn_names)
 
 
 @no_parallel
-@pytest.mark.parametrize("overwrite", [True, False])
-def test_build_mf6io_tex_from_dfn(overwrite):
-    build_mf6io_tex_from_dfn(overwrite=overwrite)
+@pytest.mark.parametrize("force", [True, False])
+def test_build_mf6io_tex(force):
+    build_mf6io_tex(force=force)
 
 
-def build_mf6io_tex_example(
+def build_usage_example_tex(
     workspace_path: PathLike, bin_path: PathLike, example_model_path: PathLike
 ):
+    """
+    Build LaTeX files for the MF6 usage example in the MF6IO guide.
+    Runs MF6 to capture the output and insert into the document.
+    """
+
     workspace_path = Path(workspace_path) / "workspace"
     bin_path = Path(bin_path).expanduser().absolute()
     mf6_exe_path = bin_path / f"mf6{EXE_EXT}"
@@ -386,12 +404,14 @@ def build_mf6io_tex_example(
             f.write("}\n")
 
 
-def build_pdfs_from_tex(
+def build_pdfs(
     tex_paths: List[PathLike],
     output_path: PathLike,
     passes: int = 3,
     overwrite: bool = False,
 ):
+    """Build PDF documents from LaTeX files."""
+
     print("Building PDFs from LaTex:")
     pprint(tex_paths)
 
@@ -454,39 +474,41 @@ def test_build_pdfs_from_tex(tmp_path):
         DOCS_PATH / "ConverterGuide" / "converter_mf5to6.bbl",
     ]
 
-    build_pdfs_from_tex(tex_paths, tmp_path)
+    build_pdfs(tex_paths, tmp_path)
     for p in tex_paths[:-1] + bbl_paths:
         assert p.is_file()
 
 
 def build_documentation(
     bin_path: PathLike,
+    force: bool = False,
     full: bool = False,
+    models: Optional[List[str]] = None,
     output_path: Optional[PathLike] = None,
-    overwrite: bool = False,
     repo_owner: str = "MODFLOW-USGS",
-    models: Optional[List[str]] = None,
 ):
+    """Build documentation for a MODFLOW 6 distribution."""
+
     print(f"Building {'full' if full else 'minimal'} documentation")
 
     bin_path = Path(bin_path).expanduser().absolute()
     output_path = Path(output_path).expanduser().absolute()
 
-    if (output_path / "mf6io.pdf").is_file() and not overwrite:
+    if (output_path / "mf6io.pdf").is_file() and not force:
         print(f"{output_path / 'mf6io.pdf'} already exists")
         return
 
     # make sure output directory exists
     output_path.mkdir(parents=True, exist_ok=True)
 
     # build LaTex input/output docs from DFN files
-    build_mf6io_tex_from_dfn(overwrite=overwrite, models=models)
+    build_mf6io_tex(force=force, models=models)
 
     # build LaTeX input/output example model docs
     with TemporaryDirectory() as temp:
-        build_mf6io_tex_example(
-            workspace_path=Path(temp),
+        build_usage_example_tex(
             bin_path=bin_path,
+            workspace_path=Path(temp),
             example_model_path=PROJ_ROOT_PATH / ".mf6minsim",
         )
 
@@ -496,12 +518,12 @@ def build_documentation(
     if full:
         # convert benchmarks to LaTex, running them first if necessary
         build_benchmark_tex(
-            output_path=output_path, overwrite=overwrite, repo_owner=repo_owner
+            output_path=output_path, force=force, repo_owner=repo_owner
         )
 
         # download example docs
         pdf_name = "mf6examples.pdf"
-        if overwrite or not (output_path / pdf_name).is_file():
+        if force or not (output_path / pdf_name).is_file():
             latest = get_release(f"{repo_owner}/modflow6-examples", "latest")
             assets = latest["assets"]
             asset = next(
@@ -524,17 +546,17 @@ def build_documentation(
                     raise
 
         # convert LaTex to PDF
-        build_pdfs_from_tex(
+        build_pdfs(
             tex_paths=TEX_PATHS["full"],
             output_path=output_path,
-            overwrite=overwrite,
+            overwrite=force,
         )
     else:
         # just convert LaTeX to PDF
-        build_pdfs_from_tex(
+        build_pdfs(
             tex_paths=TEX_PATHS["minimal"],
             output_path=output_path,
-            overwrite=overwrite,
+            overwrite=force,
         )
 
     # enforce os line endings on all text files
@@ -563,7 +585,6 @@ def test_build_documentation(tmp_path):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        prog="Convert LaTeX docs to PDFs",
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=textwrap.dedent(
             """\
@@ -625,9 +646,9 @@ def test_build_documentation(tmp_path):
     models = args.model if args.model else DEFAULT_MODELS
     build_documentation(
         bin_path=bin_path,
+        force=args.force,
         full=args.full,
+        models=models,
         output_path=output_path,
-        overwrite=args.force,
         repo_owner=args.repo_owner,
-        models=models,
     )