diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8bd2059d..ec84af8d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,6 +46,9 @@ jobs: uses: ./.github/actions/setup-idaes with: install-target: -r requirements.txt + - name: Install pandoc + run: | + conda install --yes pandoc - name: Run pytest (unit) run: | pytest --verbose -m unit tests/ @@ -82,7 +85,7 @@ jobs: install-target: -r requirements.txt - name: Install pandoc run: | - sudo apt-get install --quiet --yes pandoc + conda install --yes pandoc - name: Check installed versions run: | idaes --version diff --git a/build-ci.yml b/build-ci.yml index a2f79d9d..7f91c2b2 100644 --- a/build-ci.yml +++ b/build-ci.yml @@ -15,7 +15,6 @@ notebook: num_workers: 1 # continue on errors (otherwise stop) continue_on_error: true - test_mode: true timeout: 600 # where to put error files. special values: '__stdout__', '__stderr__' error_file: ci-test-errors.txt @@ -42,7 +41,7 @@ notebook_index: output_file: src/notebook_index.ipynb # Settings for running Sphinx build sphinx: - args: "-b html -T docs_test docs_test/_build/html" + args: "-b html -T {output} {output}/{html}" error_file: sphinx-errors.txt # Directory to create and use for linkchecker output linkcheck_dir: lc \ No newline at end of file diff --git a/build.py b/build.py index 02f4bef1..d64f851d 100755 --- a/build.py +++ b/build.py @@ -57,9 +57,6 @@ For example command lines see the README.md in this directory. """ -import time - -_import_timings = [(None, time.time())] # stdlib from abc import ABC, abstractmethod import argparse @@ -68,19 +65,22 @@ import glob from io import StringIO import logging +import multiprocessing as mproc import os from pathlib import Path +import signal import shutil import subprocess import re from string import Template import sys import tempfile -from typing import List, TextIO, Tuple, Optional +import time +from typing import TextIO, Optional import urllib.parse import yaml -_import_timings.append(("stdlib", time.time())) +_import_timings = [("stdlib", time.time())] # third-party import nbconvert @@ -99,7 +99,6 @@ if _script_dir not in sys.path: print("Add script's directory to sys.path") sys.path.insert(0, _script_dir) -from build_util import bossy _import_timings.append(("local-directory", time.time())) @@ -107,10 +106,9 @@ _log = logging.getLogger("build_notebooks") _hnd = logging.StreamHandler() _hnd.setLevel(logging.NOTSET) -_hnd.setFormatter(logging.Formatter("%(asctime)s %(levelname)s - %(message)s")) +_hnd.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) _log.addHandler(_hnd) _log.propagate = False -_log.setLevel(logging.INFO) # This is a workaround for a bug in some versions of Tornado on Windows for Python 3.8 @@ -170,7 +168,18 @@ class IndexPageInputFile(IndexPageError): # Images to copy to the build directory IMAGE_SUFFIXES = ".jpg", ".jpeg", ".png", ".gif", ".svg", ".pdf" # These should catch all data files in the same directory as the notebook, needed for execution. -DATA_SUFFIXES = ".csv", ".json", ".json.gz", ".gz", ".svg", ".xls", ".xlsx", ".txt", ".zip", ".pdf" +DATA_SUFFIXES = ( + ".csv", + ".json", + ".json.gz", + ".gz", + ".svg", + ".xls", + ".xlsx", + ".txt", + ".zip", + ".pdf", +) CODE_SUFFIXES = (".py",) NOTEBOOK_SUFFIX = ".ipynb" TERM_WIDTH = 60 # for notification message underlines @@ -223,8 +232,7 @@ def __str__(self): return str(self.d) def set_default_section(self, section: str): - """Set default section for get/set. - """ + """Set default section for get/set.""" self._dsec = section def get(self, key: str, default=_NULL): @@ -281,8 +289,7 @@ def _subst_paths(self, value): class Builder(ABC): - """Abstract base class for notebook and sphinx builders. - """ + """Abstract base class for notebook and sphinx builders.""" def __init__(self, settings): self.s = settings @@ -301,15 +308,13 @@ def _merge_options(self, options): class NotebookBuilder(Builder): - """Run Jupyter notebooks and render them for viewing. - """ + """Run Jupyter notebooks and render them for viewing.""" TEST_SUFFIXES = ("_test", "_testing") # test notebook suffixes HTML_IMAGE_DIR = "_images" class Results: - """Stores results from build(). - """ + """Stores results from build().""" def __init__(self): self.failed, self.cached = [], [] @@ -335,8 +340,8 @@ def __init__(self, *args): self._nb_error_file = None # error file, for notebook execution failures self._results = None # record results here self._nb_remove_config = None # traitlets.Config for removing test cells - self._test_mode, self._num_workers = None, 1 - self._match_expr = None + self._test_mode, self._copy_mode, self._num_workers = None, None, 1 + self._match_expr, self._timeout, self._pool = None, None, None # Lists of entries (or just one, for outdir) for each subdirectory self.notebooks_to_convert, self.data_files, self.outdir, self.depth = ( {}, @@ -348,8 +353,12 @@ def __init__(self, *args): def build(self, options): self.s.set_default_section("notebook") self._num_workers = self.s.get("num_workers", default=2) + self._timeout = self.s.get("timeout", default=60) self._merge_options(options) - self._test_mode = self.s.get("test_mode") + self._test_mode = self.s.get("test_mode", False) + self._copy_mode = self.s.get("copy_mode", False) + if self._test_mode and self._copy_mode: + raise ValueError("Cannot set both test_mode and copy_mode in options") self._open_error_file() self._ep = self._create_preprocessor() self._imgdir = ( @@ -458,6 +467,7 @@ def _create_preprocessor(self): def _read_template(self): nb_template_path = self.root_path / self.s.get("template") + notify(f"Reading notebook template from path: {nb_template_path}", level=1) try: with nb_template_path.open("r") as f: nb_template = Template(f.read()) @@ -494,8 +504,7 @@ def discover_tree(self, info: dict): self._results.dirs_processed.append(srcdir) def discover_subtree(self, srcdir: Path, outdir: Path, depth: int): - """Discover all notebooks in a given directory. - """ + """Discover all notebooks in a given directory.""" _log.debug(f"Discover.begin subtree={srcdir}") # Iterate through directory and get list of notebooks to convert (and data files) @@ -557,60 +566,88 @@ def convert_discovered_notebooks(self): # process list of jobs, in parallel _log.info(f"Process {len(jobs)} notebooks") num_workers = min(self._num_workers, len(jobs)) + self._results.num_workers = num_workers + # Run conversion in parallel - _log.info(f"Convert notebooks with {num_workers} worker(s)") + + # notify user of num. workers and timeout + timeout = self._timeout + if timeout < 60: + # force at least 10 second timeout + timeout = max(timeout, 10) + wait_time = f"{timeout} seconds" + else: + if timeout // 60 * 60 == timeout: + wait_time = f"{timeout // 60} minute{'' if timeout == 60 else 's'}" + else: + sec = timeout - (timeout // 60 * 60) + wait_time = ( + f"{timeout // 60} minute{'' if timeout == 60 else 's'}, " + f"{sec} second{'' if sec == 1 else 's'}" + ) + notify( + f"Convert notebooks with {num_workers} " + f"worker{'' if num_workers == 1 else 's'}. Timeout after {wait_time}." + ) + + # Create workers worker = ParallelNotebookWorker( processor=self._ep, wrapper_template=self.s.get("Template"), remove_config=self._nb_remove_config, test_mode=self._test_mode, - # TODO we should DRY timeout to an attr so that the same value is consistently used here and in _create_preprocessor() - timeout=self.s.get("timeout") + copy_mode=self._copy_mode, + timeout=self._timeout, ) - b = bossy.Bossy( - jobs, - num_workers=num_workers, - worker_function=worker.convert, - output_log=_log, + pool = mproc.Pool(num_workers) + num_jobs = len(jobs) + log_level = _log.getEffectiveLevel() + ar = pool.map_async( + worker.convert, + ((i + 1, jobs[i], log_level) for i in range(num_jobs)), + callback=self._convert_success, + error_callback=self._convert_failure, ) - results = b.run() - # Report results, which returns summary - successes, failures = self._report_results(results) - # Clean up any temporary directories - for tmpdir in temporary_dirs.values(): - _log.debug(f"remove temporary directory at '{tmpdir.name}'") - try: - shutil.rmtree(str(tmpdir)) - except Exception as err: - _log.error(f"could not remove temporary directory '{tmpdir}': {err}") - # Record summary of success/fail and return - _log.debug(f"Convert.end {successes}/{successes + failures}") - self._results.n_fail += failures - self._results.n_success += successes - # Record total work time so we can calculate speedup - self._results.worker_time = sum((r[1].dur for r in results)) - self._results.num_workers = num_workers + pool.close() # no more tasks can be added - def _report_results( - self, result_list: List[Tuple[int, "ParallelNotebookWorker.ConversionResult"]] - ) -> Tuple[int, int]: - # print(f"@@ result list: {result_list}") - s, f = 0, 0 - for worker_id, result in result_list: - if result.ok: - s += 1 - else: - f += 1 - filename = str(result.entry) - self._write_notebook_error(self._nb_error_file, filename, result.why) - self._results.failed.append(filename) - return s, f + # Set up signal handler + self._pool = pool # for signal handler + self._set_signals(self.abort) + + # Run workers + try: + ar.get(timeout) + except mproc.TimeoutError: + notify(f"Timeout occurred, terminating", level=1) + pool.terminate() + finally: + pool.join() + + def abort(self, *args): + notify(f"Interrupt, terminating") + self._pool.terminate() + time.sleep(2) + self._pool.join() + time.sleep(2) + sys.exit(1) @staticmethod - def _write_notebook_error(error_file, nb_filename, error): - error_file.write(f"\n====> File: {nb_filename}\n") - error_file.write(str(error)) - error_file.flush() # in case someone is tailing the file + def _set_signals(func): + if hasattr(signal, "SIGINT"): + signal.signal(signal.SIGINT, func) + if hasattr(signal, "SIGBREAK"): + signal.signal(signal.SIGBREAK, func) + + def _convert_success(self, result): + self._results.n_success += len(result) + self._results.worker_time += sum((r.dur for r in result)) + + def _convert_failure(self, exc): + self._results.n_fail += 1 + self._nb_error_file.write(str(exc)) + self._nb_error_file.flush() + filename = f"{exc}" + self._results.failed.append(filename) class ParallelNotebookWorker: @@ -619,7 +656,8 @@ class ParallelNotebookWorker: State is avoided where possible to ensure that the ForkingPickler succeeds. Main method is `convert`. - """ + """ + # Map format to file extension FORMATS = {"html": ".html", "rst": ".rst"} @@ -659,42 +697,42 @@ def __init__( wrapper_template: Optional[Template] = None, remove_config: Optional[Config] = None, test_mode: bool = False, - timeout = None, + copy_mode: bool = False, + timeout=None, ): self.processor = processor self.template, self.rm_config = ( wrapper_template, remove_config, ) - self.test_mode = test_mode - self.log_q, self.id_ = None, 0 + self.test_mode, self.copy_mode = test_mode, copy_mode + self.id_ = 0 self._timeout = timeout - # Logging utility functions - - def log(self, level, msg): - self.log_q.put((level, f"[Worker {self.id_}] {msg}")) - - def log_error(self, msg): - return self.log(logging.ERROR, msg) - - def log_warning(self, msg): - return self.log(logging.WARNING, msg) - - def log_info(self, msg): - return self.log(logging.INFO, msg) - - def log_debug(self, msg): - return self.log(logging.DEBUG, msg) - # Main function - def convert(self, id_, job, log_q) -> ConversionResult: - """Parallel 'worker' to convert a single notebook. - """ - self.log_q, self.id_ = log_q, id_ + def convert(self, args) -> ConversionResult: + """Parallel 'worker' to convert a single notebook.""" + # Handle signals for worker + if hasattr(signal, "SIGINT"): + signal.signal(signal.SIGINT, self.abort) + if hasattr(signal, "SIGBREAK"): + signal.signal(signal.SIGBREAK, self.abort) + + job_num, job, log_level = args + self.id_ = os.getpid() + # set up logging for this worker + _log.setLevel(log_level) + if len(_log.handlers) > 0: + handler = _log.handlers[0] + handler.setFormatter( + logging.Formatter( + f"%(asctime)s [%(levelname)s] {self.id_:<5d}/{job_num}: " + f"%(message)s" + ) + ) - self.log_info(f"Convert notebook name={job.nb}: begin") + _log.info(f"Convert notebook name={job.nb}: begin") time_start = time.time() ok, why = True, "" @@ -703,7 +741,7 @@ def convert(self, id_, job, log_q) -> ConversionResult: job.outdir.mkdir(parents=True) # build, if the output file is missing/stale verb = "Running" if self.test_mode else "Converting" - self.log_info(f"{verb}: {job.nb.name}") + _log.info(f"{verb}: {job.nb.name}") # continue_on_err = self.s.get("continue_on_error", None) converted = False try: @@ -713,15 +751,13 @@ def convert(self, id_, job, log_q) -> ConversionResult: except NotebookExecError as err: ok, why = False, err self._write_failed_marker(job) - self.log_error( - f"Execution failed: generating partial output for '{job.nb}'" - ) + _log.error(f"Execution failed: generating partial output for '{job.nb}'") except NotebookError as err: ok, why = False, f"NotebookError: {err}" - self.log_error(f"Failed to convert {job.nb}: {err}") + _log.error(f"Failed to convert {job.nb}: {err}") except Exception as err: ok, why = False, f"Unknown error: {err}" - self.log_error(f"Failed due to error: {err}") + _log.error(f"Failed due to error: {err}") time_end = time.time() @@ -729,17 +765,19 @@ def convert(self, id_, job, log_q) -> ConversionResult: # remove failed marker, if there was one from a previous execution failed_marker = self._get_failed_marker(job) if failed_marker: - self.log_info( - f"Remove stale marker of failed execution: {failed_marker}" - ) + _log.info(f"Remove stale marker of failed execution: {failed_marker}") failed_marker.unlink() duration = time_end - time_start - self.log_info( - f"Convert notebook name={job.nb}: end, ok={ok} duration={duration:.1f}s" + _log.info( + f"Convert notebook name={job.nb}: " f"end, ok={ok} duration={duration:.1f}s" ) - return self.ConversionResult(id_, ok, converted, why, job.nb, duration) + return self.ConversionResult(self.id_, ok, converted, why, job.nb, duration) + + def abort(self, *args): + _log.error(f"W{self.id_}: Abort on interrupt") + sys.exit(1) def _convert(self, job) -> bool: """Convert a notebook. @@ -747,12 +785,18 @@ def _convert(self, job) -> bool: Returns: True if conversion was performed, False if no conversion was needed """ - info, dbg = logging.INFO, logging.DEBUG # aliases + converted = True + verb = ( + "running" + if self.test_mode + else ("copying" if self.copy_mode else "converting") + ) + notify(f"{verb.title()} notebook {job.nb.name}", level=1) # strip special cells. if self._has_tagged_cells(job.nb, set(self.CELL_TAGS.values())): - self.log_debug(f"notebook '{job.nb.name}' has test cell(s)") + _log.debug(f"notebook '{job.nb.name}' has test cell(s)") entry = self._strip_tagged_cells(job, ("remove", "exercise"), "testing") - self.log_info(f"Stripped tags from: {job.nb.name}") + _log.info(f"Stripped tags from: {job.nb.name}") else: # copy to temporary directory just to protect from output cruft entry = job.tmpdir / job.nb.name @@ -762,27 +806,37 @@ def _convert(self, job) -> bool: # Stop if failure marker is newer than source file. failed_time = self._previously_failed(job) if failed_time is not None: - self.log_info( + _log.info( f"Skip notebook conversion, failure marker is newer, for: {entry.name}" ) failed_datetime = datetime.fromtimestamp(failed_time) + if self.copy_mode: + notify(f"Skip copying notebook: previous execution failed", level=2) + return False raise NotebookPreviouslyFailedError(f"at {failed_datetime}") - # Stop if converted result is newer than source file. + # Do not execute if converted result is newer than source file. if self._previously_converted(job, entry): - self.log_info( - f"Skip notebook conversion, output is newer, for: {entry.name}" - ) - return False - self.log_info(f"Running notebook: {entry.name}") - try: - nb = self._parse_and_execute(entry) - except NotebookExecError as err: - self.log_error(f"Notebook execution failed: {err}") - raise - if self.test_mode: # don't do export in test mode - return True - - self.log_info(f"Exporting notebook '{entry.name}' to directory {job.outdir}") + _log.info(f"Skip notebook conversion, output is newer, for: {entry.name}") + converted = False + nb = self._parse_notebook(entry) + else: + nb = self._parse_notebook(entry) + # Do not execute in copy_mode + if not self.copy_mode: + _log.info(f"Running notebook: {entry.name}") + try: + self._execute_notebook(nb, entry) + except NotebookExecError as err: + _log.error(f"Notebook execution failed: {err}") + raise + # Export notebooks in output formats + if not self.test_mode: # don't do export in test mode + self._export_notebook(nb, entry, job) + return converted + + def _export_notebook(self, nb, entry, job): + """Export notebooks in output formats.""" + _log.info(f"Exporting notebook '{entry.name}' to directory {job.outdir}") wrt = FilesWriter() # export each notebook into multiple target formats created_wrapper = False @@ -790,22 +844,19 @@ def _convert(self, job) -> bool: (RSTExporter(), self._postprocess_rst, ()), (HTMLExporter(), self._postprocess_html, (job.depth,)), ): - self.log_debug(f"export '{job.nb}' with {exp} to notebook '{entry}'") + _log.debug(f"export '{job.nb}' with {exp} to notebook '{entry}'") (body, resources) = exp.from_notebook_node(nb) body = post_process_func(body, *pp_args) wrt.build_directory = str(job.outdir) wrt.write(body, resources, notebook_name=entry.stem) # create a 'wrapper' page if not created_wrapper: - self.log_debug( - f"create wrapper page for '{entry.name}' in '{job.outdir}'" - ) + _log.debug(f"create wrapper page for '{entry.name}' in '{job.outdir}'") self._create_notebook_wrapper_page(job, entry.stem) created_wrapper = True # move notebooks into docs directory - self.log_debug(f"move notebook '{entry} to output directory: {job.outdir}") + _log.debug(f"move notebook '{entry} to output directory: {job.outdir}") shutil.copy(entry, job.outdir / entry.name) - return True def _has_tagged_cells(self, entry: Path, tags: set) -> bool: """Quickly check whether this notebook has any cells with the given tag(s). @@ -825,7 +876,7 @@ def _has_tagged_cells(self, entry: Path, tags: set) -> bool: if "tags" in c.metadata: for tag in tags: if tag in c.metadata.tags: - self.log_debug(f"Found tag '{tag}' in cell {i}") + _log.debug(f"Found tag '{tag}' in cell {i}") return True # can stop now, one is enough # no tagged cells return False @@ -842,7 +893,7 @@ def _previously_failed(self, job: Job) -> Optional[float]: failed_time = failed_file.stat().st_mtime ac = failed_time > source_time if ac: - self.log_info( + _log.info( f"Notebook '{orig.stem}.ipynb' unchanged since previous failed conversion" ) return failed_time @@ -865,7 +916,7 @@ def _previously_converted(self, job: Job, dest: Path) -> bool: failed_time = failed_file.stat().st_ctime ac = failed_time > source_time if ac: - self.log_info( + _log.info( f"Notebook '{orig.stem}.ipynb' unchanged since previous failed conversion" ) return ac @@ -874,7 +925,7 @@ def _previously_converted(self, job: Job, dest: Path) -> bool: # older than the source file (in which case it's NOT converted) for fmt, ext in self.FORMATS.items(): output_file = job.outdir / f"{dest.stem}{ext}" - self.log_debug(f"checking if cached: {output_file} src={orig}") + _log.debug(f"checking if cached: {output_file} src={orig}") if not output_file.exists(): return False if source_time >= output_file.stat().st_mtime: @@ -891,12 +942,12 @@ def _write_failed_marker(self, job: Job): try: job.outdir.mkdir(parents=True) except Exception as err: - self.log_error( + _log.error( f"Could not write failed marker '{marker}' for entry={job.nb} " f"outdir={job.outdir}: {err}" ) return # oh, well - self.log_debug( + _log.debug( f"write failed marker '{marker}' for entry={job.nb} outdir={job.outdir}" ) marker.open("w").write( @@ -906,9 +957,9 @@ def _write_failed_marker(self, job: Job): def _get_failed_marker(self, job: Job) -> Optional[Path]: marker = job.outdir / (job.nb.stem + ".failed") if marker.exists(): - self.log_debug(f"Found 'failed' marker: {marker}") + _log.debug(f"Found 'failed' marker: {marker}") return marker - self.log_debug( + _log.debug( f"No 'failed' marker for notebook '{job.nb}' in directory '{job.outdir}'" ) return None @@ -927,7 +978,7 @@ def _strip_tagged_cells(self, job: Job, tags, remove_name: str): Returns: stripped-entry, original-entry - both in the temporary directory """ - self.log_debug(f"run notebook in temporary directory: {job.tmpdir}") + _log.debug(f"run notebook in temporary directory: {job.tmpdir}") # Copy notebook to temporary directory tmp_nb = job.tmpdir / job.nb.name shutil.copy(job.nb, tmp_nb) @@ -935,7 +986,7 @@ def _strip_tagged_cells(self, job: Job, tags, remove_name: str): # Configure tag removal tag_names = [self.CELL_TAGS[t] for t in tags] self.rm_config.TagRemovePreprocessor.remove_cell_tags = tag_names - self.log_debug( + _log.debug( f"removing tag(s) <{', '.join(tag_names)}'> from notebook: {job.nb.name}" ) (body, resources) = NotebookExporter(config=self.rm_config).from_filename( @@ -952,24 +1003,24 @@ def _strip_tagged_cells(self, job: Job, tags, remove_name: str): # Create the new notebook wrt = nbconvert.writers.FilesWriter() wrt.build_directory = str(job.tmpdir) - self.log_debug(f"writing stripped notebook: {nb_name}") + _log.debug(f"writing stripped notebook: {nb_name}") wrt.write(body, resources, notebook_name=nb_name) # Return both notebook names, and temporary directory (for cleanup) stripped_entry = job.tmpdir / f"{nb_name}.ipynb" return stripped_entry - def _parse_and_execute(self, entry): - # parse - self.log_debug(f"parsing '{entry}'") + def _parse_notebook(self, entry): + _log.debug(f"parsing '{entry}'") try: nb = nbformat.read(str(entry), as_version=self.JUPYTER_NB_VERSION) except nbformat.reader.NotJSONError: raise NotebookFormatError(f"'{entry}' is not JSON") except AttributeError: raise NotebookFormatError(f"'{entry}' has invalid format") + return nb - # execute - self.log_debug(f"executing '{entry}'") + def _execute_notebook(self, nb, entry): + _log.debug(f"executing '{entry}'") t0 = time.time() try: metadata = {"metadata": {"path": str(entry.parent)}} @@ -979,11 +1030,9 @@ def _parse_and_execute(self, entry): except TimeoutError as err: dur, timeout = time.time() - t0, self._timeout raise NotebookError(f"timeout for '{entry}': {dur}s > {timeout}s") - return nb def _create_notebook_wrapper_page(self, job: Job, nb_file: str): - """Generate a Sphinx documentation page for the Module. - """ + """Generate a Sphinx documentation page for the Module.""" # interpret some characters in filename differently for title title = nb_file.replace("_", " ").title() title_under = "=" * len(title) @@ -994,18 +1043,17 @@ def _create_notebook_wrapper_page(self, job: Job, nb_file: str): # write out the new doc doc_rst = job.outdir / (nb_file + "_doc.rst") with doc_rst.open("w") as f: - self.log_info(f"generate Sphinx doc wrapper for {nb_file} => {doc_rst}") + _log.info(f"generate Sphinx doc wrapper for {nb_file} => {doc_rst}") f.write(doc) def _postprocess_rst(self, body): return self._replace_image_refs(body) - IMAGE_IDS = '0123456789abcdefghijklmnopqrstuvwxyz' + IMAGE_IDS = "0123456789abcdefghijklmnopqrstuvwxyz" def _replace_image_refs(self, body): - """Replace duplicate |image| references and associated directives with successive numbers. - """ - m = re.search(r"(.*)\n=+\n", body, flags=re.M) + """Replace duplicate |image| references and associated directives with successive numbers.""" + m = re.search(r"(.*)\n=+\n", body, flags=re.M) title = "unknown" if m is None else m.group(1) chars = list(body) # easy to manipulate this way body_pos, n = 0, 0 @@ -1045,8 +1093,7 @@ def _replace_image_refs(self, body): return "".join(chars) def _postprocess_html(self, body, depth): - """Change path on image refs to point into HTML build dir. - """ + """Change path on image refs to point into HTML build dir.""" # create prefix for attribute values, which is a relative path # to the (single) images directory, from within the HTML build tree prefix = Path("") @@ -1060,7 +1107,9 @@ def _postprocess_html(self, body, depth): orig = splits[i] prefix_unix = "/".join(prefix.parts) splits[i] = f' 1: real_notebook_names.sort(key=len) # shortest first nb_name = Path(real_notebook_names[0]).name @@ -1397,7 +1449,8 @@ def _write_markdown_contents(self, contents, depth, path, tutorials=True): for suffix in "exercise", "solution": self._write(f"[[{suffix}]({url})] ") elif tutorials: - # for tutorials, default link is exercise, but provide both in brackets at end + # for tutorials, default link is exercise, but provide both + # in brackets at end url = urllib.parse.quote(str(path) + f"/{key}_exercise.ipynb") self._write(f" * [{key}]({url}) - {value} ") for suffix in "exercise", "solution": @@ -1425,18 +1478,23 @@ def write_notebook(self, output_path): sections.append(new_section) cur = new_section cur.append(line) - sections.append([ - "## Contact info\n" - "General, background and overview information is available at the [IDAES main website](https://idaes.org).\n" - "Framework development happens at our GitHub repo where you can report issues/bugs or make contributions.\n" - "For further enquiries, send an email to: idaes-support@idaes.org\n" - ]) + sections.append( + [ + "## Contact info\n" + "General, background and overview information is available at the " + "[IDAES main website](https://idaes.org).\n" + "Framework development happens at our GitHub repo where you can report " + "issues/bugs or make contributions.\n" + "For further enquiries, send an email to: idaes-support@idaes.org\n" + ] + ) nb = nbformat.NotebookNode( metadata={"kernel_info": {}}, nbformat=4, nbformat_minor=0, cells=[ - nbformat.NotebookNode(cell_type="markdown", metadata={}, source=section) for section in sections + nbformat.NotebookNode(cell_type="markdown", metadata={}, source=section) + for section in sections ], ) # print(nb.cells) @@ -1444,8 +1502,7 @@ def write_notebook(self, output_path): nbformat.write(nb, notebook_file) def write_listing(self, output_path): - """Write a text version of the page just listing the files to `output_path`. - """ + """Write a text version of the page just listing the files to `output_path`.""" try: self._of = open(output_path, "w") except Exception as err: @@ -1498,8 +1555,7 @@ class Color: def notify(message, level=0): - """Multicolored, indented, messages to the user. - """ + """Multicolored, indented, messages to the user.""" c = [Color.MAGENTA, Color.GREEN, Color.CYAN, Color.WHITE][min(level, 3)] indent = " " * level if level == 0: @@ -1525,13 +1581,9 @@ def get_git_branch(): def print_usage(): - """Print a detailed usage message. - """ + """Print a detailed usage message.""" command = "python build.py" message = ( - "\n" - "# tl;dr To convert notebooks and build docs, use this command:\n" - "{command} -crd\n" "\n" "The build.py command is used to create the documentation from\n" "the Jupyter Notebooks and hand-written '.rst' files in this\n" @@ -1551,25 +1603,21 @@ def print_usage(): "{command} --remove\n" "{command} -r # <-- short option\n" "\n" - "# Convert Jupyter notebooks. Only those notebooks\n" - "# that have not changed since the last time this was run will\n" - "# be re-executed. Converted notebooks are stored in the 'docs'\n" - "# directory, in locations configured in the 'build.yml'\n" - "# configuration file.\n" - "{command} --convert\n" - "{command} -c # <-- short option\n" + "# Execute Jupyter notebooks. This is slow.\n" + "# Only those notebooks that have not changed since the last time\n" + "# this was run will be re-executed.\n" + "{command} --exec\n" + "{command} -e # <-- short option\n" "\n" - "# Convert Jupyter notebooks, as in previous command,\n" - "# then build Sphinx documentation.\n" - "# This can be combined with -r/--remove to convert all notebooks.\n" - "{command} -cd\n" + "# Copy Jupyter notebooks into docs. This is quick.\n" + "{command} --copy\n" + "{command} -y # <-- short option\n" "\n" - "# Run notebooks, but do not convert them into any other form.\n" - "# This can be combined with -r/--remove to run all notebooks.\n" - "{command} --test\n" - "{command} -t # <-- short option\n" + "# Convert Jupyter notebooks. Convert means execute and copy into docs.\n" + "{command} --convert\n" + "{command} -c # <-- short option\n" "\n" - "# Generate various versions of the notebook index page from the YAML input file\n" + "# Generate the notebook index page from the YAML input file\n" "{command} --index-input nb_index.yaml --index-output nb_index.md" "\n" "# Run with at different levels of verbosity\n" @@ -1577,6 +1625,12 @@ def print_usage(): "{command} -v # Add informational (info) messages\n" "{command} -vv # Add debug messages\n" "\n" + "* To fully re-run all notebooks and build docs, use this command:\n" + "{command} -crd\n" + "\n" + "* To generate docs without the long delay of running them:\n" + "{command} -dy\n" + "\n" ) print(message.format(command=command)) @@ -1600,14 +1654,24 @@ def main(): ) ap.add_argument("--docs", "-d", action="store_true", help="Build documentation") ap.add_argument( - "--convert", "-c", action="store_true", help="Convert Jupyter notebooks", + "--convert", + "-c", + action="store_true", + help="Convert Jupyter notebooks", ) ap.add_argument( - "--test", - "-t", + "--exec", + "-e", dest="test_mode", action="store_true", - help="Run notebooks but do not convert them.", + help="Execute notebooks (do not copy them into docs)", + ) + ap.add_argument( + "--copy", + "-y", + dest="copy_mode", + action="store_true", + help="Copy notebooks into docs (do not run them)", ) ap.add_argument( "-v", @@ -1642,7 +1706,7 @@ def main(): default=False, action="store_true", dest="index_dev_mode", - help="For the Jupyter Notebook index, generate links in 'dev' mode to the un-stripped notebook names" + help="For the Jupyter Notebook index, generate links in 'dev' mode to the un-stripped notebook names", ) ap.add_argument( "--workers", @@ -1673,11 +1737,11 @@ def main(): # Check for confusing option combinations if args.convert and args.test_mode: - ap.error("-t/--test conflicts with notebook conversion -c/--convert; pick one") - if args.docs and args.test_mode: - ap.error( - "-t/--test should not be used with -d/--docs, as it does not convert any notebooks" - ) + ap.error(f"-e/--exec conflicts with -c/--convert; pick one") + if args.convert and args.copy_mode: + ap.error(f"-y/--copy conflicts with -c/--convert; pick one") + if args.test_mode and args.copy_mode: + ap.error(f"-y/--copy conflicts with -e/--exec; pick one") # If building docs, check for working Sphinx command if args.docs: @@ -1728,7 +1792,7 @@ def main(): run_notebooks = args.convert build_docs = args.docs clean_files = args.remove - test_mode = args.test_mode + test_mode, copy_mode = args.test_mode, args.copy_mode # Clean first, if requested if clean_files: @@ -1738,12 +1802,12 @@ def main(): status_code = 0 # start with success - if run_notebooks or test_mode: - verb = "Run" if test_mode else "Convert" + if run_notebooks or test_mode or copy_mode: + verb = "Run" if test_mode else ("Copy" if copy_mode else "Convert") notify(f"{verb} Jupyter notebooks") nbb = NotebookBuilder(settings) try: - nbb.build({"test_mode": test_mode}) + nbb.build({"test_mode": test_mode, "copy_mode": copy_mode}) except NotebookError as err: _log.fatal(f"Could not build notebooks: {err}") return -1 @@ -1762,6 +1826,7 @@ def main(): if args.build_index: notify("Build index page") + ix_status, ix_err = 0, "" if args.index_input is None: index_input = settings.get("notebook_index.input_file") else: @@ -1780,11 +1845,23 @@ def main(): ix_page = IndexPage(input_path) ix_page.convert(output_path, dev_mode=dev_mode) except IndexPageInputFile as err: - _log.fatal(f"Error reading from intput file: {err}") - status_code = 2 + ix_status, ix_err = 1, f"Error reading from intput file: {err}" + _log.fatal(ix_err) except IndexPageOutputFile as err: - _log.fatal(f"Error writing to output file: {err}") - status_code = 2 + ix_status, ix_err = 2, f"Error writing to output file: {err}" + _log.fatal(ix_err) + except ValueError as err: + ix_status, ix_err = 3, f"Unknown error: {err}" + _log.fatal(ix_err) + if ix_status != 0: + status_code = ix_status + notify(f"Building index page failed:", level=1) + notify(f"{ix_err}", level=2) + + if args.build_linkcheck: + notify("Run Sphinx linkchecker") + builder = SphinxLinkcheckBuilder(settings) + builder.build({"hide_output": args.hide_sphinx_output}) if args.build_linkcheck: notify("Run Sphinx linkchecker") diff --git a/build_util/__init__.py b/build_util/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/build_util/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/build_util/bossy.py b/build_util/bossy.py deleted file mode 100644 index e471a45d..00000000 --- a/build_util/bossy.py +++ /dev/null @@ -1,237 +0,0 @@ -""" -Utility class to run things in parallel. -""" -import logging -from multiprocessing import Process, Queue -from queue import Empty -from random import randint -import signal -import sys -import threading -import time -from typing import List - - -class WorkerInterrupted(Exception): - def __init__(self, why): - super().__init__(f"Worker interrupted: {why}") - - -def sleepy(id_, item, logq, **kwargs): - """Example worker function that interprets its input as an integer and - just sleeps for that amount of time (in seconds). - """ - logq.put((logging.INFO, f"{id_}: Begin sleep {item}s")) - time.sleep(item) - logq.put((logging.INFO, f"{id_}: End sleep {item}s")) - return "z" * item - - -class Bossy: - def __init__( - self, - work=None, - num_workers=2, - worker_function=sleepy, - output_log=None, - **kwargs, - ): - self.n = num_workers - # create queues - self.work_q = Queue() - self.log_q = Queue() - self.done_q = Queue() - self.result_q = Queue() - self.worker_fn = worker_function - self.log = output_log - self.is_done = False - self._worker_kwargs = kwargs - self.processes, self.logging_thread = None, None - # add work, including sentinels to stop processes, and wait for it all to be processed - self._add_work(work) - sentinels = [None for i in range(self.n)] - self._add_work(sentinels) - self._handle_signals() - - def _handle_signals(self): - # pylint: disable=no-member - if hasattr(signal, 'SIGINT'): - signal.signal(signal.SIGINT, self.signal_handler) - if hasattr(signal, 'SIGBREAK'): - signal.signal(signal.SIGBREAK, self.signal_handler) - - def signal_handler(self, sig, frame): - """User interrupted the program with a Control-C or by sending it a signal (UNIX). - """ - self.log.warning("Interrupted. Killing child processes.") - if self.processes: - self.log.warning(f"Killing {len(self.processes)} processes") - for p in self.processes: - p.kill() - - def run(self) -> List: - self.log.info("Run workers: begin") - results = [] - if self.processes or self.logging_thread: - return results - # start workers and thread to tail the log messages - self.processes = self._create_worker_processes(self._worker_kwargs) - self.logging_thread = self._create_logging_thread() - self._join_worker_processes() - self.processes = None - # stop logging thread - self.is_done = True - self._join_logging_thread() - self.logging_thread = None - # return results as a list of tuples (worker, result) - self.log.debug(f"Collect {self.n} results: begin") - while not self.result_q.empty(): - id_, result_list = self.result_q.get_nowait() - self.log.debug(f"Worker [{id_}]: got result") - for r in result_list: - results.append((id_, r)) - self.log.debug(f"Worker [{id_}]: Recorded result") - self.log.debug(f"Collect {self.n} results: end") - self.log.info("Run workers: end") - return results - - def _add_work(self, items): - for item in items: - self.work_q.put(item) - - def _create_worker_processes(self, kwargs): - self.log.debug(f"Create worker processes. kwargs={kwargs}") - processes = [] - g_log = self.log - for i in range(self.n): - p = Process( - target=self.worker, - args=( - i, - self.work_q, - self.log_q, - self.result_q, - self.done_q, - self.worker_fn, - {}, - ), - ) # kwargs - self.log.debug(f"Worker [{i + 1}]: Starting process {p}") - p.start() - processes.append(p) - return processes - - @staticmethod - def worker(id_, q, log_q, result_q, done_q, func, kwargs): - pfx = f"[Worker {id_} Main-Loop]" - - def log_info(m, pfx=pfx): - log_q.put((logging.INFO, f"{pfx}: {m}")) - - def log_debug(m, pfx=pfx): - log_q.put((logging.DEBUG, f"{pfx}: {m}")) - - def log_error(m, pfx=pfx): - log_q.put((logging.ERROR, f"{pfx}: {m}")) - - log_info("begin") - result_list = [] - while True: - item = q.get() - log_debug("Got next item of work from queue") - if item is None: # sentinel - log_info("No more work: Stop.") - break - try: - log_debug(f"Run worker function: begin") - result = func(id_, item, log_q, **kwargs) - log_debug(f"Run worker function: end") - result_list.append(result) - except KeyboardInterrupt: - log_debug(f"Run worker function: end (keyboard interrupt)") - raise WorkerInterrupted("Keyboard interrupt") - except Exception as err: - log_error(f"Run worker function: end (exception): {err}") - # Put results on the queue - if result_list: - result_q.put((id_, result_list)) - done_q.put(id_) - - def _create_logging_thread(self): - t = threading.Thread(target=self._tail_messages, args=(), daemon=True) - t.start() - return t - - def _tail_messages(self): - while True: - try: - level, msg = self.log_q.get(True, 2) - self.log.log(level, msg) - except Empty: - if self.is_done: - return - - def _join_worker_processes(self): - """Unfortunately, simply joining processes doesn't seem to do the trick all the time. - Instead, we use a special queue to keep track of which workers have finished, then, - if they have not stopped on their own, forcibly terminate the associated processes - in order to join() them. If even after that the join fails, we mark the process as - 'unjoinable' and give up (!) - """ - num_joined, num_unjoinable, num_proc = 0, 0, len(self.processes) - while num_joined + num_unjoinable < num_proc: - self.log.debug(f"Waiting for {num_proc - num_joined - num_unjoinable} processes to finish") - try: - id_ = self.done_q.get(timeout=60) - except Empty: - # Interruptible wait allowing for control-c - time.sleep(1) - continue - proc = self.processes[id_] - if proc.is_alive(): - self.log.info(f"Terminating process: {proc}") - proc.terminate() - t0 = time.time() - while proc.is_alive(): - time.sleep(1) - if time.time() - t0 > 10: - break - if proc.is_alive(): - self.log.error(f"Could not terminate process: {proc}") - num_unjoinable += 1 - else: - self.log.debug(f"Joining process: {proc}") - proc.join() - num_joined += 1 - if num_unjoinable > 0: - self.log.error(f"{num_unjoinable} processes could not be joined") - - def _join_logging_thread(self): - self.logging_thread.join() - - -if __name__ == "__main__": - import sys - - if len(sys.argv) != 2: - print("usage: bossy.py ") - sys.exit(1) - try: - n = int(sys.argv[1]) - if n < 1: - raise ValueError() - except ValueError: - print(f"{sys.argv[1]} is not a positive integer") - print("usage: qtest.py ") - sys.exit(1) - - delays = [randint(3, 8) for i in range(n)] - olog = logging.getLogger() - h = logging.StreamHandler() - olog.addHandler(h) - olog.setLevel(logging.INFO) - b = Bossy(work=delays, num_workers=n, output_log=olog) - r = b.run() - print(f"Results: {r}") - - sys.exit(0) diff --git a/notebook_index.yml b/notebook_index.yml index aac949fe..6ba36580 100644 --- a/notebook_index.yml +++ b/notebook_index.yml @@ -3,12 +3,11 @@ meta: front_matter: - title: Introduction - text: >-2 + text: >- The [IDAES](https://www.idaes.org) integrated platform ships with a number of examples which can be run on the user's own computer. This page provides links to these examples and provides some guidance in the order in which to try them. - The IDAES examples are contained in Jupyter Notebooks. In order to view and use this content, you need to open the files with the Jupyter notebook executable (which may be configured on your system as the default application for files of this @@ -23,7 +22,7 @@ front_matter: [IDAES-PSE documentation](https://idaes-pse.readthedocs.io/en/stable/index.html). - title: Usage - text: >-2 + text: >- The example notebooks contained in this folder are divided into two sub-types, each with their own folder: * `Tutorials`: Notebooks that are written as tutorials complete with guided exercises * `Examples`: Notebooks that do not have tutorial content. @@ -49,7 +48,7 @@ front_matter: contents: - name: Tutorials - description: >-2 + description: >- All the notebooks in this folder have three different files that represent different variations on the same content. The suffix of the filename indicates something about the variation contained in that file. For example, if the notebook is named "a_notebook", then @@ -114,7 +113,7 @@ contents: an ideal property package - mixer: Mixer unit model with ideal property package - pump: Pump unit model with iapws property package - - heat exchanger 0D: Heat Exchanger 0D unit model heating a benzene-toluene mixture using steam + - heat_exchanger_0D: Heat Exchanger 0D unit model heating a benzene-toluene mixture using steam - name: Tools title: Tools for working with IDAES @@ -181,22 +180,22 @@ contents: - name: MatOpt title: Materials optimization - description: >- + description: > Examples of the MatOpt interface for representing material properties and specifying optimization problems. notebooks: - monometallic_nanocluster_design: Minimization of cohesive energy in nanoclusters - - bimetallic_nanocluster_design: >- + - bimetallic_nanocluster_design: > Optimize a bimetallic cluster by "labelling" the sites of a pre-defined monometallic cluster - - surface_design: >- + - surface_design: > MatOpt example optimization problem of designing a monometallic nanostructured catalyst surface - - bifunctional_surface_design: >- + - bifunctional_surface_design: > Example optimization problem of designing a nanostructured bifunctional catalyst - - metal_oxide_bulk_design: >- + - metal_oxide_bulk_design: > How to optimally place dopant in a perovskite lattice - name: Pecos title: Data quality control and fault detection - description: >- + description: > Examples of Pecos interface for data quality control and fault detection notebooks: - data_quality_control: Simple data quality control example diff --git a/src/Examples/UnitModels/heater_testing.ipynb b/src/Examples/UnitModels/heater_testing.ipynb index bffe0271..9a982d93 100644 --- a/src/Examples/UnitModels/heater_testing.ipynb +++ b/src/Examples/UnitModels/heater_testing.ipynb @@ -440,4 +440,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/src/Tutorials/Basics/HDA_flowsheet_solution_testing.ipynb b/src/Tutorials/Basics/HDA_flowsheet_solution_testing.ipynb index 13ee371b..66319121 100644 --- a/src/Tutorials/Basics/HDA_flowsheet_solution_testing.ipynb +++ b/src/Tutorials/Basics/HDA_flowsheet_solution_testing.ipynb @@ -1550,4 +1550,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/src/Tutorials/Basics/flash_unit_solution_testing.ipynb b/src/Tutorials/Basics/flash_unit_solution_testing.ipynb index 51cb09a5..0344659e 100644 --- a/src/Tutorials/Basics/flash_unit_solution_testing.ipynb +++ b/src/Tutorials/Basics/flash_unit_solution_testing.ipynb @@ -962,4 +962,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/src/notebook_index.ipynb b/src/notebook_index.ipynb index ad85a133..928a4ac8 100644 --- a/src/notebook_index.ipynb +++ b/src/notebook_index.ipynb @@ -12,6 +12,7 @@ "metadata": {}, "source": [ "## Introduction\n", + "The [IDAES](https://www.idaes.org) integrated platform ships with a number of examples which can be run on the user's own computer. This page provides links to these examples and provides some guidance in the order in which to try them.\n", "The [IDAES](https://idaes.org) integrated platform ships with a number of examples which can be run on the user's own computer. This page provides links to these examples and provides some guidance in the order in which to try them.\n", "\n", "The IDAES examples are contained in Jupyter Notebooks. In order to view and use this content, you need to open the files with the Jupyter notebook executable (which may be configured on your system as the default application for files of this type). To get started with Jupyter, please see the [Jupyter website](https://jupyter.org) or jump directly to the [official Jupyter Notebook documentation pages](https://jupyter-notebook.readthedocs.io/en/stable/).\n", @@ -112,7 +113,7 @@ "\n", " * [mixer](Examples/UnitModels/mixer.ipynb) - Mixer unit model with ideal property package\n", " * [pump](Examples/UnitModels/pump.ipynb) - Pump unit model with iapws property package\n", - " * [heat exchanger 0D](Examples/UnitModels/heat%20exchanger%200D.ipynb) - Heat Exchanger 0D unit model heating a benzene-toluene mixture using steam\n", + " * [heat_exchanger_0D](Examples/UnitModels/heat_exchanger_0D.ipynb) - Heat Exchanger 0D unit model heating a benzene-toluene mixture using steam\n", "\n", "\n", "\n", @@ -181,16 +182,22 @@ "\n", "### Materials optimization\n", "Examples of the MatOpt interface for representing material properties and specifying optimization problems.\n", + "\n", " * [monometallic_nanocluster_design](Examples/MatOpt/monometallic_nanocluster_design.ipynb) - Minimization of cohesive energy in nanoclusters\n", " * [bimetallic_nanocluster_design](Examples/MatOpt/bimetallic_nanocluster_design.ipynb) - Optimize a bimetallic cluster by \"labelling\" the sites of a pre-defined monometallic cluster\n", + "\n", " * [surface_design](Examples/MatOpt/surface_design.ipynb) - MatOpt example optimization problem of designing a monometallic nanostructured catalyst surface\n", + "\n", " * [bifunctional_surface_design](Examples/MatOpt/bifunctional_surface_design.ipynb) - Example optimization problem of designing a nanostructured bifunctional catalyst\n", + "\n", " * [metal_oxide_bulk_design](Examples/MatOpt/metal_oxide_bulk_design.ipynb) - How to optimally place dopant in a perovskite lattice\n", "\n", + "\n", "\n", "\n", "### Data quality control and fault detection\n", "Examples of Pecos interface for data quality control and fault detection\n", + "\n", " * [data_quality_control](Examples/Pecos/data_quality_control.ipynb) - Simple data quality control example\n", "\n" ] diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py index b107b661..d3d7accb 100644 --- a/tests/test_notebooks.py +++ b/tests/test_notebooks.py @@ -2,20 +2,26 @@ Test notebooks """ # stdlib +import json import logging import os +from pathlib import Path +import re import subprocess import sys # third-party import nbformat import pytest +import yaml # import "build" module from top dir _root = os.path.join(os.path.dirname(__file__), "..") sys.path.insert(0, _root) import build +newline = "\n" # useful for f-strings + @pytest.fixture(scope="module") def settings_ci(): @@ -23,17 +29,6 @@ def settings_ci(): return build.Settings(open("build-ci.yml", "r")) -@pytest.mark.component -def test_convert_some_notebooks(settings_ci): - build._log.setLevel(logging.DEBUG) # otherwise DEBUG for some reason - os.chdir(_root) - nb = build.NotebookBuilder(settings_ci) - nb.build({"rebuild": True}) - total, num_failed = nb.report() - assert total > 0 - assert num_failed == 0 - - @pytest.mark.unit def test_parse_notebook(notebook): """The parameter 'notebook' is parameterized in `conftest.py`, so that @@ -51,7 +46,67 @@ def test_run_all_notebooks(): proc.wait() assert proc.returncode == 0 # now run - cmd = ["python", "build.py", "--config", "build-ci.yml", "--test"] + cmd = ["python", "build.py", "--config", get_build_config(), "--exec"] proc = subprocess.Popen(cmd) proc.wait() assert proc.returncode == 0 + find_broken_links() + + +@pytest.mark.component +def test_broken_links(): + find_broken_links() + + +def find_broken_links(rebuild=True): + """Run the Sphinx link checker. + + This was created in response to a number of broken links in Jupyter notebook + cells, but would also find broken links in any documentation pages. + """ + os.chdir(_root) + config = get_build_config() + config_dict = load_build_config(config) + # Copy notebooks to docs. -S suppresses Sphinx output. + args = ["python", "build.py", "--config", config, "-Sy"] + proc = subprocess.Popen(args) + rc = proc.wait() + assert rc == 0, "Copying notebooks to docs failed" + # Run linkchecker (-l). -S suppresses Sphinx output. + # output will be in dir configured in sphinx.linkcheck_dir (see below) + proc = subprocess.Popen(["python", "build.py", "--config", config, "-Sl"]) + rc = proc.wait() + assert rc == 0, "Linkchecker process failed" + # find links marked [broken], report them + link_file = Path(".") / config_dict["sphinx"]["linkcheck_dir"] / "output.json" + assert link_file.exists() + links = [] + for line in link_file.open(mode="r", encoding="utf-8"): + obj = json.loads(line) + if obj["status"] == "broken": + num = len(links) + 1 + links.append(f"{num}) {obj['filename']}:{obj['lineno']} -> {obj['uri']}") + # fail if there were any broken links + assert len(links) == 0, f"{len(links)} broken links:\n" f"{newline.join(links)}" + + +def test_index_page(): + config = get_build_config() + config_dict = load_build_config(config) + args = ["python", "build.py", "--config", config, "--index", "--index-dev"] + print(f"Build index page (in 'dev' mode) with command: {' '.join(args)}") + proc = subprocess.Popen(args) + rc = proc.wait() + assert rc == 0, "Failed to build Jupyter notebook index page" + +# Utility + + +def get_build_config(): + if os.environ.get("GITHUB_ACTIONS", False): + return "build-ci.yml" + return "build.yml" + + +def load_build_config(config): + return yaml.safe_load(open(config, "r")) \ No newline at end of file