diff --git a/docs/src/_templates/imagehash.html b/docs/src/_templates/imagehash.html new file mode 100644 index 0000000000..8b0dac0cce --- /dev/null +++ b/docs/src/_templates/imagehash.html @@ -0,0 +1,15 @@ +{% extends "!layout.html" %} + +{% block body %} + +

Test: {{ test }}

+ + +{% for hash, file in hashfiles %} +
+

{{hash}}

+ +
+{% endfor %} + +{% endblock %} \ No newline at end of file diff --git a/docs/src/conf.py b/docs/src/conf.py index e13f12a13a..3362a9a023 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -158,6 +158,7 @@ def _dotv(version): "sphinx_gallery.gen_gallery", "matplotlib.sphinxext.mathmpl", "matplotlib.sphinxext.plot_directive", + "image_test_output", ] if skip_api == "1": diff --git a/docs/src/developers_guide/contributing_graphics_tests.rst b/docs/src/developers_guide/contributing_graphics_tests.rst index 53e895f440..1268aa2686 100644 --- a/docs/src/developers_guide/contributing_graphics_tests.rst +++ b/docs/src/developers_guide/contributing_graphics_tests.rst @@ -63,6 +63,8 @@ This consists of: developer to easily compare proposed new **acceptable** result images against the existing accepted reference images, for each failing test. +The acceptable images for each test can be viewed online. The :ref:`testing.imagehash_index` lists all the graphical tests in the test suite and +shows the known acceptable result images for comparison. Reviewing Failing Tests ======================= diff --git a/docs/src/developers_guide/contributing_testing_index.rst b/docs/src/developers_guide/contributing_testing_index.rst index 517111507b..c5cf1b997b 100644 --- a/docs/src/developers_guide/contributing_testing_index.rst +++ b/docs/src/developers_guide/contributing_testing_index.rst @@ -8,5 +8,6 @@ Testing contributing_testing contributing_graphics_tests + imagehash_index contributing_running_tests contributing_ci_tests diff --git a/docs/src/developers_guide/imagehash_index.rst b/docs/src/developers_guide/imagehash_index.rst new file mode 100644 index 0000000000..a11ae8a531 --- /dev/null +++ b/docs/src/developers_guide/imagehash_index.rst @@ -0,0 +1,20 @@ +.. include:: ../common_links.inc + +.. _testing.imagehash_index: + +Graphical Test Hash Index +************************* + +The iris test suite produces plots of data using matplotlib and cartopy. +The images produced are compared to known "good" output, the images for +which are kept in `scitools/test-iris-imagehash `_. + +For an overview of iris' graphics tests, see :ref:`testing.graphics` + +Typically running the iris test suite will output the rendered +images to ``$PROJECT_DIR/iris_image_test_output``. +The known good output for each test can be seen at the links below +for comparison. + + +.. imagetest-list:: \ No newline at end of file diff --git a/docs/src/sphinxext/image_test_output.py b/docs/src/sphinxext/image_test_output.py new file mode 100644 index 0000000000..9e492a5be9 --- /dev/null +++ b/docs/src/sphinxext/image_test_output.py @@ -0,0 +1,78 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. + +import json +import re +from typing import Dict, List + +from docutils import nodes +from sphinx.application import Sphinx +from sphinx.util.docutils import SphinxDirective + +ImageRepo = Dict[str, List[str]] + +HASH_MATCH = re.compile(r"([^\/]+)\.png$") + + +def hash_from_url(url: str) -> str: + match = HASH_MATCH.search(url) + if not match: + raise ValueError(f"url {url} does not match form `http...hash.png`") + else: + return match.groups()[0] + + +class ImageTestDirective(SphinxDirective): + def run(self): + with open(self.config["image_test_json"], "r") as fh: + imagerepo = json.load(fh) + enum_list = nodes.enumerated_list() + nodelist = [] + nodelist.append(enum_list) + for test in sorted(imagerepo): + link_node = nodes.raw( + "", + f'{test}', + format="html", + ) + li_node = nodes.list_item("") + li_node += link_node + enum_list += li_node + return nodelist + + +def collect_imagehash_pages(app: Sphinx): + """Generate pages for each entry in the imagerepo.json""" + with open(app.config["image_test_json"], "r") as fh: + imagerepo: ImageRepo = json.load(fh) + pages = [] + for test, hashfiles in imagerepo.items(): + hashstrs = [hash_from_url(h) for h in hashfiles] + pages.append( + ( + f"generated/image_test/{test}", + {"test": test, "hashfiles": zip(hashstrs, hashfiles)}, + "imagehash.html", + ) + ) + return pages + + +def setup(app: Sphinx): + app.add_config_value( + "image_test_json", + "../../lib/iris/tests/results/imagerepo.json", + "html", + ) + + app.add_directive("imagetest-list", ImageTestDirective) + app.connect("html-collect-pages", collect_imagehash_pages) + + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + }