Show acceptable image test results in the docs (#4392)

* Generates pages with links to imagehash images

* Added imagehash index to toc trees etc

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* License header on image_test_output.py

* flaked, blackened and isorted

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

docs/src/_templates/imagehash.html

@@ -0,0 +1,15 @@
+{% extends "!layout.html" %}
+
+{% block body %}
+
+<h1>Test: {{ test }}</h1>
+
+
+{% for hash, file in hashfiles %}
+<div>
+    <h3>{{hash}}</h3>
+    <img src="{{file}}" />
+</div>
+{% endfor %}
+
+{% endblock %}

docs/src/conf.py

@@ -158,6 +158,7 @@ def _dotv(version):
     "sphinx_gallery.gen_gallery",
     "matplotlib.sphinxext.mathmpl",
     "matplotlib.sphinxext.plot_directive",
+    "image_test_output",
 ]
 
 if skip_api == "1":

docs/src/developers_guide/contributing_graphics_tests.rst

@@ -63,6 +63,8 @@ This consists of:
    developer to easily compare proposed new **acceptable** result images
    against the existing accepted reference images, for each failing test.
 
+The acceptable images for each test can be viewed online. The :ref:`testing.imagehash_index` lists all the graphical tests in the test suite and
+shows the known acceptable result images for comparison.
 
 Reviewing Failing Tests
 =======================

docs/src/developers_guide/contributing_testing_index.rst

@@ -8,5 +8,6 @@ Testing
 
    contributing_testing
    contributing_graphics_tests
+   imagehash_index
    contributing_running_tests
    contributing_ci_tests

docs/src/developers_guide/imagehash_index.rst

@@ -0,0 +1,20 @@
+.. include:: ../common_links.inc
+
+.. _testing.imagehash_index:
+
+Graphical Test Hash Index
+*************************
+
+The iris test suite produces plots of data using matplotlib and cartopy.
+The images produced are compared to known "good" output, the images for
+which are kept in `scitools/test-iris-imagehash <https://github.com/scitools/test-iris-imagehash>`_.
+
+For an overview of iris' graphics tests, see :ref:`testing.graphics`
+
+Typically running the iris test suite will output the rendered
+images to ``$PROJECT_DIR/iris_image_test_output``.
+The known good output for each test can be seen at the links below
+for comparison.
+
+
+.. imagetest-list::

docs/src/sphinxext/image_test_output.py

@@ -0,0 +1,78 @@
+# Copyright Iris contributors
+#
+# This file is part of Iris and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+
+import json
+import re
+from typing import Dict, List
+
+from docutils import nodes
+from sphinx.application import Sphinx
+from sphinx.util.docutils import SphinxDirective
+
+ImageRepo = Dict[str, List[str]]
+
+HASH_MATCH = re.compile(r"([^\/]+)\.png$")
+
+
+def hash_from_url(url: str) -> str:
+    match = HASH_MATCH.search(url)
+    if not match:
+        raise ValueError(f"url {url} does not match form `http...hash.png`")
+    else:
+        return match.groups()[0]
+
+
+class ImageTestDirective(SphinxDirective):
+    def run(self):
+        with open(self.config["image_test_json"], "r") as fh:
+            imagerepo = json.load(fh)
+        enum_list = nodes.enumerated_list()
+        nodelist = []
+        nodelist.append(enum_list)
+        for test in sorted(imagerepo):
+            link_node = nodes.raw(
+                "",
+                f'<a href="{self.config["html_baseurl"]}/generated/image_test/{test}.html" />{test}</a>',
+                format="html",
+            )
+            li_node = nodes.list_item("")
+            li_node += link_node
+            enum_list += li_node
+        return nodelist
+
+
+def collect_imagehash_pages(app: Sphinx):
+    """Generate pages for each entry in the imagerepo.json"""
+    with open(app.config["image_test_json"], "r") as fh:
+        imagerepo: ImageRepo = json.load(fh)
+    pages = []
+    for test, hashfiles in imagerepo.items():
+        hashstrs = [hash_from_url(h) for h in hashfiles]
+        pages.append(
+            (
+                f"generated/image_test/{test}",
+                {"test": test, "hashfiles": zip(hashstrs, hashfiles)},
+                "imagehash.html",
+            )
+        )
+    return pages
+
+
+def setup(app: Sphinx):
+    app.add_config_value(
+        "image_test_json",
+        "../../lib/iris/tests/results/imagerepo.json",
+        "html",
+    )
+
+    app.add_directive("imagetest-list", ImageTestDirective)
+    app.connect("html-collect-pages", collect_imagehash_pages)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }