From 92dc411dd42a43fdb634fea5ac4db879c22e2b2f Mon Sep 17 00:00:00 2001
From: s-weigand <s.weigand.phy@gmail.com>
Date: Sun, 29 Aug 2021 01:06:28 +0200
Subject: [PATCH] =?UTF-8?q?=F0=9F=91=8C=20Add=20option=20to=20specify=20pa?=
 =?UTF-8?q?th=20for=20local=20comparison?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Using the environment variable COMPARE_RESULTS_LOCAL which can be set in tox.ini a local path can be specified to look for the results folder to use as a reference in lieu of the (gold standard) comparison-results branch in the pyglotaran-examples repository

Added some comments about which tolerances are used and their origin.
---
 .github/test_result_consistency.py | 38 ++++++++++++++++++++++++++----
 tox.ini                            |  8 ++++++-
 2 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/.github/test_result_consistency.py b/.github/test_result_consistency.py
index 3c35a05b5..d68cf71a8 100644
--- a/.github/test_result_consistency.py
+++ b/.github/test_result_consistency.py
@@ -7,6 +7,7 @@
 from collections import defaultdict
 from functools import lru_cache
 from pathlib import Path
+from textwrap import dedent
 from typing import TYPE_CHECKING
 from typing import Protocol
 from warnings import warn
@@ -170,13 +171,13 @@ def data_var_test(
     current_values = current_data
 
     eps = np.finfo(np.float32).eps
-    rtol = 1e-5
+    rtol = 1e-5  # default value of allclose
     if expected_var_name.endswith("residual"):  # type:ignore[operator]
         eps = expected_result["data"].values.max() * 1e-8
 
     if "singular_vectors" in expected_var_name:  # type:ignore[operator]
-        rtol = 1e-4
-        eps = 1e-5
+        rtol = 1e-4  # instead of 1e-5
+        eps = 1e-5  # instead of ~1.2e-7
         pre_fix = SVD_PATTERN.match(expected_var_name).group(  # type:ignore[operator]
             "pre_fix"
         )
@@ -224,7 +225,34 @@ def data_var_test(
 def map_result_files(file_glob_pattern: str) -> dict[str, list[tuple[Path, Path]]]:
     """Load all datasets and map them in a dict."""
     result_map = defaultdict(list)
-    compare_results_path = get_compare_results_path()
+    if os.getenv("COMPARE_RESULTS_LOCAL"):
+        compare_results_path = Path(os.getenv(key="COMPARE_RESULTS_LOCAL"))
+        warn(
+            dedent(
+                f"""
+                Using Path in environment variable COMPARE_RESULTS_LOCAL:
+                {compare_results_path.as_posix()}
+                """
+            )
+        )
+        try:
+            if not compare_results_path.exists():
+                raise FileNotFoundError(
+                    dedent(
+                        f"""
+                        Path in COMPARE_RESULTS_LOCAL not valid:
+                        {compare_results_path}  <- does not exist
+                        """
+                    )
+                )
+        except OSError as exception:
+            if str(compare_results_path).startswith(('"', "'")):
+                raise Exception(
+                    "Path in COMPARE_RESULTS_LOCAL should not start with ' or \""
+                ) from exception
+            raise exception
+    else:
+        compare_results_path = get_compare_results_path()
     current_result_path = get_current_result_path()
     for expected_result_file in compare_results_path.rglob(file_glob_pattern):
         key = (
@@ -334,7 +362,7 @@ def test_result_attr_consistency(
             ), f"Missing result attribute: {expected_attr_name!r} in {file_name!r}"
 
             assert allclose(
-                expected_attr_value, current.attrs[expected_attr_name], rtol=1e-5, print_fail=20
+                expected_attr_value, current.attrs[expected_attr_name], print_fail=20
             ), f"Result attr value mismatch: {expected_attr_name!r} in {file_name!r}"
 
 
diff --git a/tox.ini b/tox.ini
index 4a4dfeee2..71ac77a40 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,12 +7,18 @@ envlist = py{38}, pre-commit, docs, docs-notebooks, docs-links
 [pytest]
 ; Uncomment the following lines to deactivate pyglotaran all plugins
 ; env =
-;     DEACTIVATE_GTA_PLUGINS=1
+;    DEACTIVATE_GTA_PLUGINS=1
+; Uncomment "env =" and "COMPARE_RESULTS_LOCAL" and set it to a local folder
+; with results to use as a reference in lieu of the comparison-results branch
+; in the pyglotaran-examples git repository
+;    COMPARE_RESULTS_LOCAL=~/local_results/ ; On *nix
+;    COMPARE_RESULTS_LOCAL=%USERPROFILE%/local_results/ ; On Windows
 ; Uncomment to ignore deprecation warnings coming from pyglotaran
 ; (this helps to see the warnings from dependencies)
 ; filterwarnings =
 ;     ignore:.+glotaran:GlotaranApiDeprecationWarning
 
+
 [flake8]
 extend-ignore = E231, E203
 max-line-length = 99