feat: catch exception at verification stage as ORT backend can return…

… None
ise-uiuc · Oct 24, 2022 · 6e96cb5 · 6e96cb5 · ganler · Oct 25, 2022
1 parent 5270f9b
commit 6e96cb5
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 2 deletions.
diff --git a/nnsmith/backends/factory.py b/nnsmith/backends/factory.py
@@ -226,6 +226,15 @@ def verify_results(
                 log=traceback.format_exc(),
                 version=self.version,
             )
+        except Exception:
+            return BugReport(
+                testcase=testcase,
+                system=self.system_name,
+                symptom=Symptom.EXCEPTION,
+                stage=Stage.VERIFICATION,
+                log=traceback.format_exc(),
+                version=self.version,
+            )
 
     def verify_testcase(
         self, testcase: TestCase, equal_nan=True

diff --git a/nnsmith/difftest.py b/nnsmith/difftest.py
@@ -19,9 +19,20 @@ def assert_allclose(
         raise KeyError(f"{actual_name}: {akeys} != {oracle_name}: {dkeys}")
 
     for key in akeys:
+        lhs = actual[key]
+        rhs = desired[key]
+
+        # check if lhs is np.ndarray
+        if not isinstance(lhs, np.ndarray):
+            raise TypeError(f"{actual_name}[{key}] is not np.ndarray but {type(lhs)}")
+
+        # check if rhs is np.ndarray
+        if not isinstance(rhs, np.ndarray):
+            raise TypeError(f"{oracle_name}[{key}] is not np.ndarray but {type(rhs)}")
+
         testing.assert_allclose(
-            actual[key],
-            desired[key],
+            lhs,
+            rhs,
             equal_nan=equal_nan,
             rtol=rtol,
             atol=atol,