diff --git a/tools/wptrunner/wptrunner/executors/base.py b/tools/wptrunner/wptrunner/executors/base.py index 7119c0b019dbcc..763b6fcb19fb32 100644 --- a/tools/wptrunner/wptrunner/executors/base.py +++ b/tools/wptrunner/wptrunner/executors/base.py @@ -90,9 +90,9 @@ def __call__(self, test, result, extra=None): """Convert a JSON result into a (TestResult, [SubtestResult]) tuple""" result_url, status, message, stack, subtest_results = result assert result_url == test.url, (f"Got results from {result_url}, expected {test.url}") - harness_result = test.result_cls(self.harness_codes[status], message, extra=extra, stack=stack) + harness_result = test.make_result(self.harness_codes[status], message, extra=extra, stack=stack) return (harness_result, - [test.subtest_result_cls(st_name, self.test_codes[st_status], st_message, st_stack) + [test.make_subtest_result(st_name, self.test_codes[st_status], st_message, st_stack) for st_name, st_status, st_message, st_stack in subtest_results]) @@ -152,7 +152,7 @@ def get_pages(ranges_value, total_pages): def reftest_result_converter(self, test, result): extra = result.get("extra", {}) _ensure_hash_in_reftest_screenshots(extra) - return (test.result_cls( + return (test.make_result( result["status"], result["message"], extra=extra, @@ -165,14 +165,14 @@ def pytest_result_converter(self, test, data): if subtest_data is None: subtest_data = [] - harness_result = test.result_cls(*harness_data) - subtest_results = [test.subtest_result_cls(*item) for item in subtest_data] + harness_result = test.make_result(*harness_data) + subtest_results = [test.make_subtest_result(*item) for item in subtest_data] return (harness_result, subtest_results) def crashtest_result_converter(self, test, result): - return test.result_cls(**result), [] + return test.make_result(**result), [] class ExecutorException(Exception): @@ -352,7 +352,7 @@ def result_from_exception(self, test, e, exception_string): if message: message += "\n" message += exception_string - return test.result_cls(status, message), [] + return test.make_result(status, message), [] def wait(self): return self.protocol.base.wait() @@ -648,7 +648,7 @@ def do_test(self, test): if success: return self.convert_result(test, data) - return (test.result_cls(*data), []) + return (test.make_result(*data), []) def do_wdspec(self, path, timeout): session_config = {"host": self.browser.host, diff --git a/tools/wptrunner/wptrunner/executors/executorchrome.py b/tools/wptrunner/wptrunner/executors/executorchrome.py index 7a54eb097f9afb..f5641471562b65 100644 --- a/tools/wptrunner/wptrunner/executors/executorchrome.py +++ b/tools/wptrunner/wptrunner/executors/executorchrome.py @@ -46,7 +46,7 @@ def convert_from_crashtest_result(test, result): status = result["status"] if status == "PASS": status = "OK" - harness_result = test.result_cls(status, result["message"]) + harness_result = test.make_result(status, result["message"]) # Don't report subtests. return harness_result, [] # `crashtest` statuses are a subset of `(print-)reftest` diff --git a/tools/wptrunner/wptrunner/executors/executorcontentshell.py b/tools/wptrunner/wptrunner/executors/executorcontentshell.py index 5c4068c7acf5d8..82a6aebcdb17dd 100644 --- a/tools/wptrunner/wptrunner/executors/executorcontentshell.py +++ b/tools/wptrunner/wptrunner/executors/executorcontentshell.py @@ -213,14 +213,14 @@ def _convert_exception(test, exception, errors): """Converts our TimeoutError and CrashError exceptions into test results. """ if isinstance(exception, TimeoutError): - return (test.result_cls("EXTERNAL-TIMEOUT", errors), []) + return (test.make_result("EXTERNAL-TIMEOUT", errors), []) if isinstance(exception, CrashError): - return (test.result_cls("CRASH", errors), []) + return (test.make_result("CRASH", errors), []) if isinstance(exception, LeakError): # TODO: the internal error is to force a restart, but it doesn't correctly # describe what the issue is. Need to find a way to return a "FAIL", # and restart the content_shell after the test run. - return (test.result_cls("INTERNAL-ERROR", errors), []) + return (test.make_result("INTERNAL-ERROR", errors), []) raise exception @@ -312,7 +312,7 @@ def do_test(self, test): timeout_for_test(self, test)) errors = self.protocol.content_shell_errors.read_errors() if not text: - return (test.result_cls("ERROR", errors), []) + return (test.make_result("ERROR", errors), []) result_url, status, message, stack, subtest_results = json.loads(text) if result_url != test.url: diff --git a/tools/wptrunner/wptrunner/executors/executormarionette.py b/tools/wptrunner/wptrunner/executors/executormarionette.py index 21be5e852bf62a..a5bf61d4053a0d 100644 --- a/tools/wptrunner/wptrunner/executors/executormarionette.py +++ b/tools/wptrunner/wptrunner/executors/executormarionette.py @@ -979,7 +979,7 @@ def do_test(self, test): if success: return self.convert_result(test, data, extra=extra) - return (test.result_cls(extra=extra, *data), []) + return (test.make_result(extra=extra, *data), []) def do_testharness(self, protocol, url, timeout): parent_window = protocol.testharness.close_old_windows(self.last_environment["protocol"]) @@ -1277,7 +1277,7 @@ def do_test(self, test): if success: return self.convert_result(test, data) - return (test.result_cls(extra=extra, *data), []) + return (test.make_result(extra=extra, *data), []) def do_crashtest(self, protocol, url, timeout): if self.protocol.coverage.is_enabled: diff --git a/tools/wptrunner/wptrunner/executors/executorselenium.py b/tools/wptrunner/wptrunner/executors/executorselenium.py index 128e64804646f9..cf5ac2a22f02c5 100644 --- a/tools/wptrunner/wptrunner/executors/executorselenium.py +++ b/tools/wptrunner/wptrunner/executors/executorselenium.py @@ -369,7 +369,7 @@ def do_test(self, test): if success: return self.convert_result(test, data) - return (test.result_cls(*data), []) + return (test.make_result(*data), []) def do_testharness(self, protocol, url, timeout): format_map = {"url": strip_server(url)} diff --git a/tools/wptrunner/wptrunner/executors/executorservo.py b/tools/wptrunner/wptrunner/executors/executorservo.py index dd1ea1a3b36951..9c938b6e75e27c 100644 --- a/tools/wptrunner/wptrunner/executors/executorservo.py +++ b/tools/wptrunner/wptrunner/executors/executorservo.py @@ -136,10 +136,10 @@ def do_test(self, test): result = self.convert_result(test, self.result_data) else: self.proc.wait() - result = (test.result_cls("CRASH", None), []) + result = (test.make_result("CRASH", None), []) proc_is_running = False else: - result = (test.result_cls("TIMEOUT", None), []) + result = (test.make_result("TIMEOUT", None), []) if proc_is_running: if self.pause_after_test: @@ -312,7 +312,7 @@ def do_test(self, test): if success: return self.convert_result(test, data) - return (test.result_cls(*data), []) + return (test.make_result(*data), []) def do_crashtest(self, protocol, url, timeout): self.command = self.build_servo_command(self.test, extra_args=["-x"]) diff --git a/tools/wptrunner/wptrunner/executors/executorservodriver.py b/tools/wptrunner/wptrunner/executors/executorservodriver.py index 0a939c5251e218..5d7d55f30b551f 100644 --- a/tools/wptrunner/wptrunner/executors/executorservodriver.py +++ b/tools/wptrunner/wptrunner/executors/executorservodriver.py @@ -201,7 +201,7 @@ def do_test(self, test): if success: return self.convert_result(test, data) - return (test.result_cls(*data), []) + return (test.make_result(*data), []) def do_testharness(self, session, url, timeout): session.url = url @@ -257,15 +257,15 @@ def do_test(self, test): result = self.implementation.run_test(test) return self.convert_result(test, result) except OSError: - return test.result_cls("CRASH", None), [] + return test.make_result("CRASH", None), [] except TimeoutError: - return test.result_cls("TIMEOUT", None), [] + return test.make_result("TIMEOUT", None), [] except Exception as e: message = getattr(e, "message", "") if message: message += "\n" message += traceback.format_exc() - return test.result_cls("INTERNAL-ERROR", message), [] + return test.make_result("INTERNAL-ERROR", message), [] def screenshot(self, test, viewport_size, dpi, page_ranges): # https://github.com/web-platform-tests/wpt/issues/7135 diff --git a/tools/wptrunner/wptrunner/executors/executorwebdriver.py b/tools/wptrunner/wptrunner/executors/executorwebdriver.py index 05a26bc162f385..b49b9e2b574a7e 100644 --- a/tools/wptrunner/wptrunner/executors/executorwebdriver.py +++ b/tools/wptrunner/wptrunner/executors/executorwebdriver.py @@ -586,7 +586,7 @@ def do_test(self, test): if success: return self.convert_result(test, data) - return (test.result_cls(*data), []) + return (test.make_result(*data), []) def do_testharness(self, protocol, url, timeout): # The previous test may not have closed its old windows (if something @@ -752,7 +752,7 @@ def do_test(self, test): if success: return self.convert_result(test, data) - return (test.result_cls(*data), []) + return (test.make_result(*data), []) def do_crashtest(self, protocol, url, timeout): protocol.base.load(url) diff --git a/tools/wptrunner/wptrunner/executors/executorwktr.py b/tools/wptrunner/wptrunner/executors/executorwktr.py index b240bca556a454..3c02fa799e083b 100644 --- a/tools/wptrunner/wptrunner/executors/executorwktr.py +++ b/tools/wptrunner/wptrunner/executors/executorwktr.py @@ -172,9 +172,9 @@ def _convert_exception(test, exception, errors): """Converts our TimeoutError and CrashError exceptions into test results. """ if isinstance(exception, TimeoutError): - return (test.result_cls("EXTERNAL-TIMEOUT", errors), []) + return (test.make_result("EXTERNAL-TIMEOUT", errors), []) if isinstance(exception, CrashError): - return (test.result_cls("CRASH", errors), []) + return (test.make_result("CRASH", errors), []) raise exception @@ -245,7 +245,7 @@ def do_test(self, test): errors = self.protocol.wktr_errors.read_errors() if not text: - return (test.result_cls("ERROR", errors), []) + return (test.make_result("ERROR", errors), []) output = None output_prefix = "CONSOLE MESSAGE: WPTRUNNER OUTPUT:" @@ -255,10 +255,10 @@ def do_test(self, test): if output is None: output = line[len(output_prefix):] else: - return (test.result_cls("ERROR", "multiple wptrunner outputs"), []) + return (test.make_result("ERROR", "multiple wptrunner outputs"), []) if output is None: - return (test.result_cls("ERROR", "no wptrunner output"), []) + return (test.make_result("ERROR", "no wptrunner output"), []) return self.convert_result(test, json.loads(output)) except BaseException as exception: diff --git a/tools/wptrunner/wptrunner/testrunner.py b/tools/wptrunner/wptrunner/testrunner.py index 63f856d1c64335..28d06f88ee1f90 100644 --- a/tools/wptrunner/wptrunner/testrunner.py +++ b/tools/wptrunner/wptrunner/testrunner.py @@ -663,9 +663,9 @@ def _timeout(self): self.inject_message( "test_ended", test, - (test.result_cls("EXTERNAL-TIMEOUT", - "TestRunner hit external timeout " - "(this may indicate a hang)"), []), + (test.make_result("EXTERNAL-TIMEOUT", + "TestRunner hit external timeout " + "(this may indicate a hang)"), []), ) def test_ended(self, test, results): @@ -691,8 +691,8 @@ def test_ended(self, test, results): for result in test_results: if test.disabled(result.name): continue - expected = test.expected(result.name) - known_intermittent = test.known_intermittent(result.name) + expected = result.expected + known_intermittent = result.known_intermittent is_unexpected = expected != result.status and result.status not in known_intermittent is_expected_notrun = (expected == "NOTRUN" or "NOTRUN" in known_intermittent) @@ -728,8 +728,8 @@ def test_ended(self, test, results): stack=result.stack, subsuite=self.state.subsuite) - expected = test.expected() - known_intermittent = test.known_intermittent() + expected = file_result.expected + known_intermittent = file_result.known_intermittent status = file_result.status if self.browser.check_crash(test.id) and status != "CRASH": diff --git a/tools/wptrunner/wptrunner/wpttest.py b/tools/wptrunner/wptrunner/wpttest.py index 6df1c4cf2cd250..2e3fd974d4d43e 100644 --- a/tools/wptrunner/wptrunner/wpttest.py +++ b/tools/wptrunner/wptrunner/wpttest.py @@ -4,7 +4,7 @@ import sys from abc import ABC from collections import defaultdict -from typing import Any, ClassVar, Dict, Optional, Type +from typing import Any, ClassVar, Dict, Optional, Set, Type from urllib.parse import urljoin from .wptmanifest.parser import atoms @@ -14,6 +14,9 @@ class Result(ABC): + default_expected: ClassVar[str] + statuses: Set[str] + def __init__(self, status, message, @@ -25,7 +28,7 @@ def __init__(self, raise ValueError("Unrecognised status %s" % status) self.status = status self.message = message - self.expected = expected + self.expected = expected if expected is not None else self.default_expected self.known_intermittent = known_intermittent if known_intermittent is not None else [] self.extra = extra if extra is not None else {} self.stack = stack @@ -238,6 +241,25 @@ def __eq__(self, other): def __ne__(self, other): return not self.__eq__(other) + def make_result(self, + status, + message, + expected=None, + extra=None, + stack=None, + known_intermittent=None): + if expected is None: + expected = self.expected() + known_intermittent = self.known_intermittent() + return self.result_cls(status, message, expected, extra, stack, known_intermittent) + + def make_subtest_result(self, name, status, message, stack=None, expected=None, + known_intermittent=None): + if expected is None: + expected = self.expected(name) + known_intermittent = self.known_intermittent(name) + return self.subtest_result_cls(name, status, message, stack, expected, known_intermittent) + def update_metadata(self, metadata=None): if metadata is None: metadata = {}