Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for tests that should fail at runtime #418

Merged
merged 1 commit into from
Sep 8, 2023
Merged

Add support for tests that should fail at runtime #418

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion libcudacxx/.upstream-tests/test/lit.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ if 'PYLINT_IMPORT' in os.environ:
config.name = 'libcu++'

# suffixes: A list of file extensions to treat as test files.
config.suffixes = ['.pass.cpp', '.fail.cpp', '.sh.cpp']
config.suffixes = ['.pass.cpp', '.fail.cpp', '.runfail.cpp', '.sh.cpp']

# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)
Expand Down
19 changes: 15 additions & 4 deletions libcudacxx/.upstream-tests/utils/libcudacxx/test/format.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ class LibcxxTestFormat(object):
FOO.pass.cpp - Executable test which should compile, run, and exit with
code 0.
FOO.fail.cpp - Negative test case which is expected to fail compilation.
FOO.runfail.cpp - Negative test case which is expected to compile, run,
and exit with non-zero exit code.
FOO.sh.cpp - A test that uses LIT's ShTest format.
"""

Expand Down Expand Up @@ -88,6 +90,7 @@ def _execute(self, test, lit_config):
is_sh_test = name_root.endswith('.sh')
is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm')
is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm')
is_runfail_test = name.endswith('.runfail.cpp') or name.endswith('.runfail.mm')
is_objcxx_test = name.endswith('.mm')
is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \
name.endswith('.arc.fail.mm')
Expand Down Expand Up @@ -166,6 +169,10 @@ def _execute(self, test, lit_config):
elif is_pass_test:
return self._evaluate_pass_test(test, tmpBase, lit_config,
test_cxx, parsers)
elif is_runfail_test:
return self._evaluate_pass_test(test, tmpBase, lit_config,
test_cxx, parsers,
run_should_pass=False)
else:
# No other test type is supported
assert False
Expand All @@ -174,7 +181,7 @@ def _clean(self, exec_path): # pylint: disable=no-self-use
libcudacxx.util.cleanFile(exec_path)

def _evaluate_pass_test(self, test, tmpBase, lit_config,
test_cxx, parsers):
test_cxx, parsers, run_should_pass=True):
execDir = os.path.dirname(test.getExecPath())
source_path = test.getSourcePath()
exec_path = tmpBase + '.exe'
Expand Down Expand Up @@ -210,14 +217,18 @@ def _evaluate_pass_test(self, test, tmpBase, lit_config,
env)
report = "Compiled With: '%s'\n" % ' '.join(compile_cmd)
report += libcudacxx.util.makeReport(cmd, out, err, rc)
if rc == 0:
result_expected = (rc == 0) == run_should_pass
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One day I will rewrite this as a switch over the number of possibilities but that day is not today

if result_expected:
res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
return lit.Test.Result(res, report)
# Rarely devices are unavailable, so just restart the test to avoid false negatives.
elif rc != 0 and "cudaErrorDevicesUnavailable" in out and max_retry <= 5:
max_retry += 1
elif rc != 0 and retry_count + 1 >= max_retry:
report += "Compiled test failed unexpectedly!"
elif retry_count + 1 == max_retry:
if run_should_pass:
report += "Compiled test failed unexpectedly!"
else:
report += "Compiled test succeeded unexpectedly!"
return lit.Test.Result(lit.Test.FAIL, report)

assert False # Unreachable
Expand Down
22 changes: 16 additions & 6 deletions libcudacxx/libcxx/utils/libcxx/test/format.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ class LibcxxTestFormat(object):
FOO.pass.cpp - Executable test which should compile, run, and exit with
code 0.
FOO.fail.cpp - Negative test case which is expected to fail compilation.
FOO.runfail.cpp - Negative test case which is expected to compile, run,
and exit with non-zero exit code.
FOO.sh.cpp - A test that uses LIT's ShTest format.
"""

Expand Down Expand Up @@ -88,6 +90,7 @@ def _execute(self, test, lit_config):
is_sh_test = name_root.endswith('.sh')
is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm')
is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm')
is_runfail_test = name.endswith('.runfail.cpp') or name.endswith('.runfail.mm')
is_objcxx_test = name.endswith('.mm')
is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \
name.endswith('.arc.fail.mm')
Expand Down Expand Up @@ -163,15 +166,18 @@ def _execute(self, test, lit_config):
elif is_pass_test:
return self._evaluate_pass_test(test, tmpBase, lit_config,
test_cxx, parsers)
elif is_runfail_test:
return self._evaluate_pass_test(test, tmpBase, lit_config,
test_cxx, parsers,
run_should_pass=False)
else:
# No other test type is supported
assert False
assert False and "no other test" # No other test type is supported

def _clean(self, exec_path): # pylint: disable=no-self-use
libcxx.util.cleanFile(exec_path)

def _evaluate_pass_test(self, test, tmpBase, lit_config,
test_cxx, parsers):
test_cxx, parsers, run_should_pass=True):
execDir = os.path.dirname(test.getExecPath())
source_path = test.getSourcePath()
exec_path = tmpBase + '.exe'
Expand Down Expand Up @@ -207,11 +213,15 @@ def _evaluate_pass_test(self, test, tmpBase, lit_config,
env)
report = "Compiled With: '%s'\n" % ' '.join(compile_cmd)
report += libcxx.util.makeReport(cmd, out, err, rc)
if rc == 0:
result_expected = (rc == 0) == run_should_pass
if result_expected:
res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
return lit.Test.Result(res, report)
elif rc != 0 and retry_count + 1 == max_retry:
report += "Compiled test failed unexpectedly!"
elif retry_count + 1 == max_retry:
if run_should_pass:
report += "Compiled test failed unexpectedly!"
else:
report += "Compiled test succeeded unexpectedly!"
return lit.Test.Result(lit.Test.FAIL, report)

assert False # Unreachable
Expand Down