Allow for adding or removing test file path prefix (#495)

.github/workflows/publish.yml

@@ -111,6 +111,7 @@ jobs:
             -e "INPUT_XUNIT_FILES" \
             -e "INPUT_TRX_FILES" \
             -e "INPUT_TIME_UNIT" \
+            -e "INPUT_TEST_FILE_PREFIX" \
             -e "INPUT_REPORT_INDIVIDUAL_RUNS" \
             -e "INPUT_REPORT_SUITE_LOGS" \
             -e "INPUT_DEDUPLICATE_CLASSES_BY_FILE_NAME" \

README.md

@@ -208,6 +208,9 @@ and the changed files section of related pull requests:
 
 ![annotations example changed files](misc/github-pull-request-changes-annotation.png)
 
+***Note:** Annotations for test files are only supported when test file paths in test result files are relative to the repository root.
+Use option `test_file_prefix` to add a prefix to, or remove a prefix from these file paths. See [Configuration](#configuration) section for details.*
+
 ***Note:** Only the first failure of a test is shown. If you want to see all failures, set `report_individual_runs: "true"`.*
 
 ### GitHub Actions job summary
@@ -290,8 +293,9 @@ The list of most notable options:
 
 |Option|Default Value|Description|
 |:-----|:-----:|:----------|
-|`time_unit`|`seconds`|Time values in the XML files have this unit. Supports `seconds` and `milliseconds`.|
-|`job_summary`|`true`| Set to `true`, the results are published as part of the [job summary page](https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/) of the workflow run.|
+|`time_unit`|`seconds`|Time values in the test result files have this unit. Supports `seconds` and `milliseconds`.|
+|`test_file_prefix`|`none`|Paths in the test result files should be relative to the git repository for annotations to work best. This prefix is added to (if starting with "+"), or remove from (if starting with "-") test file paths. Examples: "+src/" or "-/opt/actions-runner".|
+|`job_summary`|`true`|Set to `true`, the results are published as part of the [job summary page](https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/) of the workflow run.|
 |`compare_to_earlier_commit`|`true`|Test results are compared to results of earlier commits to show changes:<br/>`false` - disable comparison, `true` - compare across commits.'|
 |`test_changes_limit`|`10`|Limits the number of removed or skipped tests reported on pull request comments. This report can be disabled with a value of `0`.|
 |`report_individual_runs`|`false`|Individual runs of the same test may see different failures. Reports all individual failures when set `true`, and the first failure only otherwise.|

action.yml

@@ -60,6 +60,9 @@ inputs:
     description: 'Time values in the test result files have this unit. Supports "seconds" and "milliseconds".'
     default: 'seconds'
     required: false
+  test_file_prefix:
+    description: 'Paths in the test result files should be relative to the git repository for annotations to work best. This prefix is added to (if starting with "+"), or remove from (if starting with "-") test file paths. Examples: "+src/" or "-/opt/actions-runner".'
+    required: false
   report_individual_runs:
     description: 'Individual runs of the same test may see different failures. Reports all individual failures when set "true" or the first only otherwise.'
     required: false
@@ -135,7 +138,6 @@ inputs:
     description: 'Prior to v2.6.0, the action used the "/search/issues" REST API to find pull requests related to a commit. If you need to restore that behaviour, set this to "true". Defaults to "false".'
     default: 'false'
     required: false
-
 outputs:
   json:
     description: "Test results as JSON"

composite/action.yml

@@ -60,6 +60,9 @@ inputs:
     description: 'Time values in the test result files have this unit. Supports "seconds" and "milliseconds".'
     default: 'seconds'
     required: false
+  test_file_prefix:
+    description: 'Paths in the test result files should be relative to the git repository for annotations to work best. This prefix is added to (if starting with "+"), or remove from (if starting with "-") test file paths. Examples: "+src/" or "-/opt/actions-runner".'
+    required: false
   report_individual_runs:
     description: 'Individual runs of the same test may see different failures. Reports all individual failures when set "true" or the first only otherwise.'
     required: false
@@ -135,6 +138,7 @@ inputs:
     description: 'Prior to v2.6.0, the action used the "/search/issues" REST API to find pull requests related to a commit. If you need to restore that behaviour, set this to "true". Defaults to "false".'
     default: 'false'
     required: false
+
 outputs:
   json:
     description: "Test results as JSON"
@@ -226,6 +230,7 @@ runs:
         XUNIT_FILES: ${{ inputs.xunit_files }}
         TRX_FILES: ${{ inputs.trx_files }}
         TIME_UNIT: ${{ inputs.time_unit }}
+        TEST_FILE_PREFIX: ${{ inputs.test_file_prefix }}
         REPORT_INDIVIDUAL_RUNS: ${{ inputs.report_individual_runs }}
         REPORT_SUITE_LOGS: ${{ inputs.report_suite_logs }}
         DEDUPLICATE_CLASSES_BY_FILE_NAME: ${{ inputs.deduplicate_classes_by_file_name }}

python/publish/junit.py

@@ -175,7 +175,24 @@ def parse(path: str) -> JUnitTree:
     return progress_safe_parse_xml_file(files, parse, progress)
 
 
-def process_junit_xml_elems(trees: Iterable[ParsedJUnitFile], time_factor: float = 1.0, add_suite_details: bool = False) -> ParsedUnitTestResults:
+def adjust_prefix(file: Optional[str], prefix: Optional[str]) -> Optional[str]:
+    if prefix is None or file is None:
+        return file
+
+    # prefix starts either with '+' or '-'
+    if prefix.startswith('+'):
+        # add prefix
+        return "".join([prefix[1:], file])
+
+    # remove prefix
+    return file[len(prefix)-1:] if file.startswith(prefix[1:]) else file
+
+
+def process_junit_xml_elems(trees: Iterable[ParsedJUnitFile],
+                            *,
+                            time_factor: float = 1.0,
+                            test_file_prefix: Optional[str] = None,
+                            add_suite_details: bool = False) -> ParsedUnitTestResults:
     def create_junitxml(filepath: str, tree: JUnitTree) -> JUnitXmlOrParseError:
         try:
             instance = JUnitXml.fromroot(tree.getroot())
@@ -265,7 +282,7 @@ def get_text(elem, tag):
     cases = [
         UnitTestCase(
             result_file=result_file,
-            test_file=case._elem.get('file'),
+            test_file=adjust_prefix(case._elem.get('file'), test_file_prefix),
             line=int_opt(case._elem.get('line')),
             class_name=case.classname,
             test_name=case.name,

python/publish/publisher.py

@@ -54,6 +54,7 @@ class Settings:
     nunit_files_glob: Optional[str]
     xunit_files_glob: Optional[str]
     trx_files_glob: Optional[str]
+    test_file_prefix: Optional[str]
     time_factor: float
     check_name: str
     comment_title: str

python/publish_test_results.py

@@ -208,6 +208,7 @@ def parse_files(settings: Settings, gha: GithubAction) -> ParsedUnitTestResultsW
     return process_junit_xml_elems(
         elems,
         time_factor=settings.time_factor,
+        test_file_prefix=settings.test_file_prefix,
         add_suite_details=settings.report_suite_out_logs or settings.report_suite_err_logs or settings.json_suite_details
     ).with_commit(settings.commit)
 
@@ -464,6 +465,7 @@ def get_settings(options: dict, gha: GithubAction) -> Settings:
         xunit_files_glob=get_var('XUNIT_FILES', options),
         trx_files_glob=get_var('TRX_FILES', options),
         time_factor=time_factor,
+        test_file_prefix=get_var('TEST_FILE_PREFIX', options) or None,
         check_name=check_name,
         comment_title=get_var('COMMENT_TITLE', options) or check_name,
         comment_mode=comment_mode,
@@ -481,12 +483,16 @@ def get_settings(options: dict, gha: GithubAction) -> Settings:
         seconds_between_github_reads=float(seconds_between_github_reads),
         seconds_between_github_writes=float(seconds_between_github_writes),
         secondary_rate_limit_wait_seconds=float(secondary_rate_limit_wait_seconds),
-        search_pull_requests=get_bool_var('SEARCH_PULL_REQUESTS', options, default=False)
+        search_pull_requests=get_bool_var('SEARCH_PULL_REQUESTS', options, default=False),
     )
 
     check_var(settings.token, 'GITHUB_TOKEN', 'GitHub token')
     check_var(settings.repo, 'GITHUB_REPOSITORY', 'GitHub repository')
     check_var(settings.commit, 'COMMIT, GITHUB_SHA or event file', 'Commit SHA')
+    check_var_condition(
+        settings.test_file_prefix is None or any([settings.test_file_prefix.startswith(sign) for sign in ['-', '+']]),
+        f"TEST_FILE_PREFIX is optional, but when given, it must start with '-' or '+': {settings.test_file_prefix}"
+    )
     check_var(settings.comment_mode, 'COMMENT_MODE', 'Comment mode', comment_modes)
     check_var(settings.pull_request_build, 'PULL_REQUEST_BUILD', 'Pull Request build', pull_request_build_modes)
     check_var(suite_logs_mode, 'REPORT_SUITE_LOGS', 'Report suite logs mode', available_report_suite_logs)

python/test/test_action_script.py

@@ -180,6 +180,7 @@ def get_settings(token='token',
                      xunit_files_glob='xunit-files',
                      trx_files_glob='trx-files',
                      time_factor=1.0,
+                     test_file_prefix=None,
                      check_name='check name',
                      comment_title='title',
                      comment_mode=comment_mode_always,
@@ -228,6 +229,7 @@ def get_settings(token='token',
             xunit_files_glob=xunit_files_glob,
             trx_files_glob=trx_files_glob,
             time_factor=time_factor,
+            test_file_prefix=test_file_prefix,
             check_name=check_name,
             comment_title=comment_title,
             comment_mode=comment_mode,
@@ -245,7 +247,7 @@ def get_settings(token='token',
             seconds_between_github_reads=seconds_between_github_reads,
             seconds_between_github_writes=seconds_between_github_writes,
             secondary_rate_limit_wait_seconds=secondary_rate_limit_wait_seconds,
-            search_pull_requests=search_pull_requests
+            search_pull_requests=search_pull_requests,
         )
 
     def test_get_settings(self):
@@ -353,6 +355,16 @@ def test_get_settings_time_unit(self):
         self.assertIn('TIME_UNIT minutes is not supported. It is optional, '
                       'but when given must be one of these values: seconds, milliseconds', re.exception.args)
 
+    def test_get_settings_test_file_prefix(self):
+        self.do_test_get_settings(TEST_FILE_PREFIX=None, expected=self.get_settings(test_file_prefix=None))
+        self.do_test_get_settings(TEST_FILE_PREFIX='', expected=self.get_settings(test_file_prefix=None))
+        self.do_test_get_settings(TEST_FILE_PREFIX='+src/', expected=self.get_settings(test_file_prefix='+src/'))
+        self.do_test_get_settings(TEST_FILE_PREFIX='-./', expected=self.get_settings(test_file_prefix='-./'))
+
+        with self.assertRaises(RuntimeError) as re:
+            self.do_test_get_settings(TEST_FILE_PREFIX='path/', expected=None)
+        self.assertIn("TEST_FILE_PREFIX is optional, but when given, it must start with '-' or '+': path/", re.exception.args)
+
     def test_get_settings_commit(self):
         event = {'pull_request': {'head': {'sha': 'sha2'}}}
         self.do_test_get_settings(INPUT_COMMIT='sha', GITHUB_EVENT_NAME='pull_request', event=event, GITHUB_SHA='default', expected=self.get_settings(commit='sha', event=event, event_name='pull_request', is_fork=True))

python/test/test_junit.py

@@ -19,8 +19,8 @@
 sys.path.append(str(pathlib.Path(__file__).resolve().parent))
 
 from publish import __version__, available_annotations, none_annotations
-from publish.junit import is_junit, parse_junit_xml_files, process_junit_xml_elems, get_results, get_result, get_content, \
-    get_message, Disabled, JUnitTreeOrParseError, ParseError
+from publish.junit import is_junit, parse_junit_xml_files, adjust_prefix, process_junit_xml_elems, get_results, \
+    get_result, get_content,  get_message, Disabled, JUnitTreeOrParseError, ParseError
 from publish.unittestresults import ParsedUnitTestResults, UnitTestCase
 from publish_test_results import get_test_results, get_stats, get_conclusion
 from publish.publisher import Publisher
@@ -97,6 +97,21 @@ def shorten_filename(cls, filename, prefix=None):
         else:
             return filename
 
+    def test_adjust_prefix(self):
+        self.assertEqual(adjust_prefix("file", "+"), "file")
+        self.assertEqual(adjust_prefix("file", "+."), ".file")
+        self.assertEqual(adjust_prefix("file", "+./"), "./file")
+        self.assertEqual(adjust_prefix("file", "+path/"), "path/file")
+
+        self.assertEqual(adjust_prefix("file", "-"), "file")
+        self.assertEqual(adjust_prefix(".file", "-."), "file")
+        self.assertEqual(adjust_prefix("./file", "-./"), "file")
+        self.assertEqual(adjust_prefix("path/file", "-path/"), "file")
+        self.assertEqual(adjust_prefix("file", "-"), "file")
+        self.assertEqual(adjust_prefix("file", "-."), "file")
+        self.assertEqual(adjust_prefix("file", "-./"), "file")
+        self.assertEqual(adjust_prefix("file", "-path/"), "file")
+
     def do_test_parse_and_process_files(self, filename: str):
         for locale in [None, 'en_US.UTF-8', 'de_DE.UTF-8']:
             with self.test.subTest(file=self.shorten_filename(filename), locale=locale):
@@ -299,7 +314,7 @@ def test_process_parse_junit_xml_files_with_time_factor(self):
         for time_factor in [1.0, 10.0, 60.0, 0.1, 0.001]:
             with self.subTest(time_factor=time_factor):
                 self.assertEqual(
-                    process_junit_xml_elems(parse_junit_xml_files([result_file], False, False), time_factor),
+                    process_junit_xml_elems(parse_junit_xml_files([result_file], False, False), time_factor=time_factor),
                     ParsedUnitTestResults(
                         files=1,
                         errors=[],
@@ -379,6 +394,32 @@ def test_process_parse_junit_xml_files_with_time_factor(self):
                         ]
                     ))
 
+    def test_process_parse_junit_xml_files_with_test_file_prefix(self):
+        result_file = str(test_files_path / 'pytest' / 'junit.fail.xml')
+        for prefix in ["+python/", "-test/", "-src"]:
+            with self.subTest(prefix=prefix):
+                test_file = adjust_prefix('test/test_spark.py', prefix)
+                self.assertEqual(
+                    process_junit_xml_elems(parse_junit_xml_files([result_file], False, False), test_file_prefix=prefix),
+                    ParsedUnitTestResults(
+                        files=1,
+                        errors=[],
+                        suites=1,
+                        suite_tests=5,
+                        suite_skipped=1,
+                        suite_failures=1,
+                        suite_errors=0,
+                        suite_time=2,
+                        suite_details=[],
+                        cases=[
+                            UnitTestCase(result_file=result_file, test_file=test_file, line=1412, class_name='test.test_spark.SparkTests', test_name='test_check_shape_compatibility', result='success', message=None, content=None, stdout=None, stderr=None, time=6.435),
+                            UnitTestCase(result_file=result_file, test_file=test_file, line=1641, class_name='test.test_spark.SparkTests', test_name='test_get_available_devices', result='skipped', message='get_available_devices only supported in Spark 3.0 and above', content='/horovod/test/test_spark.py:1642: get_available_devices only\n                supported in Spark 3.0 and above\n            ', stdout=None, stderr=None, time=0.001),
+                            UnitTestCase(result_file=result_file, test_file=test_file, line=1102, class_name='test.test_spark.SparkTests', test_name='test_get_col_info', result='success', message=None, content=None, stdout=None, stderr=None, time=6.417),
+                            UnitTestCase(result_file=result_file, test_file=test_file, line=819, class_name='test.test_spark.SparkTests', test_name='test_rsh_events', result='failure', message='self = <test_spark.SparkTests testMethod=test_rsh_events>      def test_rsh_events(self): >       self.do_test_rsh_events(3)  test_spark.py:821:  _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _  test_spark.py:836: in do_test_rsh_events     self.do_test_rsh(command, 143, events=events) test_spark.py:852: in do_test_rsh     self.assertEqual(expected_result, res) E   AssertionError: 143 != 0', content='self = <test_spark.SparkTests testMethod=test_rsh_events>\n\n                def test_rsh_events(self):\n                > self.do_test_rsh_events(3)\n\n                test_spark.py:821:\n                _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _\n                test_spark.py:836: in do_test_rsh_events\n                self.do_test_rsh(command, 143, events=events)\n                test_spark.py:852: in do_test_rsh\n                self.assertEqual(expected_result, res)\n                E AssertionError: 143 != 0\n            ', stdout=None, stderr=None, time=7.541),
+                            UnitTestCase(result_file=result_file, test_file=test_file, line=813, class_name='test.test_spark.SparkTests', test_name='test_rsh_with_non_zero_exit_code', result='success', message=None, content=None, stdout=None, stderr=None, time=1.514)
+                        ]
+                    ))
+
     def test_get_results(self):
         success = TestElement('success')
         skipped = TestElement('skipped')

python/test/test_publisher.py

@@ -120,6 +120,7 @@ def create_settings(actor='actor',
             xunit_files_glob=None,
             trx_files_glob=None,
             time_factor=1.0,
+            test_file_prefix=None,
             check_name='Check Name',
             comment_title='Comment Title',
             comment_mode=comment_mode,
@@ -137,7 +138,7 @@ def create_settings(actor='actor',
             seconds_between_github_reads=1.5,
             seconds_between_github_writes=2.5,
             secondary_rate_limit_wait_seconds=6.0,
-            search_pull_requests=search_pull_requests
+            search_pull_requests=search_pull_requests,
         )
 
     stats = UnitTestRunResults(