Merge pull request #6536 from jon-turney/cross-testing-refactor

Refactor CI cross-testing

.travis.yml

@@ -33,10 +33,10 @@ matrix:
   # Test cross builds separately, they do not use the global compiler
   - os: linux
     compiler: gcc
-    env: RUN_TESTS_ARGS="--cross"
+    env: RUN_TESTS_ARGS="--cross ubuntu-armhf.txt --cross linux-mingw-w64-64bit.txt"
   - os: linux
     compiler: gcc
-    env: RUN_TESTS_ARGS="--cross" MESON_ARGS="--unity=on"
+    env: RUN_TESTS_ARGS="--cross ubuntu-armhf.txt --cross linux-mingw-w64-64bit.txt" MESON_ARGS="--unity=on"
 
 before_install:
   - python ./skip_ci.py --base-branch-env=TRAVIS_BRANCH --is-pull-env=TRAVIS_PULL_REQUEST

run_cross_test.py

@@ -15,43 +15,25 @@
 # limitations under the License.
 
 '''Runs the basic test suite through a cross compiler.
-Not part of the main test suite because of two reasons:
 
-1) setup of the cross build is platform specific
-2) it can be slow (e.g. when invoking test apps via wine)
+This is now just a wrapper around run_project_tests.py with specific arguments
+'''
 
-Eventually migrate to something fancier.'''
-
-import sys
-import os
-from pathlib import Path
 import argparse
-
-from run_project_tests import gather_tests, run_tests, StopException, setup_commands
-from run_project_tests import failing_logs
+import subprocess
+import sys
+from mesonbuild import mesonlib
 
 def runtests(cross_file, failfast):
-    commontests = [('common', gather_tests(Path('test cases', 'common')), False)]
-    try:
-        (passing_tests, failing_tests, skipped_tests) = \
-            run_tests(commontests, 'meson-cross-test-run', failfast, ['--cross-file', cross_file])
-    except StopException:
-        pass
-    print('\nTotal passed cross tests:', passing_tests)
-    print('Total failed cross tests:', failing_tests)
-    print('Total skipped cross tests:', skipped_tests)
-    if failing_tests > 0 and ('CI' in os.environ):
-        print('\nMesonlogs of failing tests\n')
-        for log in failing_logs:
-            print(log, '\n')
-    return failing_tests
+    tests = ['--only', 'common']
+    cmd = mesonlib.python_command + ['run_project_tests.py', '--backend', 'ninja'] + (['--failfast'] if failfast else []) + tests + ['--cross-file', cross_file]
+    return subprocess.call(cmd)
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--failfast', action='store_true')
     parser.add_argument('cross_file')
     options = parser.parse_args()
-    setup_commands('ninja')
     return runtests(options.cross_file, options.failfast)
 
 if __name__ == '__main__':

run_project_tests.py

@@ -871,13 +871,13 @@ def check_format():
                     continue
                 check_file(root / file)
 
-def check_meson_commands_work():
+def check_meson_commands_work(options):
     global backend, compile_commands, test_commands, install_commands
     testdir = PurePath('test cases', 'common', '1 trivial').as_posix()
     meson_commands = mesonlib.python_command + [get_meson_script()]
     with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir:
         print('Checking that configuring works...')
-        gen_cmd = meson_commands + [testdir, build_dir] + backend_flags
+        gen_cmd = meson_commands + [testdir, build_dir] + backend_flags + options.extra_args
         pc, o, e = Popen_safe(gen_cmd)
         if pc.returncode != 0:
             raise RuntimeError('Failed to configure {!r}:\n{}\n{}'.format(testdir, e, o))
@@ -897,11 +897,14 @@ def check_meson_commands_work():
                 raise RuntimeError('Failed to install {!r}:\n{}\n{}'.format(testdir, e, o))
 
 
-def detect_system_compiler():
+def detect_system_compiler(options):
     global system_compiler
 
     with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir:
-        env = environment.Environment(None, build_dir, get_fake_options('/'))
+        fake_opts = get_fake_options('/')
+        if options.cross_file:
+            fake_opts.cross_file = [options.cross_file]
+        env = environment.Environment(None, build_dir, fake_opts)
         print()
         for lang in sorted(compilers.all_languages):
             try:
@@ -962,16 +965,19 @@ def get_version(t: dict) -> str:
     parser.add_argument('--no-unittests', action='store_true',
                         help='Not used, only here to simplify run_tests.py')
     parser.add_argument('--only', help='name of test(s) to run', nargs='+', choices=ALL_TESTS)
+    parser.add_argument('--cross-file', action='store', help='File describing cross compilation environment.')
     options = parser.parse_args()
-    setup_commands(options.backend)
+    if options.cross_file:
+        options.extra_args += ['--cross-file', options.cross_file]
 
-    detect_system_compiler()
+    setup_commands(options.backend)
+    detect_system_compiler(options)
     print_tool_versions()
     script_dir = os.path.split(__file__)[0]
     if script_dir != '':
         os.chdir(script_dir)
     check_format()
-    check_meson_commands_work()
+    check_meson_commands_work(options)
     try:
         all_tests = detect_tests_to_run(options.only)
         (passing_tests, failing_tests, skipped_tests) = run_tests(all_tests, 'meson-test-run', options.failfast, options.extra_args)

run_tests.py

@@ -312,14 +312,15 @@ def print_system_info():
     print('Processor:', platform.processor())
     print('System:', platform.system())
     print('')
+    print(flush=True)
 
 def main():
     print_system_info()
     parser = argparse.ArgumentParser()
     parser.add_argument('--cov', action='store_true')
     parser.add_argument('--backend', default=None, dest='backend',
                         choices=backendlist)
-    parser.add_argument('--cross', default=False, dest='cross', action='store_true')
+    parser.add_argument('--cross', default=[], dest='cross', action='append')
     parser.add_argument('--failfast', action='store_true')
     parser.add_argument('--no-unittests', action='store_true', default=False)
     (options, _) = parser.parse_known_args()
@@ -352,8 +353,6 @@ def main():
     if 'APPVEYOR' in os.environ and os.environ['arch'] == 'x86':
         os.environ.pop('platform')
     # Run tests
-    print(mlog.bold('Running unittests.').get_text(mlog.colorize_console))
-    print(flush=True)
     # Can't pass arguments to unit tests, so set the backend to use in the environment
     env = os.environ.copy()
     env['MESON_UNIT_TEST_BACKEND'] = backend.name
@@ -377,8 +376,11 @@ def main():
                 return returncode
             if no_unittests:
                 print('Skipping all unit tests.')
+                print(flush=True)
                 returncode = 0
             else:
+                print(mlog.bold('Running unittests.').get_text(mlog.colorize_console))
+                print(flush=True)
                 cmd = mesonlib.python_command + ['run_unittests.py', '-v']
                 if options.failfast:
                     cmd += ['--failfast']
@@ -389,21 +391,15 @@ def main():
             returncode += subprocess.call(cmd, env=env)
         else:
             cross_test_args = mesonlib.python_command + ['run_cross_test.py']
-            print(mlog.bold('Running armhf cross tests.').get_text(mlog.colorize_console))
-            print(flush=True)
-            cmd = cross_test_args + ['cross/ubuntu-armhf.txt']
-            if options.failfast:
-                cmd += ['--failfast']
-            returncode += subprocess.call(cmd, env=env)
-            if options.failfast and returncode != 0:
-                return returncode
-            print(mlog.bold('Running mingw-w64 64-bit cross tests.')
-                  .get_text(mlog.colorize_console))
-            print(flush=True)
-            cmd = cross_test_args + ['cross/linux-mingw-w64-64bit.txt']
-            if options.failfast:
-                cmd += ['--failfast']
-            returncode += subprocess.call(cmd, env=env)
+            for cf in options.cross:
+                print(mlog.bold('Running {} cross tests.'.format(cf)).get_text(mlog.colorize_console))
+                print(flush=True)
+                cmd = cross_test_args + ['cross/' + cf]
+                if options.failfast:
+                    cmd += ['--failfast']
+                returncode += subprocess.call(cmd, env=env)
+                if options.failfast and returncode != 0:
+                    return returncode
     return returncode
 
 if __name__ == '__main__':