From 869f7b447d850f8b9607ea93e6f891954a781237 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 4 Aug 2021 21:27:16 -0700 Subject: [PATCH] Add an xfailed test for issue #9852 (#10932) As part of doing this I upgraded the pep561 tests to support incremental tests, and removed at least a little of the duplication between different incremental test suites. --- mypy/test/data.py | 1 - mypy/test/helpers.py | 24 +++++++++- mypy/test/testcheck.py | 21 ++------- mypy/test/testfinegrained.py | 18 ++------ mypy/test/testpep561.py | 65 ++++++++++++++-------------- mypyc/test-data/run-multimodule.test | 1 + mypyc/test/test_run.py | 18 +++----- test-data/unit/pep561.test | 20 +++++++++ 8 files changed, 88 insertions(+), 80 deletions(-) diff --git a/mypy/test/data.py b/mypy/test/data.py index 5de121f09290..a04c87f60a8d 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -335,7 +335,6 @@ def module_from_path(path: str) -> str: path = re.sub(r'\.pyi?$', '', path) # We can have a mix of Unix-style and Windows-style separators. parts = re.split(r'[/\\]', path) - assert parts[0] == test_temp_dir del parts[0] module = '.'.join(parts) module = re.sub(r'\.__init__$', '', module) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index ef9834ed4fa4..2d7d5e59430c 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -5,7 +5,7 @@ import shutil import contextlib -from typing import List, Iterable, Dict, Tuple, Callable, Any, Iterator +from typing import List, Iterable, Dict, Tuple, Callable, Any, Iterator, Union from mypy import defaults import mypy.api as api @@ -18,7 +18,9 @@ from mypy.main import process_options from mypy.options import Options -from mypy.test.data import DataDrivenTestCase, fix_cobertura_filename +from mypy.test.data import ( + DataDrivenTestCase, fix_cobertura_filename, UpdateFile, DeleteFile +) from mypy.test.config import test_temp_dir import mypy.version @@ -423,6 +425,24 @@ def copy_and_fudge_mtime(source_path: str, target_path: str) -> None: os.utime(target_path, times=(new_time, new_time)) +def perform_file_operations( + operations: List[Union[UpdateFile, DeleteFile]]) -> None: + for op in operations: + if isinstance(op, UpdateFile): + # Modify/create file + copy_and_fudge_mtime(op.source_path, op.target_path) + else: + # Delete file/directory + if os.path.isdir(op.path): + # Sanity check to avoid unexpected deletions + assert op.path.startswith('tmp') + shutil.rmtree(op.path) + else: + # Use retries to work around potential flakiness on Windows (AppVeyor). + path = op.path + retry_on_error(lambda: os.remove(path)) + + def check_test_output_files(testcase: DataDrivenTestCase, step: int, strip_prefix: str = '') -> None: diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 5e14a03d58cb..7b63f60addc1 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -2,7 +2,6 @@ import os import re -import shutil import sys from typing import Dict, List, Set, Tuple @@ -12,12 +11,12 @@ from mypy.modulefinder import BuildSource, SearchPaths, FindModuleCache from mypy.test.config import test_temp_dir, test_data_prefix from mypy.test.data import ( - DataDrivenTestCase, DataSuite, FileOperation, UpdateFile, module_from_path + DataDrivenTestCase, DataSuite, FileOperation, module_from_path ) from mypy.test.helpers import ( assert_string_arrays_equal, normalize_error_messages, assert_module_equivalence, - retry_on_error, update_testcase_output, parse_options, - copy_and_fudge_mtime, assert_target_equivalence, check_test_output_files + update_testcase_output, parse_options, + assert_target_equivalence, check_test_output_files, perform_file_operations, ) from mypy.errors import CompileError from mypy.semanal_main import core_modules @@ -156,19 +155,7 @@ def run_case_once(self, testcase: DataDrivenTestCase, break elif incremental_step > 1: # In runs 2+, copy *.[num] files to * files. - for op in operations: - if isinstance(op, UpdateFile): - # Modify/create file - copy_and_fudge_mtime(op.source_path, op.target_path) - else: - # Delete file/directory - path = op.path - if os.path.isdir(path): - # Sanity check to avoid unexpected deletions - assert path.startswith('tmp') - shutil.rmtree(path) - # Use retries to work around potential flakiness on Windows (AppVeyor). - retry_on_error(lambda: os.remove(path)) + perform_file_operations(operations) # Parse options after moving files (in case mypy.ini is being moved). options = parse_options(original_program_text, testcase, incremental_step) diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index 11c77afe218b..ed99b4aeae13 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -14,7 +14,6 @@ import os import re -import shutil from typing import List, Dict, Any, Tuple, Union, cast @@ -27,8 +26,8 @@ DataDrivenTestCase, DataSuite, UpdateFile, DeleteFile ) from mypy.test.helpers import ( - assert_string_arrays_equal, parse_options, copy_and_fudge_mtime, assert_module_equivalence, - assert_target_equivalence + assert_string_arrays_equal, parse_options, assert_module_equivalence, + assert_target_equivalence, perform_file_operations, ) from mypy.server.mergecheck import check_consistency from mypy.dmypy_util import DEFAULT_STATUS_FILE @@ -211,18 +210,7 @@ def perform_step(self, Return (mypy output, triggered targets). """ - for op in operations: - if isinstance(op, UpdateFile): - # Modify/create file - copy_and_fudge_mtime(op.source_path, op.target_path) - else: - # Delete file/directory - if os.path.isdir(op.path): - # Sanity check to avoid unexpected deletions - assert op.path.startswith('tmp') - shutil.rmtree(op.path) - else: - os.remove(op.path) + perform_file_operations(operations) sources = self.parse_sources(main_src, step, options) if step <= num_regular_incremental_steps: diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index ba5be621e066..e82cd2d1169d 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -13,7 +13,7 @@ from mypy.util import try_find_python2_interpreter from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.config import test_temp_dir -from mypy.test.helpers import assert_string_arrays_equal +from mypy.test.helpers import assert_string_arrays_equal, perform_file_operations # NOTE: options.use_builtins_fixtures should not be set in these @@ -38,6 +38,7 @@ class PEP561Suite(DataSuite): files = [ 'pep561.test', ] + base_path = '.' def run_case(self, test_case: DataDrivenTestCase) -> None: test_pep561(test_case) @@ -102,6 +103,7 @@ def test_pep561(testcase: DataDrivenTestCase) -> None: pytest.skip() else: python = sys.executable + assert python is not None, "Should be impossible" pkgs, pip_args = parse_pkgs(testcase.input[0]) mypy_args = parse_mypy_args(testcase.input[1]) @@ -118,34 +120,30 @@ def test_pep561(testcase: DataDrivenTestCase) -> None: for pkg in pkgs: install_package(pkg, python_executable, use_pip, editable) - if venv_dir is not None: - old_dir = os.getcwd() - os.chdir(venv_dir) - try: - cmd_line = list(mypy_args) - has_program = not ('-p' in cmd_line or '--package' in cmd_line) - if has_program: - program = testcase.name + '.py' - with open(program, 'w', encoding='utf-8') as f: - for s in testcase.input: - f.write('{}\n'.format(s)) - cmd_line.append(program) - cmd_line.extend(['--no-incremental', '--no-error-summary']) - if python_executable != sys.executable: - cmd_line.append('--python-executable={}'.format(python_executable)) - if testcase.files != []: - for name, content in testcase.files: - if 'mypy.ini' in name: - with open('mypy.ini', 'w') as m: - m.write(content) - elif 'pyproject.toml' in name: - with open('pyproject.toml', 'w') as m: - m.write(content) + cmd_line = list(mypy_args) + has_program = not ('-p' in cmd_line or '--package' in cmd_line) + if has_program: + program = testcase.name + '.py' + with open(program, 'w', encoding='utf-8') as f: + for s in testcase.input: + f.write('{}\n'.format(s)) + cmd_line.append(program) + + cmd_line.extend(['--no-error-summary']) + if python_executable != sys.executable: + cmd_line.append('--python-executable={}'.format(python_executable)) + + steps = testcase.find_steps() + if steps != [[]]: + steps = [[]] + steps # type: ignore[operator,assignment] + + for i, operations in enumerate(steps): + perform_file_operations(operations) + output = [] # Type check the module out, err, returncode = mypy.api.run(cmd_line) - if has_program: - os.remove(program) + # split lines, remove newlines, and remove directory of test case for line in (out + err).splitlines(): if line.startswith(test_temp_dir + os.sep): @@ -154,12 +152,15 @@ def test_pep561(testcase: DataDrivenTestCase) -> None: # Normalize paths so that the output is the same on Windows and Linux/macOS. line = line.replace(test_temp_dir + os.sep, test_temp_dir + '/') output.append(line.rstrip("\r\n")) - assert_string_arrays_equal([line for line in testcase.output], output, - 'Invalid output ({}, line {})'.format( - testcase.file, testcase.line)) - finally: - if venv_dir is not None: - os.chdir(old_dir) + iter_count = '' if i == 0 else ' on iteration {}'.format(i + 1) + expected = testcase.output if i == 0 else testcase.output2.get(i + 1, []) + + assert_string_arrays_equal(expected, output, + 'Invalid output ({}, line {}){}'.format( + testcase.file, testcase.line, iter_count)) + + if has_program: + os.remove(program) def parse_pkgs(comment: str) -> Tuple[List[str], List[str]]: diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 352611ce0cc4..1444254b7595 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -780,6 +780,7 @@ assert other_a.foo() == 10 [file other_a.py] def foo() -> int: return 10 +[file build/__native_other_a.c] [delete build/__native_other_a.c.2] diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 8ed1ac3c3dc6..731f53c6a3f5 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -12,11 +12,11 @@ from typing import Any, Iterator, List, cast from mypy import build -from mypy.test.data import DataDrivenTestCase, UpdateFile +from mypy.test.data import DataDrivenTestCase from mypy.test.config import test_temp_dir from mypy.errors import CompileError from mypy.options import Options -from mypy.test.helpers import copy_and_fudge_mtime, assert_module_equivalence +from mypy.test.helpers import assert_module_equivalence, perform_file_operations from mypyc.codegen import emitmodule from mypyc.options import CompilerOptions @@ -135,7 +135,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: self.run_case_inner(testcase) def run_case_inner(self, testcase: DataDrivenTestCase) -> None: - os.mkdir(WORKDIR) + if not os.path.isdir(WORKDIR): # (one test puts something in build...) + os.mkdir(WORKDIR) text = '\n'.join(testcase.input) @@ -161,16 +162,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None: step += 1 with chdir_manager('..'): - for op in operations: - if isinstance(op, UpdateFile): - # Modify/create file - copy_and_fudge_mtime(op.source_path, op.target_path) - else: - # Delete file - try: - os.remove(op.path) - except FileNotFoundError: - pass + perform_file_operations(operations) self.run_case_step(testcase, step) def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> None: diff --git a/test-data/unit/pep561.test b/test-data/unit/pep561.test index a4f96ede9f41..bd3fa024c5d8 100644 --- a/test-data/unit/pep561.test +++ b/test-data/unit/pep561.test @@ -175,3 +175,23 @@ testTypedPkgNamespaceRegImport.py:4: error: Cannot find implementation or librar testTypedPkgNamespaceRegImport.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports testTypedPkgNamespaceRegImport.py:10: error: Argument 1 has incompatible type "bool"; expected "str" testTypedPkgNamespaceRegImport.py:11: error: Argument 1 has incompatible type "int"; expected "bool" + +-- This is really testing the test framework to make sure incremental works +[case testPep561TestIncremental] +# pkgs: typedpkg +import a +[file a.py] +[file a.py.2] +1 + 'no' +[out] +[out2] +a.py:1: error: Unsupported operand types for + ("int" and "str") + +-- Test for issue #9852, among others +[case testTypedPkgNamespaceRegFromImportTwice-xfail] +# pkgs: typedpkg, typedpkg_ns +from typedpkg_ns import ns +-- dummy should trigger a second iteration +[file dummy.py.2] +[out] +[out2]