From 8d44536029fa0616c0957f8106fc4970beab0a57 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Mon, 4 Nov 2019 14:22:03 -0800 Subject: [PATCH] [mypyc] Support disabling global optimizations (#7869) Currently all of the global optimizations are based on the children list, so the strategy here is to disable tracking that in separate compilation mode. Consumers of that information then need to handle this case. --- mypyc/emit.py | 10 +++---- mypyc/genops.py | 15 ++++++++--- mypyc/ops.py | 39 +++++++++++++++++++++------- mypyc/options.py | 4 ++- mypyc/test-data/run-multimodule.test | 9 +++++++ mypyc/test/test_run.py | 2 +- mypyc/test/test_serialization.py | 2 +- 7 files changed, 59 insertions(+), 22 deletions(-) diff --git a/mypyc/emit.py b/mypyc/emit.py index cc233df8d879..5b7dbc8f3599 100644 --- a/mypyc/emit.py +++ b/mypyc/emit.py @@ -428,18 +428,18 @@ def emit_cast(self, src: str, dest: str, typ: RType, declare_dest: bool = False, if declare_dest: self.emit_line('PyObject *{};'.format(dest)) concrete = all_concrete_classes(typ.class_ir) - n_types = len(concrete) # If there are too many concrete subclasses or we can't find any - # (meaning the code ought to be dead), fall back to a normal typecheck. + # (meaning the code ought to be dead or we aren't doing global opts), + # fall back to a normal typecheck. # Otherwise check all the subclasses. - if n_types == 0 or n_types > FAST_ISINSTANCE_MAX_SUBCLASSES + 1: + if not concrete or len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1: check = '(PyObject_TypeCheck({}, {}))'.format( src, self.type_struct_name(typ.class_ir)) else: full_str = '(Py_TYPE({src}) == {targets[0]})' - for i in range(1, n_types): + for i in range(1, len(concrete)): full_str += ' || (Py_TYPE({src}) == {targets[%d]})' % i - if n_types > 1: + if len(concrete) > 1: full_str = '(%s)' % full_str check = full_str.format( src=src, targets=[self.type_struct_name(ir) for ir in concrete]) diff --git a/mypyc/genops.py b/mypyc/genops.py index 0a1c1b69a8e9..1ad4864c7822 100644 --- a/mypyc/genops.py +++ b/mypyc/genops.py @@ -120,6 +120,7 @@ def build_type_map(mapper: 'Mapper', modules: List[MypyFile], graph: Graph, types: Dict[Expression, Type], + options: CompilerOptions, errors: Errors) -> None: # Collect all classes defined in everything we are compiling classes = [] @@ -132,6 +133,9 @@ def build_type_map(mapper: 'Mapper', for module, cdef in classes: class_ir = ClassIR(cdef.name, module.fullname(), is_trait(cdef), is_abstract=cdef.info.is_abstract) + # If global optimizations are disabled, turn of tracking of class children + if not options.global_opts: + class_ir.children = None mapper.type_to_ir[cdef.info] = class_ir # Figure out which classes need to be compiled as non-extension classes. @@ -167,7 +171,7 @@ def build_ir(modules: List[MypyFile], options: CompilerOptions, errors: Errors) -> ModuleIRs: - build_type_map(mapper, modules, graph, types, errors) + build_type_map(mapper, modules, graph, types, options, errors) result = OrderedDict() # type: ModuleIRs @@ -654,7 +658,8 @@ def prepare_class_def(path: str, module_name: str, cdef: ClassDef, ir.base_mro = base_mro for base in bases: - base.children.append(ir) + if base.children is not None: + base.children.append(ir) if is_dataclass(cdef): ir.is_augmented = True @@ -1358,7 +1363,9 @@ def populate_non_ext_bases(self, cdef: ClassDef) -> Value: continue # Add the current class to the base classes list of concrete subclasses if cls in self.mapper.type_to_ir: - self.mapper.type_to_ir[cls].children.append(ir) + base_ir = self.mapper.type_to_ir[cls] + if base_ir.children is not None: + base_ir.children.append(ir) base = self.load_global_str(cls.name(), cdef.line) bases.append(base) @@ -3064,7 +3071,7 @@ def isinstance_native(self, obj: Value, class_ir: ClassIR, line: int) -> Value: its children, use even faster type comparison checks `type(obj) is typ`. """ concrete = all_concrete_classes(class_ir) - if len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1: + if concrete is None or len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1: return self.primitive_op(fast_isinstance_op, [obj, self.get_native_type(class_ir)], line) diff --git a/mypyc/ops.py b/mypyc/ops.py index bff2f73daa9b..f0444779f580 100644 --- a/mypyc/ops.py +++ b/mypyc/ops.py @@ -1792,7 +1792,8 @@ def __init__(self, name: str, module_name: str, is_trait: bool = False, self.base_mro = [self] # type: List[ClassIR] # Direct subclasses of this class (use subclasses() to also incude non-direct ones) - self.children = [] # type: List[ClassIR] + # None if separate compilation prevents this from working + self.children = [] # type: Optional[List[ClassIR]] @property def fullname(self) -> str: @@ -1837,15 +1838,19 @@ def has_method(self, name: str) -> bool: return True def is_method_final(self, name: str) -> bool: + subs = self.subclasses() + if subs is None: + # TODO: Look at the final attribute! + return False + if self.has_method(name): method_decl = self.method_decl(name) - for subc in self.subclasses(): + for subc in subs: if subc.method_decl(name) != method_decl: return False return True else: - return not any(subc.has_method(name) - for subc in self.subclasses()) + return not any(subc.has_method(name) for subc in subs) def has_attr(self, name: str) -> bool: try: @@ -1871,24 +1876,32 @@ def get_method(self, name: str) -> Optional[FuncIR]: res = self.get_method_and_class(name) return res[0] if res else None - def subclasses(self) -> Set['ClassIR']: + def subclasses(self) -> Optional[Set['ClassIR']]: """Return all subclassses of this class, both direct and indirect.""" + if self.children is None: + return None result = set(self.children) for child in self.children: if child.children: - result.update(child.subclasses()) + child_subs = child.subclasses() + if child_subs is None: + return None + result.update(child_subs) return result - def concrete_subclasses(self) -> List['ClassIR']: + def concrete_subclasses(self) -> Optional[List['ClassIR']]: """Return all concrete (i.e. non-trait and non-abstract) subclasses. Include both direct and indirect subclasses. Place classes with no children first. """ - concrete = {c for c in self.subclasses() if not (c.is_trait or c.is_abstract)} + subs = self.subclasses() + if subs is None: + return None + concrete = {c for c in subs if not (c.is_trait or c.is_abstract)} # We place classes with no children first because they are more likely # to appear in various isinstance() checks. We then sort leafs by name # to get stable order. - return sorted(concrete, key=lambda c: (len(c.children), c.name)) + return sorted(concrete, key=lambda c: (len(c.children or []), c.name)) def serialize(self) -> JsonDict: return { @@ -1932,6 +1945,9 @@ def serialize(self) -> JsonDict: 'traits': [cir.fullname for cir in self.traits], 'mro': [cir.fullname for cir in self.mro], 'base_mro': [cir.fullname for cir in self.base_mro], + 'children': [ + cir.fullname for cir in self.children + ] if self.children is not None else None, } @classmethod @@ -1977,6 +1993,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> 'ClassIR': ir.traits = [ctx.classes[s] for s in data['traits']] ir.mro = [ctx.classes[s] for s in data['mro']] ir.base_mro = [ctx.classes[s] for s in data['base_mro']] + ir.children = data['children'] and [ctx.classes[s] for s in data['children']] return ir @@ -2221,9 +2238,11 @@ def format_modules(modules: ModuleIRs) -> List[str]: return ops -def all_concrete_classes(class_ir: ClassIR) -> List[ClassIR]: +def all_concrete_classes(class_ir: ClassIR) -> Optional[List[ClassIR]]: """Return all concrete classes among the class itself and its subclasses.""" concrete = class_ir.concrete_subclasses() + if concrete is None: + return None if not (class_ir.is_abstract or class_ir.is_trait): concrete.append(class_ir) return concrete diff --git a/mypyc/options.py b/mypyc/options.py index 365a81d3f9ab..2f04eb6ef6b2 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -1,6 +1,8 @@ class CompilerOptions: def __init__(self, strip_asserts: bool = False, multi_file: bool = False, - verbose: bool = False) -> None: + verbose: bool = False, separate: bool = False) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file self.verbose = verbose + self.separate = separate + self.global_opts = not separate diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 8db3f17cde23..e6ca12ab3d37 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -124,6 +124,10 @@ class C: def f(self) -> int: return 2 + + def check(self) -> None: + assert isinstance(self, C) + [file driver.py] from native import f, D from other import C @@ -138,6 +142,11 @@ except TypeError: else: assert False +assert isinstance(D(10), C) + +c.check() +D(10).check() + [case testMultiModuleSpecialize] from other import A diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 24eb458c68ba..7f8ea2dc3670 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -172,7 +172,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None: options=options, alt_lib_path='.') errors = Errors() - compiler_options = CompilerOptions(multi_file=self.multi_file) + compiler_options = CompilerOptions(multi_file=self.multi_file, separate=self.separate) ir, cfiles = emitmodule.compile_modules_to_c( result, compiler_options=compiler_options, diff --git a/mypyc/test/test_serialization.py b/mypyc/test/test_serialization.py index fe7f7998cd9c..4a6e26b0ceb5 100644 --- a/mypyc/test/test_serialization.py +++ b/mypyc/test/test_serialization.py @@ -46,7 +46,7 @@ def assert_blobs_same(x: Any, y: Any, trail: Tuple[Any, ...]) -> None: assert type(x) is type(y), ("Type mismatch at {}".format(trail), type(x), type(y)) if isinstance(x, (FuncDecl, FuncIR, ClassIR)): - assert x.fullname == y.fullname + assert x.fullname == y.fullname, "Name mismatch at {}".format(trail) elif isinstance(x, OrderedDict): assert len(x.keys()) == len(y.keys()), "Keys mismatch at {}".format(trail) for (xk, xv), (yk, yv) in zip(x.items(), y.items()):