[mypyc] Support disabling global optimizations (#7869)

Currently all of the global optimizations are based on the children
list, so the strategy here is to disable tracking that in separate
compilation mode. Consumers of that information then need to handle
this case.

mypyc/emit.py

@@ -428,18 +428,18 @@ def emit_cast(self, src: str, dest: str, typ: RType, declare_dest: bool = False,
             if declare_dest:
                 self.emit_line('PyObject *{};'.format(dest))
             concrete = all_concrete_classes(typ.class_ir)
-            n_types = len(concrete)
             # If there are too many concrete subclasses or we can't find any
-            # (meaning the code ought to be dead), fall back to a normal typecheck.
+            # (meaning the code ought to be dead or we aren't doing global opts),
+            # fall back to a normal typecheck.
             # Otherwise check all the subclasses.
-            if n_types == 0 or n_types > FAST_ISINSTANCE_MAX_SUBCLASSES + 1:
+            if not concrete or len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1:
                 check = '(PyObject_TypeCheck({}, {}))'.format(
                     src, self.type_struct_name(typ.class_ir))
             else:
                 full_str = '(Py_TYPE({src}) == {targets[0]})'
-                for i in range(1, n_types):
+                for i in range(1, len(concrete)):
                     full_str += ' || (Py_TYPE({src}) == {targets[%d]})' % i
-                if n_types > 1:
+                if len(concrete) > 1:
                     full_str = '(%s)' % full_str
                 check = full_str.format(
                     src=src, targets=[self.type_struct_name(ir) for ir in concrete])

mypyc/genops.py

@@ -120,6 +120,7 @@ def build_type_map(mapper: 'Mapper',
                    modules: List[MypyFile],
                    graph: Graph,
                    types: Dict[Expression, Type],
+                   options: CompilerOptions,
                    errors: Errors) -> None:
     # Collect all classes defined in everything we are compiling
     classes = []
@@ -132,6 +133,9 @@ def build_type_map(mapper: 'Mapper',
     for module, cdef in classes:
         class_ir = ClassIR(cdef.name, module.fullname(), is_trait(cdef),
                            is_abstract=cdef.info.is_abstract)
+        # If global optimizations are disabled, turn of tracking of class children
+        if not options.global_opts:
+            class_ir.children = None
         mapper.type_to_ir[cdef.info] = class_ir
 
     # Figure out which classes need to be compiled as non-extension classes.
@@ -167,7 +171,7 @@ def build_ir(modules: List[MypyFile],
              options: CompilerOptions,
              errors: Errors) -> ModuleIRs:
 
-    build_type_map(mapper, modules, graph, types, errors)
+    build_type_map(mapper, modules, graph, types, options, errors)
 
     result = OrderedDict()  # type: ModuleIRs
 
@@ -654,7 +658,8 @@ def prepare_class_def(path: str, module_name: str, cdef: ClassDef,
     ir.base_mro = base_mro
 
     for base in bases:
-        base.children.append(ir)
+        if base.children is not None:
+            base.children.append(ir)
 
     if is_dataclass(cdef):
         ir.is_augmented = True
@@ -1358,7 +1363,9 @@ def populate_non_ext_bases(self, cdef: ClassDef) -> Value:
                 continue
             # Add the current class to the base classes list of concrete subclasses
             if cls in self.mapper.type_to_ir:
-                self.mapper.type_to_ir[cls].children.append(ir)
+                base_ir = self.mapper.type_to_ir[cls]
+                if base_ir.children is not None:
+                    base_ir.children.append(ir)
 
             base = self.load_global_str(cls.name(), cdef.line)
             bases.append(base)
@@ -3064,7 +3071,7 @@ def isinstance_native(self, obj: Value, class_ir: ClassIR, line: int) -> Value:
         its children, use even faster type comparison checks `type(obj) is typ`.
         """
         concrete = all_concrete_classes(class_ir)
-        if len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1:
+        if concrete is None or len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1:
             return self.primitive_op(fast_isinstance_op,
                                      [obj, self.get_native_type(class_ir)],
                                      line)

mypyc/ops.py

@@ -1792,7 +1792,8 @@ def __init__(self, name: str, module_name: str, is_trait: bool = False,
         self.base_mro = [self]  # type: List[ClassIR]
 
         # Direct subclasses of this class (use subclasses() to also incude non-direct ones)
-        self.children = []  # type: List[ClassIR]
+        # None if separate compilation prevents this from working
+        self.children = []  # type: Optional[List[ClassIR]]
 
     @property
     def fullname(self) -> str:
@@ -1837,15 +1838,19 @@ def has_method(self, name: str) -> bool:
         return True
 
     def is_method_final(self, name: str) -> bool:
+        subs = self.subclasses()
+        if subs is None:
+            # TODO: Look at the final attribute!
+            return False
+
         if self.has_method(name):
             method_decl = self.method_decl(name)
-            for subc in self.subclasses():
+            for subc in subs:
                 if subc.method_decl(name) != method_decl:
                     return False
             return True
         else:
-            return not any(subc.has_method(name)
-                for subc in self.subclasses())
+            return not any(subc.has_method(name) for subc in subs)
 
     def has_attr(self, name: str) -> bool:
         try:
@@ -1871,24 +1876,32 @@ def get_method(self, name: str) -> Optional[FuncIR]:
         res = self.get_method_and_class(name)
         return res[0] if res else None
 
-    def subclasses(self) -> Set['ClassIR']:
+    def subclasses(self) -> Optional[Set['ClassIR']]:
         """Return all subclassses of this class, both direct and indirect."""
+        if self.children is None:
+            return None
         result = set(self.children)
         for child in self.children:
             if child.children:
-                result.update(child.subclasses())
+                child_subs = child.subclasses()
+                if child_subs is None:
+                    return None
+                result.update(child_subs)
         return result
 
-    def concrete_subclasses(self) -> List['ClassIR']:
+    def concrete_subclasses(self) -> Optional[List['ClassIR']]:
         """Return all concrete (i.e. non-trait and non-abstract) subclasses.
 
         Include both direct and indirect subclasses. Place classes with no children first.
         """
-        concrete = {c for c in self.subclasses() if not (c.is_trait or c.is_abstract)}
+        subs = self.subclasses()
+        if subs is None:
+            return None
+        concrete = {c for c in subs if not (c.is_trait or c.is_abstract)}
         # We place classes with no children first because they are more likely
         # to appear in various isinstance() checks. We then sort leafs by name
         # to get stable order.
-        return sorted(concrete, key=lambda c: (len(c.children), c.name))
+        return sorted(concrete, key=lambda c: (len(c.children or []), c.name))
 
     def serialize(self) -> JsonDict:
         return {
@@ -1932,6 +1945,9 @@ def serialize(self) -> JsonDict:
             'traits': [cir.fullname for cir in self.traits],
             'mro': [cir.fullname for cir in self.mro],
             'base_mro': [cir.fullname for cir in self.base_mro],
+            'children': [
+                cir.fullname for cir in self.children
+            ] if self.children is not None else None,
         }
 
     @classmethod
@@ -1977,6 +1993,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> 'ClassIR':
         ir.traits = [ctx.classes[s] for s in data['traits']]
         ir.mro = [ctx.classes[s] for s in data['mro']]
         ir.base_mro = [ctx.classes[s] for s in data['base_mro']]
+        ir.children = data['children'] and [ctx.classes[s] for s in data['children']]
 
         return ir
 
@@ -2221,9 +2238,11 @@ def format_modules(modules: ModuleIRs) -> List[str]:
     return ops
 
 
-def all_concrete_classes(class_ir: ClassIR) -> List[ClassIR]:
+def all_concrete_classes(class_ir: ClassIR) -> Optional[List[ClassIR]]:
     """Return all concrete classes among the class itself and its subclasses."""
     concrete = class_ir.concrete_subclasses()
+    if concrete is None:
+        return None
     if not (class_ir.is_abstract or class_ir.is_trait):
         concrete.append(class_ir)
     return concrete

mypyc/options.py

@@ -1,6 +1,8 @@
 class CompilerOptions:
     def __init__(self, strip_asserts: bool = False, multi_file: bool = False,
-                 verbose: bool = False) -> None:
+                 verbose: bool = False, separate: bool = False) -> None:
         self.strip_asserts = strip_asserts
         self.multi_file = multi_file
         self.verbose = verbose
+        self.separate = separate
+        self.global_opts = not separate

mypyc/test-data/run-multimodule.test

@@ -124,6 +124,10 @@ class C:
 
     def f(self) -> int:
         return 2
+
+    def check(self) -> None:
+        assert isinstance(self, C)
+
 [file driver.py]
 from native import f, D
 from other import C
@@ -138,6 +142,11 @@ except TypeError:
 else:
     assert False
 
+assert isinstance(D(10), C)
+
+c.check()
+D(10).check()
+
 [case testMultiModuleSpecialize]
 from other import A
 

mypyc/test/test_run.py

@@ -172,7 +172,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
                 options=options,
                 alt_lib_path='.')
             errors = Errors()
-            compiler_options = CompilerOptions(multi_file=self.multi_file)
+            compiler_options = CompilerOptions(multi_file=self.multi_file, separate=self.separate)
             ir, cfiles = emitmodule.compile_modules_to_c(
                 result,
                 compiler_options=compiler_options,

mypyc/test/test_serialization.py

@@ -46,7 +46,7 @@ def assert_blobs_same(x: Any, y: Any, trail: Tuple[Any, ...]) -> None:
 
     assert type(x) is type(y), ("Type mismatch at {}".format(trail), type(x), type(y))
     if isinstance(x, (FuncDecl, FuncIR, ClassIR)):
-        assert x.fullname == y.fullname
+        assert x.fullname == y.fullname, "Name mismatch at {}".format(trail)
     elif isinstance(x, OrderedDict):
         assert len(x.keys()) == len(y.keys()), "Keys mismatch at {}".format(trail)
         for (xk, xv), (yk, yv) in zip(x.items(), y.items()):