From 5d94c2ba71ce3b9d01f677d4705959c4edd146cb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 16 Mar 2020 23:00:18 +0000 Subject: [PATCH] [mypyc] Irbuild docstring and comment updates, and minor refactoring (#8542) Add and update docstrings and comments in mypyc.irbuild. Also try to make the style of docstrings and comments more consistent and cleaner looking. For example, generally make documentation lines shorter than the max line length, for improved readability. Group some related functions close to each other. --- mypyc/irbuild/callable_class.py | 94 ++++--- mypyc/irbuild/classdef.py | 58 +++-- mypyc/irbuild/context.py | 14 +- mypyc/irbuild/env_class.py | 76 ++++-- mypyc/irbuild/for_helpers.py | 25 +- mypyc/irbuild/function.py | 68 +++-- mypyc/irbuild/generator.py | 35 ++- mypyc/irbuild/ll_builder.py | 416 +++++++++++++++++-------------- mypyc/irbuild/main.py | 10 +- mypyc/irbuild/mapper.py | 5 + mypyc/irbuild/nonlocalcontrol.py | 45 +++- mypyc/irbuild/prebuildvisitor.py | 139 ++++++----- mypyc/irbuild/prepare.py | 13 + mypyc/irbuild/statement.py | 8 + mypyc/irbuild/util.py | 2 + mypyc/irbuild/vtable.py | 2 + 16 files changed, 614 insertions(+), 396 deletions(-) diff --git a/mypyc/irbuild/callable_class.py b/mypyc/irbuild/callable_class.py index c6d6de94232c..c0f845543f2c 100644 --- a/mypyc/irbuild/callable_class.py +++ b/mypyc/irbuild/callable_class.py @@ -1,7 +1,7 @@ """Generate a class that represents a nested function. -The class defines __call__ for calling the function and allows access to variables -defined in outer scopes. +The class defines __call__ for calling the function and allows access to +non-local variables defined in outer scopes. """ from typing import List @@ -20,21 +20,28 @@ def setup_callable_class(builder: IRBuilder) -> None: - """Generates a callable class representing a nested function or a function within a - non-extension class and sets up the 'self' variable for that class. + """Generate an (incomplete) callable class representing function. - This takes the most recently visited function and returns a ClassIR to represent that - function. Each callable class contains an environment attribute with points to another - ClassIR representing the environment class where some of its variables can be accessed. - Note that its '__call__' method is not yet implemented, and is implemented in the - add_call_to_callable_class function. + This can be a nested function or a function within a non-extension + class. Also set up the 'self' variable for that class. - Returns a newly constructed ClassIR representing the callable class for the nested - function. - """ + This takes the most recently visited function and returns a + ClassIR to represent that function. Each callable class contains + an environment attribute which points to another ClassIR + representing the environment class where some of its variables can + be accessed. - # Check to see that the name has not already been taken. If so, rename the class. We allow - # multiple uses of the same function name because this is valid in if-else blocks. Example: + Note that some methods, such as '__call__', are not yet + created here. Use additional functions, such as + add_call_to_callable_class(), to add them. + + Return a newly constructed ClassIR representing the callable + class for the nested function. + """ + # Check to see that the name has not already been taken. If so, + # rename the class. We allow multiple uses of the same function + # name because this is valid in if-else blocks. Example: + # # if True: # def foo(): ----> foo_obj() # return True @@ -48,12 +55,14 @@ def setup_callable_class(builder: IRBuilder) -> None: count += 1 builder.callable_class_names.add(name) - # Define the actual callable class ClassIR, and set its environment to point at the - # previously defined environment class. + # Define the actual callable class ClassIR, and set its + # environment to point at the previously defined environment + # class. callable_class_ir = ClassIR(name, builder.module_name, is_generated=True) - # The functools @wraps decorator attempts to call setattr on nested functions, so - # we create a dict for these nested functions. + # The functools @wraps decorator attempts to call setattr on + # nested functions, so we create a dict for these nested + # functions. # https://github.com/python/cpython/blob/3.7/Lib/functools.py#L58 if builder.fn_info.is_nested: callable_class_ir.has_dict = True @@ -68,8 +77,8 @@ def setup_callable_class(builder: IRBuilder) -> None: builder.fn_info.callable_class = ImplicitClass(callable_class_ir) builder.classes.append(callable_class_ir) - # Add a 'self' variable to the callable class' environment, and store that variable in a - # register to be accessed later. + # Add a 'self' variable to the environment of the callable class, + # and store that variable in a register to be accessed later. self_target = add_self_to_env(builder.environment, callable_class_ir) builder.fn_info.callable_class.self_reg = builder.read(self_target, builder.fn_info.fitem.line) @@ -79,13 +88,14 @@ def add_call_to_callable_class(builder: IRBuilder, sig: FuncSignature, env: Environment, fn_info: FuncInfo) -> FuncIR: - """Generates a '__call__' method for a callable class representing a nested function. + """Generate a '__call__' method for a callable class representing a nested function. - This takes the blocks, signature, and environment associated with a function definition and - uses those to build the '__call__' method of a given callable class, used to represent that - function. Note that a 'self' parameter is added to its list of arguments, as the nested - function becomes a class method. + This takes the blocks, signature, and environment associated with + a function definition and uses those to build the '__call__' + method of a given callable class, used to represent that + function. """ + # Since we create a method, we also add a 'self' parameter. sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args, sig.ret_type) call_fn_decl = FuncDecl('__call__', fn_info.callable_class.ir.name, builder.module_name, sig) call_fn_ir = FuncIR(call_fn_decl, blocks, env, @@ -95,7 +105,7 @@ def add_call_to_callable_class(builder: IRBuilder, def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None: - """Generates the '__get__' method for a callable class.""" + """Generate the '__get__' method for a callable class.""" line = fn_info.fitem.line builder.enter(fn_info) @@ -133,22 +143,30 @@ def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None: def instantiate_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> Value: - """ - Assigns a callable class to a register named after the given function definition. Note - that fn_info refers to the function being assigned, whereas builder.fn_info refers to the - function encapsulating the function being turned into a callable class. + """Create an instance of a callable class for a function. + + Calls to the function will actually call this instance. + + Note that fn_info refers to the function being assigned, whereas + builder.fn_info refers to the function encapsulating the function + being turned into a callable class. """ fitem = fn_info.fitem func_reg = builder.add(Call(fn_info.callable_class.ir.ctor, [], fitem.line)) - # Set the callable class' environment attribute to point at the environment class - # defined in the callable class' immediate outer scope. Note that there are three possible - # environment class registers we may use. If the encapsulating function is: - # - a generator function, then the callable class is instantiated from the generator class' - # __next__' function, and hence the generator class' environment register is used. - # - a nested function, then the callable class is instantiated from the current callable - # class' '__call__' function, and hence the callable class' environment register is used. - # - neither, then we use the environment register of the original function. + # Set the environment attribute of the callable class to point at + # the environment class defined in the callable class' immediate + # outer scope. Note that there are three possible environment + # class registers we may use. This depends on what the encapsulating + # (parent) function is: + # + # - A nested function: the callable class is instantiated + # from the current callable class' '__call__' function, and hence + # the callable class' environment register is used. + # - A generator function: the callable class is instantiated + # from the '__next__' method of the generator class, and hence the + # environment of the generator class is used. + # - Regular function: we use the environment of the original function. curr_env_reg = None if builder.fn_info.is_generator: curr_env_reg = builder.fn_info.generator_class.curr_env_reg diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index c5506dd8704a..89d7a16c5566 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -1,3 +1,5 @@ +"""Transform class definitions from the mypy AST form to IR.""" + from typing import List, Optional from mypy.nodes import ( @@ -29,6 +31,17 @@ def transform_class_def(builder: IRBuilder, cdef: ClassDef) -> None: + """Create IR for a class definition. + + This can generate both extension (native) and non-extension + classes. These are generated in very different ways. In the + latter case we construct a Python type object at runtime by doing + the equivalent of "type(name, bases, dict)" in IR. Extension + classes are defined via C structs that are generated later in + mypyc.codegen.emitclass. + + This is the main entry point to this module. + """ ir = builder.mapper.type_to_ir[cdef.info] # We do this check here because the base field of parent @@ -188,9 +201,9 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value: def populate_non_ext_bases(builder: IRBuilder, cdef: ClassDef) -> Value: - """ - Populate the base-class tuple passed to the metaclass constructor - for non-extension classes. + """Create base class tuple of a non-extension class. + + The tuple is passed to the metaclass constructor. """ ir = builder.mapper.type_to_ir[cdef.info] bases = [] @@ -222,11 +235,10 @@ def setup_non_ext_dict(builder: IRBuilder, cdef: ClassDef, metaclass: Value, bases: Value) -> Value: - """ - Initialize the class dictionary for a non-extension class. This class dictionary - is passed to the metaclass constructor. - """ + """Initialize the class dictionary for a non-extension class. + This class dictionary is passed to the metaclass constructor. + """ # Check if the metaclass defines a __prepare__ method, and if so, call it. has_prepare = builder.primitive_op(py_hasattr_op, [metaclass, @@ -252,14 +264,16 @@ def setup_non_ext_dict(builder: IRBuilder, return non_ext_dict -def add_non_ext_class_attr(builder: IRBuilder, non_ext: NonExtClassInfo, lvalue: NameExpr, - stmt: AssignmentStmt, cdef: ClassDef, +def add_non_ext_class_attr(builder: IRBuilder, + non_ext: NonExtClassInfo, + lvalue: NameExpr, + stmt: AssignmentStmt, + cdef: ClassDef, attr_to_cache: List[Lvalue]) -> None: - """ - Add a class attribute to __annotations__ of a non-extension class. If the - attribute is assigned to a value, it is also added to __dict__. - """ + """Add a class attribute to __annotations__ of a non-extension class. + If the attribute is initialized with a value, also add it to __dict__. + """ # We populate __annotations__ because dataclasses uses it to determine # which attributes to compute on. # TODO: Maybe generate more precise types for annotations @@ -284,7 +298,7 @@ def add_non_ext_class_attr(builder: IRBuilder, non_ext: NonExtClassInfo, lvalue: def generate_attr_defaults(builder: IRBuilder, cdef: ClassDef) -> None: - """Generate an initialization method for default attr values (from class vars)""" + """Generate an initialization method for default attr values (from class vars).""" cls = builder.mapper.type_to_ir[cdef.info] if cls.builtin_base: return @@ -347,6 +361,7 @@ def generate_attr_defaults(builder: IRBuilder, cdef: ClassDef) -> None: def create_ne_from_eq(builder: IRBuilder, cdef: ClassDef) -> None: + """Create a "__ne__" method from a "__eq__" method (if only latter exists).""" cls = builder.mapper.type_to_ir[cdef.info] if cls.has_method('__eq__') and not cls.has_method('__ne__'): f = gen_glue_ne_method(builder, cls, cdef.line) @@ -356,7 +371,7 @@ def create_ne_from_eq(builder: IRBuilder, cdef: ClassDef) -> None: def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> FuncIR: - """Generate a __ne__ method from a __eq__ method. """ + """Generate a "__ne__" method from a "__eq__" method. """ builder.enter() rt_args = (RuntimeArg("self", RInstance(cls)), RuntimeArg("rhs", object_rprimitive)) @@ -417,10 +432,13 @@ def load_non_ext_class(builder: IRBuilder, def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) -> Value: - """ - Given a decorated ClassDef and a register containing a non-extension representation of the - ClassDef created via the type constructor, applies the corresponding decorator functions - on that decorated ClassDef and returns a register containing the decorated ClassDef. + """Apply class decorators to create a decorated (non-extension) class object. + + Given a decorated ClassDef and a register containing a + non-extension representation of the ClassDef created via the type + constructor, applies the corresponding decorator functions on that + decorated ClassDef and returns a register containing the decorated + ClassDef. """ decorators = cdef.decorators dec_class = type_obj @@ -432,7 +450,7 @@ def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) -> def cache_class_attrs(builder: IRBuilder, attrs_to_cache: List[Lvalue], cdef: ClassDef) -> None: - """Add class attributes to be cached to the global cache""" + """Add class attributes to be cached to the global cache.""" typ = builder.load_native_type_object(cdef.fullname) for lval in attrs_to_cache: assert isinstance(lval, NameExpr) diff --git a/mypyc/irbuild/context.py b/mypyc/irbuild/context.py index b6b0ad3a7947..ac7521cf930c 100644 --- a/mypyc/irbuild/context.py +++ b/mypyc/irbuild/context.py @@ -1,3 +1,5 @@ +"""Helpers that store information about functions and the related classes.""" + from typing import List, Optional, Tuple from mypy.nodes import FuncItem @@ -10,6 +12,7 @@ class FuncInfo: """Contains information about functions as they are generated.""" + def __init__(self, fitem: FuncItem = INVALID_FUNC_DEF, name: str = '', @@ -87,9 +90,14 @@ def curr_env_reg(self) -> Value: class ImplicitClass: - """Contains information regarding classes that are generated as a result of nested functions or - generated functions, but not explicitly defined in the source code. + """Contains information regarding implicitly generated classes. + + Implicit classes are generated for nested functions and generator + functions. They are not explicitly defined in the source code. + + NOTE: This is both a concrete class and used as a base class. """ + def __init__(self, ir: ClassIR) -> None: # The ClassIR instance associated with this class. self.ir = ir @@ -131,6 +139,8 @@ def prev_env_reg(self, reg: Value) -> None: class GeneratorClass(ImplicitClass): + """Contains information about implicit generator function classes.""" + def __init__(self, ir: ClassIR) -> None: super().__init__(ir) # This register holds the label number that the '__next__' function should go to the next diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 6ecb29735564..87a72b4385e4 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -1,4 +1,19 @@ -"""Generate classes representing function environments (+ related operations).""" +"""Generate classes representing function environments (+ related operations). + +If we have a nested function that has non-local (free) variables, access to the +non-locals is via an instance of an environment class. Example: + + def f() -> int: + x = 0 # Make 'x' an attribute of an environment class instance + + def g() -> int: + # We have access to the environment class instance to + # allow accessing 'x' + return x + 2 + + x + 1 # Modify the attribute + return g() +""" from typing import Optional, Union @@ -13,16 +28,19 @@ def setup_env_class(builder: IRBuilder) -> ClassIR: - """Generates a class representing a function environment. - - Note that the variables in the function environment are not actually populated here. This - is because when the environment class is generated, the function environment has not yet - been visited. This behavior is allowed so that when the compiler visits nested functions, - it can use the returned ClassIR instance to figure out free variables it needs to access. - The remaining attributes of the environment class are populated when the environment - registers are loaded. - - Returns a ClassIR representing an environment for a function containing a nested function. + """Generate a class representing a function environment. + + Note that the variables in the function environment are not + actually populated here. This is because when the environment + class is generated, the function environment has not yet been + visited. This behavior is allowed so that when the compiler visits + nested functions, it can use the returned ClassIR instance to + figure out free variables it needs to access. The remaining + attributes of the environment class are populated when the + environment registers are loaded. + + Return a ClassIR representing an environment for a function + containing a nested function. """ env_class = ClassIR('{}_env'.format(builder.fn_info.namespaced_name()), builder.module_name, is_generated=True) @@ -38,8 +56,7 @@ def setup_env_class(builder: IRBuilder) -> ClassIR: def finalize_env_class(builder: IRBuilder) -> None: - """Generates, instantiates, and sets up the environment of an environment class.""" - + """Generate, instantiate, and set up the environment of an environment class.""" instantiate_env_class(builder) # Iterate through the function arguments and replace local definitions (using registers) @@ -52,7 +69,7 @@ def finalize_env_class(builder: IRBuilder) -> None: def instantiate_env_class(builder: IRBuilder) -> Value: - """Assigns an environment class to a register named after the given function definition.""" + """Assign an environment class to a register named after the given function definition.""" curr_env_reg = builder.add( Call(builder.fn_info.env_class.ctor, [], builder.fn_info.fitem.line) ) @@ -70,11 +87,12 @@ def instantiate_env_class(builder: IRBuilder) -> Value: def load_env_registers(builder: IRBuilder) -> None: - """Loads the registers for the current FuncItem being visited. + """Load the registers for the current FuncItem being visited. - Adds the arguments of the FuncItem to the environment. If the FuncItem is nested inside of - another function, then this also loads all of the outer environments of the FuncItem into - registers so that they can be used when accessing free variables. + Adds the arguments of the FuncItem to the environment. If the + FuncItem is nested inside of another function, then this also + loads all of the outer environments of the FuncItem into registers + so that they can be used when accessing free variables. """ add_args_to_env(builder, local=True) @@ -89,12 +107,14 @@ def load_env_registers(builder: IRBuilder) -> None: def load_outer_env(builder: IRBuilder, base: Value, outer_env: Environment) -> Value: - """Loads the environment class for a given base into a register. + """Load the environment class for a given base into a register. - Additionally, iterates through all of the SymbolNode and AssignmentTarget instances of the - environment at the given index's symtable, and adds those instances to the environment of - the current environment. This is done so that the current environment can access outer - environment variables without having to reload all of the environment registers. + Additionally, iterates through all of the SymbolNode and + AssignmentTarget instances of the environment at the given index's + symtable, and adds those instances to the environment of the + current environment. This is done so that the current environment + can access outer environment variables without having to reload + all of the environment registers. Returns the register where the environment class was loaded. """ @@ -150,10 +170,12 @@ def add_args_to_env(builder: IRBuilder, def setup_func_for_recursive_call(builder: IRBuilder, fdef: FuncDef, base: ImplicitClass) -> None: - """ - Adds the instance of the callable class representing the given FuncDef to a register in the - environment so that the function can be called recursively. Note that this needs to be done - only for nested functions. + """Enable calling a nested function (with a callable class) recursively. + + Adds the instance of the callable class representing the given + FuncDef to a register in the environment so that the function can + be called recursively. Note that this needs to be done only for + nested functions. """ # First, set the attribute of the environment class so that GetAttr can be called on it. prev_env = builder.fn_infos[-2].env_class diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 0a12f2e92ecc..9a56eb2ce4be 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -96,12 +96,13 @@ def comprehension_helper(builder: IRBuilder, line: int) -> None: """Helper function for list comprehensions. - "loop_params" is a list of (index, expr, [conditions]) tuples defining nested loops: - - "index" is the Lvalue indexing that loop; - - "expr" is the expression for the object to be iterated over; - - "conditions" is a list of conditions, evaluated in order with short-circuiting, - that must all be true for the loop body to be executed - "gen_inner_stmts" is a function to generate the IR for the body of the innermost loop + Args: + loop_params: a list of (index, expr, [conditions]) tuples defining nested loops: + - "index" is the Lvalue indexing that loop; + - "expr" is the expression for the object to be iterated over; + - "conditions" is a list of conditions, evaluated in order with short-circuiting, + that must all be true for the loop body to be executed + gen_inner_stmts: function to generate the IR for the body of the innermost loop """ def handle_loop(loop_params: List[Tuple[Lvalue, Expression, List[Expression]]]) -> None: """Generate IR for a loop. @@ -120,10 +121,11 @@ def loop_contents( ) -> None: """Generate the body of the loop. - "conds" is a list of conditions to be evaluated (in order, with short circuiting) - to gate the body of the loop. - "remaining_loop_params" is the parameters for any further nested loops; if it's empty - we'll instead evaluate the "gen_inner_stmts" function. + Args: + conds: a list of conditions to be evaluated (in order, with short circuiting) + to gate the body of the loop + remaining_loop_params: the parameters for any further nested loops; if it's empty + we'll instead evaluate the "gen_inner_stmts" function """ # Check conditions, in order, short circuiting them. for cond in conds: @@ -356,7 +358,8 @@ def unsafe_index( class ForSequence(ForGenerator): """Generate optimized IR for a for loop over a sequence. - Supports iterating in both forward and reverse.""" + Supports iterating in both forward and reverse. + """ def init(self, expr_reg: Value, target_type: RType, reverse: bool) -> None: builder = self.builder diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index 028a88ff8855..f791c7f8eafd 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -1,6 +1,13 @@ """Transform mypy AST functions to IR (and related things). -This also deals with generators, async functions and nested functions. +Normal functions are translated into a list of basic blocks +containing various IR ops (defined in mypyc.ir.ops). + +This also deals with generators, async functions and nested +functions. All of these are transformed into callable classes. These +have a custom __call__ method that implements the call, and state, such +as an environment containing non-local variables, is stored in the +instance of the callable class. """ from typing import Optional, List, Tuple, Union @@ -152,22 +159,22 @@ def gen_func_item(builder: IRBuilder, sig: FuncSignature, cdef: Optional[ClassDef] = None, ) -> Tuple[FuncIR, Optional[Value]]: - # TODO: do something about abstract methods. + """Generate and return the FuncIR for a given FuncDef. - """Generates and returns the FuncIR for a given FuncDef. + If the given FuncItem is a nested function, then we generate a + callable class representing the function and use that instead of + the actual function. if the given FuncItem contains a nested + function, then we generate an environment class so that inner + nested functions can access the environment of the given FuncDef. - If the given FuncItem is a nested function, then we generate a callable class representing - the function and use that instead of the actual function. if the given FuncItem contains a - nested function, then we generate an environment class so that inner nested functions can - access the environment of the given FuncDef. + Consider the following nested function: - Consider the following nested function. - def a() -> None: - def b() -> None: - def c() -> None: + def a() -> None: + def b() -> None: + def c() -> None: + return None return None return None - return None The classes generated would look something like the following. @@ -186,6 +193,8 @@ def c() -> None: +-------+ """ + # TODO: do something about abstract methods. + func_reg = None # type: Optional[Value] # We treat lambdas as always being nested because we always generate @@ -293,9 +302,12 @@ def gen_func_ir(builder: IRBuilder, env: Environment, fn_info: FuncInfo, cdef: Optional[ClassDef]) -> Tuple[FuncIR, Optional[Value]]: - """Generates the FuncIR for a function given the blocks, environment, and function info of - a particular function and returns it. If the function is nested, also returns the register - containing the instance of the corresponding callable class. + """Generate the FuncIR for a function. + + This takes the basic blocks, environment, and function info of a + particular function and returns the IR. If the function is nested, + also returns the register containing the instance of the + corresponding callable class. """ func_reg = None # type: Optional[Value] if fn_info.is_nested or fn_info.in_non_ext: @@ -445,7 +457,7 @@ def calculate_arg_defaults(builder: IRBuilder, def gen_func_ns(builder: IRBuilder) -> str: - """Generates a namespace for a nested function using its outer function names.""" + """Generate a namespace for a nested function using its outer function names.""" return '_'.join(info.name + ('' if not info.class_name else '_' + info.class_name) for info in builder.fn_infos if info.name and info.name != '') @@ -560,11 +572,12 @@ def else_body() -> None: def load_decorated_func(builder: IRBuilder, fdef: FuncDef, orig_func_reg: Value) -> Value: - """ - Given a decorated FuncDef and the register containing an instance of the callable class - representing that FuncDef, applies the corresponding decorator functions on that decorated - FuncDef and returns a register containing an instance of the callable class representing - the decorated function. + """Apply decorators to a function. + + Given a decorated FuncDef and an instance of the callable class + representing that FuncDef, apply the corresponding decorator + functions on that decorated FuncDef and return the decorated + function. """ if not is_decorated(builder, fdef): # If there are no decorators associated with the function, then just return the @@ -591,7 +604,8 @@ def gen_glue(builder: IRBuilder, sig: FuncSignature, target: FuncIR, ) -> FuncIR: """Generate glue methods that mediate between different method types in subclasses. - Works on both properties and methods. See gen_glue_methods below for more details. + Works on both properties and methods. See gen_glue_methods below + for more details. If do_py_ops is True, then the glue methods should use generic C API operations instead of direct calls, to enable generating @@ -676,7 +690,7 @@ def gen_glue_property(builder: IRBuilder, properties also require glue. However, this only requires the return type to change. Further, instead of a method call, an attribute get is performed. - If do_pygetattr is True, then get the attribute using the C + If do_pygetattr is True, then get the attribute using the Python C API instead of a native call. """ builder.enter() @@ -699,10 +713,10 @@ def gen_glue_property(builder: IRBuilder, def get_func_target(builder: IRBuilder, fdef: FuncDef) -> AssignmentTarget: - """ - Given a FuncDef, return the target associated the instance of its callable class. If the - function was not already defined somewhere, then define it and add it to the current - environment. + """Given a FuncDef, return the target for the instance of its callable class. + + If the function was not already defined somewhere, then define it + and add it to the current environment. """ if fdef.original_def: # Get the target associated with the previously defined FuncDef. diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index aaf54d51d70d..e09711b7e1f7 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -1,7 +1,11 @@ """Generate IR for generator functions. -A generator function is represented by a class that implements the generator protocol -and keeps track of the generator state, including local variables. +A generator function is represented by a class that implements the +generator protocol and keeps track of the generator state, including +local variables. + +The top-level logic for dealing with generator functions is in +mypyc.irbuild.function. """ from typing import List @@ -93,9 +97,11 @@ def populate_switch_for_generator_class(builder: IRBuilder) -> None: def add_raise_exception_blocks_to_generator_class(builder: IRBuilder, line: int) -> None: - """ - Generates blocks to check if error flags are set while calling the helper method for - generator functions, and raises an exception if those flags are set. + """Add error handling blocks to a generator class. + + Generates blocks to check if error flags are set while calling the + helper method for generator functions, and raises an exception if + those flags are set. """ cls = builder.fn_info.generator_class assert cls.exc_regs is not None @@ -225,8 +231,9 @@ def add_throw_to_generator_class(builder: IRBuilder, val = builder.environment.add_local_reg(Var('value'), object_rprimitive, True) tb = builder.environment.add_local_reg(Var('traceback'), object_rprimitive, True) - # Because the value and traceback arguments are optional and hence can be NULL if not - # passed in, we have to assign them Py_None if they are not passed in. + # Because the value and traceback arguments are optional and hence + # can be NULL if not passed in, we have to assign them Py_None if + # they are not passed in. none_reg = builder.none_object() builder.assign_if_null(val, lambda: none_reg, builder.fn_info.fitem.line) builder.assign_if_null(tb, lambda: none_reg, builder.fn_info.fitem.line) @@ -242,9 +249,9 @@ def add_throw_to_generator_class(builder: IRBuilder, builder.add(Return(result)) blocks, env, _, fn_info = builder.leave() - # Create the FuncSignature for the throw function. NOte that the value and traceback fields - # are optional, and are assigned to if they are not passed in inside the body of the throw - # function. + # Create the FuncSignature for the throw function. Note that the + # value and traceback fields are optional, and are assigned to if + # they are not passed in inside the body of the throw function. sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive), RuntimeArg('type', object_rprimitive), RuntimeArg('value', object_rprimitive, ARG_OPT), @@ -312,8 +319,9 @@ def setup_env_for_generator_class(builder: IRBuilder) -> None: cls.self_reg = builder.read(self_target, fitem.line) cls.curr_env_reg = load_outer_env(builder, cls.self_reg, builder.environment) - # Define a variable representing the label to go to the next time the '__next__' function - # of the generator is called, and add it as an attribute to the environment class. + # Define a variable representing the label to go to the next time + # the '__next__' function of the generator is called, and add it + # as an attribute to the environment class. cls.next_label_target = builder.add_var_to_env_class( Var(NEXT_LABEL_ATTR_NAME), int_rprimitive, @@ -321,7 +329,8 @@ def setup_env_for_generator_class(builder: IRBuilder) -> None: reassign=False ) - # Add arguments from the original generator function to the generator class' environment. + # Add arguments from the original generator function to the + # environment of the generator class. add_args_to_env(builder, local=False, base=cls, reassign=False) # Set the next label register for the generator class. diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index d319cb83f481..5e3d8d9ca324 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -64,7 +64,10 @@ def __init__( # Stack of except handler entry blocks self.error_handlers = [None] # type: List[Optional[BasicBlock]] + # Basic operations + def add(self, op: Op) -> Value: + """Add an op.""" assert not self.blocks[-1].terminated, "Can't add to finished block" self.blocks[-1].ops.append(op) @@ -73,10 +76,12 @@ def add(self, op: Op) -> Value: return op def goto(self, target: BasicBlock) -> None: + """Add goto to a basic block.""" if not self.blocks[-1].terminated: self.add(Goto(target)) def activate_block(self, block: BasicBlock) -> None: + """Add a basic block and make it the active one (target of adds).""" if self.blocks: assert self.blocks[-1].terminated @@ -84,6 +89,7 @@ def activate_block(self, block: BasicBlock) -> None: self.blocks.append(block) def goto_and_activate(self, block: BasicBlock) -> None: + """Add goto a block and make it the active block.""" self.goto(block) self.activate_block(block) @@ -93,30 +99,10 @@ def push_error_handler(self, handler: Optional[BasicBlock]) -> None: def pop_error_handler(self) -> Optional[BasicBlock]: return self.error_handlers.pop() - ## - - def get_native_type(self, cls: ClassIR) -> Value: - fullname = '%s.%s' % (cls.module_name, cls.name) - return self.load_native_type_object(fullname) - - def primitive_op(self, desc: OpDescription, args: List[Value], line: int) -> Value: - assert desc.result_type is not None - coerced = [] - for i, arg in enumerate(args): - formal_type = self.op_arg_type(desc, i) - arg = self.coerce(arg, formal_type, line) - coerced.append(arg) - target = self.add(PrimitiveOp(coerced, desc, line)) - return target - def alloc_temp(self, type: RType) -> Register: return self.environment.add_temp(type) - def op_arg_type(self, desc: OpDescription, n: int) -> RType: - if n >= len(desc.arg_types): - assert desc.is_var_arg - return desc.arg_types[-1] - return desc.arg_types[n] + # Type conversions def box(self, src: Value) -> Value: if src.type.is_unboxed: @@ -158,13 +144,10 @@ def coerce(self, src: Value, target_type: RType, line: int, force: bool = False) return tmp return src - def none(self) -> Value: - return self.add(PrimitiveOp([], none_op, line=-1)) - - def none_object(self) -> Value: - return self.add(PrimitiveOp([], none_object_op, line=-1)) + # Attribute access def get_attr(self, obj: Value, attr: str, result_type: RType, line: int) -> Value: + """Get a native or Python attribute of an object.""" if (isinstance(obj.type, RInstance) and obj.type.class_ir.is_ext_class and obj.type.class_ir.has_attr(attr)): return self.add(GetAttr(obj, attr, line)) @@ -179,71 +162,22 @@ def union_get_attr(self, attr: str, result_type: RType, line: int) -> Value: + """Get an attribute of an object with a union type.""" + def get_item_attr(value: Value) -> Value: return self.get_attr(value, attr, result_type, line) return self.decompose_union_helper(obj, rtype, result_type, get_item_attr, line) - def decompose_union_helper(self, - obj: Value, - rtype: RUnion, - result_type: RType, - process_item: Callable[[Value], Value], - line: int) -> Value: - """Generate isinstance() + specialized operations for union items. - - Say, for Union[A, B] generate ops resembling this (pseudocode): - - if isinstance(obj, A): - result = - else: - result = + def py_get_attr(self, obj: Value, attr: str, line: int) -> Value: + """Get a Python attribute (slow). - Args: - obj: value with a union type - rtype: the union type - result_type: result of the operation - process_item: callback to generate op for a single union item (arg is coerced - to union item type) - line: line number + Prefer get_attr() which generates optimized code for native classes. """ - # TODO: Optimize cases where a single operation can handle multiple union items - # (say a method is implemented in a common base class) - fast_items = [] - rest_items = [] - for item in rtype.items: - if isinstance(item, RInstance): - fast_items.append(item) - else: - # For everything but RInstance we fall back to C API - rest_items.append(item) - exit_block = BasicBlock() - result = self.alloc_temp(result_type) - for i, item in enumerate(fast_items): - more_types = i < len(fast_items) - 1 or rest_items - if more_types: - # We are not at the final item so we need one more branch - op = self.isinstance_native(obj, item.class_ir, line) - true_block, false_block = BasicBlock(), BasicBlock() - self.add_bool_branch(op, true_block, false_block) - self.activate_block(true_block) - coerced = self.coerce(obj, item, line) - temp = process_item(coerced) - temp2 = self.coerce(temp, result_type, line) - self.add(Assign(result, temp2)) - self.goto(exit_block) - if more_types: - self.activate_block(false_block) - if rest_items: - # For everything else we use generic operation. Use force=True to drop the - # union type. - coerced = self.coerce(obj, object_rprimitive, line, force=True) - temp = process_item(coerced) - temp2 = self.coerce(temp, result_type, line) - self.add(Assign(result, temp2)) - self.goto(exit_block) - self.activate_block(exit_block) - return result + key = self.load_static_unicode(attr) + return self.add(PrimitiveOp([obj, key], py_getattr_op, line)) + + # isinstance() checks def isinstance_helper(self, obj: Value, class_irs: List[ClassIR], line: int) -> Value: """Fast path for isinstance() that checks against a list of native classes.""" @@ -259,8 +193,9 @@ def other() -> Value: def isinstance_native(self, obj: Value, class_ir: ClassIR, line: int) -> Value: """Fast isinstance() check for a native class. - If there three or less concrete (non-trait) classes among the class and all - its children, use even faster type comparison checks `type(obj) is typ`. + If there are three or fewer concrete (non-trait) classes among the class + and all its children, use even faster type comparison checks `type(obj) + is typ`. """ concrete = all_concrete_classes(class_ir) if concrete is None or len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1: @@ -278,9 +213,7 @@ def other() -> Value: ret = self.shortcircuit_helper('or', bool_rprimitive, lambda: ret, other, line) return ret - def py_get_attr(self, obj: Value, attr: str, line: int) -> Value: - key = self.load_static_unicode(attr) - return self.add(PrimitiveOp([obj, key], py_getattr_op, line)) + # Calls def py_call(self, function: Value, @@ -288,7 +221,10 @@ def py_call(self, line: int, arg_kinds: Optional[List[int]] = None, arg_names: Optional[Sequence[Optional[str]]] = None) -> Value: - """Use py_call_op or py_call_with_kwargs_op for function call.""" + """Call a Python function (non-native and slow). + + Use py_call_op or py_call_with_kwargs_op for Python function call. + """ # If all arguments are positional, we can use py_call_op. if (arg_kinds is None) or all(kind == ARG_POS for kind in arg_kinds): return self.primitive_op(py_call_op, [function] + arg_values, line) @@ -338,6 +274,7 @@ def py_method_call(self, line: int, arg_kinds: Optional[List[int]], arg_names: Optional[Sequence[Optional[str]]]) -> Value: + """Call a Python method (non-native and slow).""" if (arg_kinds is None) or all(kind == ARG_POS for kind in arg_kinds): method_name_reg = self.load_static_unicode(method_name) return self.primitive_op(py_method_call_op, [obj, method_name_reg] + arg_values, line) @@ -345,10 +282,13 @@ def py_method_call(self, method = self.py_get_attr(obj, method_name, line) return self.py_call(method, arg_values, line, arg_kinds=arg_kinds, arg_names=arg_names) - def call(self, decl: FuncDecl, args: Sequence[Value], + def call(self, + decl: FuncDecl, + args: Sequence[Value], arg_kinds: List[int], arg_names: Sequence[Optional[str]], line: int) -> Value: + """Call a native function.""" # Normalize args to positionals. args = self.native_args_to_positional( args, arg_kinds, arg_names, decl.sig, line) @@ -397,39 +337,86 @@ def native_args_to_positional(self, return output_args - def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: - result = None # type: Union[Value, None] - initial_items = [] # type: List[Value] - for key, value in key_value_pairs: - if key is not None: - # key:value - if result is None: - initial_items.extend((key, value)) - continue + def gen_method_call(self, + base: Value, + name: str, + arg_values: List[Value], + result_type: Optional[RType], + line: int, + arg_kinds: Optional[List[int]] = None, + arg_names: Optional[List[Optional[str]]] = None) -> Value: + """Generate either a native or Python method call.""" + # If arg_kinds contains values other than arg_pos and arg_named, then fallback to + # Python method call. + if (arg_kinds is not None + and not all(kind in (ARG_POS, ARG_NAMED) for kind in arg_kinds)): + return self.py_method_call(base, name, arg_values, base.line, arg_kinds, arg_names) - self.translate_special_method_call( - result, - '__setitem__', - [key, value], - result_type=None, - line=line) - else: - # **value - if result is None: - result = self.primitive_op(new_dict_op, initial_items, line) + # If the base type is one of ours, do a MethodCall + if (isinstance(base.type, RInstance) and base.type.class_ir.is_ext_class + and not base.type.class_ir.builtin_base): + if base.type.class_ir.has_method(name): + decl = base.type.class_ir.method_decl(name) + if arg_kinds is None: + assert arg_names is None, "arg_kinds not present but arg_names is" + arg_kinds = [ARG_POS for _ in arg_values] + arg_names = [None for _ in arg_values] + else: + assert arg_names is not None, "arg_kinds present but arg_names is not" - self.primitive_op( - dict_update_in_display_op, - [result, value], - line=line - ) + # Normalize args to positionals. + assert decl.bound_sig + arg_values = self.native_args_to_positional( + arg_values, arg_kinds, arg_names, decl.bound_sig, line) + return self.add(MethodCall(base, name, arg_values, line)) + elif base.type.class_ir.has_attr(name): + function = self.add(GetAttr(base, name, line)) + return self.py_call(function, arg_values, line, + arg_kinds=arg_kinds, arg_names=arg_names) - if result is None: - result = self.primitive_op(new_dict_op, initial_items, line) + elif isinstance(base.type, RUnion): + return self.union_method_call(base, base.type, name, arg_values, result_type, line, + arg_kinds, arg_names) - return result + # Try to do a special-cased method call + if not arg_kinds or arg_kinds == [ARG_POS] * len(arg_values): + target = self.translate_special_method_call(base, name, arg_values, result_type, line) + if target: + return target + + # Fall back to Python method call + return self.py_method_call(base, name, arg_values, line, arg_kinds, arg_names) + + def union_method_call(self, + base: Value, + obj_type: RUnion, + name: str, + arg_values: List[Value], + return_rtype: Optional[RType], + line: int, + arg_kinds: Optional[List[int]], + arg_names: Optional[List[Optional[str]]]) -> Value: + """Generate a method call with a union type for the object.""" + # Union method call needs a return_rtype for the type of the output register. + # If we don't have one, use object_rprimitive. + return_rtype = return_rtype or object_rprimitive + + def call_union_item(value: Value) -> Value: + return self.gen_method_call(value, name, arg_values, return_rtype, line, + arg_kinds, arg_names) + + return self.decompose_union_helper(base, obj_type, return_rtype, call_union_item, line) + + # Loading various values + + def none(self) -> Value: + """Load unboxed None value (type: none_rprimitive).""" + return self.add(PrimitiveOp([], none_op, line=-1)) + + def none_object(self) -> Value: + """Load Python None value (type: object_rprimitive).""" + return self.add(PrimitiveOp([], none_object_op, line=-1)) - # Loading stuff def literal_static_name(self, value: Union[int, float, complex, str, bytes]) -> str: return self.mapper.literal_static_name(self.current_module, value) @@ -468,10 +455,27 @@ def load_static_unicode(self, value: str) -> Value: def load_module(self, name: str) -> Value: return self.add(LoadStatic(object_rprimitive, name, namespace=NAMESPACE_MODULE)) + def get_native_type(self, cls: ClassIR) -> Value: + """Load native type object.""" + fullname = '%s.%s' % (cls.module_name, cls.name) + return self.load_native_type_object(fullname) + def load_native_type_object(self, fullname: str) -> Value: module, name = fullname.rsplit('.', 1) return self.add(LoadStatic(object_rprimitive, name, module, NAMESPACE_TYPE)) + # Other primitive operations + + def primitive_op(self, desc: OpDescription, args: List[Value], line: int) -> Value: + assert desc.result_type is not None + coerced = [] + for i, arg in enumerate(args): + formal_type = self.op_arg_type(desc, i) + arg = self.coerce(arg, formal_type, line) + coerced.append(arg) + target = self.add(PrimitiveOp(coerced, desc, line)) + return target + def matching_primitive_op(self, candidates: List[OpDescription], args: List[Value], @@ -529,6 +533,38 @@ def unary_op(self, assert target, 'Unsupported unary operation: %s' % expr_op return target + def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: + result = None # type: Union[Value, None] + initial_items = [] # type: List[Value] + for key, value in key_value_pairs: + if key is not None: + # key:value + if result is None: + initial_items.extend((key, value)) + continue + + self.translate_special_method_call( + result, + '__setitem__', + [key, value], + result_type=None, + line=line) + else: + # **value + if result is None: + result = self.primitive_op(new_dict_op, initial_items, line) + + self.primitive_op( + dict_update_in_display_op, + [result, value], + line=line + ) + + if result is None: + result = self.primitive_op(new_dict_op, initial_items, line) + + return result + def builtin_call(self, args: List[Value], fn_op: str, @@ -607,6 +643,75 @@ def add_bool_branch(self, value: Value, true: BasicBlock, false: BasicBlock) -> value = self.primitive_op(bool_op, [value], value.line) self.add(Branch(value, true, false, Branch.BOOL_EXPR)) + # Internal helpers + + def decompose_union_helper(self, + obj: Value, + rtype: RUnion, + result_type: RType, + process_item: Callable[[Value], Value], + line: int) -> Value: + """Generate isinstance() + specialized operations for union items. + + Say, for Union[A, B] generate ops resembling this (pseudocode): + + if isinstance(obj, A): + result = + else: + result = + + Args: + obj: value with a union type + rtype: the union type + result_type: result of the operation + process_item: callback to generate op for a single union item (arg is coerced + to union item type) + line: line number + """ + # TODO: Optimize cases where a single operation can handle multiple union items + # (say a method is implemented in a common base class) + fast_items = [] + rest_items = [] + for item in rtype.items: + if isinstance(item, RInstance): + fast_items.append(item) + else: + # For everything but RInstance we fall back to C API + rest_items.append(item) + exit_block = BasicBlock() + result = self.alloc_temp(result_type) + for i, item in enumerate(fast_items): + more_types = i < len(fast_items) - 1 or rest_items + if more_types: + # We are not at the final item so we need one more branch + op = self.isinstance_native(obj, item.class_ir, line) + true_block, false_block = BasicBlock(), BasicBlock() + self.add_bool_branch(op, true_block, false_block) + self.activate_block(true_block) + coerced = self.coerce(obj, item, line) + temp = process_item(coerced) + temp2 = self.coerce(temp, result_type, line) + self.add(Assign(result, temp2)) + self.goto(exit_block) + if more_types: + self.activate_block(false_block) + if rest_items: + # For everything else we use generic operation. Use force=True to drop the + # union type. + coerced = self.coerce(obj, object_rprimitive, line, force=True) + temp = process_item(coerced) + temp2 = self.coerce(temp, result_type, line) + self.add(Assign(result, temp2)) + self.goto(exit_block) + self.activate_block(exit_block) + return result + + def op_arg_type(self, desc: OpDescription, n: int) -> RType: + if n >= len(desc.arg_types): + assert desc.is_var_arg + return desc.arg_types[-1] + return desc.arg_types[n] + def translate_special_method_call(self, base_reg: Value, name: str, @@ -629,6 +734,11 @@ def translate_eq_cmp(self, rreg: Value, expr_op: str, line: int) -> Optional[Value]: + """Add a equality comparison operation. + + Args: + expr_op: either '==' or '!=' + """ ltype = lreg.type rtype = rreg.type if not (isinstance(ltype, RInstance) and ltype == rtype): @@ -662,71 +772,3 @@ def translate_eq_cmp(self, ltype, line ) - - def gen_method_call(self, - base: Value, - name: str, - arg_values: List[Value], - result_type: Optional[RType], - line: int, - arg_kinds: Optional[List[int]] = None, - arg_names: Optional[List[Optional[str]]] = None) -> Value: - # If arg_kinds contains values other than arg_pos and arg_named, then fallback to - # Python method call. - if (arg_kinds is not None - and not all(kind in (ARG_POS, ARG_NAMED) for kind in arg_kinds)): - return self.py_method_call(base, name, arg_values, base.line, arg_kinds, arg_names) - - # If the base type is one of ours, do a MethodCall - if (isinstance(base.type, RInstance) and base.type.class_ir.is_ext_class - and not base.type.class_ir.builtin_base): - if base.type.class_ir.has_method(name): - decl = base.type.class_ir.method_decl(name) - if arg_kinds is None: - assert arg_names is None, "arg_kinds not present but arg_names is" - arg_kinds = [ARG_POS for _ in arg_values] - arg_names = [None for _ in arg_values] - else: - assert arg_names is not None, "arg_kinds present but arg_names is not" - - # Normalize args to positionals. - assert decl.bound_sig - arg_values = self.native_args_to_positional( - arg_values, arg_kinds, arg_names, decl.bound_sig, line) - return self.add(MethodCall(base, name, arg_values, line)) - elif base.type.class_ir.has_attr(name): - function = self.add(GetAttr(base, name, line)) - return self.py_call(function, arg_values, line, - arg_kinds=arg_kinds, arg_names=arg_names) - - elif isinstance(base.type, RUnion): - return self.union_method_call(base, base.type, name, arg_values, result_type, line, - arg_kinds, arg_names) - - # Try to do a special-cased method call - if not arg_kinds or arg_kinds == [ARG_POS] * len(arg_values): - target = self.translate_special_method_call(base, name, arg_values, result_type, line) - if target: - return target - - # Fall back to Python method call - return self.py_method_call(base, name, arg_values, line, arg_kinds, arg_names) - - def union_method_call(self, - base: Value, - obj_type: RUnion, - name: str, - arg_values: List[Value], - return_rtype: Optional[RType], - line: int, - arg_kinds: Optional[List[int]], - arg_names: Optional[List[Optional[str]]]) -> Value: - # Union method call needs a return_rtype for the type of the output register. - # If we don't have one, use object_rprimitive. - return_rtype = return_rtype or object_rprimitive - - def call_union_item(value: Value) -> Value: - return self.gen_method_call(value, name, arg_values, return_rtype, line, - arg_kinds, arg_names) - - return self.decompose_union_helper(base, obj_type, return_rtype, call_union_item, line) diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index d0dc0fda5a81..30e800c651f8 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -13,10 +13,11 @@ def f(x: int) -> int: r3 = r2 + r1 :: int return r3 -The IR is implemented in mypyc.ops. +This module deals with the module-level IR transformation logic and +putting it all together. The actual IR is implemented in mypyc.ir. -For the core of the implementation, look at build_ir() below, -mypyc.irbuild.builder, and mypyc.irbuild.visitor. +For the core of the IR transform implementation, look at build_ir() +below, mypyc.irbuild.builder, and mypyc.irbuild.visitor. """ from collections import OrderedDict @@ -54,6 +55,7 @@ def build_ir(modules: List[MypyFile], mapper: 'Mapper', options: CompilerOptions, errors: Errors) -> ModuleIRs: + """Build IR for a set of modules that have been type-checked by mypy.""" build_type_map(mapper, modules, graph, types, options, errors) @@ -95,6 +97,8 @@ def build_ir(modules: List[MypyFile], def transform_mypy_file(builder: IRBuilder, mypyfile: MypyFile) -> None: + """Generate IR for a single module.""" + if mypyfile.fullname in ('typing', 'abc'): # These module are special; their contents are currently all # built-in primitives. diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index ee143fbf735e..dfa7a99753fb 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -1,3 +1,5 @@ +"""Maintain a mapping from mypy concepts to IR/compiled concepts.""" + from typing import Dict, Optional, Union from collections import OrderedDict @@ -21,6 +23,9 @@ class Mapper: """Keep track of mappings from mypy concepts to IR concepts. + For example, we keep track of how the mypy TypeInfos of compiled + classes map to class IR objects. + This state is shared across all modules being compiled in all compilation groups. """ diff --git a/mypyc/irbuild/nonlocalcontrol.py b/mypyc/irbuild/nonlocalcontrol.py index 831698bf5016..2baacd6f372a 100644 --- a/mypyc/irbuild/nonlocalcontrol.py +++ b/mypyc/irbuild/nonlocalcontrol.py @@ -1,3 +1,8 @@ +"""Helpers for dealing with nonlocal control such as 'break' and 'return'. + +Model how these behave differently in different contexts. +""" + from abc import abstractmethod from typing import Optional, Union from typing_extensions import TYPE_CHECKING @@ -13,7 +18,7 @@ class NonlocalControl: - """Represents a stack frame of constructs that modify nonlocal control flow. + """ABC representing a stack frame of constructs that modify nonlocal control flow. The nonlocal control flow constructs are break, continue, and return, and their behavior is modified by a number of other @@ -35,6 +40,8 @@ def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: pas class BaseNonlocalControl(NonlocalControl): + """Default nonlocal control outside any statements that affect it.""" + def gen_break(self, builder: 'IRBuilder', line: int) -> None: assert False, "break outside of loop" @@ -46,8 +53,12 @@ def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: class LoopNonlocalControl(NonlocalControl): - def __init__(self, outer: NonlocalControl, - continue_block: BasicBlock, break_block: BasicBlock) -> None: + """Nonlocal control within a loop.""" + + def __init__(self, + outer: NonlocalControl, + continue_block: BasicBlock, + break_block: BasicBlock) -> None: self.outer = outer self.continue_block = continue_block self.break_block = break_block @@ -63,23 +74,31 @@ def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: class GeneratorNonlocalControl(BaseNonlocalControl): + """Default nonlocal control in a generator function outside statements.""" + def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: - # Assign an invalid next label number so that the next time __next__ is called, we jump to - # the case in which StopIteration is raised. + # Assign an invalid next label number so that the next time + # __next__ is called, we jump to the case in which + # StopIteration is raised. builder.assign(builder.fn_info.generator_class.next_label_target, builder.add(LoadInt(-1)), line) - # Raise a StopIteration containing a field for the value that should be returned. Before - # doing so, create a new block without an error handler set so that the implicitly thrown - # StopIteration isn't caught by except blocks inside of the generator function. + + # Raise a StopIteration containing a field for the value that + # should be returned. Before doing so, create a new block + # without an error handler set so that the implicitly thrown + # StopIteration isn't caught by except blocks inside of the + # generator function. builder.builder.push_error_handler(None) builder.goto_and_activate(BasicBlock()) + # Skip creating a traceback frame when we raise here, because # we don't care about the traceback frame and it is kind of - # expensive since raising StopIteration is an extremely common case. - # Also we call a special internal function to set StopIteration instead of - # using RaiseStandardError because the obvious thing doesn't work if the - # value is a tuple (???). + # expensive since raising StopIteration is an extremely common + # case. Also we call a special internal function to set + # StopIteration instead of using RaiseStandardError because + # the obvious thing doesn't work if the value is a tuple + # (???). builder.primitive_op(set_stop_iteration_value, [value], NO_TRACEBACK_LINE_NO) builder.add(Unreachable()) builder.builder.pop_error_handler() @@ -108,6 +127,8 @@ def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: class TryFinallyNonlocalControl(NonlocalControl): + """Nonlocal control within try/finally.""" + def __init__(self, target: BasicBlock) -> None: self.target = target self.ret_reg = None # type: Optional[Register] diff --git a/mypyc/irbuild/prebuildvisitor.py b/mypyc/irbuild/prebuildvisitor.py index 1f21e080d098..9050920813b2 100644 --- a/mypyc/irbuild/prebuildvisitor.py +++ b/mypyc/irbuild/prebuildvisitor.py @@ -7,110 +7,137 @@ class PreBuildVisitor(TraverserVisitor): + """Mypy file AST visitor run before building the IR. + + This collects various things, including: + + * Determine relationships between nested functions and functions that + contain nested functions + * Find non-local variables (free variables) + * Find property setters + * Find decorators of functions + + The main IR build pass uses this information. """ - Class used to visit a mypy file before building the IR for that program. This is done as a - first pass so that nested functions, encapsulating functions, lambda functions, decorated - functions, and free variables can be determined before instantiating the IRBuilder. - """ + def __init__(self) -> None: super().__init__() - # Mapping from FuncItem instances to sets of variables. The FuncItem instances are where - # these variables were first declared, and these variables are free in any functions that - # are nested within the FuncItem from which they are mapped. + # Dict from a function to symbols defined directly in the + # function that are used as non-local (free) variables within a + # nested function. self.free_variables = {} # type: Dict[FuncItem, Set[SymbolNode]] - # Intermediate data structure used to map SymbolNode instances to the FuncDef in which they - # were first visited. + + # Intermediate data structure used to find the function where + # a SymbolNode is declared. Initially this may point to a + # function nested inside the function with the declaration, + # but we'll eventually update this to refer to the function + # with the declaration. self.symbols_to_funcs = {} # type: Dict[SymbolNode, FuncItem] - # Stack representing the function call stack. + + # Stack representing current function nesting. self.funcs = [] # type: List[FuncItem] - # The set of property setters + + # All property setters encountered so far. self.prop_setters = set() # type: Set[FuncDef] + # A map from any function that contains nested functions to # a set of all the functions that are nested within it. self.encapsulating_funcs = {} # type: Dict[FuncItem, List[FuncItem]] - # A map from a nested func to it's parent/encapsulating func. + + # Map nested function to its parent/encapsulating function. self.nested_funcs = {} # type: Dict[FuncItem, FuncItem] - self.funcs_to_decorators = {} # type: Dict[FuncDef, List[Expression]] - def add_free_variable(self, symbol: SymbolNode) -> None: - # Get the FuncItem instance where the free symbol was first declared, and map that FuncItem - # to the SymbolNode representing the free symbol. - func = self.symbols_to_funcs[symbol] - self.free_variables.setdefault(func, set()).add(symbol) + # Map function to its non-special decorators. + self.funcs_to_decorators = {} # type: Dict[FuncDef, List[Expression]] def visit_decorator(self, dec: Decorator) -> None: if dec.decorators: - # Only add the function being decorated if there exist decorators in the decorator - # list. Note that meaningful decorators (@property, @abstractmethod) are removed from - # this list by mypy, but functions decorated by those decorators (in addition to - # property setters) do not need to be added to the set of decorated functions for - # the IRBuilder, because they are handled in a special way. + # Only add the function being decorated if there exist + # (ordinary) decorators in the decorator list. Certain + # decorators (such as @property, @abstractmethod) are + # special cased and removed from this list by + # mypy. Functions decorated only by special decorators + # (and property setters) are not treated as decorated + # functions by the IR builder. if isinstance(dec.decorators[0], MemberExpr) and dec.decorators[0].name == 'setter': + # Property setters are not treated as decorated methods. self.prop_setters.add(dec.func) else: self.funcs_to_decorators[dec.func] = dec.decorators super().visit_decorator(dec) + def visit_func_def(self, fdef: FuncItem) -> None: + # TODO: What about overloaded functions? + self.visit_func(fdef) + + def visit_lambda_expr(self, expr: LambdaExpr) -> None: + self.visit_func(expr) + def visit_func(self, func: FuncItem) -> None: - # If there were already functions or lambda expressions defined in the function stack, then - # note the previous FuncItem as containing a nested function and the current FuncItem as - # being a nested function. + # If there were already functions or lambda expressions + # defined in the function stack, then note the previous + # FuncItem as containing a nested function and the current + # FuncItem as being a nested function. if self.funcs: - # Add the new func to the set of nested funcs within the func at top of the func stack. + # Add the new func to the set of nested funcs within the + # func at top of the func stack. self.encapsulating_funcs.setdefault(self.funcs[-1], []).append(func) - # Add the func at top of the func stack as the parent of new func. + # Add the func at top of the func stack as the parent of + # new func. self.nested_funcs[func] = self.funcs[-1] self.funcs.append(func) super().visit_func(func) self.funcs.pop() - def visit_func_def(self, fdef: FuncItem) -> None: - self.visit_func(fdef) - - def visit_lambda_expr(self, expr: LambdaExpr) -> None: - self.visit_func(expr) - def visit_name_expr(self, expr: NameExpr) -> None: if isinstance(expr.node, (Var, FuncDef)): self.visit_symbol_node(expr.node) - # Check if child is contained within fdef (possibly indirectly within - # multiple nested functions). - def is_parent(self, fitem: FuncItem, child: FuncItem) -> bool: - if child in self.nested_funcs: - parent = self.nested_funcs[child] - if parent == fitem: - return True - return self.is_parent(fitem, parent) - return False + def visit_var(self, var: Var) -> None: + self.visit_symbol_node(var) def visit_symbol_node(self, symbol: SymbolNode) -> None: if not self.funcs: - # If the list of FuncDefs is empty, then we are not inside of a function and hence do - # not need to do anything regarding free variables. + # We are not inside a function and hence do not need to do + # anything regarding free variables. return if symbol in self.symbols_to_funcs: orig_func = self.symbols_to_funcs[symbol] if self.is_parent(self.funcs[-1], orig_func): - # If the function in which the symbol was originally seen is nested - # within the function currently being visited, fix the free_variable - # and symbol_to_funcs dictionaries. + # The function in which the symbol was previously seen is + # nested within the function currently being visited. Thus + # the current function is a better candidate to contain the + # declaration. self.symbols_to_funcs[symbol] = self.funcs[-1] + # TODO: Remove from the orig_func free_variables set? self.free_variables.setdefault(self.funcs[-1], set()).add(symbol) elif self.is_parent(orig_func, self.funcs[-1]): - # If the SymbolNode instance has already been visited before, - # and it was declared in a FuncDef not nested within the current - # FuncDef being visited, then it is a free symbol because it is - # being visited again. + # The SymbolNode instance has already been visited + # before in a parent function, thus it's a non-local + # symbol. self.add_free_variable(symbol) else: - # Otherwise, this is the first time the SymbolNode is being visited. We map the - # SymbolNode to the current FuncDef being visited to note where it was first visited. + # This is the first time the SymbolNode is being + # visited. We map the SymbolNode to the current FuncDef + # being visited to note where it was first visited. self.symbols_to_funcs[symbol] = self.funcs[-1] - def visit_var(self, var: Var) -> None: - self.visit_symbol_node(var) + def is_parent(self, fitem: FuncItem, child: FuncItem) -> bool: + # Check if child is nested within fdef (possibly indirectly + # within multiple nested functions). + if child in self.nested_funcs: + parent = self.nested_funcs[child] + if parent == fitem: + return True + return self.is_parent(fitem, parent) + return False + + def add_free_variable(self, symbol: SymbolNode) -> None: + # Find the function where the symbol was (likely) first declared, + # and mark is as a non-local symbol within that function. + func = self.symbols_to_funcs[symbol] + self.free_variables.setdefault(func, set()).add(symbol) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 8069cbd29d4c..4ac752f22f5f 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -1,3 +1,16 @@ +"""Prepare for IR transform. + +This needs to run after type checking and before generating IR. + +For example, construct partially initialized FuncIR and ClassIR +objects for all functions and classes. This allows us to bind +references to functions and classes before we've generated full IR for +functions or classes. The actual IR transform will then populate all +the missing bits, such as function bodies (basic blocks). + +Also build a mapping from mypy TypeInfos to ClassIR objects. +""" + from typing import List, Dict, Iterable, Optional, Union from mypy.nodes import ( diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 028db3201584..2fc8d9bb5f4a 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -1,3 +1,11 @@ +"""Transform mypy statement ASTs to mypyc IR (Intermediate Representation). + +The top-level AST transformation logic is implemented in mypyc.irbuild.visitor +and mypyc.irbuild.builder. + +A few statements are transformed in mypyc.irbuild.function (yield, for example). +""" + from typing import Optional, List, Tuple, Sequence, Callable import importlib.util diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index 7c574576f5f2..18d8306c869e 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -1,3 +1,5 @@ +"""Various utilities that don't depend on other modules in mypyc.irbuild.""" + from typing import Dict, Any, Union, Optional from mypy.nodes import ( diff --git a/mypyc/irbuild/vtable.py b/mypyc/irbuild/vtable.py index 0ebeb5752a55..41afed8505b5 100644 --- a/mypyc/irbuild/vtable.py +++ b/mypyc/irbuild/vtable.py @@ -1,3 +1,5 @@ +"""Compute vtables of native (extension) classes.""" + import itertools from mypyc.ir.class_ir import ClassIR, VTableEntries, VTableMethod, VTableAttr