diff --git a/mypyc/irbuild/callable_class.py b/mypyc/irbuild/callable_class.py index c6d6de94232c..c0f845543f2c 100644 --- a/mypyc/irbuild/callable_class.py +++ b/mypyc/irbuild/callable_class.py @@ -1,7 +1,7 @@ """Generate a class that represents a nested function. -The class defines __call__ for calling the function and allows access to variables -defined in outer scopes. +The class defines __call__ for calling the function and allows access to +non-local variables defined in outer scopes. """ from typing import List @@ -20,21 +20,28 @@ def setup_callable_class(builder: IRBuilder) -> None: - """Generates a callable class representing a nested function or a function within a - non-extension class and sets up the 'self' variable for that class. + """Generate an (incomplete) callable class representing function. - This takes the most recently visited function and returns a ClassIR to represent that - function. Each callable class contains an environment attribute with points to another - ClassIR representing the environment class where some of its variables can be accessed. - Note that its '__call__' method is not yet implemented, and is implemented in the - add_call_to_callable_class function. + This can be a nested function or a function within a non-extension + class. Also set up the 'self' variable for that class. - Returns a newly constructed ClassIR representing the callable class for the nested - function. - """ + This takes the most recently visited function and returns a + ClassIR to represent that function. Each callable class contains + an environment attribute which points to another ClassIR + representing the environment class where some of its variables can + be accessed. - # Check to see that the name has not already been taken. If so, rename the class. We allow - # multiple uses of the same function name because this is valid in if-else blocks. Example: + Note that some methods, such as '__call__', are not yet + created here. Use additional functions, such as + add_call_to_callable_class(), to add them. + + Return a newly constructed ClassIR representing the callable + class for the nested function. + """ + # Check to see that the name has not already been taken. If so, + # rename the class. We allow multiple uses of the same function + # name because this is valid in if-else blocks. Example: + # # if True: # def foo(): ----> foo_obj() # return True @@ -48,12 +55,14 @@ def setup_callable_class(builder: IRBuilder) -> None: count += 1 builder.callable_class_names.add(name) - # Define the actual callable class ClassIR, and set its environment to point at the - # previously defined environment class. + # Define the actual callable class ClassIR, and set its + # environment to point at the previously defined environment + # class. callable_class_ir = ClassIR(name, builder.module_name, is_generated=True) - # The functools @wraps decorator attempts to call setattr on nested functions, so - # we create a dict for these nested functions. + # The functools @wraps decorator attempts to call setattr on + # nested functions, so we create a dict for these nested + # functions. # https://github.com/python/cpython/blob/3.7/Lib/functools.py#L58 if builder.fn_info.is_nested: callable_class_ir.has_dict = True @@ -68,8 +77,8 @@ def setup_callable_class(builder: IRBuilder) -> None: builder.fn_info.callable_class = ImplicitClass(callable_class_ir) builder.classes.append(callable_class_ir) - # Add a 'self' variable to the callable class' environment, and store that variable in a - # register to be accessed later. + # Add a 'self' variable to the environment of the callable class, + # and store that variable in a register to be accessed later. self_target = add_self_to_env(builder.environment, callable_class_ir) builder.fn_info.callable_class.self_reg = builder.read(self_target, builder.fn_info.fitem.line) @@ -79,13 +88,14 @@ def add_call_to_callable_class(builder: IRBuilder, sig: FuncSignature, env: Environment, fn_info: FuncInfo) -> FuncIR: - """Generates a '__call__' method for a callable class representing a nested function. + """Generate a '__call__' method for a callable class representing a nested function. - This takes the blocks, signature, and environment associated with a function definition and - uses those to build the '__call__' method of a given callable class, used to represent that - function. Note that a 'self' parameter is added to its list of arguments, as the nested - function becomes a class method. + This takes the blocks, signature, and environment associated with + a function definition and uses those to build the '__call__' + method of a given callable class, used to represent that + function. """ + # Since we create a method, we also add a 'self' parameter. sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args, sig.ret_type) call_fn_decl = FuncDecl('__call__', fn_info.callable_class.ir.name, builder.module_name, sig) call_fn_ir = FuncIR(call_fn_decl, blocks, env, @@ -95,7 +105,7 @@ def add_call_to_callable_class(builder: IRBuilder, def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None: - """Generates the '__get__' method for a callable class.""" + """Generate the '__get__' method for a callable class.""" line = fn_info.fitem.line builder.enter(fn_info) @@ -133,22 +143,30 @@ def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None: def instantiate_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> Value: - """ - Assigns a callable class to a register named after the given function definition. Note - that fn_info refers to the function being assigned, whereas builder.fn_info refers to the - function encapsulating the function being turned into a callable class. + """Create an instance of a callable class for a function. + + Calls to the function will actually call this instance. + + Note that fn_info refers to the function being assigned, whereas + builder.fn_info refers to the function encapsulating the function + being turned into a callable class. """ fitem = fn_info.fitem func_reg = builder.add(Call(fn_info.callable_class.ir.ctor, [], fitem.line)) - # Set the callable class' environment attribute to point at the environment class - # defined in the callable class' immediate outer scope. Note that there are three possible - # environment class registers we may use. If the encapsulating function is: - # - a generator function, then the callable class is instantiated from the generator class' - # __next__' function, and hence the generator class' environment register is used. - # - a nested function, then the callable class is instantiated from the current callable - # class' '__call__' function, and hence the callable class' environment register is used. - # - neither, then we use the environment register of the original function. + # Set the environment attribute of the callable class to point at + # the environment class defined in the callable class' immediate + # outer scope. Note that there are three possible environment + # class registers we may use. This depends on what the encapsulating + # (parent) function is: + # + # - A nested function: the callable class is instantiated + # from the current callable class' '__call__' function, and hence + # the callable class' environment register is used. + # - A generator function: the callable class is instantiated + # from the '__next__' method of the generator class, and hence the + # environment of the generator class is used. + # - Regular function: we use the environment of the original function. curr_env_reg = None if builder.fn_info.is_generator: curr_env_reg = builder.fn_info.generator_class.curr_env_reg diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index c5506dd8704a..89d7a16c5566 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -1,3 +1,5 @@ +"""Transform class definitions from the mypy AST form to IR.""" + from typing import List, Optional from mypy.nodes import ( @@ -29,6 +31,17 @@ def transform_class_def(builder: IRBuilder, cdef: ClassDef) -> None: + """Create IR for a class definition. + + This can generate both extension (native) and non-extension + classes. These are generated in very different ways. In the + latter case we construct a Python type object at runtime by doing + the equivalent of "type(name, bases, dict)" in IR. Extension + classes are defined via C structs that are generated later in + mypyc.codegen.emitclass. + + This is the main entry point to this module. + """ ir = builder.mapper.type_to_ir[cdef.info] # We do this check here because the base field of parent @@ -188,9 +201,9 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value: def populate_non_ext_bases(builder: IRBuilder, cdef: ClassDef) -> Value: - """ - Populate the base-class tuple passed to the metaclass constructor - for non-extension classes. + """Create base class tuple of a non-extension class. + + The tuple is passed to the metaclass constructor. """ ir = builder.mapper.type_to_ir[cdef.info] bases = [] @@ -222,11 +235,10 @@ def setup_non_ext_dict(builder: IRBuilder, cdef: ClassDef, metaclass: Value, bases: Value) -> Value: - """ - Initialize the class dictionary for a non-extension class. This class dictionary - is passed to the metaclass constructor. - """ + """Initialize the class dictionary for a non-extension class. + This class dictionary is passed to the metaclass constructor. + """ # Check if the metaclass defines a __prepare__ method, and if so, call it. has_prepare = builder.primitive_op(py_hasattr_op, [metaclass, @@ -252,14 +264,16 @@ def setup_non_ext_dict(builder: IRBuilder, return non_ext_dict -def add_non_ext_class_attr(builder: IRBuilder, non_ext: NonExtClassInfo, lvalue: NameExpr, - stmt: AssignmentStmt, cdef: ClassDef, +def add_non_ext_class_attr(builder: IRBuilder, + non_ext: NonExtClassInfo, + lvalue: NameExpr, + stmt: AssignmentStmt, + cdef: ClassDef, attr_to_cache: List[Lvalue]) -> None: - """ - Add a class attribute to __annotations__ of a non-extension class. If the - attribute is assigned to a value, it is also added to __dict__. - """ + """Add a class attribute to __annotations__ of a non-extension class. + If the attribute is initialized with a value, also add it to __dict__. + """ # We populate __annotations__ because dataclasses uses it to determine # which attributes to compute on. # TODO: Maybe generate more precise types for annotations @@ -284,7 +298,7 @@ def add_non_ext_class_attr(builder: IRBuilder, non_ext: NonExtClassInfo, lvalue: def generate_attr_defaults(builder: IRBuilder, cdef: ClassDef) -> None: - """Generate an initialization method for default attr values (from class vars)""" + """Generate an initialization method for default attr values (from class vars).""" cls = builder.mapper.type_to_ir[cdef.info] if cls.builtin_base: return @@ -347,6 +361,7 @@ def generate_attr_defaults(builder: IRBuilder, cdef: ClassDef) -> None: def create_ne_from_eq(builder: IRBuilder, cdef: ClassDef) -> None: + """Create a "__ne__" method from a "__eq__" method (if only latter exists).""" cls = builder.mapper.type_to_ir[cdef.info] if cls.has_method('__eq__') and not cls.has_method('__ne__'): f = gen_glue_ne_method(builder, cls, cdef.line) @@ -356,7 +371,7 @@ def create_ne_from_eq(builder: IRBuilder, cdef: ClassDef) -> None: def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> FuncIR: - """Generate a __ne__ method from a __eq__ method. """ + """Generate a "__ne__" method from a "__eq__" method. """ builder.enter() rt_args = (RuntimeArg("self", RInstance(cls)), RuntimeArg("rhs", object_rprimitive)) @@ -417,10 +432,13 @@ def load_non_ext_class(builder: IRBuilder, def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) -> Value: - """ - Given a decorated ClassDef and a register containing a non-extension representation of the - ClassDef created via the type constructor, applies the corresponding decorator functions - on that decorated ClassDef and returns a register containing the decorated ClassDef. + """Apply class decorators to create a decorated (non-extension) class object. + + Given a decorated ClassDef and a register containing a + non-extension representation of the ClassDef created via the type + constructor, applies the corresponding decorator functions on that + decorated ClassDef and returns a register containing the decorated + ClassDef. """ decorators = cdef.decorators dec_class = type_obj @@ -432,7 +450,7 @@ def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) -> def cache_class_attrs(builder: IRBuilder, attrs_to_cache: List[Lvalue], cdef: ClassDef) -> None: - """Add class attributes to be cached to the global cache""" + """Add class attributes to be cached to the global cache.""" typ = builder.load_native_type_object(cdef.fullname) for lval in attrs_to_cache: assert isinstance(lval, NameExpr) diff --git a/mypyc/irbuild/context.py b/mypyc/irbuild/context.py index b6b0ad3a7947..ac7521cf930c 100644 --- a/mypyc/irbuild/context.py +++ b/mypyc/irbuild/context.py @@ -1,3 +1,5 @@ +"""Helpers that store information about functions and the related classes.""" + from typing import List, Optional, Tuple from mypy.nodes import FuncItem @@ -10,6 +12,7 @@ class FuncInfo: """Contains information about functions as they are generated.""" + def __init__(self, fitem: FuncItem = INVALID_FUNC_DEF, name: str = '', @@ -87,9 +90,14 @@ def curr_env_reg(self) -> Value: class ImplicitClass: - """Contains information regarding classes that are generated as a result of nested functions or - generated functions, but not explicitly defined in the source code. + """Contains information regarding implicitly generated classes. + + Implicit classes are generated for nested functions and generator + functions. They are not explicitly defined in the source code. + + NOTE: This is both a concrete class and used as a base class. """ + def __init__(self, ir: ClassIR) -> None: # The ClassIR instance associated with this class. self.ir = ir @@ -131,6 +139,8 @@ def prev_env_reg(self, reg: Value) -> None: class GeneratorClass(ImplicitClass): + """Contains information about implicit generator function classes.""" + def __init__(self, ir: ClassIR) -> None: super().__init__(ir) # This register holds the label number that the '__next__' function should go to the next diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 6ecb29735564..87a72b4385e4 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -1,4 +1,19 @@ -"""Generate classes representing function environments (+ related operations).""" +"""Generate classes representing function environments (+ related operations). + +If we have a nested function that has non-local (free) variables, access to the +non-locals is via an instance of an environment class. Example: + + def f() -> int: + x = 0 # Make 'x' an attribute of an environment class instance + + def g() -> int: + # We have access to the environment class instance to + # allow accessing 'x' + return x + 2 + + x + 1 # Modify the attribute + return g() +""" from typing import Optional, Union @@ -13,16 +28,19 @@ def setup_env_class(builder: IRBuilder) -> ClassIR: - """Generates a class representing a function environment. - - Note that the variables in the function environment are not actually populated here. This - is because when the environment class is generated, the function environment has not yet - been visited. This behavior is allowed so that when the compiler visits nested functions, - it can use the returned ClassIR instance to figure out free variables it needs to access. - The remaining attributes of the environment class are populated when the environment - registers are loaded. - - Returns a ClassIR representing an environment for a function containing a nested function. + """Generate a class representing a function environment. + + Note that the variables in the function environment are not + actually populated here. This is because when the environment + class is generated, the function environment has not yet been + visited. This behavior is allowed so that when the compiler visits + nested functions, it can use the returned ClassIR instance to + figure out free variables it needs to access. The remaining + attributes of the environment class are populated when the + environment registers are loaded. + + Return a ClassIR representing an environment for a function + containing a nested function. """ env_class = ClassIR('{}_env'.format(builder.fn_info.namespaced_name()), builder.module_name, is_generated=True) @@ -38,8 +56,7 @@ def setup_env_class(builder: IRBuilder) -> ClassIR: def finalize_env_class(builder: IRBuilder) -> None: - """Generates, instantiates, and sets up the environment of an environment class.""" - + """Generate, instantiate, and set up the environment of an environment class.""" instantiate_env_class(builder) # Iterate through the function arguments and replace local definitions (using registers) @@ -52,7 +69,7 @@ def finalize_env_class(builder: IRBuilder) -> None: def instantiate_env_class(builder: IRBuilder) -> Value: - """Assigns an environment class to a register named after the given function definition.""" + """Assign an environment class to a register named after the given function definition.""" curr_env_reg = builder.add( Call(builder.fn_info.env_class.ctor, [], builder.fn_info.fitem.line) ) @@ -70,11 +87,12 @@ def instantiate_env_class(builder: IRBuilder) -> Value: def load_env_registers(builder: IRBuilder) -> None: - """Loads the registers for the current FuncItem being visited. + """Load the registers for the current FuncItem being visited. - Adds the arguments of the FuncItem to the environment. If the FuncItem is nested inside of - another function, then this also loads all of the outer environments of the FuncItem into - registers so that they can be used when accessing free variables. + Adds the arguments of the FuncItem to the environment. If the + FuncItem is nested inside of another function, then this also + loads all of the outer environments of the FuncItem into registers + so that they can be used when accessing free variables. """ add_args_to_env(builder, local=True) @@ -89,12 +107,14 @@ def load_env_registers(builder: IRBuilder) -> None: def load_outer_env(builder: IRBuilder, base: Value, outer_env: Environment) -> Value: - """Loads the environment class for a given base into a register. + """Load the environment class for a given base into a register. - Additionally, iterates through all of the SymbolNode and AssignmentTarget instances of the - environment at the given index's symtable, and adds those instances to the environment of - the current environment. This is done so that the current environment can access outer - environment variables without having to reload all of the environment registers. + Additionally, iterates through all of the SymbolNode and + AssignmentTarget instances of the environment at the given index's + symtable, and adds those instances to the environment of the + current environment. This is done so that the current environment + can access outer environment variables without having to reload + all of the environment registers. Returns the register where the environment class was loaded. """ @@ -150,10 +170,12 @@ def add_args_to_env(builder: IRBuilder, def setup_func_for_recursive_call(builder: IRBuilder, fdef: FuncDef, base: ImplicitClass) -> None: - """ - Adds the instance of the callable class representing the given FuncDef to a register in the - environment so that the function can be called recursively. Note that this needs to be done - only for nested functions. + """Enable calling a nested function (with a callable class) recursively. + + Adds the instance of the callable class representing the given + FuncDef to a register in the environment so that the function can + be called recursively. Note that this needs to be done only for + nested functions. """ # First, set the attribute of the environment class so that GetAttr can be called on it. prev_env = builder.fn_infos[-2].env_class diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 0a12f2e92ecc..9a56eb2ce4be 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -96,12 +96,13 @@ def comprehension_helper(builder: IRBuilder, line: int) -> None: """Helper function for list comprehensions. - "loop_params" is a list of (index, expr, [conditions]) tuples defining nested loops: - - "index" is the Lvalue indexing that loop; - - "expr" is the expression for the object to be iterated over; - - "conditions" is a list of conditions, evaluated in order with short-circuiting, - that must all be true for the loop body to be executed - "gen_inner_stmts" is a function to generate the IR for the body of the innermost loop + Args: + loop_params: a list of (index, expr, [conditions]) tuples defining nested loops: + - "index" is the Lvalue indexing that loop; + - "expr" is the expression for the object to be iterated over; + - "conditions" is a list of conditions, evaluated in order with short-circuiting, + that must all be true for the loop body to be executed + gen_inner_stmts: function to generate the IR for the body of the innermost loop """ def handle_loop(loop_params: List[Tuple[Lvalue, Expression, List[Expression]]]) -> None: """Generate IR for a loop. @@ -120,10 +121,11 @@ def loop_contents( ) -> None: """Generate the body of the loop. - "conds" is a list of conditions to be evaluated (in order, with short circuiting) - to gate the body of the loop. - "remaining_loop_params" is the parameters for any further nested loops; if it's empty - we'll instead evaluate the "gen_inner_stmts" function. + Args: + conds: a list of conditions to be evaluated (in order, with short circuiting) + to gate the body of the loop + remaining_loop_params: the parameters for any further nested loops; if it's empty + we'll instead evaluate the "gen_inner_stmts" function """ # Check conditions, in order, short circuiting them. for cond in conds: @@ -356,7 +358,8 @@ def unsafe_index( class ForSequence(ForGenerator): """Generate optimized IR for a for loop over a sequence. - Supports iterating in both forward and reverse.""" + Supports iterating in both forward and reverse. + """ def init(self, expr_reg: Value, target_type: RType, reverse: bool) -> None: builder = self.builder diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index 028a88ff8855..f791c7f8eafd 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -1,6 +1,13 @@ """Transform mypy AST functions to IR (and related things). -This also deals with generators, async functions and nested functions. +Normal functions are translated into a list of basic blocks +containing various IR ops (defined in mypyc.ir.ops). + +This also deals with generators, async functions and nested +functions. All of these are transformed into callable classes. These +have a custom __call__ method that implements the call, and state, such +as an environment containing non-local variables, is stored in the +instance of the callable class. """ from typing import Optional, List, Tuple, Union @@ -152,22 +159,22 @@ def gen_func_item(builder: IRBuilder, sig: FuncSignature, cdef: Optional[ClassDef] = None, ) -> Tuple[FuncIR, Optional[Value]]: - # TODO: do something about abstract methods. + """Generate and return the FuncIR for a given FuncDef. - """Generates and returns the FuncIR for a given FuncDef. + If the given FuncItem is a nested function, then we generate a + callable class representing the function and use that instead of + the actual function. if the given FuncItem contains a nested + function, then we generate an environment class so that inner + nested functions can access the environment of the given FuncDef. - If the given FuncItem is a nested function, then we generate a callable class representing - the function and use that instead of the actual function. if the given FuncItem contains a - nested function, then we generate an environment class so that inner nested functions can - access the environment of the given FuncDef. + Consider the following nested function: - Consider the following nested function. - def a() -> None: - def b() -> None: - def c() -> None: + def a() -> None: + def b() -> None: + def c() -> None: + return None return None return None - return None The classes generated would look something like the following. @@ -186,6 +193,8 @@ def c() -> None: +-------+ """ + # TODO: do something about abstract methods. + func_reg = None # type: Optional[Value] # We treat lambdas as always being nested because we always generate @@ -293,9 +302,12 @@ def gen_func_ir(builder: IRBuilder, env: Environment, fn_info: FuncInfo, cdef: Optional[ClassDef]) -> Tuple[FuncIR, Optional[Value]]: - """Generates the FuncIR for a function given the blocks, environment, and function info of - a particular function and returns it. If the function is nested, also returns the register - containing the instance of the corresponding callable class. + """Generate the FuncIR for a function. + + This takes the basic blocks, environment, and function info of a + particular function and returns the IR. If the function is nested, + also returns the register containing the instance of the + corresponding callable class. """ func_reg = None # type: Optional[Value] if fn_info.is_nested or fn_info.in_non_ext: @@ -445,7 +457,7 @@ def calculate_arg_defaults(builder: IRBuilder, def gen_func_ns(builder: IRBuilder) -> str: - """Generates a namespace for a nested function using its outer function names.""" + """Generate a namespace for a nested function using its outer function names.""" return '_'.join(info.name + ('' if not info.class_name else '_' + info.class_name) for info in builder.fn_infos if info.name and info.name != '') @@ -560,11 +572,12 @@ def else_body() -> None: def load_decorated_func(builder: IRBuilder, fdef: FuncDef, orig_func_reg: Value) -> Value: - """ - Given a decorated FuncDef and the register containing an instance of the callable class - representing that FuncDef, applies the corresponding decorator functions on that decorated - FuncDef and returns a register containing an instance of the callable class representing - the decorated function. + """Apply decorators to a function. + + Given a decorated FuncDef and an instance of the callable class + representing that FuncDef, apply the corresponding decorator + functions on that decorated FuncDef and return the decorated + function. """ if not is_decorated(builder, fdef): # If there are no decorators associated with the function, then just return the @@ -591,7 +604,8 @@ def gen_glue(builder: IRBuilder, sig: FuncSignature, target: FuncIR, ) -> FuncIR: """Generate glue methods that mediate between different method types in subclasses. - Works on both properties and methods. See gen_glue_methods below for more details. + Works on both properties and methods. See gen_glue_methods below + for more details. If do_py_ops is True, then the glue methods should use generic C API operations instead of direct calls, to enable generating @@ -676,7 +690,7 @@ def gen_glue_property(builder: IRBuilder, properties also require glue. However, this only requires the return type to change. Further, instead of a method call, an attribute get is performed. - If do_pygetattr is True, then get the attribute using the C + If do_pygetattr is True, then get the attribute using the Python C API instead of a native call. """ builder.enter() @@ -699,10 +713,10 @@ def gen_glue_property(builder: IRBuilder, def get_func_target(builder: IRBuilder, fdef: FuncDef) -> AssignmentTarget: - """ - Given a FuncDef, return the target associated the instance of its callable class. If the - function was not already defined somewhere, then define it and add it to the current - environment. + """Given a FuncDef, return the target for the instance of its callable class. + + If the function was not already defined somewhere, then define it + and add it to the current environment. """ if fdef.original_def: # Get the target associated with the previously defined FuncDef. diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index aaf54d51d70d..e09711b7e1f7 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -1,7 +1,11 @@ """Generate IR for generator functions. -A generator function is represented by a class that implements the generator protocol -and keeps track of the generator state, including local variables. +A generator function is represented by a class that implements the +generator protocol and keeps track of the generator state, including +local variables. + +The top-level logic for dealing with generator functions is in +mypyc.irbuild.function. """ from typing import List @@ -93,9 +97,11 @@ def populate_switch_for_generator_class(builder: IRBuilder) -> None: def add_raise_exception_blocks_to_generator_class(builder: IRBuilder, line: int) -> None: - """ - Generates blocks to check if error flags are set while calling the helper method for - generator functions, and raises an exception if those flags are set. + """Add error handling blocks to a generator class. + + Generates blocks to check if error flags are set while calling the + helper method for generator functions, and raises an exception if + those flags are set. """ cls = builder.fn_info.generator_class assert cls.exc_regs is not None @@ -225,8 +231,9 @@ def add_throw_to_generator_class(builder: IRBuilder, val = builder.environment.add_local_reg(Var('value'), object_rprimitive, True) tb = builder.environment.add_local_reg(Var('traceback'), object_rprimitive, True) - # Because the value and traceback arguments are optional and hence can be NULL if not - # passed in, we have to assign them Py_None if they are not passed in. + # Because the value and traceback arguments are optional and hence + # can be NULL if not passed in, we have to assign them Py_None if + # they are not passed in. none_reg = builder.none_object() builder.assign_if_null(val, lambda: none_reg, builder.fn_info.fitem.line) builder.assign_if_null(tb, lambda: none_reg, builder.fn_info.fitem.line) @@ -242,9 +249,9 @@ def add_throw_to_generator_class(builder: IRBuilder, builder.add(Return(result)) blocks, env, _, fn_info = builder.leave() - # Create the FuncSignature for the throw function. NOte that the value and traceback fields - # are optional, and are assigned to if they are not passed in inside the body of the throw - # function. + # Create the FuncSignature for the throw function. Note that the + # value and traceback fields are optional, and are assigned to if + # they are not passed in inside the body of the throw function. sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive), RuntimeArg('type', object_rprimitive), RuntimeArg('value', object_rprimitive, ARG_OPT), @@ -312,8 +319,9 @@ def setup_env_for_generator_class(builder: IRBuilder) -> None: cls.self_reg = builder.read(self_target, fitem.line) cls.curr_env_reg = load_outer_env(builder, cls.self_reg, builder.environment) - # Define a variable representing the label to go to the next time the '__next__' function - # of the generator is called, and add it as an attribute to the environment class. + # Define a variable representing the label to go to the next time + # the '__next__' function of the generator is called, and add it + # as an attribute to the environment class. cls.next_label_target = builder.add_var_to_env_class( Var(NEXT_LABEL_ATTR_NAME), int_rprimitive, @@ -321,7 +329,8 @@ def setup_env_for_generator_class(builder: IRBuilder) -> None: reassign=False ) - # Add arguments from the original generator function to the generator class' environment. + # Add arguments from the original generator function to the + # environment of the generator class. add_args_to_env(builder, local=False, base=cls, reassign=False) # Set the next label register for the generator class. diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index d319cb83f481..5e3d8d9ca324 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -64,7 +64,10 @@ def __init__( # Stack of except handler entry blocks self.error_handlers = [None] # type: List[Optional[BasicBlock]] + # Basic operations + def add(self, op: Op) -> Value: + """Add an op.""" assert not self.blocks[-1].terminated, "Can't add to finished block" self.blocks[-1].ops.append(op) @@ -73,10 +76,12 @@ def add(self, op: Op) -> Value: return op def goto(self, target: BasicBlock) -> None: + """Add goto to a basic block.""" if not self.blocks[-1].terminated: self.add(Goto(target)) def activate_block(self, block: BasicBlock) -> None: + """Add a basic block and make it the active one (target of adds).""" if self.blocks: assert self.blocks[-1].terminated @@ -84,6 +89,7 @@ def activate_block(self, block: BasicBlock) -> None: self.blocks.append(block) def goto_and_activate(self, block: BasicBlock) -> None: + """Add goto a block and make it the active block.""" self.goto(block) self.activate_block(block) @@ -93,30 +99,10 @@ def push_error_handler(self, handler: Optional[BasicBlock]) -> None: def pop_error_handler(self) -> Optional[BasicBlock]: return self.error_handlers.pop() - ## - - def get_native_type(self, cls: ClassIR) -> Value: - fullname = '%s.%s' % (cls.module_name, cls.name) - return self.load_native_type_object(fullname) - - def primitive_op(self, desc: OpDescription, args: List[Value], line: int) -> Value: - assert desc.result_type is not None - coerced = [] - for i, arg in enumerate(args): - formal_type = self.op_arg_type(desc, i) - arg = self.coerce(arg, formal_type, line) - coerced.append(arg) - target = self.add(PrimitiveOp(coerced, desc, line)) - return target - def alloc_temp(self, type: RType) -> Register: return self.environment.add_temp(type) - def op_arg_type(self, desc: OpDescription, n: int) -> RType: - if n >= len(desc.arg_types): - assert desc.is_var_arg - return desc.arg_types[-1] - return desc.arg_types[n] + # Type conversions def box(self, src: Value) -> Value: if src.type.is_unboxed: @@ -158,13 +144,10 @@ def coerce(self, src: Value, target_type: RType, line: int, force: bool = False) return tmp return src - def none(self) -> Value: - return self.add(PrimitiveOp([], none_op, line=-1)) - - def none_object(self) -> Value: - return self.add(PrimitiveOp([], none_object_op, line=-1)) + # Attribute access def get_attr(self, obj: Value, attr: str, result_type: RType, line: int) -> Value: + """Get a native or Python attribute of an object.""" if (isinstance(obj.type, RInstance) and obj.type.class_ir.is_ext_class and obj.type.class_ir.has_attr(attr)): return self.add(GetAttr(obj, attr, line)) @@ -179,71 +162,22 @@ def union_get_attr(self, attr: str, result_type: RType, line: int) -> Value: + """Get an attribute of an object with a union type.""" + def get_item_attr(value: Value) -> Value: return self.get_attr(value, attr, result_type, line) return self.decompose_union_helper(obj, rtype, result_type, get_item_attr, line) - def decompose_union_helper(self, - obj: Value, - rtype: RUnion, - result_type: RType, - process_item: Callable[[Value], Value], - line: int) -> Value: - """Generate isinstance() + specialized operations for union items. - - Say, for Union[A, B] generate ops resembling this (pseudocode): - - if isinstance(obj, A): - result = - else: - result = + def py_get_attr(self, obj: Value, attr: str, line: int) -> Value: + """Get a Python attribute (slow). - Args: - obj: value with a union type - rtype: the union type - result_type: result of the operation - process_item: callback to generate op for a single union item (arg is coerced - to union item type) - line: line number + Prefer get_attr() which generates optimized code for native classes. """ - # TODO: Optimize cases where a single operation can handle multiple union items - # (say a method is implemented in a common base class) - fast_items = [] - rest_items = [] - for item in rtype.items: - if isinstance(item, RInstance): - fast_items.append(item) - else: - # For everything but RInstance we fall back to C API - rest_items.append(item) - exit_block = BasicBlock() - result = self.alloc_temp(result_type) - for i, item in enumerate(fast_items): - more_types = i < len(fast_items) - 1 or rest_items - if more_types: - # We are not at the final item so we need one more branch - op = self.isinstance_native(obj, item.class_ir, line) - true_block, false_block = BasicBlock(), BasicBlock() - self.add_bool_branch(op, true_block, false_block) - self.activate_block(true_block) - coerced = self.coerce(obj, item, line) - temp = process_item(coerced) - temp2 = self.coerce(temp, result_type, line) - self.add(Assign(result, temp2)) - self.goto(exit_block) - if more_types: - self.activate_block(false_block) - if rest_items: - # For everything else we use generic operation. Use force=True to drop the - # union type. - coerced = self.coerce(obj, object_rprimitive, line, force=True) - temp = process_item(coerced) - temp2 = self.coerce(temp, result_type, line) - self.add(Assign(result, temp2)) - self.goto(exit_block) - self.activate_block(exit_block) - return result + key = self.load_static_unicode(attr) + return self.add(PrimitiveOp([obj, key], py_getattr_op, line)) + + # isinstance() checks def isinstance_helper(self, obj: Value, class_irs: List[ClassIR], line: int) -> Value: """Fast path for isinstance() that checks against a list of native classes.""" @@ -259,8 +193,9 @@ def other() -> Value: def isinstance_native(self, obj: Value, class_ir: ClassIR, line: int) -> Value: """Fast isinstance() check for a native class. - If there three or less concrete (non-trait) classes among the class and all - its children, use even faster type comparison checks `type(obj) is typ`. + If there are three or fewer concrete (non-trait) classes among the class + and all its children, use even faster type comparison checks `type(obj) + is typ`. """ concrete = all_concrete_classes(class_ir) if concrete is None or len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1: @@ -278,9 +213,7 @@ def other() -> Value: ret = self.shortcircuit_helper('or', bool_rprimitive, lambda: ret, other, line) return ret - def py_get_attr(self, obj: Value, attr: str, line: int) -> Value: - key = self.load_static_unicode(attr) - return self.add(PrimitiveOp([obj, key], py_getattr_op, line)) + # Calls def py_call(self, function: Value, @@ -288,7 +221,10 @@ def py_call(self, line: int, arg_kinds: Optional[List[int]] = None, arg_names: Optional[Sequence[Optional[str]]] = None) -> Value: - """Use py_call_op or py_call_with_kwargs_op for function call.""" + """Call a Python function (non-native and slow). + + Use py_call_op or py_call_with_kwargs_op for Python function call. + """ # If all arguments are positional, we can use py_call_op. if (arg_kinds is None) or all(kind == ARG_POS for kind in arg_kinds): return self.primitive_op(py_call_op, [function] + arg_values, line) @@ -338,6 +274,7 @@ def py_method_call(self, line: int, arg_kinds: Optional[List[int]], arg_names: Optional[Sequence[Optional[str]]]) -> Value: + """Call a Python method (non-native and slow).""" if (arg_kinds is None) or all(kind == ARG_POS for kind in arg_kinds): method_name_reg = self.load_static_unicode(method_name) return self.primitive_op(py_method_call_op, [obj, method_name_reg] + arg_values, line) @@ -345,10 +282,13 @@ def py_method_call(self, method = self.py_get_attr(obj, method_name, line) return self.py_call(method, arg_values, line, arg_kinds=arg_kinds, arg_names=arg_names) - def call(self, decl: FuncDecl, args: Sequence[Value], + def call(self, + decl: FuncDecl, + args: Sequence[Value], arg_kinds: List[int], arg_names: Sequence[Optional[str]], line: int) -> Value: + """Call a native function.""" # Normalize args to positionals. args = self.native_args_to_positional( args, arg_kinds, arg_names, decl.sig, line) @@ -397,39 +337,86 @@ def native_args_to_positional(self, return output_args - def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: - result = None # type: Union[Value, None] - initial_items = [] # type: List[Value] - for key, value in key_value_pairs: - if key is not None: - # key:value - if result is None: - initial_items.extend((key, value)) - continue + def gen_method_call(self, + base: Value, + name: str, + arg_values: List[Value], + result_type: Optional[RType], + line: int, + arg_kinds: Optional[List[int]] = None, + arg_names: Optional[List[Optional[str]]] = None) -> Value: + """Generate either a native or Python method call.""" + # If arg_kinds contains values other than arg_pos and arg_named, then fallback to + # Python method call. + if (arg_kinds is not None + and not all(kind in (ARG_POS, ARG_NAMED) for kind in arg_kinds)): + return self.py_method_call(base, name, arg_values, base.line, arg_kinds, arg_names) - self.translate_special_method_call( - result, - '__setitem__', - [key, value], - result_type=None, - line=line) - else: - # **value - if result is None: - result = self.primitive_op(new_dict_op, initial_items, line) + # If the base type is one of ours, do a MethodCall + if (isinstance(base.type, RInstance) and base.type.class_ir.is_ext_class + and not base.type.class_ir.builtin_base): + if base.type.class_ir.has_method(name): + decl = base.type.class_ir.method_decl(name) + if arg_kinds is None: + assert arg_names is None, "arg_kinds not present but arg_names is" + arg_kinds = [ARG_POS for _ in arg_values] + arg_names = [None for _ in arg_values] + else: + assert arg_names is not None, "arg_kinds present but arg_names is not" - self.primitive_op( - dict_update_in_display_op, - [result, value], - line=line - ) + # Normalize args to positionals. + assert decl.bound_sig + arg_values = self.native_args_to_positional( + arg_values, arg_kinds, arg_names, decl.bound_sig, line) + return self.add(MethodCall(base, name, arg_values, line)) + elif base.type.class_ir.has_attr(name): + function = self.add(GetAttr(base, name, line)) + return self.py_call(function, arg_values, line, + arg_kinds=arg_kinds, arg_names=arg_names) - if result is None: - result = self.primitive_op(new_dict_op, initial_items, line) + elif isinstance(base.type, RUnion): + return self.union_method_call(base, base.type, name, arg_values, result_type, line, + arg_kinds, arg_names) - return result + # Try to do a special-cased method call + if not arg_kinds or arg_kinds == [ARG_POS] * len(arg_values): + target = self.translate_special_method_call(base, name, arg_values, result_type, line) + if target: + return target + + # Fall back to Python method call + return self.py_method_call(base, name, arg_values, line, arg_kinds, arg_names) + + def union_method_call(self, + base: Value, + obj_type: RUnion, + name: str, + arg_values: List[Value], + return_rtype: Optional[RType], + line: int, + arg_kinds: Optional[List[int]], + arg_names: Optional[List[Optional[str]]]) -> Value: + """Generate a method call with a union type for the object.""" + # Union method call needs a return_rtype for the type of the output register. + # If we don't have one, use object_rprimitive. + return_rtype = return_rtype or object_rprimitive + + def call_union_item(value: Value) -> Value: + return self.gen_method_call(value, name, arg_values, return_rtype, line, + arg_kinds, arg_names) + + return self.decompose_union_helper(base, obj_type, return_rtype, call_union_item, line) + + # Loading various values + + def none(self) -> Value: + """Load unboxed None value (type: none_rprimitive).""" + return self.add(PrimitiveOp([], none_op, line=-1)) + + def none_object(self) -> Value: + """Load Python None value (type: object_rprimitive).""" + return self.add(PrimitiveOp([], none_object_op, line=-1)) - # Loading stuff def literal_static_name(self, value: Union[int, float, complex, str, bytes]) -> str: return self.mapper.literal_static_name(self.current_module, value) @@ -468,10 +455,27 @@ def load_static_unicode(self, value: str) -> Value: def load_module(self, name: str) -> Value: return self.add(LoadStatic(object_rprimitive, name, namespace=NAMESPACE_MODULE)) + def get_native_type(self, cls: ClassIR) -> Value: + """Load native type object.""" + fullname = '%s.%s' % (cls.module_name, cls.name) + return self.load_native_type_object(fullname) + def load_native_type_object(self, fullname: str) -> Value: module, name = fullname.rsplit('.', 1) return self.add(LoadStatic(object_rprimitive, name, module, NAMESPACE_TYPE)) + # Other primitive operations + + def primitive_op(self, desc: OpDescription, args: List[Value], line: int) -> Value: + assert desc.result_type is not None + coerced = [] + for i, arg in enumerate(args): + formal_type = self.op_arg_type(desc, i) + arg = self.coerce(arg, formal_type, line) + coerced.append(arg) + target = self.add(PrimitiveOp(coerced, desc, line)) + return target + def matching_primitive_op(self, candidates: List[OpDescription], args: List[Value], @@ -529,6 +533,38 @@ def unary_op(self, assert target, 'Unsupported unary operation: %s' % expr_op return target + def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: + result = None # type: Union[Value, None] + initial_items = [] # type: List[Value] + for key, value in key_value_pairs: + if key is not None: + # key:value + if result is None: + initial_items.extend((key, value)) + continue + + self.translate_special_method_call( + result, + '__setitem__', + [key, value], + result_type=None, + line=line) + else: + # **value + if result is None: + result = self.primitive_op(new_dict_op, initial_items, line) + + self.primitive_op( + dict_update_in_display_op, + [result, value], + line=line + ) + + if result is None: + result = self.primitive_op(new_dict_op, initial_items, line) + + return result + def builtin_call(self, args: List[Value], fn_op: str, @@ -607,6 +643,75 @@ def add_bool_branch(self, value: Value, true: BasicBlock, false: BasicBlock) -> value = self.primitive_op(bool_op, [value], value.line) self.add(Branch(value, true, false, Branch.BOOL_EXPR)) + # Internal helpers + + def decompose_union_helper(self, + obj: Value, + rtype: RUnion, + result_type: RType, + process_item: Callable[[Value], Value], + line: int) -> Value: + """Generate isinstance() + specialized operations for union items. + + Say, for Union[A, B] generate ops resembling this (pseudocode): + + if isinstance(obj, A): + result = + else: + result = + + Args: + obj: value with a union type + rtype: the union type + result_type: result of the operation + process_item: callback to generate op for a single union item (arg is coerced + to union item type) + line: line number + """ + # TODO: Optimize cases where a single operation can handle multiple union items + # (say a method is implemented in a common base class) + fast_items = [] + rest_items = [] + for item in rtype.items: + if isinstance(item, RInstance): + fast_items.append(item) + else: + # For everything but RInstance we fall back to C API + rest_items.append(item) + exit_block = BasicBlock() + result = self.alloc_temp(result_type) + for i, item in enumerate(fast_items): + more_types = i < len(fast_items) - 1 or rest_items + if more_types: + # We are not at the final item so we need one more branch + op = self.isinstance_native(obj, item.class_ir, line) + true_block, false_block = BasicBlock(), BasicBlock() + self.add_bool_branch(op, true_block, false_block) + self.activate_block(true_block) + coerced = self.coerce(obj, item, line) + temp = process_item(coerced) + temp2 = self.coerce(temp, result_type, line) + self.add(Assign(result, temp2)) + self.goto(exit_block) + if more_types: + self.activate_block(false_block) + if rest_items: + # For everything else we use generic operation. Use force=True to drop the + # union type. + coerced = self.coerce(obj, object_rprimitive, line, force=True) + temp = process_item(coerced) + temp2 = self.coerce(temp, result_type, line) + self.add(Assign(result, temp2)) + self.goto(exit_block) + self.activate_block(exit_block) + return result + + def op_arg_type(self, desc: OpDescription, n: int) -> RType: + if n >= len(desc.arg_types): + assert desc.is_var_arg + return desc.arg_types[-1] + return desc.arg_types[n] + def translate_special_method_call(self, base_reg: Value, name: str, @@ -629,6 +734,11 @@ def translate_eq_cmp(self, rreg: Value, expr_op: str, line: int) -> Optional[Value]: + """Add a equality comparison operation. + + Args: + expr_op: either '==' or '!=' + """ ltype = lreg.type rtype = rreg.type if not (isinstance(ltype, RInstance) and ltype == rtype): @@ -662,71 +772,3 @@ def translate_eq_cmp(self, ltype, line ) - - def gen_method_call(self, - base: Value, - name: str, - arg_values: List[Value], - result_type: Optional[RType], - line: int, - arg_kinds: Optional[List[int]] = None, - arg_names: Optional[List[Optional[str]]] = None) -> Value: - # If arg_kinds contains values other than arg_pos and arg_named, then fallback to - # Python method call. - if (arg_kinds is not None - and not all(kind in (ARG_POS, ARG_NAMED) for kind in arg_kinds)): - return self.py_method_call(base, name, arg_values, base.line, arg_kinds, arg_names) - - # If the base type is one of ours, do a MethodCall - if (isinstance(base.type, RInstance) and base.type.class_ir.is_ext_class - and not base.type.class_ir.builtin_base): - if base.type.class_ir.has_method(name): - decl = base.type.class_ir.method_decl(name) - if arg_kinds is None: - assert arg_names is None, "arg_kinds not present but arg_names is" - arg_kinds = [ARG_POS for _ in arg_values] - arg_names = [None for _ in arg_values] - else: - assert arg_names is not None, "arg_kinds present but arg_names is not" - - # Normalize args to positionals. - assert decl.bound_sig - arg_values = self.native_args_to_positional( - arg_values, arg_kinds, arg_names, decl.bound_sig, line) - return self.add(MethodCall(base, name, arg_values, line)) - elif base.type.class_ir.has_attr(name): - function = self.add(GetAttr(base, name, line)) - return self.py_call(function, arg_values, line, - arg_kinds=arg_kinds, arg_names=arg_names) - - elif isinstance(base.type, RUnion): - return self.union_method_call(base, base.type, name, arg_values, result_type, line, - arg_kinds, arg_names) - - # Try to do a special-cased method call - if not arg_kinds or arg_kinds == [ARG_POS] * len(arg_values): - target = self.translate_special_method_call(base, name, arg_values, result_type, line) - if target: - return target - - # Fall back to Python method call - return self.py_method_call(base, name, arg_values, line, arg_kinds, arg_names) - - def union_method_call(self, - base: Value, - obj_type: RUnion, - name: str, - arg_values: List[Value], - return_rtype: Optional[RType], - line: int, - arg_kinds: Optional[List[int]], - arg_names: Optional[List[Optional[str]]]) -> Value: - # Union method call needs a return_rtype for the type of the output register. - # If we don't have one, use object_rprimitive. - return_rtype = return_rtype or object_rprimitive - - def call_union_item(value: Value) -> Value: - return self.gen_method_call(value, name, arg_values, return_rtype, line, - arg_kinds, arg_names) - - return self.decompose_union_helper(base, obj_type, return_rtype, call_union_item, line) diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index d0dc0fda5a81..30e800c651f8 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -13,10 +13,11 @@ def f(x: int) -> int: r3 = r2 + r1 :: int return r3 -The IR is implemented in mypyc.ops. +This module deals with the module-level IR transformation logic and +putting it all together. The actual IR is implemented in mypyc.ir. -For the core of the implementation, look at build_ir() below, -mypyc.irbuild.builder, and mypyc.irbuild.visitor. +For the core of the IR transform implementation, look at build_ir() +below, mypyc.irbuild.builder, and mypyc.irbuild.visitor. """ from collections import OrderedDict @@ -54,6 +55,7 @@ def build_ir(modules: List[MypyFile], mapper: 'Mapper', options: CompilerOptions, errors: Errors) -> ModuleIRs: + """Build IR for a set of modules that have been type-checked by mypy.""" build_type_map(mapper, modules, graph, types, options, errors) @@ -95,6 +97,8 @@ def build_ir(modules: List[MypyFile], def transform_mypy_file(builder: IRBuilder, mypyfile: MypyFile) -> None: + """Generate IR for a single module.""" + if mypyfile.fullname in ('typing', 'abc'): # These module are special; their contents are currently all # built-in primitives. diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index ee143fbf735e..dfa7a99753fb 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -1,3 +1,5 @@ +"""Maintain a mapping from mypy concepts to IR/compiled concepts.""" + from typing import Dict, Optional, Union from collections import OrderedDict @@ -21,6 +23,9 @@ class Mapper: """Keep track of mappings from mypy concepts to IR concepts. + For example, we keep track of how the mypy TypeInfos of compiled + classes map to class IR objects. + This state is shared across all modules being compiled in all compilation groups. """ diff --git a/mypyc/irbuild/nonlocalcontrol.py b/mypyc/irbuild/nonlocalcontrol.py index 831698bf5016..2baacd6f372a 100644 --- a/mypyc/irbuild/nonlocalcontrol.py +++ b/mypyc/irbuild/nonlocalcontrol.py @@ -1,3 +1,8 @@ +"""Helpers for dealing with nonlocal control such as 'break' and 'return'. + +Model how these behave differently in different contexts. +""" + from abc import abstractmethod from typing import Optional, Union from typing_extensions import TYPE_CHECKING @@ -13,7 +18,7 @@ class NonlocalControl: - """Represents a stack frame of constructs that modify nonlocal control flow. + """ABC representing a stack frame of constructs that modify nonlocal control flow. The nonlocal control flow constructs are break, continue, and return, and their behavior is modified by a number of other @@ -35,6 +40,8 @@ def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: pas class BaseNonlocalControl(NonlocalControl): + """Default nonlocal control outside any statements that affect it.""" + def gen_break(self, builder: 'IRBuilder', line: int) -> None: assert False, "break outside of loop" @@ -46,8 +53,12 @@ def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: class LoopNonlocalControl(NonlocalControl): - def __init__(self, outer: NonlocalControl, - continue_block: BasicBlock, break_block: BasicBlock) -> None: + """Nonlocal control within a loop.""" + + def __init__(self, + outer: NonlocalControl, + continue_block: BasicBlock, + break_block: BasicBlock) -> None: self.outer = outer self.continue_block = continue_block self.break_block = break_block @@ -63,23 +74,31 @@ def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: class GeneratorNonlocalControl(BaseNonlocalControl): + """Default nonlocal control in a generator function outside statements.""" + def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: - # Assign an invalid next label number so that the next time __next__ is called, we jump to - # the case in which StopIteration is raised. + # Assign an invalid next label number so that the next time + # __next__ is called, we jump to the case in which + # StopIteration is raised. builder.assign(builder.fn_info.generator_class.next_label_target, builder.add(LoadInt(-1)), line) - # Raise a StopIteration containing a field for the value that should be returned. Before - # doing so, create a new block without an error handler set so that the implicitly thrown - # StopIteration isn't caught by except blocks inside of the generator function. + + # Raise a StopIteration containing a field for the value that + # should be returned. Before doing so, create a new block + # without an error handler set so that the implicitly thrown + # StopIteration isn't caught by except blocks inside of the + # generator function. builder.builder.push_error_handler(None) builder.goto_and_activate(BasicBlock()) + # Skip creating a traceback frame when we raise here, because # we don't care about the traceback frame and it is kind of - # expensive since raising StopIteration is an extremely common case. - # Also we call a special internal function to set StopIteration instead of - # using RaiseStandardError because the obvious thing doesn't work if the - # value is a tuple (???). + # expensive since raising StopIteration is an extremely common + # case. Also we call a special internal function to set + # StopIteration instead of using RaiseStandardError because + # the obvious thing doesn't work if the value is a tuple + # (???). builder.primitive_op(set_stop_iteration_value, [value], NO_TRACEBACK_LINE_NO) builder.add(Unreachable()) builder.builder.pop_error_handler() @@ -108,6 +127,8 @@ def gen_return(self, builder: 'IRBuilder', value: Value, line: int) -> None: class TryFinallyNonlocalControl(NonlocalControl): + """Nonlocal control within try/finally.""" + def __init__(self, target: BasicBlock) -> None: self.target = target self.ret_reg = None # type: Optional[Register] diff --git a/mypyc/irbuild/prebuildvisitor.py b/mypyc/irbuild/prebuildvisitor.py index 1f21e080d098..9050920813b2 100644 --- a/mypyc/irbuild/prebuildvisitor.py +++ b/mypyc/irbuild/prebuildvisitor.py @@ -7,110 +7,137 @@ class PreBuildVisitor(TraverserVisitor): + """Mypy file AST visitor run before building the IR. + + This collects various things, including: + + * Determine relationships between nested functions and functions that + contain nested functions + * Find non-local variables (free variables) + * Find property setters + * Find decorators of functions + + The main IR build pass uses this information. """ - Class used to visit a mypy file before building the IR for that program. This is done as a - first pass so that nested functions, encapsulating functions, lambda functions, decorated - functions, and free variables can be determined before instantiating the IRBuilder. - """ + def __init__(self) -> None: super().__init__() - # Mapping from FuncItem instances to sets of variables. The FuncItem instances are where - # these variables were first declared, and these variables are free in any functions that - # are nested within the FuncItem from which they are mapped. + # Dict from a function to symbols defined directly in the + # function that are used as non-local (free) variables within a + # nested function. self.free_variables = {} # type: Dict[FuncItem, Set[SymbolNode]] - # Intermediate data structure used to map SymbolNode instances to the FuncDef in which they - # were first visited. + + # Intermediate data structure used to find the function where + # a SymbolNode is declared. Initially this may point to a + # function nested inside the function with the declaration, + # but we'll eventually update this to refer to the function + # with the declaration. self.symbols_to_funcs = {} # type: Dict[SymbolNode, FuncItem] - # Stack representing the function call stack. + + # Stack representing current function nesting. self.funcs = [] # type: List[FuncItem] - # The set of property setters + + # All property setters encountered so far. self.prop_setters = set() # type: Set[FuncDef] + # A map from any function that contains nested functions to # a set of all the functions that are nested within it. self.encapsulating_funcs = {} # type: Dict[FuncItem, List[FuncItem]] - # A map from a nested func to it's parent/encapsulating func. + + # Map nested function to its parent/encapsulating function. self.nested_funcs = {} # type: Dict[FuncItem, FuncItem] - self.funcs_to_decorators = {} # type: Dict[FuncDef, List[Expression]] - def add_free_variable(self, symbol: SymbolNode) -> None: - # Get the FuncItem instance where the free symbol was first declared, and map that FuncItem - # to the SymbolNode representing the free symbol. - func = self.symbols_to_funcs[symbol] - self.free_variables.setdefault(func, set()).add(symbol) + # Map function to its non-special decorators. + self.funcs_to_decorators = {} # type: Dict[FuncDef, List[Expression]] def visit_decorator(self, dec: Decorator) -> None: if dec.decorators: - # Only add the function being decorated if there exist decorators in the decorator - # list. Note that meaningful decorators (@property, @abstractmethod) are removed from - # this list by mypy, but functions decorated by those decorators (in addition to - # property setters) do not need to be added to the set of decorated functions for - # the IRBuilder, because they are handled in a special way. + # Only add the function being decorated if there exist + # (ordinary) decorators in the decorator list. Certain + # decorators (such as @property, @abstractmethod) are + # special cased and removed from this list by + # mypy. Functions decorated only by special decorators + # (and property setters) are not treated as decorated + # functions by the IR builder. if isinstance(dec.decorators[0], MemberExpr) and dec.decorators[0].name == 'setter': + # Property setters are not treated as decorated methods. self.prop_setters.add(dec.func) else: self.funcs_to_decorators[dec.func] = dec.decorators super().visit_decorator(dec) + def visit_func_def(self, fdef: FuncItem) -> None: + # TODO: What about overloaded functions? + self.visit_func(fdef) + + def visit_lambda_expr(self, expr: LambdaExpr) -> None: + self.visit_func(expr) + def visit_func(self, func: FuncItem) -> None: - # If there were already functions or lambda expressions defined in the function stack, then - # note the previous FuncItem as containing a nested function and the current FuncItem as - # being a nested function. + # If there were already functions or lambda expressions + # defined in the function stack, then note the previous + # FuncItem as containing a nested function and the current + # FuncItem as being a nested function. if self.funcs: - # Add the new func to the set of nested funcs within the func at top of the func stack. + # Add the new func to the set of nested funcs within the + # func at top of the func stack. self.encapsulating_funcs.setdefault(self.funcs[-1], []).append(func) - # Add the func at top of the func stack as the parent of new func. + # Add the func at top of the func stack as the parent of + # new func. self.nested_funcs[func] = self.funcs[-1] self.funcs.append(func) super().visit_func(func) self.funcs.pop() - def visit_func_def(self, fdef: FuncItem) -> None: - self.visit_func(fdef) - - def visit_lambda_expr(self, expr: LambdaExpr) -> None: - self.visit_func(expr) - def visit_name_expr(self, expr: NameExpr) -> None: if isinstance(expr.node, (Var, FuncDef)): self.visit_symbol_node(expr.node) - # Check if child is contained within fdef (possibly indirectly within - # multiple nested functions). - def is_parent(self, fitem: FuncItem, child: FuncItem) -> bool: - if child in self.nested_funcs: - parent = self.nested_funcs[child] - if parent == fitem: - return True - return self.is_parent(fitem, parent) - return False + def visit_var(self, var: Var) -> None: + self.visit_symbol_node(var) def visit_symbol_node(self, symbol: SymbolNode) -> None: if not self.funcs: - # If the list of FuncDefs is empty, then we are not inside of a function and hence do - # not need to do anything regarding free variables. + # We are not inside a function and hence do not need to do + # anything regarding free variables. return if symbol in self.symbols_to_funcs: orig_func = self.symbols_to_funcs[symbol] if self.is_parent(self.funcs[-1], orig_func): - # If the function in which the symbol was originally seen is nested - # within the function currently being visited, fix the free_variable - # and symbol_to_funcs dictionaries. + # The function in which the symbol was previously seen is + # nested within the function currently being visited. Thus + # the current function is a better candidate to contain the + # declaration. self.symbols_to_funcs[symbol] = self.funcs[-1] + # TODO: Remove from the orig_func free_variables set? self.free_variables.setdefault(self.funcs[-1], set()).add(symbol) elif self.is_parent(orig_func, self.funcs[-1]): - # If the SymbolNode instance has already been visited before, - # and it was declared in a FuncDef not nested within the current - # FuncDef being visited, then it is a free symbol because it is - # being visited again. + # The SymbolNode instance has already been visited + # before in a parent function, thus it's a non-local + # symbol. self.add_free_variable(symbol) else: - # Otherwise, this is the first time the SymbolNode is being visited. We map the - # SymbolNode to the current FuncDef being visited to note where it was first visited. + # This is the first time the SymbolNode is being + # visited. We map the SymbolNode to the current FuncDef + # being visited to note where it was first visited. self.symbols_to_funcs[symbol] = self.funcs[-1] - def visit_var(self, var: Var) -> None: - self.visit_symbol_node(var) + def is_parent(self, fitem: FuncItem, child: FuncItem) -> bool: + # Check if child is nested within fdef (possibly indirectly + # within multiple nested functions). + if child in self.nested_funcs: + parent = self.nested_funcs[child] + if parent == fitem: + return True + return self.is_parent(fitem, parent) + return False + + def add_free_variable(self, symbol: SymbolNode) -> None: + # Find the function where the symbol was (likely) first declared, + # and mark is as a non-local symbol within that function. + func = self.symbols_to_funcs[symbol] + self.free_variables.setdefault(func, set()).add(symbol) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 8069cbd29d4c..4ac752f22f5f 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -1,3 +1,16 @@ +"""Prepare for IR transform. + +This needs to run after type checking and before generating IR. + +For example, construct partially initialized FuncIR and ClassIR +objects for all functions and classes. This allows us to bind +references to functions and classes before we've generated full IR for +functions or classes. The actual IR transform will then populate all +the missing bits, such as function bodies (basic blocks). + +Also build a mapping from mypy TypeInfos to ClassIR objects. +""" + from typing import List, Dict, Iterable, Optional, Union from mypy.nodes import ( diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 028db3201584..2fc8d9bb5f4a 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -1,3 +1,11 @@ +"""Transform mypy statement ASTs to mypyc IR (Intermediate Representation). + +The top-level AST transformation logic is implemented in mypyc.irbuild.visitor +and mypyc.irbuild.builder. + +A few statements are transformed in mypyc.irbuild.function (yield, for example). +""" + from typing import Optional, List, Tuple, Sequence, Callable import importlib.util diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index 7c574576f5f2..18d8306c869e 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -1,3 +1,5 @@ +"""Various utilities that don't depend on other modules in mypyc.irbuild.""" + from typing import Dict, Any, Union, Optional from mypy.nodes import ( diff --git a/mypyc/irbuild/vtable.py b/mypyc/irbuild/vtable.py index 0ebeb5752a55..41afed8505b5 100644 --- a/mypyc/irbuild/vtable.py +++ b/mypyc/irbuild/vtable.py @@ -1,3 +1,5 @@ +"""Compute vtables of native (extension) classes.""" + import itertools from mypyc.ir.class_ir import ClassIR, VTableEntries, VTableMethod, VTableAttr