[mypyc] Implement builtins.len primitive for list (#9271)

* implement list len primitive

* use pointer type

* rename

* add PyObject

* use CPyPtr

* revert RStruct design and fix tests

* list_len helper and updates according to review comments

* fix

* remove size_t_to_short_int

mypyc/codegen/emitfunc.py

@@ -477,7 +477,8 @@ def visit_get_element_ptr(self, op: GetElementPtr) -> None:
         # TODO: support tuple type
         assert isinstance(op.src_type, RStruct)
         assert op.field in op.src_type.names, "Invalid field name."
-        self.emit_line('%s = &%s.%s;' % (dest, src, op.field))
+        self.emit_line('%s = (%s)&((%s *)%s)->%s;' % (dest, op.type._ctype, op.src_type.name,
+                                                      src, op.field))
 
     # Helpers
 

mypyc/ir/ops.py

@@ -25,8 +25,7 @@
 from mypyc.ir.rtypes import (
     RType, RInstance, RTuple, RVoid, is_bool_rprimitive, is_int_rprimitive,
     is_short_int_rprimitive, is_none_rprimitive, object_rprimitive, bool_rprimitive,
-    short_int_rprimitive, int_rprimitive, void_rtype, is_c_py_ssize_t_rprimitive,
-    c_pyssize_t_rprimitive
+    short_int_rprimitive, int_rprimitive, void_rtype, pointer_rprimitive, is_pointer_rprimitive
 )
 from mypyc.common import short_name
 
@@ -1360,7 +1359,7 @@ def __init__(self, type: RType, src: Value, line: int = -1) -> None:
         self.type = type
         # TODO: for now we enforce that the src memory address should be Py_ssize_t
         #       later we should also support same width unsigned int
-        assert is_c_py_ssize_t_rprimitive(src.type)
+        assert is_pointer_rprimitive(src.type)
         self.src = src
 
     def sources(self) -> List[Value]:
@@ -1379,7 +1378,7 @@ class GetElementPtr(RegisterOp):
 
     def __init__(self, src: Value, src_type: RType, field: str, line: int = -1) -> None:
         super().__init__(line)
-        self.type = c_pyssize_t_rprimitive
+        self.type = pointer_rprimitive
         self.src = src
         self.src_type = src_type
         self.field = field
@@ -1388,7 +1387,7 @@ def sources(self) -> List[Value]:
         return [self.src]
 
     def to_str(self, env: Environment) -> str:
-        return env.format("%r = get_element_ptr %r %r :: %r", self, self.src,
+        return env.format("%r = get_element_ptr %r %s :: %r", self, self.src,
                           self.field, self.src_type)
 
     def accept(self, visitor: 'OpVisitor[T]') -> T:

mypyc/ir/rtypes.py

@@ -182,8 +182,10 @@ def __init__(self,
         self.size = size
         # TODO: For low-level integers, they actually don't have undefined values
         #       we need to figure out some way to represent here.
-        if ctype in ('CPyTagged', 'int32_t', 'int64_t'):
+        if ctype == 'CPyTagged':
             self.c_undefined = 'CPY_INT_TAG'
+        elif ctype in ('int32_t', 'int64_t', 'CPyPtr'):
+            self.c_undefined = '0'
         elif ctype == 'PyObject *':
             # Boxed types use the null pointer as the error value.
             self.c_undefined = 'NULL'
@@ -254,6 +256,10 @@ def __repr__(self) -> str:
 else:
     c_pyssize_t_rprimitive = int64_rprimitive
 
+# low level pointer, represented as integer in C backends
+pointer_rprimitive = RPrimitive('ptr', is_unboxed=True, is_refcounted=False,
+                              ctype='CPyPtr')  # type: Final
+
 # Floats are represent as 'float' PyObject * values. (In the future
 # we'll likely switch to a more efficient, unboxed representation.)
 float_rprimitive = RPrimitive('builtins.float', is_unboxed=False,
@@ -311,6 +317,10 @@ def is_c_py_ssize_t_rprimitive(rtype: RType) -> bool:
     return rtype is c_pyssize_t_rprimitive
 
 
+def is_pointer_rprimitive(rtype: RType) -> bool:
+    return rtype is pointer_rprimitive
+
+
 def is_float_rprimitive(rtype: RType) -> bool:
     return isinstance(rtype, RPrimitive) and rtype.name == 'builtins.float'
 
@@ -514,12 +524,8 @@ def compute_aligned_offsets_and_size(types: List[RType]) -> Tuple[List[int], int
     return offsets, final_size
 
 
-class StructInfo:
-    """Struct-like type Infomation
-
-    StructInfo should work with registry to ensure constraints like the unique naming
-    constraint for struct type
-    """
+class RStruct(RType):
+    """Represent CPython structs"""
     def __init__(self,
                  name: str,
                  names: List[str],
@@ -532,31 +538,7 @@ def __init__(self,
             for i in range(len(self.types) - len(self.names)):
                 self.names.append('_item' + str(i))
         self.offsets, self.size = compute_aligned_offsets_and_size(types)
-
-
-class RStruct(RType):
-    """Represent CPython structs"""
-    def __init__(self,
-                 info: StructInfo) -> None:
-        self.info = info
-        self.name = self.info.name
-        self._ctype = self.info.name
-
-    @property
-    def names(self) -> List[str]:
-        return self.info.names
-
-    @property
-    def types(self) -> List[RType]:
-        return self.info.types
-
-    @property
-    def offsets(self) -> List[int]:
-        return self.info.offsets
-
-    @property
-    def size(self) -> int:
-        return self.info.size
+        self._ctype = name
 
     def accept(self, visitor: 'RTypeVisitor[T]') -> T:
         return visitor.visit_rstruct(self)
@@ -571,10 +553,11 @@ def __repr__(self) -> str:
                                                           in zip(self.names, self.types)))
 
     def __eq__(self, other: object) -> bool:
-        return isinstance(other, RStruct) and self.info == other.info
+        return (isinstance(other, RStruct) and self.name == other.name
+                and self.names == other.names and self.types == other.types)
 
     def __hash__(self) -> int:
-        return hash(self.info)
+        return hash((self.name, tuple(self.names), tuple(self.types)))
 
     def serialize(self) -> JsonDict:
         assert False
@@ -687,3 +670,14 @@ def optional_value_type(rtype: RType) -> Optional[RType]:
 def is_optional_type(rtype: RType) -> bool:
     """Is rtype an optional type with exactly two union items?"""
     return optional_value_type(rtype) is not None
+
+
+PyObject = RStruct(
+    name='PyObject',
+    names=['ob_refcnt', 'ob_type'],
+    types=[c_pyssize_t_rprimitive, pointer_rprimitive])
+
+PyVarObject = RStruct(
+    name='PyVarObject',
+    names=['ob_base', 'ob_size'],
+    types=[PyObject, c_pyssize_t_rprimitive])

mypyc/irbuild/builder.py

@@ -44,7 +44,7 @@
 from mypyc.ir.func_ir import FuncIR, INVALID_FUNC_DEF
 from mypyc.ir.class_ir import ClassIR, NonExtClassInfo
 from mypyc.primitives.registry import func_ops, CFunctionDescription, c_function_ops
-from mypyc.primitives.list_ops import list_len_op, to_list, list_pop_last
+from mypyc.primitives.list_ops import to_list, list_pop_last
 from mypyc.primitives.dict_ops import dict_get_item_op, dict_set_item_op
 from mypyc.primitives.generic_ops import py_setattr_op, iter_op, next_op
 from mypyc.primitives.misc_ops import true_op, false_op, import_op
@@ -238,6 +238,9 @@ def binary_int_op(self, type: RType, lhs: Value, rhs: Value, op: int, line: int)
     def compare_tagged(self, lhs: Value, rhs: Value, op: str, line: int) -> Value:
         return self.builder.compare_tagged(lhs, rhs, op, line)
 
+    def list_len(self, val: Value, line: int) -> Value:
+        return self.builder.list_len(val, line)
+
     @property
     def environment(self) -> Environment:
         return self.builder.environment
@@ -508,7 +511,7 @@ def process_iterator_tuple_assignment(self,
         if target.star_idx is not None:
             post_star_vals = target.items[split_idx + 1:]
             iter_list = self.call_c(to_list, [iterator], line)
-            iter_list_len = self.primitive_op(list_len_op, [iter_list], line)
+            iter_list_len = self.list_len(iter_list, line)
             post_star_len = self.add(LoadInt(len(post_star_vals)))
             condition = self.binary_op(post_star_len, iter_list_len, '<=', line)
 

mypyc/irbuild/for_helpers.py

@@ -29,7 +29,6 @@
 from mypyc.primitives.exc_ops import no_err_occurred_op
 from mypyc.irbuild.builder import IRBuilder
 
-
 GenFunc = Callable[[], None]
 
 
@@ -333,6 +332,9 @@ def gen_cleanup(self) -> None:
 
     def load_len(self, expr: Union[Value, AssignmentTarget]) -> Value:
         """A helper to get collection length, used by several subclasses."""
+        val = self.builder.read(expr, self.line)
+        if is_list_rprimitive(val.type):
+            return self.builder.builder.list_len(self.builder.read(expr, self.line), self.line)
         return self.builder.builder.builtin_call(
             [self.builder.read(expr, self.line)],
             'builtins.len',

mypyc/irbuild/ll_builder.py

@@ -21,12 +21,13 @@
     Assign, Branch, Goto, Call, Box, Unbox, Cast, GetAttr,
     LoadStatic, MethodCall, PrimitiveOp, OpDescription, RegisterOp, CallC, Truncate,
     RaiseStandardError, Unreachable, LoadErrorValue, LoadGlobal,
-    NAMESPACE_TYPE, NAMESPACE_MODULE, NAMESPACE_STATIC, BinaryIntOp
+    NAMESPACE_TYPE, NAMESPACE_MODULE, NAMESPACE_STATIC, BinaryIntOp, GetElementPtr,
+    LoadMem
 )
 from mypyc.ir.rtypes import (
     RType, RUnion, RInstance, optional_value_type, int_rprimitive, float_rprimitive,
     bool_rprimitive, list_rprimitive, str_rprimitive, is_none_rprimitive, object_rprimitive,
-    c_pyssize_t_rprimitive, is_short_int_rprimitive, is_tagged
+    c_pyssize_t_rprimitive, is_short_int_rprimitive, is_tagged, PyVarObject, short_int_rprimitive
 )
 from mypyc.ir.func_ir import FuncDecl, FuncSignature
 from mypyc.ir.class_ir import ClassIR, all_concrete_classes
@@ -40,7 +41,7 @@
     c_binary_ops, c_unary_ops
 )
 from mypyc.primitives.list_ops import (
-    list_extend_op, list_len_op, new_list_op
+    list_extend_op, new_list_op
 )
 from mypyc.primitives.tuple_ops import list_tuple_op, new_tuple_op
 from mypyc.primitives.dict_ops import (
@@ -703,7 +704,7 @@ def add_bool_branch(self, value: Value, true: BasicBlock, false: BasicBlock) ->
             zero = self.add(LoadInt(0))
             value = self.binary_op(value, zero, '!=', value.line)
         elif is_same_type(value.type, list_rprimitive):
-            length = self.primitive_op(list_len_op, [value], value.line)
+            length = self.list_len(value, value.line)
             zero = self.add(LoadInt(0))
             value = self.binary_op(length, zero, '!=', value.line)
         elif (isinstance(value.type, RInstance) and value.type.class_ir.is_ext_class
@@ -810,6 +811,12 @@ def matching_call_c(self,
     def binary_int_op(self, type: RType, lhs: Value, rhs: Value, op: int, line: int) -> Value:
         return self.add(BinaryIntOp(type, lhs, rhs, op, line))
 
+    def list_len(self, val: Value, line: int) -> Value:
+        elem_address = self.add(GetElementPtr(val, PyVarObject, 'ob_size'))
+        size_value = self.add(LoadMem(c_pyssize_t_rprimitive, elem_address))
+        offset = self.add(LoadInt(1, -1, rtype=c_pyssize_t_rprimitive))
+        return self.binary_int_op(short_int_rprimitive, size_value, offset,
+                                  BinaryIntOp.LEFT_SHIFT, -1)
     # Internal helpers
 
     def decompose_union_helper(self,

mypyc/irbuild/specialize.py

@@ -18,11 +18,11 @@
 from mypy.types import AnyType, TypeOfAny
 
 from mypyc.ir.ops import (
-    Value, BasicBlock, LoadInt, RaiseStandardError, Unreachable, OpDescription
+    Value, BasicBlock, LoadInt, RaiseStandardError, Unreachable, OpDescription,
 )
 from mypyc.ir.rtypes import (
     RType, RTuple, str_rprimitive, list_rprimitive, dict_rprimitive, set_rprimitive,
-    bool_rprimitive, is_dict_rprimitive
+    bool_rprimitive, is_dict_rprimitive, is_list_rprimitive,
 )
 from mypyc.primitives.dict_ops import dict_keys_op, dict_values_op, dict_items_op
 from mypyc.primitives.misc_ops import true_op, false_op
@@ -75,6 +75,9 @@ def translate_len(
             # though we still need to evaluate it.
             builder.accept(expr.args[0])
             return builder.add(LoadInt(len(expr_rtype.types)))
+        elif is_list_rprimitive(expr_rtype):
+            obj = builder.accept(expr.args[0])
+            return builder.list_len(obj, -1)
     return None
 
 

mypyc/lib-rt/mypyc_util.h

@@ -32,6 +32,7 @@
 #define CPy_XDECREF(p) Py_XDECREF(p)
 
 typedef size_t CPyTagged;
+typedef size_t CPyPtr;
 
 #define CPY_INT_BITS (CHAR_BIT * sizeof(CPyTagged))
 

mypyc/primitives/list_ops.py

@@ -8,7 +8,7 @@
     c_int_rprimitive
 )
 from mypyc.primitives.registry import (
-    name_ref_op, func_op, custom_op, name_emit,
+    name_ref_op, custom_op, name_emit,
     call_emit, c_function_op, c_binary_op, c_method_op
 )
 
@@ -146,11 +146,3 @@ def emit_len(emitter: EmitterInterface, args: List[str], dest: str) -> None:
     emitter.emit_declaration('Py_ssize_t %s;' % temp)
     emitter.emit_line('%s = PyList_GET_SIZE(%s);' % (temp, args[0]))
     emitter.emit_line('%s = CPyTagged_ShortFromSsize_t(%s);' % (dest, temp))
-
-
-# len(list)
-list_len_op = func_op(name='builtins.len',
-                      arg_types=[list_rprimitive],
-                      result_type=short_int_rprimitive,
-                      error_kind=ERR_NEVER,
-                      emit=emit_len)