From 5601e359593439a9e1cc1f38768020bbceb7f576 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Tue, 22 Jun 2021 04:14:37 +0800 Subject: [PATCH 01/14] Add branch for str --- mypyc/irbuild/ll_builder.py | 9 ++++++++- mypyc/primitives/str_ops.py | 10 +++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index ef2e482e45c2..11a3563a8c7e 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -63,7 +63,7 @@ ) from mypyc.primitives.int_ops import int_comparison_op_mapping from mypyc.primitives.exc_ops import err_occurred_op, keep_propagating_op -from mypyc.primitives.str_ops import unicode_compare, str_check_if_true +from mypyc.primitives.str_ops import unicode_compare, str_check_if_true, str_size_op from mypyc.primitives.set_ops import new_set_op from mypyc.rt_subtype import is_runtime_subtype from mypyc.subtype import is_subtype @@ -1132,6 +1132,13 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val offset = Integer(1, c_pyssize_t_rprimitive, line) return self.int_op(short_int_rprimitive, size_value, offset, IntOp.LEFT_SHIFT, line) + elif is_str_rprimitive(typ): + size_value = self.call_c(str_size_op, [val], line) + if use_pyssize_t: + return size_value + offset = Integer(1, c_pyssize_t_rprimitive, line) + return self.int_op(short_int_rprimitive, size_value, offset, + IntOp.LEFT_SHIFT, line) elif is_set_rprimitive(typ): elem_address = self.add(GetElementPtr(val, PySetObject, 'used')) size_value = self.add(LoadMem(c_pyssize_t_rprimitive, elem_address)) diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index 4e56d885528b..9e9706b2d7e6 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -5,7 +5,8 @@ from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( RType, object_rprimitive, str_rprimitive, int_rprimitive, list_rprimitive, - c_int_rprimitive, pointer_rprimitive, bool_rprimitive, bit_rprimitive + c_int_rprimitive, pointer_rprimitive, bool_rprimitive, bit_rprimitive, + c_pyssize_t_rprimitive ) from mypyc.primitives.registry import ( method_op, binary_op, function_op, @@ -134,3 +135,10 @@ c_function_name="CPyStr_IsTrue", error_kind=ERR_NEVER, ) + +str_size_op = custom_op( + arg_types=[str_rprimitive], + return_type=c_pyssize_t_rprimitive, + c_function_name='PyUnicode_Size', + error_kind=ERR_NEVER) + From 96213011be8ef3b0ee14e8dc8d78083124c1d573 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Tue, 22 Jun 2021 04:20:54 +0800 Subject: [PATCH 02/14] Fix method name --- mypyc/primitives/str_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index 9e9706b2d7e6..c12827513927 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -139,6 +139,6 @@ str_size_op = custom_op( arg_types=[str_rprimitive], return_type=c_pyssize_t_rprimitive, - c_function_name='PyUnicode_Size', + c_function_name='PyUnicode_GET_LENGTH', error_kind=ERR_NEVER) From cc98072e00cef78c7c2fc8ec230daaeb87e8bb1b Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Tue, 22 Jun 2021 14:06:39 +0800 Subject: [PATCH 03/14] Support str in builtin_len --- mypyc/irbuild/ll_builder.py | 17 ++++++++++------- mypyc/primitives/dict_ops.py | 2 +- mypyc/primitives/str_ops.py | 2 +- mypyc/test-data/irbuild-str.test | 11 +++++++---- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 11a3563a8c7e..43e15fc7bee9 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -51,7 +51,7 @@ list_tuple_op, new_tuple_op, new_tuple_with_length_op ) from mypyc.primitives.dict_ops import ( - dict_update_in_display_op, dict_new_op, dict_build_op, dict_size_op + dict_update_in_display_op, dict_new_op, dict_build_op, dict_ssize_t_size_op ) from mypyc.primitives.generic_ops import ( py_getattr_op, py_call_op, py_call_with_kwargs_op, py_method_call_op, @@ -63,7 +63,9 @@ ) from mypyc.primitives.int_ops import int_comparison_op_mapping from mypyc.primitives.exc_ops import err_occurred_op, keep_propagating_op -from mypyc.primitives.str_ops import unicode_compare, str_check_if_true, str_size_op +from mypyc.primitives.str_ops import ( + unicode_compare, str_check_if_true, str_ssize_t_size_op +) from mypyc.primitives.set_ops import new_set_op from mypyc.rt_subtype import is_runtime_subtype from mypyc.subtype import is_subtype @@ -982,10 +984,11 @@ def add_bool_branch(self, value: Value, true: BasicBlock, false: BasicBlock) -> zero = Integer(0, short_int_rprimitive) self.compare_tagged_condition(value, zero, '!=', true, false, value.line) return - elif is_same_type(value.type, str_rprimitive): - value = self.call_c(str_check_if_true, [value], value.line) + # elif is_same_type(value.type, str_rprimitive): + # value = self.call_c(str_check_if_true, [value], value.line) elif (is_same_type(value.type, list_rprimitive) - or is_same_type(value.type, dict_rprimitive)): + or is_same_type(value.type, dict_rprimitive) + or is_same_type(value.type, str_rprimitive)): length = self.builtin_len(value, value.line) zero = Integer(0) value = self.binary_op(length, zero, '!=', value.line) @@ -1126,14 +1129,14 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val return self.int_op(short_int_rprimitive, size_value, offset, IntOp.LEFT_SHIFT, line) elif is_dict_rprimitive(typ): - size_value = self.call_c(dict_size_op, [val], line) + size_value = self.call_c(dict_ssize_t_size_op, [val], line) if use_pyssize_t: return size_value offset = Integer(1, c_pyssize_t_rprimitive, line) return self.int_op(short_int_rprimitive, size_value, offset, IntOp.LEFT_SHIFT, line) elif is_str_rprimitive(typ): - size_value = self.call_c(str_size_op, [val], line) + size_value = self.call_c(str_ssize_t_size_op, [val], line) if use_pyssize_t: return size_value offset = Integer(1, c_pyssize_t_rprimitive, line) diff --git a/mypyc/primitives/dict_ops.py b/mypyc/primitives/dict_ops.py index e176ffcac316..5ae96a174904 100644 --- a/mypyc/primitives/dict_ops.py +++ b/mypyc/primitives/dict_ops.py @@ -240,7 +240,7 @@ c_function_name='CPyDict_CheckSize', error_kind=ERR_FALSE) -dict_size_op = custom_op( +dict_ssize_t_size_op = custom_op( arg_types=[dict_rprimitive], return_type=c_pyssize_t_rprimitive, c_function_name='PyDict_Size', diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index c12827513927..a94cd480ec8b 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -136,7 +136,7 @@ error_kind=ERR_NEVER, ) -str_size_op = custom_op( +str_ssize_t_size_op = custom_op( arg_types=[str_rprimitive], return_type=c_pyssize_t_rprimitive, c_function_name='PyUnicode_GET_LENGTH', diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index 3de91be40ea2..ce98c746c474 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -148,14 +148,17 @@ def is_true(x: str) -> bool: [out] def is_true(x): x :: str - r0 :: bit + r0 :: native_int + r1 :: short_int + r2 :: bit L0: - r0 = CPyStr_IsTrue(x) - if r0 goto L1 else goto L2 :: bool + r0 = PyUnicode_GET_LENGTH(x) + r1 = r0 << 1 + r2 = r1 != 0 + if r2 goto L1 else goto L2 :: bool L1: return 1 L2: return 0 L3: unreachable - From 52bd1ef1e74d743a3ab1a8d18d1382e93f8310db Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Tue, 22 Jun 2021 14:37:06 +0800 Subject: [PATCH 04/14] Revert and simplify --- mypyc/irbuild/ll_builder.py | 46 ++++++++++++-------------------- mypyc/test-data/irbuild-str.test | 10 +++---- 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 43e15fc7bee9..3ad1eed91303 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -984,11 +984,10 @@ def add_bool_branch(self, value: Value, true: BasicBlock, false: BasicBlock) -> zero = Integer(0, short_int_rprimitive) self.compare_tagged_condition(value, zero, '!=', true, false, value.line) return - # elif is_same_type(value.type, str_rprimitive): - # value = self.call_c(str_check_if_true, [value], value.line) + elif is_same_type(value.type, str_rprimitive): + value = self.call_c(str_check_if_true, [value], value.line) elif (is_same_type(value.type, list_rprimitive) - or is_same_type(value.type, dict_rprimitive) - or is_same_type(value.type, str_rprimitive)): + or is_same_type(value.type, dict_rprimitive)): length = self.builtin_len(value, value.line) zero = Integer(0) value = self.binary_op(length, zero, '!=', value.line) @@ -1119,39 +1118,28 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val Return c_pyssize_t if use_pyssize_t is true (unshifted). """ typ = val.type + size_value = None if is_list_rprimitive(typ) or is_tuple_rprimitive(typ): elem_address = self.add(GetElementPtr(val, PyVarObject, 'ob_size')) size_value = self.add(LoadMem(c_pyssize_t_rprimitive, elem_address)) self.add(KeepAlive([val])) - if use_pyssize_t: - return size_value - offset = Integer(1, c_pyssize_t_rprimitive, line) - return self.int_op(short_int_rprimitive, size_value, offset, - IntOp.LEFT_SHIFT, line) - elif is_dict_rprimitive(typ): - size_value = self.call_c(dict_ssize_t_size_op, [val], line) - if use_pyssize_t: - return size_value - offset = Integer(1, c_pyssize_t_rprimitive, line) - return self.int_op(short_int_rprimitive, size_value, offset, - IntOp.LEFT_SHIFT, line) - elif is_str_rprimitive(typ): - size_value = self.call_c(str_ssize_t_size_op, [val], line) - if use_pyssize_t: - return size_value - offset = Integer(1, c_pyssize_t_rprimitive, line) - return self.int_op(short_int_rprimitive, size_value, offset, - IntOp.LEFT_SHIFT, line) elif is_set_rprimitive(typ): elem_address = self.add(GetElementPtr(val, PySetObject, 'used')) size_value = self.add(LoadMem(c_pyssize_t_rprimitive, elem_address)) self.add(KeepAlive([val])) + elif is_dict_rprimitive(typ): + size_value = self.call_c(dict_ssize_t_size_op, [val], line) + elif is_str_rprimitive(typ): + size_value = self.call_c(str_ssize_t_size_op, [val], line) + + if size_value is not None: if use_pyssize_t: return size_value offset = Integer(1, c_pyssize_t_rprimitive, line) return self.int_op(short_int_rprimitive, size_value, offset, IntOp.LEFT_SHIFT, line) - elif isinstance(typ, RInstance): + + if isinstance(typ, RInstance): # TODO: Support use_pyssize_t assert not use_pyssize_t length = self.gen_method_call(val, '__len__', [], int_rprimitive, line) @@ -1165,12 +1153,12 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val self.add(Unreachable()) self.activate_block(ok) return length + + # generic case + if use_pyssize_t: + return self.call_c(generic_ssize_t_len_op, [val], line) else: - # generic case - if use_pyssize_t: - return self.call_c(generic_ssize_t_len_op, [val], line) - else: - return self.call_c(generic_len_op, [val], line) + return self.call_c(generic_len_op, [val], line) def new_tuple(self, items: List[Value], line: int) -> Value: size = Integer(len(items), c_pyssize_t_rprimitive) # type: Value diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index ce98c746c474..936cd4b0b4f0 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -148,14 +148,10 @@ def is_true(x: str) -> bool: [out] def is_true(x): x :: str - r0 :: native_int - r1 :: short_int - r2 :: bit + r0 :: bit L0: - r0 = PyUnicode_GET_LENGTH(x) - r1 = r0 << 1 - r2 = r1 != 0 - if r2 goto L1 else goto L2 :: bool + r0 = CPyStr_IsTrue(x) + if r0 goto L1 else goto L2 :: bool L1: return 1 L2: From 2a880c75cb63278c9fe507b40c56448814511b29 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Tue, 22 Jun 2021 15:03:16 +0800 Subject: [PATCH 05/14] Add test about list comprehension for str --- mypyc/irbuild/for_helpers.py | 5 +++-- mypyc/test-data/run-lists.test | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 2e44ec3afaf5..98878cea64c6 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -16,7 +16,7 @@ ) from mypyc.ir.rtypes import ( RType, is_short_int_rprimitive, is_list_rprimitive, is_sequence_rprimitive, - is_tuple_rprimitive, is_dict_rprimitive, + is_tuple_rprimitive, is_dict_rprimitive, is_str_rprimitive, RTuple, short_int_rprimitive, int_rprimitive ) from mypyc.primitives.registry import CFunctionDescription @@ -164,7 +164,8 @@ def sequence_from_generator_preallocate_helper( """ if len(gen.sequences) == 1 and len(gen.indices) == 1 and len(gen.condlists[0]) == 0: rtype = builder.node_type(gen.sequences[0]) - if is_list_rprimitive(rtype) or is_tuple_rprimitive(rtype): + if (is_list_rprimitive(rtype) or is_tuple_rprimitive(rtype) + or is_str_rprimitive(rtype)): sequence = builder.accept(gen.sequences[0]) length = builder.builder.builtin_len(sequence, gen.line, use_pyssize_t=True) target_op = empty_op_llbuilder(length, gen.line) diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index a78b2d65de4b..9ef337c4e075 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -357,3 +357,7 @@ def test() -> None: source_e = [0, 1, 2] e = list((x ** 2) for x in (y + 2 for y in source_e)) assert e == [4, 9, 16] + source_str = "abcd" + f = list("str:" + x for x in source_str) + assert f == ["str:a", "str:b", "str:c", "str:d"] + From 3b6b9342b9c6d212bda09c0c9aeddee6b47c7b49 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Fri, 25 Jun 2021 16:32:28 +0800 Subject: [PATCH 06/14] Modify irbuild test and add run test for tuple creation --- mypyc/irbuild/specialize.py | 4 +- mypyc/test-data/irbuild-tuple.test | 87 ++++++++++-------------------- mypyc/test-data/run-tuples.test | 4 ++ 3 files changed, 34 insertions(+), 61 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index ef2986cda0b1..367a2fc33268 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -121,7 +121,7 @@ def dict_methods_fast_path( def translate_list_from_generator_call( builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: # Special case for simplest list comprehension, for example - # list(f(x) for x in other_list/other_tuple) + # list(f(x) for x in some_list/some_tuple/some_str) # translate_list_comprehension() would take care of other cases if this fails. if (len(expr.args) == 1 and expr.arg_kinds[0] == ARG_POS @@ -137,7 +137,7 @@ def translate_list_from_generator_call( def translate_tuple_from_generator_call( builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: # Special case for simplest tuple creation from a generator, for example - # tuple(f(x) for x in other_list/other_tuple) + # tuple(f(x) for x in some_list/some_tuple/some_str) # translate_safe_generator_call() would take care of other cases if this fails. if (len(expr.args) == 1 and expr.arg_kinds[0] == ARG_POS diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 20590645d5d3..00fb66aa6a02 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -350,14 +350,13 @@ L4: return 1 -[case testTupleBuiltFromList2] +[case testTupleBuiltFromStr] def f2(val: str) -> str: return val + "f2" def test() -> None: - source = ["a", "b", "c"] + source = "abc" a = tuple(f2(x) for x in source) - print(a) [out] def f2(val): val, r0, r1 :: str @@ -366,71 +365,41 @@ L0: r1 = PyUnicode_Concat(val, r0) return r1 def test(): - r0, r1, r2 :: str - r3 :: list - r4, r5, r6, r7 :: ptr - source :: list - r8 :: ptr - r9 :: native_int - r10 :: tuple - r11 :: short_int - r12 :: ptr - r13 :: native_int - r14 :: short_int - r15 :: bit - r16 :: object - r17, x, r18 :: str - r19 :: bit - r20 :: short_int + r0, source :: str + r1 :: native_int + r2 :: tuple + r3 :: short_int + r4 :: native_int + r5 :: short_int + r6 :: bit + r7, x, r8 :: str + r9 :: bit + r10 :: short_int a :: tuple - r21 :: object - r22 :: str - r23, r24 :: object L0: - r0 = 'a' - r1 = 'b' - r2 = 'c' - r3 = PyList_New(3) - r4 = get_element_ptr r3 ob_item :: PyListObject - r5 = load_mem r4 :: ptr* - set_mem r5, r0 :: builtins.object* - r6 = r5 + WORD_SIZE*1 - set_mem r6, r1 :: builtins.object* - r7 = r5 + WORD_SIZE*2 - set_mem r7, r2 :: builtins.object* - keep_alive r3 - source = r3 - r8 = get_element_ptr source ob_size :: PyVarObject - r9 = load_mem r8 :: native_int* - keep_alive source - r10 = PyTuple_New(r9) - r11 = 0 + r0 = 'abc' + source = r0 + r1 = PyUnicode_GET_LENGTH(source) + r2 = PyTuple_New(r1) + r3 = 0 L1: - r12 = get_element_ptr source ob_size :: PyVarObject - r13 = load_mem r12 :: native_int* - keep_alive source - r14 = r13 << 1 - r15 = r11 < r14 :: signed - if r15 goto L2 else goto L4 :: bool + r4 = PyUnicode_GET_LENGTH(source) + r5 = r4 << 1 + r6 = r3 < r5 :: signed + if r6 goto L2 else goto L4 :: bool L2: - r16 = CPyList_GetItemUnsafe(source, r11) - r17 = cast(str, r16) - x = r17 - r18 = f2(x) - r19 = CPySequenceTuple_SetItemUnsafe(r10, r11, r18) + r7 = CPyStr_GetItem(source, r3) + x = r7 + r8 = f2(x) + r9 = CPySequenceTuple_SetItemUnsafe(r2, r3, r8) L3: - r20 = r11 + 2 - r11 = r20 + r10 = r3 + 2 + r3 = r10 goto L1 L4: - a = r10 - r21 = builtins :: module - r22 = 'print' - r23 = CPyObject_GetAttr(r21, r22) - r24 = PyObject_CallFunctionObjArgs(r23, a, 0) + a = r2 return 1 - [case testTupleBuiltFromVariableLengthTuple] from typing import Tuple diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test index c4abfb1d3a36..5ef243b10aa0 100644 --- a/mypyc/test-data/run-tuples.test +++ b/mypyc/test-data/run-tuples.test @@ -196,6 +196,10 @@ def test_sequence_generator() -> None: a = tuple(f8(x) for x in source_fixed_length_tuple) assert a == (False, True, False, True) + source_str = "abbc" + a = tuple("s:" + x for x in source_str) + assert a == ("s:a", "s:b", "s:b", "s:c") + TUPLE: Final[Tuple[str, ...]] = ('x', 'y') def test_final_boxed_tuple() -> None: From 253a99b7ff15d7b653e68fd1b20678c0d1cd86a6 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Mon, 28 Jun 2021 03:01:07 +0800 Subject: [PATCH 07/14] Add a wrapper function --- mypyc/lib-rt/str_ops.c | 8 ++++++++ mypyc/primitives/str_ops.py | 7 +++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 7208a9b7e206..a933b16b0b82 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -102,6 +102,14 @@ PyObject *CPyStr_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { } return CPyObject_GetSlice(obj, start, end); } + +Py_ssize_t CPyStr_Size_size_t(PyObject *str) { + if (PyUnicode_READY(str) != -1) { + return PyUnicode_GET_LENGTH(str); + } + return -1; +} + /* Check if the given string is true (i.e. it's length isn't zero) */ bool CPyStr_IsTrue(PyObject *obj) { Py_ssize_t length = PyUnicode_GET_LENGTH(obj); diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index a94cd480ec8b..4ffb0e7dc8c7 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -10,7 +10,7 @@ ) from mypyc.primitives.registry import ( method_op, binary_op, function_op, - load_address_op, custom_op + load_address_op, custom_op, ERR_NEG_INT ) @@ -139,6 +139,5 @@ str_ssize_t_size_op = custom_op( arg_types=[str_rprimitive], return_type=c_pyssize_t_rprimitive, - c_function_name='PyUnicode_GET_LENGTH', - error_kind=ERR_NEVER) - + c_function_name='CPyStr_Size_size_t', + error_kind=ERR_NEG_INT) From 4c717faa4ea4f4cf24aed0583d4e8da5ad664ccf Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Tue, 29 Jun 2021 00:31:26 +0800 Subject: [PATCH 08/14] Fix test --- mypyc/lib-rt/str_ops.c | 7 ------- mypyc/primitives/str_ops.py | 10 +++++----- mypyc/test-data/run-tuples.test | 6 +++--- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index a933b16b0b82..328020d9fb17 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -103,13 +103,6 @@ PyObject *CPyStr_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { return CPyObject_GetSlice(obj, start, end); } -Py_ssize_t CPyStr_Size_size_t(PyObject *str) { - if (PyUnicode_READY(str) != -1) { - return PyUnicode_GET_LENGTH(str); - } - return -1; -} - /* Check if the given string is true (i.e. it's length isn't zero) */ bool CPyStr_IsTrue(PyObject *obj) { Py_ssize_t length = PyUnicode_GET_LENGTH(obj); diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index 4ffb0e7dc8c7..1a3efcaead48 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -89,7 +89,7 @@ # str1 += str2 # -# PyUnicodeAppend makes an effort to reuse the LHS when the refcount +# PyUnicode_Append makes an effort to reuse the LHS when the refcount # is 1. This is super dodgy but oh well, the interpreter does it. binary_op(name='+=', arg_types=[str_rprimitive, str_rprimitive], @@ -116,7 +116,7 @@ name='replace', arg_types=[str_rprimitive, str_rprimitive, str_rprimitive], return_type=str_rprimitive, - c_function_name="PyUnicode_Replace", + c_function_name='PyUnicode_Replace', error_kind=ERR_MAGIC, extra_int_constants=[(-1, c_int_rprimitive)]) @@ -125,19 +125,19 @@ name='replace', arg_types=[str_rprimitive, str_rprimitive, str_rprimitive, int_rprimitive], return_type=str_rprimitive, - c_function_name="CPyStr_Replace", + c_function_name='CPyStr_Replace', error_kind=ERR_MAGIC) # check if a string is true (isn't an empty string) str_check_if_true = custom_op( arg_types=[str_rprimitive], return_type=bit_rprimitive, - c_function_name="CPyStr_IsTrue", + c_function_name='CPyStr_IsTrue', error_kind=ERR_NEVER, ) str_ssize_t_size_op = custom_op( arg_types=[str_rprimitive], return_type=c_pyssize_t_rprimitive, - c_function_name='CPyStr_Size_size_t', + c_function_name='PyUnicode_GET_LENGTH', error_kind=ERR_NEG_INT) diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test index 5ef243b10aa0..759177342fa9 100644 --- a/mypyc/test-data/run-tuples.test +++ b/mypyc/test-data/run-tuples.test @@ -196,9 +196,9 @@ def test_sequence_generator() -> None: a = tuple(f8(x) for x in source_fixed_length_tuple) assert a == (False, True, False, True) - source_str = "abbc" - a = tuple("s:" + x for x in source_str) - assert a == ("s:a", "s:b", "s:b", "s:c") + source_str = 'abbc' + b = tuple('s:' + x for x in source_str) + assert b == ('s:a', 's:b', 's:b', 's:c') TUPLE: Final[Tuple[str, ...]] = ('x', 'y') From 872fd8258610f5860c5187be6103a0d5d401c96c Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Tue, 29 Jun 2021 01:02:09 +0800 Subject: [PATCH 09/14] Fix irbuild test --- mypyc/test-data/irbuild-tuple.test | 44 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 00fb66aa6a02..81996109644f 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -367,37 +367,41 @@ L0: def test(): r0, source :: str r1 :: native_int - r2 :: tuple - r3 :: short_int - r4 :: native_int - r5 :: short_int + r2 :: bit + r3 :: tuple + r4 :: short_int + r5 :: native_int r6 :: bit - r7, x, r8 :: str - r9 :: bit - r10 :: short_int + r7 :: short_int + r8 :: bit + r9, x, r10 :: str + r11 :: bit + r12 :: short_int a :: tuple L0: r0 = 'abc' source = r0 r1 = PyUnicode_GET_LENGTH(source) - r2 = PyTuple_New(r1) - r3 = 0 + r2 = r1 >= 0 :: signed + r3 = PyTuple_New(r1) + r4 = 0 L1: - r4 = PyUnicode_GET_LENGTH(source) - r5 = r4 << 1 - r6 = r3 < r5 :: signed - if r6 goto L2 else goto L4 :: bool + r5 = PyUnicode_GET_LENGTH(source) + r6 = r5 >= 0 :: signed + r7 = r5 << 1 + r8 = r4 < r7 :: signed + if r8 goto L2 else goto L4 :: bool L2: - r7 = CPyStr_GetItem(source, r3) - x = r7 - r8 = f2(x) - r9 = CPySequenceTuple_SetItemUnsafe(r2, r3, r8) + r9 = CPyStr_GetItem(source, r4) + x = r9 + r10 = f2(x) + r11 = CPySequenceTuple_SetItemUnsafe(r3, r4, r10) L3: - r10 = r3 + 2 - r3 = r10 + r12 = r4 + 2 + r4 = r12 goto L1 L4: - a = r2 + a = r3 return 1 [case testTupleBuiltFromVariableLengthTuple] From 0689db6d9b87fc49f7bfeb87cb0036add49c51f0 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Tue, 29 Jun 2021 22:39:33 +0800 Subject: [PATCH 10/14] Add a wrapper function --- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/str_ops.c | 4 ++++ mypyc/primitives/str_ops.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index f1a342527b11..cd57380160ce 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -391,6 +391,7 @@ PyObject *CPyStr_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); bool CPyStr_Startswith(PyObject *self, PyObject *subobj); bool CPyStr_Endswith(PyObject *self, PyObject *subobj); bool CPyStr_IsTrue(PyObject *obj); +Py_ssize_t CPyStr_Size_size_t(PyObject *str); // Set operations diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 328020d9fb17..60b462b76fa2 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -108,3 +108,7 @@ bool CPyStr_IsTrue(PyObject *obj) { Py_ssize_t length = PyUnicode_GET_LENGTH(obj); return length != 0; } + +Py_ssize_t CPyStr_Size_size_t(PyObject *str) { + return PyUnicode_GET_LENGTH(str); +} \ No newline at end of file diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index 1a3efcaead48..b8480a2831c2 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -139,5 +139,5 @@ str_ssize_t_size_op = custom_op( arg_types=[str_rprimitive], return_type=c_pyssize_t_rprimitive, - c_function_name='PyUnicode_GET_LENGTH', + c_function_name='CPyStr_Size_size_t', error_kind=ERR_NEG_INT) From f46d88306fd71c96850b5f3183a23210369653b5 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Tue, 29 Jun 2021 22:49:10 +0800 Subject: [PATCH 11/14] Fix --- mypyc/primitives/str_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index b8480a2831c2..23013c40eb26 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -10,7 +10,7 @@ ) from mypyc.primitives.registry import ( method_op, binary_op, function_op, - load_address_op, custom_op, ERR_NEG_INT + load_address_op, custom_op ) @@ -140,4 +140,4 @@ arg_types=[str_rprimitive], return_type=c_pyssize_t_rprimitive, c_function_name='CPyStr_Size_size_t', - error_kind=ERR_NEG_INT) + error_kind=ERR_NEVER) From 6148994b30c3844a4c585a34e307239018cacdf2 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Wed, 30 Jun 2021 00:23:11 +0800 Subject: [PATCH 12/14] Fix irbuild test --- mypyc/test-data/irbuild-tuple.test | 46 ++++++++++++++---------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 81996109644f..28ceea1e026d 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -367,41 +367,37 @@ L0: def test(): r0, source :: str r1 :: native_int - r2 :: bit - r3 :: tuple - r4 :: short_int - r5 :: native_int + r2 :: tuple + r3 :: short_int + r4 :: native_int + r5 :: short_int r6 :: bit - r7 :: short_int - r8 :: bit - r9, x, r10 :: str - r11 :: bit - r12 :: short_int + r7, x, r8 :: str + r9 :: bit + r10 :: short_int a :: tuple L0: r0 = 'abc' source = r0 - r1 = PyUnicode_GET_LENGTH(source) - r2 = r1 >= 0 :: signed - r3 = PyTuple_New(r1) - r4 = 0 + r1 = CPyStr_Size_size_t(source) + r2 = PyTuple_New(r1) + r3 = 0 L1: - r5 = PyUnicode_GET_LENGTH(source) - r6 = r5 >= 0 :: signed - r7 = r5 << 1 - r8 = r4 < r7 :: signed - if r8 goto L2 else goto L4 :: bool + r4 = CPyStr_Size_size_t(source) + r5 = r4 << 1 + r6 = r3 < r5 :: signed + if r6 goto L2 else goto L4 :: bool L2: - r9 = CPyStr_GetItem(source, r4) - x = r9 - r10 = f2(x) - r11 = CPySequenceTuple_SetItemUnsafe(r3, r4, r10) + r7 = CPyStr_GetItem(source, r3) + x = r7 + r8 = f2(x) + r9 = CPySequenceTuple_SetItemUnsafe(r2, r3, r8) L3: - r12 = r4 + 2 - r4 = r12 + r10 = r3 + 2 + r3 = r10 goto L1 L4: - a = r3 + a = r2 return 1 [case testTupleBuiltFromVariableLengthTuple] From 2f2a959a23e5f497b8229d5ed8071802261f4167 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Wed, 30 Jun 2021 23:59:38 +0800 Subject: [PATCH 13/14] Revert previous revert --- mypyc/lib-rt/str_ops.c | 5 ++++- mypyc/primitives/str_ops.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 60b462b76fa2..1b1454bed714 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -110,5 +110,8 @@ bool CPyStr_IsTrue(PyObject *obj) { } Py_ssize_t CPyStr_Size_size_t(PyObject *str) { - return PyUnicode_GET_LENGTH(str); + if (PyUnicode_READY(str) != -1) { + return PyUnicode_GET_LENGTH(str); + } + return -1; } \ No newline at end of file diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index 23013c40eb26..b8480a2831c2 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -10,7 +10,7 @@ ) from mypyc.primitives.registry import ( method_op, binary_op, function_op, - load_address_op, custom_op + load_address_op, custom_op, ERR_NEG_INT ) @@ -140,4 +140,4 @@ arg_types=[str_rprimitive], return_type=c_pyssize_t_rprimitive, c_function_name='CPyStr_Size_size_t', - error_kind=ERR_NEVER) + error_kind=ERR_NEG_INT) From 70f6e25614d4bdc78050166d3667f3e9fb40733b Mon Sep 17 00:00:00 2001 From: Jingchen Ye <97littleleaf11@gmail.com> Date: Thu, 1 Jul 2021 02:39:11 +0800 Subject: [PATCH 14/14] Fix irbuild test --- mypyc/test-data/irbuild-tuple.test | 44 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 28ceea1e026d..564a4bf74d50 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -367,37 +367,41 @@ L0: def test(): r0, source :: str r1 :: native_int - r2 :: tuple - r3 :: short_int - r4 :: native_int - r5 :: short_int + r2 :: bit + r3 :: tuple + r4 :: short_int + r5 :: native_int r6 :: bit - r7, x, r8 :: str - r9 :: bit - r10 :: short_int + r7 :: short_int + r8 :: bit + r9, x, r10 :: str + r11 :: bit + r12 :: short_int a :: tuple L0: r0 = 'abc' source = r0 r1 = CPyStr_Size_size_t(source) - r2 = PyTuple_New(r1) - r3 = 0 + r2 = r1 >= 0 :: signed + r3 = PyTuple_New(r1) + r4 = 0 L1: - r4 = CPyStr_Size_size_t(source) - r5 = r4 << 1 - r6 = r3 < r5 :: signed - if r6 goto L2 else goto L4 :: bool + r5 = CPyStr_Size_size_t(source) + r6 = r5 >= 0 :: signed + r7 = r5 << 1 + r8 = r4 < r7 :: signed + if r8 goto L2 else goto L4 :: bool L2: - r7 = CPyStr_GetItem(source, r3) - x = r7 - r8 = f2(x) - r9 = CPySequenceTuple_SetItemUnsafe(r2, r3, r8) + r9 = CPyStr_GetItem(source, r4) + x = r9 + r10 = f2(x) + r11 = CPySequenceTuple_SetItemUnsafe(r3, r4, r10) L3: - r10 = r3 + 2 - r3 = r10 + r12 = r4 + 2 + r4 = r12 goto L1 L4: - a = r2 + a = r3 return 1 [case testTupleBuiltFromVariableLengthTuple]