From ce29a3887c23b5436c5e50710ef9f16a9ade894d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Feb 2025 20:24:12 +0800 Subject: [PATCH] make gcc work --- Python/bytecodes.c | 166 +- Python/ceval_macros.h | 3 +- Python/generated_cases.c.h | 23841 ++++++++++++------- Tools/cases_generator/generators_common.py | 2 +- Tools/cases_generator/tier1_generator.py | 12 +- 5 files changed, 15143 insertions(+), 8881 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 84d0da86f52869..183cabdc65b672 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5213,23 +5213,31 @@ dummy_func( } label(pop_4_error) { - STACK_SHRINK(4); - JUMP_TO_LABEL(error); + { + STACK_SHRINK(4); + } + TAIL_CALL(error); } label(pop_3_error) { - STACK_SHRINK(3); - JUMP_TO_LABEL(error); + { + STACK_SHRINK(3); + } + TAIL_CALL(error); } label(pop_2_error) { - STACK_SHRINK(2); - JUMP_TO_LABEL(error); + { + STACK_SHRINK(2); + } + TAIL_CALL(error); } label(pop_1_error) { - STACK_SHRINK(1); - JUMP_TO_LABEL(error); + { + STACK_SHRINK(1); + } + TAIL_CALL(error); } label(error) { @@ -5252,61 +5260,65 @@ dummy_func( } } _PyEval_MonitorRaise(tstate, frame, next_instr-1); - JUMP_TO_LABEL(exception_unwind); + TAIL_CALL(exception_unwind); } - spilled label(exception_unwind) { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); - if (handled == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (frame->stackpointer > stackbase) { + spilled label(exception_unwind) + { + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET() - 1; + int level, handler, lasti; + int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, + &level, &handler, &lasti); + if (handled == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (frame->stackpointer > stackbase) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); + } + monitor_unwind(tstate, frame, next_instr - 1); + TAIL_CALL(exit_unwind); + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + assert(frame->stackpointer >= new_top); + while (frame->stackpointer > new_top) { _PyStackRef ref = _PyFrame_StackPop(frame); PyStackRef_XCLOSE(ref); } - monitor_unwind(tstate, frame, next_instr-1); - JUMP_TO_LABEL(exit_unwind); - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - assert(frame->stackpointer >= new_top); - while (frame->stackpointer > new_top) { - _PyStackRef ref = _PyFrame_StackPop(frame); - PyStackRef_XCLOSE(ref); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - JUMP_TO_LABEL(exception_unwind); + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + TAIL_CALL(exception_unwind); + } + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); } - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); - } - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; - int err = monitor_handled(tstate, frame, next_instr, exc); - if (err < 0) { - JUMP_TO_LABEL(exception_unwind); - } - /* Resume normal execution */ -#ifdef LLTRACE - if (frame->lltrace >= 5) { - lltrace_resume_frame(frame); + int err = monitor_handled(tstate, frame, next_instr, exc); + if (err < 0) { + TAIL_CALL(exception_unwind); + } + /* Resume normal execution */ + #ifdef LLTRACE + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + RELOAD_STACK(); } -#endif - RELOAD_STACK(); #ifdef Py_TAIL_CALL_INTERP int opcode; #endif @@ -5314,6 +5326,7 @@ dummy_func( } spilled label(exit_unwind) { + { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); @@ -5330,33 +5343,36 @@ dummy_func( } next_instr = frame->instr_ptr; RELOAD_STACK(); - JUMP_TO_LABEL(error); + } + TAIL_CALL(error); } spilled label(start_frame) { - int too_deep = _Py_EnterRecursivePy(tstate); - if (too_deep) { - JUMP_TO_LABEL(exit_unwind); - } - next_instr = frame->instr_ptr; - - #ifdef LLTRACE { - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + TAIL_CALL(exit_unwind); } - } - #endif + next_instr = frame->instr_ptr; - #ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); - #endif - RELOAD_STACK(); +#ifdef LLTRACE + { + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + TAIL_CALL(exit_unwind); + } + } +#endif + +#ifdef Py_DEBUG + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); +#endif + RELOAD_STACK(); + } #ifdef Py_TAIL_CALL_INTERP int opcode; #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 44bb52a09aab5f..4de5fb914ab398 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -84,7 +84,8 @@ do { \ Py_MUSTTAIL return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ } while (0) -# define JUMP_TO_LABEL(name) \ +# define JUMP_TO_LABEL(name) goto tail_call_##name; +# define TAIL_CALL(name) \ do { \ Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ } while (0) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 88fa838fe0bc36..7437869156ebb1 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -244,12 +244,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, + { +_PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, PyUnstable_InterpreterFrame_GetLine(frame), opcode); -JUMP_TO_LABEL(error); +} +TAIL_CALL(error); } static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { @@ -527,55 +529,83 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED_BINARY_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED_BINARY_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + /* Skip 4 cache entries */ + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - /* Skip 4 cache entries */ - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_OP_ADD_FLOAT) { @@ -583,55 +613,83 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyFloat_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_OP_ADD_INT) { @@ -639,54 +697,82 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyLong_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_OP_ADD_UNICODE) { @@ -694,54 +780,82 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_UNICODE { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyUnicode_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_UNICODE - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_OP_EXTEND) { @@ -749,55 +863,83 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_EXTEND); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_BINARY_OP_EXTEND { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - assert(d && d->guard); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = d->guard(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!res) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - /* Skip -4 cache entry */ - // _BINARY_OP_EXTEND - { - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = d->action(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { @@ -805,84 +947,112 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - // _GUARD_BOTH_UNICODE { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyUnicode_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - } - /* Skip 5 cache entries */ - // _BINARY_OP_INPLACE_ADD_UNICODE - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = CURRENT_OPERAND0(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left_o) >= 2); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyUnicode_Append(&temp, right_o); - *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) { - JUMP_TO_LABEL(pop_2_error); + /* Skip 5 cache entries */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left_o) >= 2); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (PyStackRef_IsNull(*target_local)) { + JUMP_TO_LABEL(pop_2_error); + } + #if TIER_ONE + // The STORE_FAST is already done. This is done here in tier one, + // and during trace projection in tier two: + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif } - #if TIER_ONE - // The STORE_FAST is already done. This is done here in tier one, - // and during trace projection in tier two: - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - #endif + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_OP_MULTIPLY_FLOAT) { @@ -890,55 +1060,83 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyFloat_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_OP_MULTIPLY_INT) { @@ -946,54 +1144,82 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - if (!PyLong_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_OP_SUBTRACT_FLOAT) { @@ -1001,55 +1227,83 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyFloat_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_OP_SUBTRACT_INT) { @@ -1057,54 +1311,82 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } - if (!PyLong_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_SLICE) { @@ -1112,59 +1394,87 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE { - // Placeholder until we implement BINARY_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res_o = NULL; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + // Placeholder until we implement BINARY_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - Py_DECREF(slice); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; + PyObject *res_o; + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(container); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer += -3; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(container); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_SUBSCR) { @@ -1172,260 +1482,372 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED_BINARY_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef res; - // _SPECIALIZE_BINARY_SUBSCR { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - assert(frame->stackpointer == NULL); - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED_BINARY_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef res; + // _SPECIALIZE_BINARY_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + assert(frame->stackpointer == NULL); + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinarySubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _BINARY_SUBSCR + { + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinarySubscr(container, sub, next_instr); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - OPCODE_DEFERRED_INC(BINARY_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - // _BINARY_SUBSCR + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(BINARY_SUBSCR_DICT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ { - PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); - PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_GetItem(container_o, sub_o); + int rc = PyDict_GetItemRef(dict, sub, &res_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (res_o == NULL) { + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); + if (rc <= 0) { JUMP_TO_LABEL(pop_2_error); } + // not found or error res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(BINARY_SUBSCR_DICT) { + TARGET(BINARY_SUBSCR_GETITEM) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef dict_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(dict_st); - PyStackRef_CLOSE(sub_st); - if (rc <= 0) { - JUMP_TO_LABEL(pop_2_error); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + } + // _BINARY_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_SUBSCR, hit); + } + // _BINARY_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = 2 ; + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } } - // not found or error - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(BINARY_SUBSCR_GETITEM) { + TARGET(BINARY_SUBSCR_LIST_INT) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef sub; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 { - if (tstate->interp->eval_frame) { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); JUMP_TO_PREDICTED(BINARY_SUBSCR); } - } - // _BINARY_SUBSCR_CHECK_FUNC - { - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + if (!PyList_CheckExact(list)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); JUMP_TO_PREDICTED(BINARY_SUBSCR); } - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - if (getitem_o == NULL) { + // Deopt unless 0 <= sub < PyList_Size(list) + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); JUMP_TO_PREDICTED(BINARY_SUBSCR); } - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); JUMP_TO_PREDICTED(BINARY_SUBSCR); } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + STAT_INC(BINARY_SUBSCR, hit); + #else + if (index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); JUMP_TO_PREDICTED(BINARY_SUBSCR); } - getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); - } - // _BINARY_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = 2 ; - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); + #endif + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + PyStackRef_CLOSE(list_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - } - - TARGET(BINARY_SUBSCR_LIST_INT) { #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - if (!PyList_CheckExact(list)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - STAT_INC(BINARY_SUBSCR, hit); - #else - if (index >= PyList_GET_SIZE(list)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - Py_INCREF(res_o); - #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_SUBSCR_STR_INT) { @@ -1433,63 +1855,91 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef str_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - str_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - if (!PyUnicode_CheckExact(str)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!PyUnicode_CheckExact(str)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BINARY_SUBSCR_TUPLE_INT) { @@ -1497,59 +1947,87 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef tuple_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - if (!PyTuple_CheckExact(tuple)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - JUMP_TO_PREDICTED(BINARY_SUBSCR); - } - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(tuple_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!PyTuple_CheckExact(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + // Deopt unless 0 <= sub < PyTuple_Size(list) + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(tuple_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BUILD_LIST) { @@ -1557,21 +2035,49 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - _PyStackRef *values; - _PyStackRef list; - values = &stack_pointer[-oparg]; - PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - if (list_o == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + if (list_o == NULL) { + JUMP_TO_LABEL(error); + } + list = PyStackRef_FromPyObjectSteal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - list = PyStackRef_FromPyObjectSteal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BUILD_MAP) { @@ -1579,41 +2085,69 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - _PyStackRef *values; - _PyStackRef map; - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - if (map_o == NULL) { - stack_pointer += -oparg*2; + if (map_o == NULL) { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + map = PyStackRef_FromPyObjectSteal(map_o); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); } - map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BUILD_SET) { @@ -1621,47 +2155,75 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - _PyStackRef *values; - _PyStackRef set; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + int err = 0; + for (int i = 0; i < oparg; i++) { + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - int err = 0; - for (int i = 0; i < oparg; i++) { - if (err == 0) { + if (err != 0) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } - } - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - if (err != 0) { - stack_pointer += -oparg; + set = PyStackRef_FromPyObjectSteal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - set = PyStackRef_FromPyObjectSteal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BUILD_SLICE) { @@ -1669,29 +2231,57 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef *args; - _PyStackRef slice; - args = &stack_pointer[-oparg]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); - PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (slice_o == NULL) { - stack_pointer += -oparg; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + _PyStackRef *args; + _PyStackRef slice; + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (slice_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + slice = PyStackRef_FromPyObjectSteal(slice_o); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); } - slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BUILD_STRING) { @@ -1699,36 +2289,64 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - _PyStackRef *pieces; - _PyStackRef str; - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); } - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); - } - if (str_o == NULL) { - stack_pointer += -oparg; + if (str_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(BUILD_TUPLE) { @@ -1736,21 +2354,49 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - _PyStackRef *values; - _PyStackRef tup; - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); - if (tup_o == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + JUMP_TO_LABEL(error); + } + tup = PyStackRef_FromPyObjectSteal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - tup = PyStackRef_FromPyObjectSteal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CACHE) { @@ -1758,182 +2404,238 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - assert(0 && "Executing a cache."); - Py_FatalError("Executing a cache."); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + } DISPATCH(); - } - - TARGET(CALL) { #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(CALL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED_CALL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD { - args = &stack_pointer[-oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + // _DO_CALL { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, total_args, NULL, frame - ); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - JUMP_TO_LABEL(error); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); } - else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_ALLOC_AND_ENTER_INIT) { @@ -1941,125 +2643,153 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *args; - _PyStackRef *init; - _PyStackRef *self; - _PyInterpreterFrame *init_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - } - // _CHECK_AND_ALLOCATE_OBJECT { - args = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - init = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyStackRef_IsNull(null[0])) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *args; + _PyStackRef *init; + _PyStackRef *self; + _PyInterpreterFrame *init_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - JUMP_TO_LABEL(error); + // _CHECK_AND_ALLOCATE_OBJECT + { + args = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + JUMP_TO_LABEL(error); + } + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - self[0] = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable[0]; - init[0] = PyStackRef_FromPyObjectNew(init_func); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _CREATE_INIT_FRAME - { - args = &stack_pointer[-oparg]; - self = &stack_pointer[-1 - oparg]; - init = &stack_pointer[-2 - oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - stack_pointer = _PyFrame_GetStackPointer(frame); - /* Push self onto stack of shim */ - shim->localsplus[0] = PyStackRef_DUP(self[0]); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init[0], NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - _PyEval_FrameClearAndPop(tstate, shim); - JUMP_TO_LABEL(error); + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + stack_pointer = _PyFrame_GetStackPointer(frame); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + JUMP_TO_LABEL(error); + } + init_frame = temp; + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + } + // _PUSH_FRAME + { + new_frame = init_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - init_frame = temp; - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - } - // _PUSH_FRAME - { - new_frame = init_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { @@ -2067,138 +2797,166 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - if (!PyStackRef_IsNull(null[0])) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null[0])); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + STAT_INC(CALL, hit); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null[0])); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - STAT_INC(CALL, hit); - self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_FUNCTION_VERSION - { - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } } - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_BOUND_METHOD_GENERAL) { @@ -2206,121 +2964,149 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - } - // _CHECK_METHOD_VERSION { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CHECK_METHOD_VERSION + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyStackRef_IsNull(self_or_null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - if (!PyStackRef_IsNull(null[0])) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // flush + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = temp; } - } - // _EXPAND_METHOD - { - self_or_null = null; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyStackRef_IsNull(self_or_null[0])); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable[0])); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - JUMP_TO_LABEL(error); + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_BUILTIN_CLASS) { @@ -2328,93 +3114,121 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - if (tp->tp_vectorcall == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_BUILTIN_FAST) { @@ -2422,99 +3236,127 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + /* res = func(self, args, nargs) */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { @@ -2522,100 +3364,128 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - /* res = func(self, arguments, nargs, kwnames) */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + /* res = func(self, arguments, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_BUILTIN_O) { @@ -2623,97 +3493,125 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_O - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyStackRef_CLOSE(callable[0]); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + if (res_o == NULL) { JUMP_TO_LABEL(error); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_FUNCTION_EX) { @@ -2721,191 +3619,219 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - opcode = CALL_FUNCTION_EX; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef kwargs_in; - _PyStackRef tuple; - _PyStackRef kwargs_out; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - kwargs_in = stack_pointer[-1]; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (PyTuple_CheckExact(callargs_o)) { - tuple = callargs; - kwargs_out = kwargs_in; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + opcode = CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in; + _PyStackRef tuple; + _PyStackRef kwargs_out; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + kwargs_out = kwargs_in; } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - JUMP_TO_LABEL(error); + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + kwargs_out = kwargs_in; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } - kwargs_out = kwargs_in; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - tuple = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = kwargs_out; - callargs_st = tuple; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } } } } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); + result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); - // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - JUMP_TO_LABEL(error); - } - assert( 1 == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); + PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + if (result_o == NULL) { JUMP_TO_LABEL(error); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_INTRINSIC_1) { @@ -2913,23 +3839,51 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_1_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_INTRINSIC_2) { @@ -2937,30 +3891,58 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - _PyStackRef value2_st; - _PyStackRef value1_st; - _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value2_st); - PyStackRef_CLOSE(value1_st); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_ISINSTANCE) { @@ -2968,62 +3950,90 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - if (total_args != 2) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - _PyStackRef cls_stackref = arguments[1]; - _PyStackRef inst_stackref = arguments[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - JUMP_TO_LABEL(error); - } - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyStackRef cls_stackref = arguments[1]; + _PyStackRef inst_stackref = arguments[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + JUMP_TO_LABEL(error); + } + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_KW) { @@ -3031,897 +4041,661 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW); - PREDICTED_CALL_KW:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL_KW; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef kwnames_out; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW { - kwnames_in = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL_KW; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - kwnames_out = kwnames_in; - } - // _DO_CALL_KW - { - kwnames = kwnames_out; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + kwnames_out = kwnames_in; } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + // _DO_CALL_KW { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); stack_pointer = _PyFrame_GetStackPointer(frame); - // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - JUMP_TO_LABEL(error); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(res_o); } - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(CALL_KW_BOUND_METHOD) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef kwnames; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); - } - } - // _CHECK_METHOD_VERSION_KW - { - null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); - } - if (!PyStackRef_IsNull(null[0])) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); - } - } - // _EXPAND_METHOD_KW - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null[0])); - _PyStackRef callable_s = callable[0]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable[0])); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. + stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - JUMP_TO_LABEL(error); - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(CALL_KW_NON_PY) { + TARGET(CALL_KW_BOUND_METHOD) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); - opcode = CALL_KW_NON_PY; - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef kwnames; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE_KW - { - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); - } - } - // _CALL_KW_NON_PY { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef kwnames; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); } - PyStackRef_CLOSE(kwnames); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + // _CHECK_METHOD_VERSION_KW + { + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null[0])); + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(CALL_KW_PY) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef kwnames; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + new_frame = temp; } - } - // _CHECK_FUNCTION_VERSION_KW - { - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); - } - } - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - JUMP_TO_LABEL(error); + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); - } - - TARGET(CALL_LEN) { #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* len(o) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - _PyStackRef arg_stackref = args[0]; - PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - JUMP_TO_LABEL(error); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - GOTO_ERROR(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); - TARGET(CALL_LIST_APPEND) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self; - _PyStackRef arg; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - assert(self_o != NULL); - if (!PyList_Check(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (!LOCK_OBJECT(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); - } - #if TIER_ONE - // Skip the following POP_TOP. This is done here in tier one, and - // during trace projection in tier two: - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); #endif - DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + TARGET(CALL_KW_NON_PY) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + opcode = CALL_KW_NON_PY; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef kwnames; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + TARGET(CALL_KW_PY) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - if (!Py_IS_TYPE(self, d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef kwnames; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + // _CHECK_FUNCTION_VERSION_KW + { + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } - if (res_o == NULL) { - stack_pointer += -2 - oparg; + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + if (temp == NULL) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + TARGET(CALL_LEN) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); + /* len(o) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3933,74 +4707,562 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { + STAT_INC(CALL, hit); + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + JUMP_TO_LABEL(error); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + GOTO_ERROR(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(CALL_LIST_APPEND) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self; + _PyStackRef arg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (meth->ml_flags != METH_NOARGS) { + assert(self_o != NULL); + if (!PyList_Check(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) { + if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(self); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { + if (err) { JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif } - // _CHECK_PERIODIC + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + /* Builtin METH_FASTCALL methods, without keywords */ + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } - stack_pointer += -1; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_METHOD_DESCRIPTOR_O) { @@ -4008,107 +5270,135 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_NON_PY_GENERAL) { @@ -4116,104 +5406,132 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - opcode = CALL_NON_PY_GENERAL; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE - { - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - } - // _CALL_NON_PY_GENERAL { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + opcode = CALL_NON_PY_GENERAL; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_PY_EXACT_ARGS) { @@ -4221,110 +5539,138 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } } - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_PY_GENERAL) { @@ -4332,95 +5678,123 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = temp; } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - JUMP_TO_LABEL(error); + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CALL_STR_1) { @@ -4428,97 +5802,227 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_STR_1 { - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - if (callable_o != (PyObject *)&PyUnicode_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(CALL_TUPLE_1) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + if (res_o == NULL) { JUMP_TO_LABEL(error); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(CALL_TUPLE_1) { + TARGET(CALL_TYPE_1) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_TUPLE_1 { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; @@ -4530,93 +6034,47 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (callable_o != (PyObject *)&PyTuple_Type) { + if (callable_o != (PyObject *)&PyType_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + stack_pointer[-3] = res; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - } - - TARGET(CALL_TYPE_1) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (callable_o != (PyObject *)&PyType_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CHECK_EG_MATCH) { @@ -4624,54 +6082,82 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - _PyStackRef exc_value_st; - _PyStackRef match_type_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - JUMP_TO_LABEL(pop_2_error); - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, - &match_o, &rest_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - if (res < 0) { - JUMP_TO_LABEL(pop_2_error); - } - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - if (!Py_IsNone(match_o)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + if (err < 0) { + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + JUMP_TO_LABEL(pop_2_error); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (res < 0) { + JUMP_TO_LABEL(pop_2_error); + } + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + if (!Py_IsNone(match_o)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[-2] = rest; + stack_pointer[-1] = match; } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[-2] = rest; - stack_pointer[-1] = match; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CHECK_EXC_MATCH) { @@ -4679,31 +6165,59 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(right); + JUMP_TO_LABEL(pop_1_error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(right); - JUMP_TO_LABEL(pop_1_error); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = b; } - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = b; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CLEANUP_THROW) { @@ -4711,47 +6225,75 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - _PyStackRef sub_iter_st; - _PyStackRef last_sent_val_st; - _PyStackRef exc_value_st; - _PyStackRef none; - _PyStackRef value; - exc_value_st = stack_pointer[-1]; - last_sent_val_st = stack_pointer[-2]; - sub_iter_st = stack_pointer[-3]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #ifndef Py_TAIL_CALL_INTERP - assert(throwflag); - #endif - assert(exc_value && PyExceptionInstance_Check(exc_value)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - none = PyStackRef_None; - value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - PyStackRef_CLOSE(sub_iter_st); - PyStackRef_CLOSE(last_sent_val_st); - PyStackRef_CLOSE(exc_value_st); - } - else { + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + _PyStackRef sub_iter_st; + _PyStackRef last_sent_val_st; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val_st = stack_pointer[-2]; + sub_iter_st = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #ifndef Py_TAIL_CALL_INTERP + assert(throwflag); + #endif + assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - JUMP_TO_LABEL(exception_unwind); + if (matches) { + none = PyStackRef_None; + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + PyStackRef_CLOSE(sub_iter_st); + PyStackRef_CLOSE(last_sent_val_st); + PyStackRef_CLOSE(exc_value_st); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMP_TO_LABEL(exception_unwind); + } + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(COMPARE_OP) { @@ -4759,68 +6301,96 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED_COMPARE_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _COMPARE_OP { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED_COMPARE_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) { - JUMP_TO_LABEL(error); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) { + JUMP_TO_LABEL(error); + } + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(COMPARE_OP_FLOAT) { @@ -4828,53 +6398,81 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); - } - if (!PyFloat_CheckExact(right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 1 cache entry */ - // _COMPARE_OP_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(COMPARE_OP_INT) { @@ -4882,160 +6480,359 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!_PyLong_IsCompact((PyLongObject *)right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(COMPARE_OP_STR) { #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; + int opcode = next_instr->op.code; + (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); - } - if (!PyLong_CheckExact(right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 1 cache entry */ - // _COMPARE_OP_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); - } - if (!_PyLong_IsCompact((PyLongObject *)right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); - } - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(COMPARE_OP_STR) { + TARGET(CONTAINS_OP) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED_CONTAINS_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) { + JUMP_TO_LABEL(pop_2_error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(CONTAINS_OP_DICT) { #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; + int opcode = next_instr->op.code; + (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_UNICODE { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + if (!PyDict_CheckExact(right_o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); } - if (!PyUnicode_CheckExact(right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) { + JUMP_TO_LABEL(pop_2_error); } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 1 cache entry */ - // _COMPARE_OP_STR - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(CONTAINS_OP) { + TARGET(CONTAINS_OP_SET) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED_CONTAINS_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); + int res = _PySet_Contains((PySetObject *)right_o, left_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); @@ -5043,96 +6840,37 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - } - - TARGET(CONTAINS_OP_DICT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyDict_CheckExact(right_o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(CONTAINS_OP); - } - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) { - JUMP_TO_LABEL(pop_2_error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); - TARGET(CONTAINS_OP_SET) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(CONTAINS_OP); - } - STAT_INC(CONTAINS_OP, hit); - // Note: both set and frozenset use the same seq_contains method! - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) { - JUMP_TO_LABEL(pop_2_error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(CONVERT_VALUE) { @@ -5140,31 +6878,59 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - _PyStackRef value; - _PyStackRef result; - value = stack_pointer[-1]; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(COPY) { @@ -5172,18 +6938,46 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(COPY_FREE_VARS) { @@ -5191,21 +6985,49 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(DELETE_ATTR) { @@ -5213,22 +7035,50 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (err) { - JUMP_TO_LABEL(pop_1_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (err) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(DELETE_DEREF) { @@ -5236,23 +7086,51 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + Py_DECREF(oldobj); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(DELETE_FAST) { @@ -5260,25 +7138,53 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(DELETE_GLOBAL) { @@ -5286,25 +7192,53 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (err == 0) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + int err = PyDict_Pop(GLOBALS(), name, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + // Can't use ERROR_IF here. + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(DELETE_NAME) { @@ -5312,32 +7246,60 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err != 0) { + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + err = PyObject_DelItem(ns, name); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + // Can't use ERROR_IF here. + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(DELETE_SUBSCR) { @@ -5345,26 +7307,54 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - _PyStackRef container; - _PyStackRef sub; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) { - JUMP_TO_LABEL(pop_2_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) { + JUMP_TO_LABEL(pop_2_error); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(DICT_MERGE) { @@ -5372,32 +7362,60 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - _PyStackRef callable; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); + int err = _PyDict_MergeEx(dict_o, update_o, 2); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(update); + JUMP_TO_LABEL(pop_1_error); + } PyStackRef_CLOSE(update); - JUMP_TO_LABEL(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(DICT_UPDATE) { @@ -5405,36 +7423,64 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + int err = PyDict_Update(dict_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { + if (err < 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(update); + JUMP_TO_LABEL(pop_1_error); } PyStackRef_CLOSE(update); - JUMP_TO_LABEL(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(END_ASYNC_FOR) { @@ -5442,36 +7488,64 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - _PyStackRef awaitable_st; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - PyStackRef_CLOSE(awaitable_st); - PyStackRef_CLOSE(exc_st); - } - else { - Py_INCREF(exc); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - JUMP_TO_LABEL(exception_unwind); + if (matches) { + PyStackRef_CLOSE(awaitable_st); + PyStackRef_CLOSE(exc_st); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMP_TO_LABEL(exception_unwind); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(END_FOR) { @@ -5479,21 +7553,49 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - _PyStackRef value; - value = stack_pointer[-1]; - /* Don't update instr_ptr, so that POP_ITER sees - * the FOR_ITER as the previous instruction. - * This has the benign side effect that if value is - * finalized it will see the location as the FOR_ITER's. - */ - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + _PyStackRef value; + value = stack_pointer[-1]; + /* Don't update instr_ptr, so that POP_ITER sees + * the FOR_ITER as the previous instruction. + * This has the benign side effect that if value is + * finalized it will see the location as the FOR_ITER's. + */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(END_SEND) { @@ -5501,21 +7603,49 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - (void)receiver; - val = value; - PyStackRef_CLOSE(receiver); - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + (void)receiver; + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(ENTER_EXECUTOR) { @@ -5523,38 +7653,66 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - opcode = ENTER_EXECUTOR; - #ifdef _Py_TIER2 - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.index == INSTR_OFFSET() - 1); - assert(executor->vm_data.code == code); - assert(executor->vm_data.valid); - assert(tstate->previous_executor == NULL); - /* If the eval breaker is set then stay in tier 1. - * This avoids any potentially infinite loops - * involving _RESUME_CHECK */ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - opcode = executor->vm_data.opcode; - oparg = (oparg & ~255) | executor->vm_data.oparg; - next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + opcode = ENTER_EXECUTOR; + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); } - DISPATCH_GOTO(); + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ } - tstate->previous_executor = Py_None; - Py_INCREF(executor); - GOTO_TIER_TWO(executor); - #else - Py_FatalError("ENTER_EXECUTOR is not supported in this build"); - #endif /* _Py_TIER2 */ DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(EXIT_INIT_CHECK) { @@ -5562,23 +7720,51 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); - if (!PyStackRef_IsNone(should_be_none)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(EXTENDED_ARG) { @@ -5586,15 +7772,43 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - opcode = EXTENDED_ARG; - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + opcode = EXTENDED_ARG; + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(FORMAT_SIMPLE) { @@ -5602,38 +7816,66 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - res = PyStackRef_FromPyObjectSteal(res_o); - } - else { - res = value; - stack_pointer += -1; + else { + res = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(FORMAT_WITH_SPEC) { @@ -5641,27 +7883,55 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - _PyStackRef value; - _PyStackRef fmt_spec; - _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(FOR_ITER) { @@ -5669,65 +7939,93 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED_FOR_ITER:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef iter; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - iter = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED_FOR_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + iter = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + _Py_Specialize_ForIter(iter, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); + DISPATCH(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Jump forward oparg, then skip following END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(FOR_ITER_GEN) { @@ -5735,67 +8033,95 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + STAT_INC(FOR_ITER, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)( 2 + oparg); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - } - // _FOR_ITER_GEN_FRAME - { - iter = stack_pointer[-1]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(gen) != &PyGen_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; - // oparg is the return offset from the next instruction. - frame->return_offset = (uint16_t)( 2 + oparg); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(FOR_ITER_LIST) { @@ -5803,62 +8129,90 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST { - iter = stack_pointer[-1]; - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - } - // _ITER_JUMP_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - #ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + iter = stack_pointer[-1]; + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); } - #endif - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); } + // _ITER_JUMP_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED + if (seq != NULL) { + it->it_seq = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - // _ITER_NEXT_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(FOR_ITER_RANGE) { @@ -5866,56 +8220,84 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE { - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } } - } - // _ITER_JUMP_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); + DISPATCH(); + } } - } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) { - JUMP_TO_LABEL(error); + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + JUMP_TO_LABEL(error); + } + next = PyStackRef_FromPyObjectSteal(res); } - next = PyStackRef_FromPyObjectSteal(res); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(FOR_ITER_TUPLE) { @@ -5923,59 +8305,87 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE { - iter = stack_pointer[-1]; - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - } - // _ITER_JUMP_TUPLE - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + iter = stack_pointer[-1]; + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); } - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); } + // _ITER_JUMP_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - // _ITER_NEXT_TUPLE - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(GET_AITER) { @@ -5983,52 +8393,80 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + JUMP_TO_LABEL(pop_1_error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - JUMP_TO_LABEL(pop_1_error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(obj); - if (iter_o == NULL) { - JUMP_TO_LABEL(pop_1_error); - } - if (Py_TYPE(iter_o)->tp_as_async == NULL || - Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + if (iter_o == NULL) { + JUMP_TO_LABEL(pop_1_error); + } + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); - Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(GET_ANEXT) { @@ -6036,23 +8474,51 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + JUMP_TO_LABEL(error); + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(GET_AWAITABLE) { @@ -6060,22 +8526,50 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) { + JUMP_TO_LABEL(pop_1_error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(GET_ITER) { @@ -6083,23 +8577,51 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) { + JUMP_TO_LABEL(pop_1_error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(GET_LEN) { @@ -6107,28 +8629,56 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - JUMP_TO_LABEL(error); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + JUMP_TO_LABEL(error); + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + JUMP_TO_LABEL(error); + } + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(GET_YIELD_FROM_ITER) { @@ -6136,46 +8686,74 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } - iter = iterable; - } - else { - if (PyGen_CheckExact(iterable_o)) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } iter = iterable; } else { - /* `iterable` is not a generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); + if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + PyStackRef_CLOSE(iterable); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - PyStackRef_CLOSE(iterable); } + stack_pointer[-1] = iter; } - stack_pointer[-1] = iter; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(IMPORT_FROM) { @@ -6183,24 +8761,52 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - _PyStackRef from; - _PyStackRef res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(IMPORT_NAME) { @@ -6208,30 +8814,58 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - _PyStackRef level; - _PyStackRef fromlist; - _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(level); - PyStackRef_CLOSE(fromlist); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_CALL) { @@ -6239,372 +8873,428 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - opcode = INSTRUMENTED_CALL; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - /* Skip 3 cache entries */ - // _MAYBE_EXPAND_METHOD - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL { - args = &stack_pointer[-oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-2 - oparg]; - int is_meth = !PyStackRef_IsNull(maybe_self[0]); - PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); - PyObject *arg0; - if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + opcode = INSTRUMENTED_CALL; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } } - else { - if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-2 - oparg]; + int is_meth = !PyStackRef_IsNull(maybe_self[0]); + PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); } else { - arg0 = &_PyInstrumentation_MISSING; + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg0 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); - } - } - // _DO_CALL - { - self_or_null = maybe_self; - callable = func; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, total_args, NULL, frame + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 ); stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { + if (err) { JUMP_TO_LABEL(error); } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + // _DO_CALL + { + self_or_null = maybe_self; + callable = func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in; + _PyStackRef tuple; + _PyStackRef kwargs_out; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + kwargs_out = kwargs_in; } else { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + kwargs_out = kwargs_in; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef kwargs_in; - _PyStackRef tuple; - _PyStackRef kwargs_out; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - kwargs_in = stack_pointer[-1]; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (PyTuple_CheckExact(callargs_o)) { - tuple = callargs; - kwargs_out = kwargs_in; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); + PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - JUMP_TO_LABEL(error); - } - kwargs_out = kwargs_in; - stack_pointer += -2; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs); + PyStackRef_CLOSE(callargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); - tuple = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer += 2; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = kwargs_out; - callargs_st = tuple; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); + PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { + if (result_o == NULL) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } + result = PyStackRef_FromPyObjectSteal(result_o); } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[0] = result; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (err != 0) { JUMP_TO_LABEL(error); } - assert( 1 == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; + stack_pointer[0] = result; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_CALL_KW) { @@ -6612,177 +9302,205 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - opcode = INSTRUMENTED_CALL_KW; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef kwnames_out; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _MONITOR_CALL_KW - { - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - int is_meth = !PyStackRef_IsNull(self_or_null[0]); - PyObject *arg; - if (is_meth) { - arg = PyStackRef_AsPyObjectBorrow(self_or_null[0]); - } - else { - if (args) { - arg = PyStackRef_AsPyObjectBorrow(args[0]); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + opcode = INSTRUMENTED_CALL_KW; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null[0]); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null[0]); } else { - arg = &_PyInstrumentation_MISSING; + if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } } - } - PyObject *function = PyStackRef_AsPyObjectBorrow(callable[0]); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); - } - } - // _MAYBE_EXPAND_METHOD_KW - { - kwnames_in = stack_pointer[-1]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + PyObject *function = PyStackRef_AsPyObjectBorrow(callable[0]); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } } - kwnames_out = kwnames_in; - } - // _DO_CALL_KW - { - kwnames = kwnames_out; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + kwnames_out = kwnames_in; } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + // _DO_CALL_KW { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); stack_pointer = _PyFrame_GetStackPointer(frame); - // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - JUMP_TO_LABEL(error); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(res_o); } - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_END_FOR) { @@ -6790,28 +9508,56 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - _PyStackRef receiver; - _PyStackRef value; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyStackRef_GenCheck(receiver)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } } + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_END_SEND) { @@ -6819,33 +9565,61 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); - } } - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_FOR_ITER) { @@ -6853,41 +9627,69 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - /* Skip 1 cache entry */ - _PyStackRef iter_stackref = TOP(); - PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next != NULL) { - PUSH(PyStackRef_FromPyObjectSteal(next)); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + /* Skip 1 cache entry */ + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next != NULL) { + PUSH(PyStackRef_FromPyObjectSteal(next)); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || + else { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Skip END_FOR */ - JUMPBY(oparg + 1); + /* Skip END_FOR */ + JUMPBY(oparg + 1); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_INSTRUCTION) { @@ -6895,26 +9697,54 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - opcode = INSTRUMENTED_INSTRUCTION; - _PyFrame_SetStackPointer(frame, stack_pointer); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) { - JUMP_TO_LABEL(error); - } - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + opcode = INSTRUMENTED_INSTRUCTION; + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_JUMP_BACKWARD) { @@ -6922,30 +9752,58 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } } } - } - // _MONITOR_JUMP_BACKWARD - { - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_JUMP_FORWARD) { @@ -6953,13 +9811,41 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_LINE) { @@ -6967,43 +9853,71 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_LINE); - opcode = INSTRUMENTED_LINE; - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; - next_instr = this_instr; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = this_instr+1; - JUMP_TO_LABEL(error); + { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + opcode = INSTRUMENTED_LINE; + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + JUMP_TO_LABEL(error); + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } } - next_instr = frame->instr_ptr; - if (next_instr != this_instr) { - DISPATCH(); + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); } + opcode = original_opcode; + DISPATCH_GOTO(); } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - /* Prevent the underlying instruction from specializing - * and overwriting the instrumentation. */ - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { @@ -7011,95 +9925,123 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _LOAD_SUPER_ATTR { - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - JUMP_TO_LABEL(pop_3_error); - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, + tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + JUMP_TO_LABEL(pop_3_error); + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } } } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) { + JUMP_TO_LABEL(pop_3_error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) { - JUMP_TO_LABEL(pop_3_error); + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - JUMP_TO_LABEL(error); - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; } - stack_pointer[0] = attr; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_NOT_TAKEN) { @@ -7107,15 +10049,43 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - (void)this_instr; // INSTRUMENTED_JUMP requires this_instr - INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + (void)this_instr; // INSTRUMENTED_JUMP requires this_instr + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_POP_ITER) { @@ -7123,21 +10093,49 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); - _PyStackRef iter; - iter = stack_pointer[-1]; - INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + _PyStackRef iter; + iter = stack_pointer[-1]; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { @@ -7145,20 +10143,48 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { @@ -7166,24 +10192,52 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { @@ -7191,22 +10245,50 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = !PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { @@ -7214,20 +10296,48 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_RESUME) { @@ -7235,82 +10345,110 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { + if (bytecode == NULL) { JUMP_TO_LABEL(error); } - next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; DISPATCH(); } + #endif } - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + DISPATCH(); } } } - } - // _MONITOR_RESUME - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + } } - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = frame->instr_ptr; + } } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_RETURN_VALUE) { @@ -7318,49 +10456,77 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef res; - // _RETURN_VALUE_EVENT { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } } - } - // _RETURN_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = retval; - stack_pointer += -1; + // _RETURN_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(INSTRUMENTED_YIELD_VALUE) { @@ -7368,94 +10534,150 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef value; - // _YIELD_VALUE_EVENT { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - DISPATCH(); + // _YIELD_VALUE + { + retval = val; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - // _YIELD_VALUE + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(INTERPRETER_EXIT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ { - retval = val; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + PyObject *result = PyStackRef_AsPyObjectSteal(retval); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); + /* Not strictly necessary, but prevents warnings */ + return result; } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); - TARGET(INTERPRETER_EXIT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - PyObject *result = PyStackRef_AsPyObjectSteal(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - /* Not strictly necessary, but prevents warnings */ - return result; + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(IS_OP) { @@ -7463,22 +10685,50 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(JUMP_BACKWARD) { @@ -7486,48 +10736,76 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - PREDICTED_JUMP_BACKWARD:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - /* Skip 1 cache entry */ - // _SPECIALIZE_JUMP_BACKWARD { - #if ENABLE_SPECIALIZATION - if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; - // Need to re-dispatch so the warmup counter isn't off by one: - next_instr = this_instr; - DISPATCH_SAME_OPARG(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + /* Skip 1 cache entry */ + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == JUMP_BACKWARD) { + this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + // Need to re-dispatch so the warmup counter isn't off by one: + next_instr = this_instr; + DISPATCH_SAME_OPARG(); + } + #endif } - #endif - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } } } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(JUMP_BACKWARD_JIT) { @@ -7535,72 +10813,100 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_JIT); - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - // _JIT { - #ifdef _Py_TIER2 - _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - while (oparg > 255) { - oparg >>= 8; - start--; - } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { JUMP_TO_LABEL(error); } } - else { + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + // _JIT + { + #ifdef _Py_TIER2 + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; + } + _PyExecutorObject *executor; _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); + int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->previous_executor == NULL); - tstate->previous_executor = Py_None; - GOTO_TIER_TWO(executor); + if (optimized <= 0) { + this_instr[1].counter = restart_backoff_counter(counter); + if (optimized < 0) { + JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + this_instr[1].counter = initial_jump_backoff_counter(); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); + } } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif } - else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - #endif } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { @@ -7608,17 +10914,45 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(JUMP_BACKWARD_NO_JIT) { @@ -7626,35 +10960,63 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } } } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(JUMP_FORWARD) { @@ -7662,11 +11024,39 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LIST_APPEND) { @@ -7674,21 +11064,49 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - _PyStackRef list; - _PyStackRef v; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LIST_EXTEND) { @@ -7696,40 +11114,68 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - _PyStackRef list_st; - _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { + if (none_val == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(iterable_st); + JUMP_TO_LABEL(pop_1_error); } + assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); - JUMP_TO_LABEL(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - assert(Py_IsNone(none_val)); - PyStackRef_CLOSE(iterable_st); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_ATTR) { @@ -7737,83 +11183,111 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED_LOAD_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *self_or_null; - // _SPECIALIZE_LOAD_ATTR { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED_LOAD_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *self_or_null; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - self_or_null = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null[0] = owner; // Transfer ownership + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + self_or_null = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null[0] = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) { + JUMP_TO_LABEL(pop_1_error); + } + self_or_null[0] = PyStackRef_NULL; + } } else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ + /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) { JUMP_TO_LABEL(pop_1_error); } - self_or_null[0] = PyStackRef_NULL; - } - } - else { - /* Classic, pushes one value. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) { - JUMP_TO_LABEL(pop_1_error); } + attr = PyStackRef_FromPyObjectSteal(attr_o); } - attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-1] = attr; + stack_pointer += (oparg&1); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = attr; - stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_ATTR_CLASS) { @@ -7821,51 +11295,79 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + PyStackRef_CLOSE(owner); } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { @@ -7873,811 +11375,1147 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - } - // _GUARD_TYPE_VERSION - { - uint32_t type_version = read_u32(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + PyStackRef_CLOSE(owner); } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - } - - TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - if (f->func_version != func_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10 ; - DISPATCH_INLINED(new_frame); + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_INSTANCE_VALUE) { + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } - #ifdef Py_GIL_DISABLED - if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10 ; + DISPATCH_INLINED(new_frame); } - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + TARGET(LOAD_ATTR_INSTANCE_VALUE) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - /* This object has a __dict__, just not yet created */ - if (dict != NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_METHOD_NO_DICT) { + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + /* This object has a __dict__, just not yet created */ + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + TARGET(LOAD_ATTR_METHOD_NO_DICT) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_MODULE) { + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - PyDictKeysObject *mod_keys; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE_PUSH_KEYS { - owner = stack_pointer[-1]; - uint32_t dict_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - mod_keys = keys; - } - // _LOAD_ATTR_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - // Clear mod_keys from stack in case we need to deopt - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } - #else - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + TARGET(LOAD_ATTR_MODULE) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + PyDictKeysObject *mod_keys; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE_PUSH_KEYS + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + mod_keys = keys; } + // _LOAD_ATTR_MODULE_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + // Clear mod_keys from stack in case we need to deopt + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[-1] = attr; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); } + stack_pointer[-1] = attr; } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[-1] = attr; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_PROPERTY) { + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_PROPERTY_FRAME { - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - if (code->co_kwonlyargcount) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - if (code->co_argcount != 1) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); } - STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + stack_pointer[-1] = attr; } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_SLOT) { + TARGET(LOAD_ATTR_PROPERTY) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + new_frame->localsplus[0] = owner; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_ATTR_WITH_HINT) { + TARGET(LOAD_ATTR_SLOT) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - PyDictObject *dict; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } - // _CHECK_ATTR_WITH_HINT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - if (dict_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - assert(PyDict_CheckExact((PyObject *)dict_o)); - dict = dict_o; - } - // _LOAD_ATTR_WITH_HINT { - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *attr_o; - if (!LOCK_OBJECT(dict)) { - if (true) { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } - if (hint >= (size_t)dict->ma_keys->dk_nentries) { - UNLOCK_OBJECT(dict); - if (true) { + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { - UNLOCK_OBJECT(dict); - if (true) { + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + PyStackRef_CLOSE(owner); } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(LOAD_ATTR_WITH_HINT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + PyDictObject *dict; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } - attr_o = ep->me_value; - if (attr_o == NULL) { - UNLOCK_OBJECT(dict); - if (true) { + // _CHECK_ATTR_WITH_HINT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + if (dict_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; } - STAT_INC(LOAD_ATTR, hit); - attr = PyStackRef_FromPyObjectNew(attr_o); - UNLOCK_OBJECT(dict); - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *attr_o; + if (!LOCK_OBJECT(dict)) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + attr_o = ep->me_value; + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + STAT_INC(LOAD_ATTR, hit); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_BUILD_CLASS) { @@ -8685,29 +12523,57 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - _PyStackRef bc; - PyObject *bc_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (bc_o == NULL) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + _PyStackRef bc; + PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_COMMON_CONSTANT) { @@ -8715,25 +12581,53 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - _PyStackRef value; - // Keep in sync with _common_constants in opcode.py - // If we ever have more than two constants, use a lookup table - PyObject *val; - if (oparg == CONSTANT_ASSERTIONERROR) { - val = PyExc_AssertionError; - } - else { - assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); - val = PyExc_NotImplementedError; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + // Keep in sync with _common_constants in opcode.py + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; + } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; + } + value = PyStackRef_FromPyObjectImmortal(val); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - value = PyStackRef_FromPyObjectImmortal(val); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_CONST) { @@ -8741,36 +12635,64 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PREDICTED_LOAD_CONST:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef value; - /* We can't do this in the bytecode compiler as - * marshalling can intern strings and make them immortal. */ - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - #if ENABLE_SPECIALIZATION_FT - #ifdef Py_GIL_DISABLED - uint8_t expected = LOAD_CONST; - if (!_Py_atomic_compare_exchange_uint8( - &this_instr->op.code, &expected, - _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) { - // We might lose a race with instrumentation, which we don't care about. - assert(expected >= MIN_INSTRUMENTED_OPCODE); - } - #else - if (this_instr->op.code == LOAD_CONST) { - this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + PREDICTED_LOAD_CONST:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef value; + /* We can't do this in the bytecode compiler as + * marshalling can intern strings and make them immortal. */ + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + #if ENABLE_SPECIALIZATION_FT + #ifdef Py_GIL_DISABLED + uint8_t expected = LOAD_CONST; + if (!_Py_atomic_compare_exchange_uint8( + &this_instr->op.code, &expected, + _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) { + // We might lose a race with instrumentation, which we don't care about. + assert(expected >= MIN_INSTRUMENTED_OPCODE); + } + #else + if (this_instr->op.code == LOAD_CONST) { + this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; + } + #endif + #endif + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - #endif - #endif - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_CONST_IMMORTAL) { @@ -8778,18 +12700,46 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - assert(_Py_IsImmortal(obj)); - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + assert(_Py_IsImmortal(obj)); + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_CONST_MORTAL) { @@ -8797,17 +12747,45 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_MORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_MORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_DEREF) { @@ -8815,23 +12793,51 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - _PyStackRef value; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_FAST) { @@ -8839,16 +12845,44 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_FAST_AND_CLEAR) { @@ -8856,16 +12890,44 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - _PyStackRef value; - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + _PyStackRef value; + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_FAST_CHECK) { @@ -8873,25 +12935,53 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - _PyStackRef value; - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_FAST_LOAD_FAST) { @@ -8899,20 +12989,48 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_DUP(GETLOCAL(oparg1)); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_FROM_DICT_OR_DEREF) { @@ -8920,44 +13038,72 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - _PyStackRef class_dict_st; - _PyStackRef value; - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { JUMP_TO_LABEL(error); } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { @@ -8965,79 +13111,107 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - _PyStackRef mod_or_class_dict; - _PyStackRef v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(mod_or_class_dict); - if (err < 0) { - JUMP_TO_LABEL(pop_1_error); - } - if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - JUMP_TO_LABEL(error); - } + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(mod_or_class_dict); + if (err < 0) { + JUMP_TO_LABEL(pop_1_error); } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); + } } - if (v_o == NULL) { - /* namespace 2: builtins */ + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { + /* namespace 2: builtins */ _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } } } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[-1] = v; } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[-1] = v; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_GLOBAL) { @@ -9045,53 +13219,81 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED_LOAD_GLOBAL:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef *res; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_GLOBAL { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED_LOAD_GLOBAL:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef *res; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + if (PyStackRef_IsNull(*res)) { + JUMP_TO_LABEL(error); + } } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) { - JUMP_TO_LABEL(error); + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; } + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_GLOBAL_BUILTIN) { @@ -9099,83 +13301,111 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *builtins_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - } - // _GUARD_BUILTINS_VERSION_PUSH_KEYS { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *builtins_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + // _GUARD_BUILTINS_VERSION_PUSH_KEYS + { + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + builtins_keys = keys; + assert(DK_IS_UNICODE(builtins_keys)); } - builtins_keys = keys; - assert(DK_IS_UNICODE(builtins_keys)); - } - // _LOAD_GLOBAL_BUILTINS_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + // _LOAD_GLOBAL_BUILTINS_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; } - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_GLOBAL_MODULE) { @@ -9183,67 +13413,95 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *globals_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION_PUSH_KEYS { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *globals_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION_PUSH_KEYS + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + globals_keys = keys; + assert(DK_IS_UNICODE(globals_keys)); } - globals_keys = keys; - assert(DK_IS_UNICODE(globals_keys)); - } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; } - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_LOCALS) { @@ -9251,23 +13509,51 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_NAME) { @@ -9275,22 +13561,50 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - _PyStackRef v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + JUMP_TO_LABEL(error); + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_SMALL_INT) { @@ -9298,17 +13612,45 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); - _PyStackRef value; - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_SPECIAL) { @@ -9316,40 +13658,68 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - owner = stack_pointer[-1]; - assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); - PyObject *name = _Py_SpecialMethods[oparg].name; - PyObject *self_or_null_o; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + owner = stack_pointer[-1]; + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, Py_TYPE(owner_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); } - JUMP_TO_LABEL(error); + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } - attr = PyStackRef_FromPyObjectSteal(attr_o); - self_or_null = self_or_null_o == NULL ? - PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(LOAD_SUPER_ATTR) { @@ -9357,290 +13727,430 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED_LOAD_SUPER_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = LOAD_SUPER_ATTR; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - JUMP_TO_LABEL(pop_3_error); - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED_LOAD_SUPER_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - else { + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, + tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + JUMP_TO_LABEL(pop_3_error); + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } } } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) { + JUMP_TO_LABEL(pop_3_error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; } + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(LOAD_SUPER_ATTR_ATTR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (super == NULL) { + if (attr == NULL) { JUMP_TO_LABEL(pop_3_error); } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(LOAD_SUPER_ATTR_METHOD) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } + if (method_found) { + self_or_null = self_st; // transfer ownership + } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[0] = attr; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_SUPER_ATTR_ATTR) { + TARGET(MAKE_CELL) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr_st; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (attr == NULL) { - JUMP_TO_LABEL(pop_3_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[-3] = attr_st; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(LOAD_SUPER_ATTR_METHOD) { + TARGET(MAKE_FUNCTION) { #ifdef Py_TAIL_CALL_INTERP int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef self_or_null; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - JUMP_TO_LABEL(error); - } - if (method_found) { - self_or_null = self_st; // transfer ownership - } else { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); + PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; + if (func_obj == NULL) { + JUMP_TO_LABEL(error); + } + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[0] = func; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - } - - TARGET(MAKE_CELL) { #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via the f_locals proxy before MAKE_CELL has run). - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - JUMP_TO_LABEL(error); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); - TARGET(MAKE_FUNCTION) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - _PyStackRef codeobj_st; - _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(codeobj_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (func_obj == NULL) { - JUMP_TO_LABEL(error); - } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[0] = func; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(MAP_ADD) { @@ -9648,32 +14158,60 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - _PyStackRef dict_st; - _PyStackRef key; - _PyStackRef value; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(pop_2_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(pop_2_error); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(MATCH_CLASS) { @@ -9681,43 +14219,71 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - _PyStackRef subject; - _PyStackRef type; - _PyStackRef names; - _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(subject); - PyStackRef_CLOSE(type); - PyStackRef_CLOSE(names); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); // Success! - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) { - JUMP_TO_LABEL(pop_3_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + JUMP_TO_LABEL(pop_3_error); + } + // Error! + attrs = PyStackRef_None; // Failure! } - // Error! - attrs = PyStackRef_None; // Failure! + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3] = attrs; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(MATCH_KEYS) { @@ -9725,27 +14291,55 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - _PyStackRef subject; - _PyStackRef keys; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) { + JUMP_TO_LABEL(error); + } + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(MATCH_MAPPING) { @@ -9753,18 +14347,46 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(MATCH_SEQUENCE) { @@ -9772,18 +14394,46 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(NOP) { @@ -9791,10 +14441,38 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(NOT_TAKEN) { @@ -9802,10 +14480,38 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(POP_EXCEPT) { @@ -9813,20 +14519,48 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, PyStackRef_IsNone(exc_value) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(POP_ITER) { @@ -9834,37 +14568,93 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_ITER); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } - TARGET(POP_JUMP_IF_FALSE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + TARGET(POP_JUMP_IF_FALSE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(POP_JUMP_IF_NONE) { @@ -9872,37 +14662,65 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); + } } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - // _POP_JUMP_IF_TRUE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(POP_JUMP_IF_NOT_NONE) { @@ -9910,37 +14728,65 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); + } } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - // _POP_JUMP_IF_FALSE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(POP_JUMP_IF_TRUE) { @@ -9948,21 +14794,49 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(POP_TOP) { @@ -9970,15 +14844,43 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(PUSH_EXC_INFO) { @@ -9986,28 +14888,56 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(PUSH_NULL) { @@ -10015,15 +14945,43 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(RAISE_VARARGS) { @@ -10031,30 +14989,58 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - _PyStackRef *args; - args = &stack_pointer[-oparg]; - assert(oparg < 3); - PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; - PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - assert(oparg == 0); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_reraise(tstate, frame, this_instr); + int err = do_raise(tstate, exc, cause); stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - JUMP_TO_LABEL(exception_unwind); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMP_TO_LABEL(exception_unwind); + } + JUMP_TO_LABEL(error); } - JUMP_TO_LABEL(error); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(RERAISE) { @@ -10062,49 +15048,77 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - _PyStackRef *values; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); - if (PyLong_Check(lasti)) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - } - else { - stack_pointer += -1; + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); + if (PyLong_Check(lasti)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + Py_DECREF(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - stack_pointer += 1; + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMP_TO_LABEL(exception_unwind); } - assert(exc && PyExceptionInstance_Check(exc)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - JUMP_TO_LABEL(exception_unwind); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(RESERVED) { @@ -10112,12 +15126,40 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - assert(0 && "Executing RESERVED instruction."); - Py_FatalError("Executing RESERVED instruction."); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(RESUME) { @@ -10125,77 +15167,105 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED_RESUME:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED_RESUME:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { + if (bytecode == NULL) { JUMP_TO_LABEL(error); } - next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; DISPATCH(); } + #endif } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + DISPATCH(); + } + } } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } } } } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(RESUME_CHECK) { @@ -10203,39 +15273,67 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(RESUME); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(RESUME); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(RESUME); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #endif } - #endif DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(RETURN_GENERATOR) { @@ -10243,38 +15341,66 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(RETURN_VALUE) { @@ -10282,31 +15408,59 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(SEND) { @@ -10314,99 +15468,127 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED_SEND:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND - { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _SEND { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED_SEND:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); + _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { - PyStackRef_CLOSE(v); - JUMP_TO_LABEL(pop_1_error); + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + PyStackRef_CLOSE(v); + JUMP_TO_LABEL(pop_1_error); + } } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer += -1; + stack_pointer[0] = retval; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer[0] = retval; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(SEND_GEN) { @@ -10414,71 +15596,99 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef receiver; - _PyStackRef v; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(SEND); + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + gen_frame->previous = frame; + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - } - // _SEND_GEN_FRAME - { - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(SEND); - } - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(SEND); - } - STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - gen_frame->previous = frame; - } - // _PUSH_FRAME - { - new_frame = gen_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(SETUP_ANNOTATIONS) { @@ -10486,46 +15696,74 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } - /* check if __annotations__ in locals()... */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) { + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } + /* check if __annotations__ in locals()... */ _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { + if (err < 0) { JUMP_TO_LABEL(error); } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(SET_ADD) { @@ -10533,24 +15771,52 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - _PyStackRef set; - _PyStackRef v; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) { - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(SET_FUNCTION_ATTRIBUTE) { @@ -10558,27 +15824,55 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - _PyStackRef attr_st; - _PyStackRef func_in; - _PyStackRef func_out; - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(SET_UPDATE) { @@ -10586,24 +15880,52 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - _PyStackRef set; - _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (err < 0) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_ATTR) { @@ -10611,50 +15933,78 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED_STORE_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED_STORE_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); - if (err) { - JUMP_TO_LABEL(pop_2_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) { + JUMP_TO_LABEL(pop_2_error); + } } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -10662,75 +16012,103 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); - } - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } } - } - // _GUARD_DORV_NO_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } } - } - // _STORE_ATTR_INSTANCE_VALUE - { - value = stack_pointer[-2]; - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_ATTR_SLOT) { @@ -10738,50 +16116,78 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - } - // _STORE_ATTR_SLOT - { - value = stack_pointer[-2]; - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); - } - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_ATTR_WITH_HINT) { @@ -10789,118 +16195,174 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); - } - } - // _STORE_ATTR_WITH_HINT - { - value = stack_pointer[-2]; - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); - } - if (!LOCK_OBJECT(dict)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); - } - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - if (true) { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } } - #endif - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { - UNLOCK_OBJECT(dict); - if (true) { + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { + if (!LOCK_OBJECT(dict)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + STAT_INC(STORE_ATTR, hit); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } + } + DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif + } + + TARGET(STORE_DEREF) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, - // when dict only holds the strong reference to value in ep->me_value. - STAT_INC(STORE_ATTR, hit); - stack_pointer += -2; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); - } + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); - TARGET(STORE_DEREF) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; - (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - _PyStackRef v; - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + #endif } TARGET(STORE_FAST) { @@ -10908,19 +16370,47 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - _PyStackRef value; - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + _PyStackRef value; + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_FAST_LOAD_FAST) { @@ -10928,22 +16418,50 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[-1] = value2; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_FAST_STORE_FAST) { @@ -10951,30 +16469,58 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - _PyStackRef value2; - _PyStackRef value1; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - tmp = GETLOCAL(oparg2); - GETLOCAL(oparg2) = value2; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + tmp = GETLOCAL(oparg2); + GETLOCAL(oparg2) = value2; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_GLOBAL) { @@ -10982,22 +16528,50 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) { - JUMP_TO_LABEL(pop_1_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_NAME) { @@ -11005,39 +16579,67 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + JUMP_TO_LABEL(pop_1_error); + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } PyStackRef_CLOSE(v); - JUMP_TO_LABEL(pop_1_error); - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(v); - if (err) { - JUMP_TO_LABEL(pop_1_error); + if (err) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_SLICE) { @@ -11045,53 +16647,81 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - _PyStackRef v; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - // _SPECIALIZE_STORE_SLICE - { - // Placeholder until we implement STORE_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(STORE_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SLICE { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + // Placeholder until we implement STORE_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - Py_DECREF(slice); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - if (err) { - JUMP_TO_LABEL(pop_4_error); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + if (err) { + JUMP_TO_LABEL(pop_4_error); + } } + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_SUBSCR) { @@ -11099,50 +16729,78 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED_STORE_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED_STORE_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + /* container[sub] = v */ _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - /* container[sub] = v */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) { - JUMP_TO_LABEL(pop_3_error); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) { + JUMP_TO_LABEL(pop_3_error); + } } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_SUBSCR_DICT) { @@ -11150,42 +16808,70 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef dict_st; - _PyStackRef sub; - /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - STAT_INC(STORE_SUBSCR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(STORE_SUBSCR_LIST_INT) { @@ -11193,67 +16879,95 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef list_st; - _PyStackRef sub_st; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - if (!PyList_CheckExact(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef list_st; + _PyStackRef sub_st; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + // Ensure nonnegative, zero-or-one-digit ints. + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(list_st); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); // unlock before decrefs! - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(SWAP) { @@ -11261,18 +16975,46 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - _PyStackRef *bottom; - _PyStackRef *top; - top = &stack_pointer[-1]; - bottom = &stack_pointer[-2 - (oparg-2)]; - _PyStackRef temp = bottom[0]; - bottom[0] = top[0]; - top[0] = temp; - assert(oparg >= 2); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + _PyStackRef *bottom; + _PyStackRef *top; + top = &stack_pointer[-1]; + bottom = &stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom[0]; + bottom[0] = top[0]; + top[0] = temp; + assert(oparg >= 2); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(TO_BOOL) { @@ -11280,45 +17022,73 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED_TO_BOOL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED_TO_BOOL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _TO_BOOL - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + PyStackRef_CLOSE(value); + if (err < 0) { + JUMP_TO_LABEL(pop_1_error); + } + res = err ? PyStackRef_True : PyStackRef_False; } - res = err ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; } - stack_pointer[-1] = res; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(TO_BOOL_ALWAYS_TRUE) { @@ -11326,36 +17096,64 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } } + // _REPLACE_WITH_TRUE + { + value = owner; + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; } - // _REPLACE_WITH_TRUE - { - value = owner; - PyStackRef_CLOSE(value); - res = PyStackRef_True; - } - stack_pointer[-1] = res; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(TO_BOOL_BOOL) { @@ -11363,25 +17161,53 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_BoolCheck(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); } - STAT_INC(TO_BOOL, hit); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(TO_BOOL_INT) { @@ -11389,36 +17215,64 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - PyStackRef_CLOSE(value); - res = PyStackRef_True; + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; } - stack_pointer[-1] = res; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(TO_BOOL_LIST) { @@ -11426,30 +17280,58 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyList_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyList_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); + stack_pointer[-1] = res; } - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - PyStackRef_CLOSE(value); - stack_pointer[-1] = res; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(TO_BOOL_NONE) { @@ -11457,29 +17339,57 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - if (!PyStackRef_IsNone(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(TO_BOOL_STR) { @@ -11487,37 +17397,65 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - assert(Py_SIZE(value_o)); - PyStackRef_CLOSE(value); - res = PyStackRef_True; + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; } - stack_pointer[-1] = res; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(UNARY_INVERT) { @@ -11525,22 +17463,50 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_1_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(UNARY_NEGATIVE) { @@ -11548,22 +17514,50 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_1_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(UNARY_NOT) { @@ -11571,17 +17565,45 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(UNPACK_EX) { @@ -11589,24 +17611,52 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - _PyStackRef seq; - _PyStackRef *right; - seq = stack_pointer[-1]; - right = &stack_pointer[(oparg & 0xFF)]; - _PyStackRef *top = right + (oparg >> 8); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) { - JUMP_TO_LABEL(pop_1_error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + _PyStackRef seq; + _PyStackRef *right; + seq = stack_pointer[-1]; + right = &stack_pointer[(oparg & 0xFF)]; + _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(UNPACK_SEQUENCE) { @@ -11614,48 +17664,76 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED_UNPACK_SEQUENCE:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef seq; - _PyStackRef *output; - // _SPECIALIZE_UNPACK_SEQUENCE { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED_UNPACK_SEQUENCE:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *output; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + output = &stack_pointer[-1]; + _PyStackRef *top = output + oparg; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) { - JUMP_TO_LABEL(pop_1_error); + PyStackRef_CLOSE(seq); + if (res == 0) { + JUMP_TO_LABEL(pop_1_error); + } } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(UNPACK_SEQUENCE_LIST) { @@ -11663,48 +17741,76 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyList_CheckExact(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - if (!LOCK_OBJECT(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - if (true) { + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyList_CheckExact(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - UNLOCK_OBJECT(seq_o); - PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(UNPACK_SEQUENCE_TUPLE) { @@ -11712,39 +17818,67 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyTuple_CheckExact(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - if (PyTuple_GET_SIZE(seq_o) != oparg) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { @@ -11752,40 +17886,68 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyTuple_CheckExact(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - if (PyTuple_GET_SIZE(seq_o) != 2) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + { + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(WITH_EXCEPT_START) { @@ -11793,57 +17955,85 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - _PyStackRef exit_func; - _PyStackRef exit_self; - _PyStackRef lasti; - _PyStackRef val; - _PyStackRef res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_self = stack_pointer[-4]; - exit_func = stack_pointer[-5]; - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_self: FOURTH = the context or NULL - - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); - PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); - assert(val_o && PyExceptionInstance_Check(val_o)); - exc = PyExceptionInstance_Class(val_o); - tb = PyException_GetTraceback(val_o); - if (tb == NULL) { - tb = Py_None; - } - else { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_self: FOURTH = the context or NULL + - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(tb); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + assert(PyStackRef_LongCheck(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(tb); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - assert(PyStackRef_LongCheck(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; - int has_self = !PyStackRef_IsNull(exit_self); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } TARGET(YIELD_VALUE) { @@ -11851,49 +18041,77 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { int opcode = next_instr->op.code; (void)(opcode); #endif /* Py_TAIL_CALL_INTERP */ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - _PyStackRef retval; - _PyStackRef value; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + #ifdef Py_TAIL_CALL_INTERP + tail_call_pop_4_error: + TAIL_CALL(pop_4_error); + + tail_call_pop_3_error: + TAIL_CALL(pop_3_error); + + tail_call_pop_2_error: + TAIL_CALL(pop_2_error); + + tail_call_pop_1_error: + TAIL_CALL(pop_1_error); + + tail_call_error: + TAIL_CALL(error); + + tail_call_exception_unwind: + TAIL_CALL(exception_unwind); + + tail_call_exit_unwind: + TAIL_CALL(exit_unwind); + + tail_call_start_frame: + TAIL_CALL(start_frame); + + #endif } /* END INSTRUCTIONS */ @@ -11906,12 +18124,14 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, + { +_PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, PyUnstable_InterpreterFrame_GetLine(frame), opcode); -JUMP_TO_LABEL(error); +} +TAIL_CALL(error); } @@ -11924,26 +18144,34 @@ JUMP_TO_LABEL(error); LABEL(pop_4_error) { - STACK_SHRINK(4); - JUMP_TO_LABEL(error); + { + STACK_SHRINK(4); + } + TAIL_CALL(error); } LABEL(pop_3_error) { - STACK_SHRINK(3); - JUMP_TO_LABEL(error); + { + STACK_SHRINK(3); + } + TAIL_CALL(error); } LABEL(pop_2_error) { - STACK_SHRINK(2); - JUMP_TO_LABEL(error); + { + STACK_SHRINK(2); + } + TAIL_CALL(error); } LABEL(pop_1_error) { - STACK_SHRINK(1); - JUMP_TO_LABEL(error); + { + STACK_SHRINK(1); + } + TAIL_CALL(error); } LABEL(error) @@ -11976,61 +18204,64 @@ JUMP_TO_LABEL(error); _PyEval_MonitorRaise(tstate, frame, next_instr-1); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - JUMP_TO_LABEL(exception_unwind); + TAIL_CALL(exception_unwind); } LABEL(exception_unwind) { /* STACK SPILLED */ - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); - if (handled == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (frame->stackpointer > stackbase) { + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET() - 1; + int level, handler, lasti; + int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, + &level, &handler, &lasti); + if (handled == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (frame->stackpointer > stackbase) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); + } + monitor_unwind(tstate, frame, next_instr - 1); + TAIL_CALL(exit_unwind); + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + assert(frame->stackpointer >= new_top); + while (frame->stackpointer > new_top) { _PyStackRef ref = _PyFrame_StackPop(frame); PyStackRef_XCLOSE(ref); } - monitor_unwind(tstate, frame, next_instr-1); - JUMP_TO_LABEL(exit_unwind); - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - assert(frame->stackpointer >= new_top); - while (frame->stackpointer > new_top) { - _PyStackRef ref = _PyFrame_StackPop(frame); - PyStackRef_XCLOSE(ref); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - JUMP_TO_LABEL(exception_unwind); + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + TAIL_CALL(exception_unwind); + } + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); + } + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + int err = monitor_handled(tstate, frame, next_instr, exc); + if (err < 0) { + TAIL_CALL(exception_unwind); } - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); - } - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - int err = monitor_handled(tstate, frame, next_instr, exc); - if (err < 0) { - JUMP_TO_LABEL(exception_unwind); - } - /* Resume normal execution */ - #ifdef LLTRACE - if (frame->lltrace >= 5) { - lltrace_resume_frame(frame); + /* Resume normal execution */ + #ifdef LLTRACE + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); } - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef Py_TAIL_CALL_INTERP int opcode; #endif @@ -12040,50 +18271,54 @@ JUMP_TO_LABEL(error); LABEL(exit_unwind) { /* STACK SPILLED */ - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); } - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TAIL_CALL(error); } LABEL(start_frame) { /* STACK SPILLED */ - int too_deep = _Py_EnterRecursivePy(tstate); - if (too_deep) { - JUMP_TO_LABEL(exit_unwind); - } - next_instr = frame->instr_ptr; - #ifdef LLTRACE { - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + TAIL_CALL(exit_unwind); } - } - #endif + next_instr = frame->instr_ptr; + #ifdef LLTRACE + { + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + TAIL_CALL(exit_unwind); + } + } + #endif - #ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef Py_DEBUG + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + } #ifdef Py_TAIL_CALL_INTERP int opcode; #endif diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 30ab0e0385b0bd..e18525d8473a87 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -626,7 +626,7 @@ def _emit_block( if tkn.text.startswith("DISPATCH"): self._print_storage(storage) reachable = False - if tkn.text.startswith("JUMP_TO_LABEL"): + if tkn.text.startswith("JUMP_TO_LABEL") or tkn.text.startswith("TAIL_CALL"): next(tkn_iter) label_tkn = next(tkn_iter) next(tkn_iter) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 8c1cb8a6a4ee4f..7d088c0699b6fa 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -141,12 +141,14 @@ def uses_this(inst: Instruction) -> tuple[bool, bool]: UNKNOWN_OPCODE_HANDLER ="""\ +{ _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, PyUnstable_InterpreterFrame_GetLine(frame), opcode); -JUMP_TO_LABEL(error); +} +TAIL_CALL(error); """ def generate_tier1( @@ -276,6 +278,7 @@ def generate_tier1_cases( out.emit(f"int opcode = next_instr->op.code;\n") out.emit(f"(void)(opcode);\n") out.emit(f"#endif /* Py_TAIL_CALL_INTERP */\n") + out.emit("{\n") needs_this, only_for_tail_call = uses_this(inst) if not tail_call_mode: if only_for_tail_call and needs_this: @@ -318,8 +321,15 @@ def generate_tier1_cases( out.start_line() stack.flush(out) + out.emit("}\n") if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") + out.emit("#ifdef Py_TAIL_CALL_INTERP\n") + for name in analysis.labels: + emitter.emit(f"tail_call_{name}:\n") + emitter.emit(f"TAIL_CALL({name});\n") + emitter.emit("\n") + out.emit("#endif\n") out.start_line() out.emit("}") out.emit("\n")