From 23db9c62272f7470aadf8f52fe3ebb42b5e5d380 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 5 Mar 2024 15:23:08 +0000 Subject: [PATCH] GH-115685: Split `_TO_BOOL_ALWAYS_TRUE` into micro-ops (GH-116352) --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 40 +++++++++++------------ Include/internal/pycore_uop_metadata.h | 4 +-- Python/bytecodes.c | 13 ++++---- Python/executor_cases.c.h | 7 +--- Python/generated_cases.c.h | 23 ++++++++----- Python/optimizer_analysis.c | 3 ++ Python/optimizer_cases.c.h | 2 +- 8 files changed, 50 insertions(+), 44 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index ab34366ab1066c..f355a496cf2e70 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1320,7 +1320,7 @@ _PyOpcode_macro_expansion[256] = { [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_LIST_INT, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, 0, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { _TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, 0, 0 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, 0, 0 } } }, [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, 0, 0 } } }, [TO_BOOL_LIST] = { .nuops = 1, .uops = { { _TO_BOOL_LIST, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 8f71eab44d914d..f1601de6ae75b6 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -220,40 +220,40 @@ extern "C" { #define _PUSH_EXC_INFO PUSH_EXC_INFO #define _PUSH_FRAME 405 #define _PUSH_NULL PUSH_NULL +#define _REPLACE_WITH_TRUE 406 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 406 -#define _SEND 407 +#define _SAVE_RETURN_OFFSET 407 +#define _SEND 408 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 408 -#define _STORE_ATTR 409 -#define _STORE_ATTR_INSTANCE_VALUE 410 -#define _STORE_ATTR_SLOT 411 +#define _START_EXECUTOR 409 +#define _STORE_ATTR 410 +#define _STORE_ATTR_INSTANCE_VALUE 411 +#define _STORE_ATTR_SLOT 412 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 412 -#define _STORE_FAST_0 413 -#define _STORE_FAST_1 414 -#define _STORE_FAST_2 415 -#define _STORE_FAST_3 416 -#define _STORE_FAST_4 417 -#define _STORE_FAST_5 418 -#define _STORE_FAST_6 419 -#define _STORE_FAST_7 420 +#define _STORE_FAST 413 +#define _STORE_FAST_0 414 +#define _STORE_FAST_1 415 +#define _STORE_FAST_2 416 +#define _STORE_FAST_3 417 +#define _STORE_FAST_4 418 +#define _STORE_FAST_5 419 +#define _STORE_FAST_6 420 +#define _STORE_FAST_7 421 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 421 +#define _STORE_SUBSCR 422 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 422 -#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE +#define _TO_BOOL 423 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -263,12 +263,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 423 +#define _UNPACK_SEQUENCE 424 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 423 +#define MAX_UOP_ID 424 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 7f921a6cd3f4c8..9452dff9d8797d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -54,7 +54,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, @@ -430,6 +430,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", + [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", @@ -461,7 +462,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", [_SWAP] = "_SWAP", [_TO_BOOL] = "_TO_BOOL", - [_TO_BOOL_ALWAYS_TRUE] = "_TO_BOOL_ALWAYS_TRUE", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", [_TO_BOOL_INT] = "_TO_BOOL_INT", [_TO_BOOL_LIST] = "_TO_BOOL_LIST", diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 396a8f09f3feca..15794cb017f9ca 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -382,15 +382,16 @@ dummy_func( } } - inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) { - // This one is a bit weird, because we expect *some* failures: - assert(version); - EXIT_IF(Py_TYPE(value)->tp_version_tag != version); - STAT_INC(TO_BOOL, hit); - DECREF_INPUTS(); + op(_REPLACE_WITH_TRUE, (value -- res)) { + Py_DECREF(value); res = Py_True; } + macro(TO_BOOL_ALWAYS_TRUE) = + unused/1 + + _GUARD_TYPE_VERSION + + _REPLACE_WITH_TRUE; + inst(UNARY_INVERT, (value -- res)) { res = PyNumber_Invert(value); DECREF_INPUTS(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 56ee93862743d5..806f748b05ebbf 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -383,15 +383,10 @@ break; } - case _TO_BOOL_ALWAYS_TRUE: { + case _REPLACE_WITH_TRUE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; - uint32_t version = (uint32_t)CURRENT_OPERAND(); - // This one is a bit weird, because we expect *some* failures: - assert(version); - if (Py_TYPE(value)->tp_version_tag != version) goto side_exit; - STAT_INC(TO_BOOL, hit); Py_DECREF(value); res = Py_True; stack_pointer[-1] = res; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e612c9e4c37632..4947c919b4ca20 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5580,17 +5580,24 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *owner; PyObject *value; PyObject *res; /* Skip 1 cache entry */ - value = stack_pointer[-1]; - uint32_t version = read_u32(&this_instr[2].cache); - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); + } + // _REPLACE_WITH_TRUE + value = owner; + { + Py_DECREF(value); + res = Py_True; + } stack_pointer[-1] = res; DISPATCH(); } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 21cccdb95de76b..18e3382839a92f 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -456,6 +456,9 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) last->opcode = _NOP; buffer[pc].opcode = NOP; } + if (last->opcode == _REPLACE_WITH_TRUE) { + last->opcode = _NOP; + } break; } case _JUMP_TO_TOP: diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 6e65e2e0b4c4ca..4ed1ac90c1e4bd 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -203,7 +203,7 @@ break; } - case _TO_BOOL_ALWAYS_TRUE: { + case _REPLACE_WITH_TRUE: { _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space;