From 6ca0d42cf668c6959c15366e232403b49895d05c Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 9 Mar 2022 16:05:34 -0800 Subject: [PATCH 01/34] Move bytecode into the code object --- Include/cpython/code.h | 22 ++-- Include/internal/pycore_code.h | 20 ++-- Include/marshal.h | 2 +- Lib/test/test_compile.py | 3 +- Objects/clinic/codeobject.c.h | 10 +- Objects/codeobject.c | 151 +++++++++++----------------- Objects/frameobject.c | 10 +- Objects/genobject.c | 17 ++-- Objects/typeobject.c | 3 +- Programs/test_frozenmain.h | 64 ++++++------ Python/ceval.c | 42 ++------ Python/marshal.c | 17 +++- Python/specialize.c | 178 ++++++++++++++++++++++++++------- Tools/scripts/deepfreeze.py | 46 ++++++--- Tools/scripts/umarshal.py | 3 +- 15 files changed, 327 insertions(+), 261 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index f3e0761d953458..003daeab4c8b97 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -26,7 +26,7 @@ typedef uint16_t _Py_CODEUNIT; /* Bytecode object */ struct PyCodeObject { - PyObject_HEAD + PyObject_VAR_HEAD /* Note only the following fields are used in hash and/or comparisons * @@ -41,9 +41,7 @@ struct PyCodeObject { * - co_code * - co_consts * - co_names - * - co_varnames - * - co_freevars - * - co_cellvars + * - co_localsplusnames * * This is done to preserve the name and line number for tracebacks * and debuggers; otherwise, constant de-duplication would collapse @@ -55,12 +53,9 @@ struct PyCodeObject { // The hottest fields (in the eval loop) are grouped here at the top. PyObject *co_consts; /* list (constants used) */ PyObject *co_names; /* list of strings (names used) */ - _Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening. - Unlike the other "hot" fields, this one is - actually derived from co_code. */ PyObject *co_exceptiontable; /* Byte string encoding exception handling table */ int co_flags; /* CO_..., see below */ - int co_warmup; /* Warmup counter for quickening */ + int co_warmup; // XXX // The rest are not so impactful on performance. int co_argcount; /* #arguments, except *args */ @@ -68,7 +63,6 @@ struct PyCodeObject { int co_kwonlyargcount; /* #keyword only arguments */ int co_stacksize; /* #entries needed for evaluation stack */ int co_firstlineno; /* first source line number */ - PyObject *co_code; /* instruction opcodes */ PyObject *co_localsplusnames; /* tuple mapping offsets to names */ PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */ PyObject *co_filename; /* unicode (where it was loaded from) */ @@ -90,10 +84,6 @@ struct PyCodeObject { int co_nplaincellvars; /* number of non-arg cell variables */ int co_ncellvars; /* total number of cell variables */ int co_nfreevars; /* number of free variables */ - // lazily-computed values - PyObject *co_varnames; /* tuple of strings (local variable names) */ - PyObject *co_cellvars; /* tuple of strings (cell variable names) */ - PyObject *co_freevars; /* tuple of strings (free variable names) */ /* The remaining fields are zeroed out on new code objects. */ @@ -105,10 +95,12 @@ struct PyCodeObject { /* Quickened instructions and cache, or NULL This should be treated as opaque by all code except the specializer and interpreter. */ - _Py_CODEUNIT *co_quickened; - + char co_quickened[1]; }; +#define _PyCode_GET_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_quickened) +#define _PyCode_GET_SIZE(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) + /* Masks for co_flags above */ #define CO_OPTIMIZED 0x0001 #define CO_NEWLOCALS 0x0002 diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0d324e9e4c0f59..dd7a7309a374c3 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -92,32 +92,26 @@ typedef struct { #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) -/* Maximum size of code to quicken, in code units. */ -#define MAX_SIZE_TO_QUICKEN 10000 - #define QUICKENING_WARMUP_DELAY 8 /* We want to compare to zero for efficiency, so we offset values accordingly */ #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) -#define QUICKENING_WARMUP_COLDEST 1 -int _Py_Quicken(PyCodeObject *code); +void _Py_Quicken(PyCodeObject *code); -/* Returns 1 if quickening occurs. - * -1 if an error occurs - * 0 otherwise */ -static inline int +static inline void _Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) { if (code->co_warmup != 0) { code->co_warmup++; if (code->co_warmup == 0) { - return _Py_Quicken(code) ? -1 : 1; + _Py_Quicken(code); } } - return 0; } +void _Py_SetCountAndUnquicken(PyCodeObject *code); + extern Py_ssize_t _Py_QuickenedCount; // Borrowed references to common callables: @@ -277,9 +271,9 @@ extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg); /* Deallocator function for static codeobjects used in deepfreeze.py */ -extern void _PyStaticCode_Dealloc(PyCodeObject *co); +extern void _PyStaticCode_Dealloc(void *code); /* Function to intern strings of codeobjects */ -extern int _PyStaticCode_InternStrings(PyCodeObject *co); +extern int _PyStaticCode_InternStrings(void *code); #ifdef Py_STATS diff --git a/Include/marshal.h b/Include/marshal.h index f8b0de80cfc38d..f773587bdd0429 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -13,7 +13,7 @@ PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); -#define Py_MARSHAL_VERSION 4 +#define Py_MARSHAL_VERSION 5 PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 79046b8615e3e0..526740092b9dc8 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -641,7 +641,7 @@ def check_same_constant(const): self.check_constant(f1, frozenset({0})) self.assertTrue(f1(0)) - # Merging equal co_linetable and co_code is not a strict requirement + # Merging equal co_linetable is not a strict requirement # for the Python semantics, it's a more an implementation detail. @support.cpython_only def test_merge_code_attrs(self): @@ -650,7 +650,6 @@ def test_merge_code_attrs(self): f2 = lambda a: a.b.c self.assertIs(f1.__code__.co_linetable, f2.__code__.co_linetable) - self.assertIs(f1.__code__.co_code, f2.__code__.co_code) # Stripping unused constants is not a strict requirement for the # Python semantics, it's a more an implementation detail. diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index ee425f61bb113d..272bcd6ea17b2a 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -203,12 +203,12 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje int co_stacksize = self->co_stacksize; int co_flags = self->co_flags; int co_firstlineno = self->co_firstlineno; - PyBytesObject *co_code = (PyBytesObject *)self->co_code; + PyBytesObject *co_code = NULL; PyObject *co_consts = self->co_consts; PyObject *co_names = self->co_names; - PyObject *co_varnames = self->co_varnames; - PyObject *co_freevars = self->co_freevars; - PyObject *co_cellvars = self->co_cellvars; + PyObject *co_varnames = NULL; + PyObject *co_freevars = NULL; + PyObject *co_cellvars = NULL; PyObject *co_filename = self->co_filename; PyObject *co_name = self->co_name; PyObject *co_qualname = self->co_qualname; @@ -456,4 +456,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=9e8c4a19474ec520 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b1b83a70ffc5b7cd input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 5279f6ce170648..21ee23ae2200c8 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -305,9 +305,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_qualname = con->qualname; co->co_flags = con->flags; - Py_INCREF(con->code); - co->co_code = con->code; - co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(con->code); co->co_firstlineno = con->firstlineno; Py_INCREF(con->linetable); co->co_linetable = con->linetable; @@ -341,16 +338,14 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_nplaincellvars = nplaincellvars; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - co->co_varnames = NULL; - co->co_cellvars = NULL; - co->co_freevars = NULL; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; - co->co_quickened = NULL; + memcpy(_PyCode_GET_CODE(co), PyBytes_AS_STRING(con->code), + PyBytes_GET_SIZE(con->code)); } /* The caller is responsible for ensuring that the given data is valid. */ @@ -386,7 +381,7 @@ _PyCode_New(struct _PyCodeConstructor *con) con->columntable = Py_None; } - PyCodeObject *co = PyObject_New(PyCodeObject, &PyCode_Type); + PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, PyBytes_GET_SIZE(con->code) / 2); if (co == NULL) { PyErr_NoMemory(); return NULL; @@ -521,13 +516,6 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, goto error; } - Py_INCREF(varnames); - co->co_varnames = varnames; - Py_INCREF(cellvars); - co->co_cellvars = cellvars; - Py_INCREF(freevars); - co->co_freevars = freevars; - error: Py_XDECREF(localsplusnames); Py_XDECREF(localspluskinds); @@ -611,7 +599,7 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) if (addrq < 0) { return co->co_firstlineno; } - assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code)); + assert(addrq >= 0 && addrq < _PyCode_GET_SIZE(co)); PyCodeAddressRange bounds; _PyCode_InitAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -639,7 +627,7 @@ _PyCode_Addr2EndLine(PyCodeObject* co, int addrq) return -1; } - assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code)); + assert(addrq >= 0 && addrq < _PyCode_GET_SIZE(co)); PyCodeAddressRange bounds; _PyCode_InitEndAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -995,7 +983,7 @@ _source_offset_converter(int* value) { static PyObject* positionsiter_next(positionsiterator* pi) { - if (pi->pi_offset >= PyBytes_GET_SIZE(pi->pi_code->co_code)) { + if (pi->pi_offset >= _PyCode_GET_SIZE(pi->pi_code)) { return NULL; } @@ -1151,46 +1139,19 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) PyObject * _PyCode_GetVarnames(PyCodeObject *co) { - if (co->co_varnames == NULL) { - // PyCodeObject owns this reference. - co->co_varnames = get_localsplus_names(co, CO_FAST_LOCAL, - co->co_nlocals); - if (co->co_varnames == NULL) { - return NULL; - } - } - Py_INCREF(co->co_varnames); - return co->co_varnames; + return get_localsplus_names(co, CO_FAST_LOCAL, co->co_nlocals); } PyObject * _PyCode_GetCellvars(PyCodeObject *co) { - if (co->co_cellvars == NULL) { - // PyCodeObject owns this reference. - co->co_cellvars = get_localsplus_names(co, CO_FAST_CELL, - co->co_ncellvars); - if (co->co_cellvars == NULL) { - return NULL; - } - } - Py_INCREF(co->co_cellvars); - return co->co_cellvars; + return get_localsplus_names(co, CO_FAST_CELL, co->co_ncellvars); } PyObject * _PyCode_GetFreevars(PyCodeObject *co) { - if (co->co_freevars == NULL) { - // PyCodeObject owns this reference. - co->co_freevars = get_localsplus_names(co, CO_FAST_FREE, - co->co_nfreevars); - if (co->co_freevars == NULL) { - return NULL; - } - } - Py_INCREF(co->co_freevars); - return co->co_freevars; + return get_localsplus_names(co, CO_FAST_FREE, co->co_nfreevars); } @@ -1348,14 +1309,10 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } - Py_XDECREF(co->co_code); Py_XDECREF(co->co_consts); Py_XDECREF(co->co_names); Py_XDECREF(co->co_localsplusnames); Py_XDECREF(co->co_localspluskinds); - Py_XDECREF(co->co_varnames); - Py_XDECREF(co->co_freevars); - Py_XDECREF(co->co_cellvars); Py_XDECREF(co->co_filename); Py_XDECREF(co->co_name); Py_XDECREF(co->co_qualname); @@ -1365,8 +1322,7 @@ code_dealloc(PyCodeObject *co) Py_XDECREF(co->co_exceptiontable); if (co->co_weakreflist != NULL) PyObject_ClearWeakRefs((PyObject*)co); - if (co->co_quickened) { - PyMem_Free(co->co_quickened); + if (co->co_warmup == 0) { _Py_QuickenedCount--; } PyObject_Free(co); @@ -1420,8 +1376,9 @@ code_richcompare(PyObject *self, PyObject *other, int op) if (!eq) goto unequal; eq = co->co_firstlineno == cp->co_firstlineno; if (!eq) goto unequal; - eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ); - if (eq <= 0) goto unequal; + // XXX: Compare code! + // eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ); + // if (eq <= 0) goto unequal; /* compare constants */ consts1 = _PyCode_ConstantKey(co->co_consts); @@ -1465,18 +1422,19 @@ code_richcompare(PyObject *self, PyObject *other, int op) static Py_hash_t code_hash(PyCodeObject *co) { - Py_hash_t h, h0, h1, h2, h3, h4; + Py_hash_t h, h0, h1, h2, h3; h0 = PyObject_Hash(co->co_name); if (h0 == -1) return -1; - h1 = PyObject_Hash(co->co_code); + // XXX + // h1 = PyObject_Hash(co->co_code); + // if (h1 == -1) return -1; + h1 = PyObject_Hash(co->co_consts); if (h1 == -1) return -1; - h2 = PyObject_Hash(co->co_consts); + h2 = PyObject_Hash(co->co_names); if (h2 == -1) return -1; - h3 = PyObject_Hash(co->co_names); + h3 = PyObject_Hash(co->co_localsplusnames); if (h3 == -1) return -1; - h4 = PyObject_Hash(co->co_localsplusnames); - if (h4 == -1) return -1; - h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ + h = h0 ^ h1 ^ h2 ^ h3 ^ co->co_argcount ^ co->co_posonlyargcount ^ co->co_kwonlyargcount ^ co->co_flags; if (h == -1) h = -2; @@ -1492,7 +1450,6 @@ static PyMemberDef code_memberlist[] = { {"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY}, {"co_stacksize",T_INT, OFF(co_stacksize), READONLY}, {"co_flags", T_INT, OFF(co_flags), READONLY}, - {"co_code", T_OBJECT, OFF(co_code), READONLY}, {"co_consts", T_OBJECT, OFF(co_consts), READONLY}, {"co_names", T_OBJECT, OFF(co_names), READONLY}, {"co_filename", T_OBJECT, OFF(co_filename), READONLY}, @@ -1503,6 +1460,7 @@ static PyMemberDef code_memberlist[] = { {"co_endlinetable", T_OBJECT, OFF(co_endlinetable), READONLY}, {"co_columntable", T_OBJECT, OFF(co_columntable), READONLY}, {"co_exceptiontable", T_OBJECT, OFF(co_exceptiontable), READONLY}, + {"co_nlocals", T_INT, OFF(co_nlocals), READONLY}, {NULL} /* Sentinel */ }; @@ -1513,12 +1471,6 @@ code_getlnotab(PyCodeObject *code, void *closure) return decode_linetable(code); } -static PyObject * -code_getnlocals(PyCodeObject *code, void *closure) -{ - return PyLong_FromLong(code->co_nlocals); -} - static PyObject * code_getvarnames(PyCodeObject *code, void *closure) { @@ -1540,21 +1492,27 @@ code_getfreevars(PyCodeObject *code, void *closure) static PyObject * code_getquickened(PyCodeObject *code, void *closure) { - if (code->co_quickened == NULL) { - Py_RETURN_NONE; - } - return PyBytes_FromStringAndSize((char *)code->co_firstinstr, - PyBytes_Size(code->co_code)); + return PyMemoryView_FromMemory(code->co_quickened, + _PyCode_GET_SIZE(code), PyBUF_READ); +} + +static PyObject * +code_getcode(PyCodeObject *code, void *closure) +{ + // XXX + _Py_SetCountAndUnquicken(code); + return PyBytes_FromStringAndSize(code->co_quickened, + _PyCode_GET_SIZE(code)); } static PyGetSetDef code_getsetlist[] = { {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, // The following old names are kept for backward compatibility. - {"co_nlocals", (getter)code_getnlocals, NULL, NULL}, {"co_varnames", (getter)code_getvarnames, NULL, NULL}, {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, {"co_freevars", (getter)code_getfreevars, NULL, NULL}, {"_co_quickened", (getter)code_getquickened, NULL, NULL}, + {"co_code", (getter)code_getcode, NULL, NULL}, {0} }; @@ -1562,7 +1520,7 @@ static PyGetSetDef code_getsetlist[] = { static PyObject * code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) { - Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co)); + Py_ssize_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co)); _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; if (co_extra != NULL) { @@ -1570,10 +1528,6 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) (co_extra->ce_size-1) * sizeof(co_extra->ce_extras[0]); } - if (co->co_quickened != NULL) { - res += PyBytes_GET_SIZE(co->co_code); - } - return PyLong_FromSsize_t(res); } @@ -1594,12 +1548,12 @@ code.replace co_stacksize: int(c_default="self->co_stacksize") = -1 co_flags: int(c_default="self->co_flags") = -1 co_firstlineno: int(c_default="self->co_firstlineno") = -1 - co_code: PyBytesObject(c_default="(PyBytesObject *)self->co_code") = None + co_code: PyBytesObject(c_default="NULL") = None co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None - co_varnames: object(subclass_of="&PyTuple_Type", c_default="self->co_varnames") = None - co_freevars: object(subclass_of="&PyTuple_Type", c_default="self->co_freevars") = None - co_cellvars: object(subclass_of="&PyTuple_Type", c_default="self->co_cellvars") = None + co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None + co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None + co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None co_filename: unicode(c_default="self->co_filename") = None co_name: unicode(c_default="self->co_name") = None co_qualname: unicode(c_default="self->co_qualname") = None @@ -1622,7 +1576,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, PyObject *co_name, PyObject *co_qualname, PyBytesObject *co_linetable, PyObject *co_endlinetable, PyObject *co_columntable, PyBytesObject *co_exceptiontable) -/*[clinic end generated code: output=f046bf0be3bab91f input=a63d09f248f00794]*/ +/*[clinic end generated code: output=f046bf0be3bab91f input=78dbe204dbd06c2f]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ @@ -1641,6 +1595,15 @@ code_replace_impl(PyCodeObject *self, int co_argcount, #undef CHECK_INT_ARG + PyObject *code = NULL; + if (co_code == NULL) { + code = code_getcode(self, NULL); + if (code == NULL) { + return NULL; + } + co_code = (PyBytesObject *)code; + } + if (PySys_Audit("code.__new__", "OOOiiiiii", co_code, co_filename, co_name, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, @@ -1694,6 +1657,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, (PyObject*)co_exceptiontable); error: + Py_XDECREF(code); Py_XDECREF(varnames); Py_XDECREF(cellvars); Py_XDECREF(freevars); @@ -1737,8 +1701,8 @@ static struct PyMethodDef code_methods[] = { PyTypeObject PyCode_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "code", - sizeof(PyCodeObject), - 0, + offsetof(PyCodeObject, co_quickened), + sizeof(_Py_CODEUNIT), (destructor)code_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ @@ -1911,17 +1875,15 @@ _PyCode_ConstantKey(PyObject *op) } void -_PyStaticCode_Dealloc(PyCodeObject *co) +_PyStaticCode_Dealloc(void *code) { - if (co->co_quickened) { - PyMem_Free(co->co_quickened); - co->co_quickened = NULL; + PyCodeObject *co = (PyCodeObject *)code; + if (co->co_warmup == 0) { _Py_QuickenedCount--; } co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; PyMem_Free(co->co_extra); co->co_extra = NULL; - co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code); if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *)co); co->co_weakreflist = NULL; @@ -1929,8 +1891,9 @@ _PyStaticCode_Dealloc(PyCodeObject *co) } int -_PyStaticCode_InternStrings(PyCodeObject *co) +_PyStaticCode_InternStrings(void *code) { + PyCodeObject *co = (PyCodeObject *)code; int res = intern_strings(co->co_names); if (res < 0) { return -1; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index eb7fdb30cd75e6..cd18da2e78a481 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -170,8 +170,7 @@ top_of_stack(int64_t stack) static int64_t * mark_stacks(PyCodeObject *code_obj, int len) { - const _Py_CODEUNIT *code = - (const _Py_CODEUNIT *)PyBytes_AS_STRING(code_obj->co_code); + const _Py_CODEUNIT *code = _PyCode_GET_CODE(code_obj); int64_t *stacks = PyMem_New(int64_t, len+1); int i, j, opcode; @@ -493,7 +492,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore /* PyCode_NewWithPosOnlyArgs limits co_code to be under INT_MAX so this * should never overflow. */ - int len = (int)(PyBytes_GET_SIZE(f->f_frame->f_code->co_code) / sizeof(_Py_CODEUNIT)); + int len = Py_SIZE(f->f_frame->f_code); int *lines = marklines(f->f_frame->f_code, len); if (lines == NULL) { return -1; @@ -838,8 +837,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, static int _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) { - const _Py_CODEUNIT *code = - (const _Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code); + const _Py_CODEUNIT *code = _PyCode_GET_CODE(frame->f_code); for (int i = 0; i < frame->f_lasti; i++) { if (_Py_OPCODE(code[i]) == opcode && _Py_OPARG(code[i]) == oparg) { return 1; @@ -862,7 +860,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) { } co = frame->f_code; fast = _PyFrame_GetLocalsArray(frame); - if (frame->f_lasti < 0 && _Py_OPCODE(co->co_firstinstr[0]) == COPY_FREE_VARS) { + if (frame->f_lasti < 0 && _Py_OPCODE(_PyCode_GET_CODE(co)[0]) == COPY_FREE_VARS) { /* Free vars have not been initialized -- Do that */ PyCodeObject *co = frame->f_code; PyObject *closure = frame->f_func->func_closure; diff --git a/Objects/genobject.c b/Objects/genobject.c index bfa1ea5c45f66e..39520b5a683101 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -349,19 +349,20 @@ _PyGen_yf(PyGenObject *gen) if (gen->gi_frame_valid) { _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - PyObject *bytecode = gen->gi_code->co_code; - unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); + _Py_CODEUNIT *code = _PyCode_GET_CODE(gen->gi_code); if (frame->f_lasti < 1) { /* Return immediately if the frame didn't start yet. SEND always come after LOAD_CONST: a code object should not start with SEND */ - assert(code[0] != SEND); + assert(_Py_OPCODE(code[0]) != SEND); return NULL; } - if (code[(frame->f_lasti-1)*sizeof(_Py_CODEUNIT)] != SEND || frame->stacktop < 0) + if (_Py_OPCODE(code[frame->f_lasti - 1]) != SEND || frame->stacktop < 0) + { return NULL; + } yf = _PyFrame_StackPeek(frame); Py_INCREF(yf); } @@ -484,12 +485,12 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, Py_DECREF(ret); /* Termination repetition of SEND loop */ assert(frame->f_lasti >= 0); - PyObject *bytecode = gen->gi_code->co_code; - unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); + _Py_CODEUNIT *code = _PyCode_GET_CODE(gen->gi_code); /* Backup to SEND */ frame->f_lasti--; - assert(code[frame->f_lasti*sizeof(_Py_CODEUNIT)] == SEND); - int jump = code[frame->f_lasti*sizeof(_Py_CODEUNIT)+1]; + assert(_Py_OPCODE(code[frame->f_lasti]) == SEND); + // XXX: This doesn't seem to handle EXTENDED_ARGs: + int jump = _Py_OPARG(code[frame->f_lasti]); frame->f_lasti += jump; if (_PyGen_FetchStopIterationValue(&val) == 0) { ret = gen_send(gen, val); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 78795150756130..7498012c9de0e9 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8949,7 +8949,8 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, // "firstarg" is a cell here unless (very unlikely) super() // was called from the C-API before the first MAKE_CELL op. if (cframe->f_lasti >= 0) { - assert(_Py_OPCODE(*co->co_firstinstr) == MAKE_CELL || _Py_OPCODE(*co->co_firstinstr) == COPY_FREE_VARS); + assert(_Py_OPCODE(_PyCode_GET_CODE(co)[0]) == MAKE_CELL || + _Py_OPCODE(_PyCode_GET_CODE(co)[0]) == COPY_FREE_VARS); assert(PyCell_Check(firstarg)); firstarg = PyCell_GET(firstarg); } diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 8cae77a4899f12..33b0a8c3222ab5 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,7 +1,37 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,115,176,0,0,0,151,0,100,0,100,1, + 0,0,0,0,0,41,8,233,0,0,0,0,78,122,18,70, + 114,111,122,101,110,32,72,101,108,108,111,32,87,111,114,108, + 100,122,8,115,121,115,46,97,114,103,118,218,6,99,111,110, + 102,105,103,41,5,90,12,112,114,111,103,114,97,109,95,110, + 97,109,101,218,10,101,120,101,99,117,116,97,98,108,101,90, + 15,117,115,101,95,101,110,118,105,114,111,110,109,101,110,116, + 90,17,99,111,110,102,105,103,117,114,101,95,99,95,115,116, + 100,105,111,90,14,98,117,102,102,101,114,101,100,95,115,116, + 100,105,111,122,7,99,111,110,102,105,103,32,122,2,58,32, + 41,7,218,3,115,121,115,90,17,95,116,101,115,116,105,110, + 116,101,114,110,97,108,99,97,112,105,218,5,112,114,105,110, + 116,218,4,97,114,103,118,90,11,103,101,116,95,99,111,110, + 102,105,103,115,114,2,0,0,0,218,3,107,101,121,169,0, + 243,0,0,0,0,250,18,116,101,115,116,95,102,114,111,122, + 101,110,109,97,105,110,46,112,121,250,8,60,109,111,100,117, + 108,101,62,114,11,0,0,0,1,0,0,0,115,18,0,0, + 0,2,128,8,3,8,1,22,2,34,1,42,1,8,1,48, + 7,4,249,115,20,0,0,0,2,128,8,3,8,1,22,2, + 34,1,42,1,2,7,4,1,2,249,52,7,115,176,0,0, + 0,0,0,1,11,1,11,1,11,1,11,1,25,1,25,1, + 25,1,25,1,6,1,6,7,27,1,28,1,28,1,28,1, + 28,1,28,1,28,1,28,1,28,1,6,1,6,7,17,19, + 22,19,27,19,27,19,27,19,27,19,27,1,28,1,28,1, + 28,1,28,1,28,1,28,1,28,1,28,10,39,10,27,10, + 39,10,39,10,39,10,39,10,39,10,41,10,41,10,41,10, + 41,10,41,10,41,10,41,42,50,10,51,10,51,10,51,10, + 51,10,51,1,7,12,2,1,42,1,42,5,8,5,10,5, + 10,11,41,21,24,11,41,11,41,28,34,35,38,28,39,28, + 39,28,39,28,39,28,39,11,41,11,41,5,42,5,42,5, + 42,5,42,5,42,5,42,5,42,5,42,5,42,1,42,1, + 42,114,9,0,0,0,88,0,0,0,151,0,100,0,100,1, 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, 100,2,166,1,0,0,171,1,0,0,0,0,0,0,0,0, 1,0,2,0,101,2,100,3,101,0,106,3,0,0,0,0, @@ -12,35 +42,5 @@ unsigned char M_test_frozenmain[] = { 68,0,93,25,90,6,2,0,101,2,100,6,101,6,155,0, 100,7,101,5,101,6,25,0,0,0,0,0,0,0,0,0, 155,0,157,4,166,1,0,0,171,1,0,0,0,0,0,0, - 0,0,1,0,113,60,100,1,83,0,41,8,233,0,0,0, - 0,78,122,18,70,114,111,122,101,110,32,72,101,108,108,111, - 32,87,111,114,108,100,122,8,115,121,115,46,97,114,103,118, - 218,6,99,111,110,102,105,103,41,5,90,12,112,114,111,103, - 114,97,109,95,110,97,109,101,218,10,101,120,101,99,117,116, - 97,98,108,101,90,15,117,115,101,95,101,110,118,105,114,111, - 110,109,101,110,116,90,17,99,111,110,102,105,103,117,114,101, - 95,99,95,115,116,100,105,111,90,14,98,117,102,102,101,114, - 101,100,95,115,116,100,105,111,122,7,99,111,110,102,105,103, - 32,122,2,58,32,41,7,218,3,115,121,115,90,17,95,116, - 101,115,116,105,110,116,101,114,110,97,108,99,97,112,105,218, - 5,112,114,105,110,116,218,4,97,114,103,118,90,11,103,101, - 116,95,99,111,110,102,105,103,115,114,2,0,0,0,218,3, - 107,101,121,169,0,243,0,0,0,0,250,18,116,101,115,116, - 95,102,114,111,122,101,110,109,97,105,110,46,112,121,250,8, - 60,109,111,100,117,108,101,62,114,11,0,0,0,1,0,0, - 0,115,18,0,0,0,2,128,8,3,8,1,22,2,34,1, - 42,1,8,1,48,7,4,249,115,20,0,0,0,2,128,8, - 3,8,1,22,2,34,1,42,1,2,7,4,1,2,249,52, - 7,115,176,0,0,0,0,0,1,11,1,11,1,11,1,11, - 1,25,1,25,1,25,1,25,1,6,1,6,7,27,1,28, - 1,28,1,28,1,28,1,28,1,28,1,28,1,28,1,6, - 1,6,7,17,19,22,19,27,19,27,19,27,19,27,19,27, - 1,28,1,28,1,28,1,28,1,28,1,28,1,28,1,28, - 10,39,10,27,10,39,10,39,10,39,10,39,10,39,10,41, - 10,41,10,41,10,41,10,41,10,41,10,41,42,50,10,51, - 10,51,10,51,10,51,10,51,1,7,12,2,1,42,1,42, - 5,8,5,10,5,10,11,41,21,24,11,41,11,41,28,34, - 35,38,28,39,28,39,28,39,28,39,28,39,11,41,11,41, - 5,42,5,42,5,42,5,42,5,42,5,42,5,42,5,42, - 5,42,1,42,1,42,114,9,0,0,0, + 0,0,1,0,113,60,100,1,83,0, }; diff --git a/Python/ceval.c b/Python/ceval.c index 83309e2c5219a7..fb90fbe8b20bae 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1325,12 +1325,6 @@ eval_frame_handle_pending(PyThreadState *tstate) #define SKIP_CALL() \ JUMPBY(INLINE_CACHE_ENTRIES_PRECALL + 1 + INLINE_CACHE_ENTRIES_CALL) -/* Get opcode and oparg from original instructions, not quickened form. */ -#define TRACING_NEXTOPARG() do { \ - _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code))[INSTR_OFFSET()]; \ - opcode = _Py_OPCODE(word); \ - oparg = _Py_OPARG(word); \ - } while (0) /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to @@ -1568,7 +1562,7 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject static int skip_backwards_over_extended_args(PyCodeObject *code, int offset) { - _Py_CODEUNIT *instrs = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code); + _Py_CODEUNIT *instrs = _PyCode_GET_CODE(code); while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) { offset--; } @@ -1660,7 +1654,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyCodeObject *co = frame->f_code; \ names = co->co_names; \ consts = co->co_consts; \ - first_instr = co->co_firstinstr; \ + first_instr = _PyCode_GET_CODE(co); \ } \ assert(frame->f_lasti >= -1); \ next_instr = first_instr + frame->f_lasti + 1; \ @@ -1732,16 +1726,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } TARGET(RESUME) { - int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code); - if (err) { - if (err < 0) { - goto error; - } - /* Update first_instr and next_instr to point to newly quickened code */ - int nexti = INSTR_OFFSET(); - first_instr = frame->f_code->co_firstinstr; - next_instr = first_instr + nexti; - } + _Py_IncrementCountAndMaybeQuicken(frame->f_code); JUMP_TO_INSTRUCTION(RESUME_QUICK); } @@ -4069,16 +4054,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(JUMP_ABSOLUTE) { PREDICTED(JUMP_ABSOLUTE); - int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code); - if (err) { - if (err < 0) { - goto error; - } - /* Update first_instr and next_instr to point to newly quickened code */ - int nexti = INSTR_OFFSET(); - first_instr = frame->f_code->co_firstinstr; - next_instr = first_instr + nexti; - } + _Py_IncrementCountAndMaybeQuicken(frame->f_code); JUMP_TO_INSTRUCTION(JUMP_ABSOLUTE_QUICK); } @@ -5443,9 +5419,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #else case DO_TRACING: { #endif + // Un-quicken in-place: + _Py_SetCountAndUnquicken(frame->f_code); int instr_prev = skip_backwards_over_extended_args(frame->f_code, frame->f_lasti); frame->f_lasti = INSTR_OFFSET(); - TRACING_NEXTOPARG(); + NEXTOPARG(); if (opcode == RESUME) { if (oparg < 2) { CHECK_EVAL_BREAKER(); @@ -5482,7 +5460,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->stacktop = -1; } } - TRACING_NEXTOPARG(); + NEXTOPARG(); PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } @@ -6725,7 +6703,7 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, then call the trace function if we're tracing source lines. */ initialize_trace_info(&tstate->trace_info, frame); - _Py_CODEUNIT prev = ((_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code))[instr_prev]; + _Py_CODEUNIT prev = _PyCode_GET_CODE(frame->f_code)[instr_prev]; int lastline; if (_Py_OPCODE(prev) == RESUME && _Py_OPARG(prev) == 0) { lastline = -1; @@ -6742,7 +6720,7 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, /* Trace backward edges (except in 'yield from') or if line number has changed */ int trace = line != lastline || (frame->f_lasti < instr_prev && - _Py_OPCODE(frame->f_code->co_firstinstr[frame->f_lasti]) != SEND); + _Py_OPCODE(_PyCode_GET_CODE(frame->f_code)[frame->f_lasti]) != SEND); if (trace) { result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); } diff --git a/Python/marshal.c b/Python/marshal.c index 44e492925cb25f..0d178ef633c843 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -551,7 +551,6 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_long(co->co_kwonlyargcount, p); w_long(co->co_stacksize, p); w_long(co->co_flags, p); - w_object(co->co_code, p); w_object(co->co_consts, p); w_object(co->co_names, p); w_object(co->co_localsplusnames, p); @@ -564,6 +563,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_endlinetable, p); w_object(co->co_columntable, p); w_object(co->co_exceptiontable, p); + w_pstring(co->co_quickened, _PyCode_GET_SIZE(co), p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ @@ -1381,9 +1381,6 @@ r_object(RFILE *p) flags = (int)r_long(p); if (PyErr_Occurred()) goto code_error; - code = r_object(p); - if (code == NULL) - goto code_error; consts = r_object(p); if (consts == NULL) goto code_error; @@ -1420,6 +1417,18 @@ r_object(RFILE *p) exceptiontable = r_object(p); if (exceptiontable == NULL) goto code_error; + n = r_long(p); + if (size == -1 && PyErr_Occurred()) { + break; + } + const char *quickened = r_pstring(n, p); + if (quickened == NULL) { + break; + } + code = PyBytes_FromStringAndSize(quickened, n); + if (code == NULL) { + goto code_error; + } struct _PyCodeConstructor con = { .filename = filename, diff --git a/Python/specialize.c b/Python/specialize.c index a11a76c4ef118b..d0ef41e71cfa4e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -275,27 +275,17 @@ _Py_PrintSpecializationStats(int to_file) #define SPECIALIZATION_FAIL(opcode, kind) ((void)0) #endif -static _Py_CODEUNIT * -allocate(int instruction_count) -{ - assert(instruction_count > 0); - void *array = PyMem_Malloc(sizeof(_Py_CODEUNIT) * instruction_count); - if (array == NULL) { - PyErr_NoMemory(); - return NULL; - } - _Py_QuickenedCount++; - return (_Py_CODEUNIT *)array; -} - // Insert adaptive instructions and superinstructions. -static void -optimize(_Py_CODEUNIT *instructions, int len) + +void +_Py_Quicken(PyCodeObject *code) { + _Py_QuickenedCount++; int previous_opcode = -1; int previous_oparg = -1; - for(int i = 0; i < len; i++) { + _Py_CODEUNIT *instructions = _PyCode_GET_CODE(code); + for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _Py_OPCODE(instructions[i]); int oparg = _Py_OPARG(instructions[i]); uint8_t adaptive_opcode = adaptive_opcodes[opcode]; @@ -351,26 +341,144 @@ optimize(_Py_CODEUNIT *instructions, int len) } } -int -_Py_Quicken(PyCodeObject *code) { - if (code->co_quickened) { - return 0; - } - Py_ssize_t size = PyBytes_GET_SIZE(code->co_code); - int instr_count = (int)(size/sizeof(_Py_CODEUNIT)); - if (instr_count > MAX_SIZE_TO_QUICKEN) { - code->co_warmup = QUICKENING_WARMUP_COLDEST; - return 0; - } - _Py_CODEUNIT *quickened = allocate(instr_count); - if (quickened == NULL) { - return -1; +void +_Py_SetCountAndUnquicken(PyCodeObject *code) +{ + code->co_warmup = QUICKENING_WARMUP_DELAY; + // if (!code->co_quickened) { + // return; + // } + _Py_CODEUNIT *instructions = _PyCode_GET_CODE(code); + for (int i = 0; i < Py_SIZE(code); i++) { + int opcode = _Py_OPCODE(instructions[i]); + // TODO: Generate a table for this: + switch (opcode) { + case BINARY_OP: + case BINARY_OP_ADAPTIVE: + case BINARY_OP_ADD_FLOAT: + case BINARY_OP_ADD_INT: + case BINARY_OP_ADD_UNICODE: + case BINARY_OP_INPLACE_ADD_UNICODE: + case BINARY_OP_MULTIPLY_FLOAT: + case BINARY_OP_MULTIPLY_INT: + case BINARY_OP_SUBTRACT_FLOAT: + case BINARY_OP_SUBTRACT_INT: + opcode = BINARY_OP; + break; + case BINARY_SUBSCR: + case BINARY_SUBSCR_ADAPTIVE: + case BINARY_SUBSCR_DICT: + case BINARY_SUBSCR_GETITEM: + case BINARY_SUBSCR_LIST_INT: + case BINARY_SUBSCR_TUPLE_INT: + opcode = BINARY_SUBSCR; + break; + case CALL: + case CALL_ADAPTIVE: + case CALL_PY_EXACT_ARGS: + case CALL_PY_WITH_DEFAULTS: + opcode = CALL; + break; + case COMPARE_OP: + case COMPARE_OP_ADAPTIVE: + case COMPARE_OP_FLOAT_JUMP: + case COMPARE_OP_INT_JUMP: + case COMPARE_OP_STR_JUMP: + opcode = COMPARE_OP; + break; + case JUMP_ABSOLUTE: + case JUMP_ABSOLUTE_QUICK: + opcode = JUMP_ABSOLUTE; + break; + case LOAD_ATTR: + case LOAD_ATTR_ADAPTIVE: + case LOAD_ATTR_INSTANCE_VALUE: + case LOAD_ATTR_MODULE: + case LOAD_ATTR_SLOT: + case LOAD_ATTR_WITH_HINT: + opcode = LOAD_ATTR; + break; + case LOAD_CONST: + case LOAD_CONST__LOAD_FAST: + opcode = LOAD_CONST; + break; + case LOAD_FAST: + case LOAD_FAST__LOAD_CONST: + case LOAD_FAST__LOAD_FAST: + opcode = LOAD_FAST; + break; + case LOAD_GLOBAL: + case LOAD_GLOBAL_ADAPTIVE: + case LOAD_GLOBAL_BUILTIN: + case LOAD_GLOBAL_MODULE: + opcode = LOAD_GLOBAL; + break; + case LOAD_METHOD: + case LOAD_METHOD_ADAPTIVE: + case LOAD_METHOD_CLASS: + case LOAD_METHOD_MODULE: + case LOAD_METHOD_NO_DICT: + case LOAD_METHOD_WITH_DICT: + case LOAD_METHOD_WITH_VALUES: + opcode = LOAD_METHOD; + break; + case PRECALL: + case PRECALL_ADAPTIVE: + case PRECALL_BOUND_METHOD: + case PRECALL_BUILTIN_CLASS: + case PRECALL_BUILTIN_FAST_WITH_KEYWORDS: + case PRECALL_NO_KW_BUILTIN_FAST: + case PRECALL_NO_KW_BUILTIN_O: + case PRECALL_NO_KW_ISINSTANCE: + case PRECALL_NO_KW_LEN: + case PRECALL_NO_KW_LIST_APPEND: + case PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST: + case PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: + case PRECALL_NO_KW_METHOD_DESCRIPTOR_O: + case PRECALL_NO_KW_STR_1: + case PRECALL_NO_KW_TUPLE_1: + case PRECALL_NO_KW_TYPE_1: + case PRECALL_PYFUNC: + opcode = PRECALL; + break; + case RESUME: + case RESUME_QUICK: + opcode = RESUME; + break; + case STORE_ATTR: + case STORE_ATTR_ADAPTIVE: + case STORE_ATTR_INSTANCE_VALUE: + case STORE_ATTR_SLOT: + case STORE_ATTR_WITH_HINT: + opcode = STORE_ATTR; + break; + case STORE_FAST: + case STORE_FAST__LOAD_FAST: + case STORE_FAST__STORE_FAST: + opcode = STORE_FAST; + break; + case STORE_SUBSCR: + case STORE_SUBSCR_ADAPTIVE: + case STORE_SUBSCR_DICT: + case STORE_SUBSCR_LIST_INT: + opcode = STORE_SUBSCR; + break; + case UNPACK_SEQUENCE: + case UNPACK_SEQUENCE_ADAPTIVE: + case UNPACK_SEQUENCE_LIST: + case UNPACK_SEQUENCE_TUPLE: + case UNPACK_SEQUENCE_TWO_TUPLE: + opcode = UNPACK_SEQUENCE; + break; + default: + assert(!_PyOpcode_InlineCacheEntries[opcode]); + continue; + } + instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instructions[i])); + for (int j = 0; j < _PyOpcode_InlineCacheEntries[opcode]; j++) { + instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); + } } - memcpy(quickened, code->co_firstinstr, size); - optimize(quickened, instr_count); - code->co_quickened = quickened; - code->co_firstinstr = quickened; - return 0; } static inline int diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 954fca81b51e99..ca12bf7d92c88b 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -229,12 +229,8 @@ def generate_unicode(self, name: str, s: str) -> str: def generate_code(self, name: str, code: types.CodeType) -> str: # The ordering here matches PyCode_NewWithPosOnlyArgs() # (but see below). - co_code = self.generate(name + "_code", code.co_code) co_consts = self.generate(name + "_consts", code.co_consts) co_names = self.generate(name + "_names", code.co_names) - co_varnames = self.generate(name + "_varnames", code.co_varnames) - co_freevars = self.generate(name + "_freevars", code.co_freevars) - co_cellvars = self.generate(name + "_cellvars", code.co_cellvars) co_filename = self.generate(name + "_filename", code.co_filename) co_name = self.generate(name + "_name", code.co_name) co_qualname = self.generate(name + "_qualname", code.co_qualname) @@ -249,14 +245,43 @@ def generate_code(self, name: str, code: types.CodeType) -> str: # Derived values nlocals, nplaincellvars, ncellvars, nfreevars = \ get_localsplus_counts(code, localsplusnames, localspluskinds) - with self.block(f"static struct PyCodeObject {name} =", ";"): - self.object_head("PyCode_Type") + self.write("static") + with self.indent(): + with self.block("struct"): + self.write("PyObject_VAR_HEAD") + self.write("PyObject *co_consts;") + self.write("PyObject *co_names;") + self.write("PyObject *co_exceptiontable;") + self.write("int co_flags;") + self.write("int co_warmup;") + self.write("int co_argcount;") + self.write("int co_posonlyargcount;") + self.write("int co_kwonlyargcount;") + self.write("int co_stacksize;") + self.write("int co_firstlineno;") + self.write("PyObject *co_localsplusnames;") + self.write("PyObject *co_localspluskinds;") + self.write("PyObject *co_filename;") + self.write("PyObject *co_name;") + self.write("PyObject *co_qualname;") + self.write("PyObject *co_linetable;") + self.write("PyObject *co_endlinetable;") + self.write("PyObject *co_columntable;") + self.write("int co_nlocalsplus;") + self.write("int co_nlocals;") + self.write("int co_nplaincellvars;") + self.write("int co_ncellvars;") + self.write("int co_nfreevars;") + self.write("PyObject *co_weakreflist;") + self.write("void *co_extra;") + self.write(f"char co_quickened[{len(code.co_code)}];") + with self.block(f"{name} =", ";"): + self.object_var_head("PyCode_Type", len(code.co_code) // 2) # But the ordering here must match that in cpython/code.h # (which is a pain because we tend to reorder those for perf) # otherwise MSVC doesn't like it. self.write(f".co_consts = {co_consts},") self.write(f".co_names = {co_names},") - self.write(f".co_firstinstr = (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval,") self.write(f".co_exceptiontable = {co_exceptiontable},") self.field(code, "co_flags") self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,") @@ -265,7 +290,6 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.field(code, "co_kwonlyargcount") self.field(code, "co_stacksize") self.field(code, "co_firstlineno") - self.write(f".co_code = {co_code},") self.write(f".co_localsplusnames = {co_localsplusnames},") self.write(f".co_localspluskinds = {co_localspluskinds},") self.write(f".co_filename = {co_filename},") @@ -279,12 +303,10 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_nplaincellvars = {nplaincellvars},") self.write(f".co_ncellvars = {ncellvars},") self.write(f".co_nfreevars = {nfreevars},") - self.write(f".co_varnames = {co_varnames},") - self.write(f".co_cellvars = {co_cellvars},") - self.write(f".co_freevars = {co_freevars},") + self.write(f".co_quickened = {make_string_literal(code.co_code)},") self.deallocs.append(f"_PyStaticCode_Dealloc(&{name});") self.interns.append(f"_PyStaticCode_InternStrings(&{name})") - return f"& {name}.ob_base" + return f"& {name}.ob_base.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: if len(t) == 0: diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py index 2eaaa7ce2d95bc..4495de651dd0ef 100644 --- a/Tools/scripts/umarshal.py +++ b/Tools/scripts/umarshal.py @@ -279,7 +279,6 @@ def R_REF(obj: Any) -> Any: retval.co_kwonlyargcount = self.r_long() retval.co_stacksize = self.r_long() retval.co_flags = self.r_long() - retval.co_code = self.r_object() retval.co_consts = self.r_object() retval.co_names = self.r_object() retval.co_localsplusnames = self.r_object() @@ -292,6 +291,8 @@ def R_REF(obj: Any) -> Any: retval.co_endlinetable = self.r_object() retval.co_columntable = self.r_object() retval.co_exceptiontable = self.r_object() + n = self.r_long() + retval.co_code = self.r_string(n * 2) return retval elif type == Type.REF: n = self.r_long() From a77a124cd57e67710bacf787404f3ddf613b4fd2 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 9 Mar 2022 16:27:18 -0800 Subject: [PATCH 02/34] Clean things up a bit --- Include/cpython/code.h | 4 ++-- Objects/codeobject.c | 13 ++++++------- Programs/test_frozenmain.h | 2 +- Python/marshal.c | 6 +++--- Tools/scripts/deepfreeze.py | 4 ++-- Tools/scripts/umarshal.py | 2 +- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 003daeab4c8b97..f9bdab9a65ea3d 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -95,10 +95,10 @@ struct PyCodeObject { /* Quickened instructions and cache, or NULL This should be treated as opaque by all code except the specializer and interpreter. */ - char co_quickened[1]; + char co_bytecode[1]; }; -#define _PyCode_GET_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_quickened) +#define _PyCode_GET_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_bytecode) #define _PyCode_GET_SIZE(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) /* Masks for co_flags above */ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 21ee23ae2200c8..680b4a2f1014bb 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1490,10 +1490,10 @@ code_getfreevars(PyCodeObject *code, void *closure) } static PyObject * -code_getquickened(PyCodeObject *code, void *closure) +code_getbytecode(PyCodeObject *code, void *closure) { - return PyMemoryView_FromMemory(code->co_quickened, - _PyCode_GET_SIZE(code), PyBUF_READ); + return PyMemoryView_FromMemory(code->co_bytecode, _PyCode_GET_SIZE(code), + PyBUF_READ); } static PyObject * @@ -1501,8 +1501,7 @@ code_getcode(PyCodeObject *code, void *closure) { // XXX _Py_SetCountAndUnquicken(code); - return PyBytes_FromStringAndSize(code->co_quickened, - _PyCode_GET_SIZE(code)); + return PyBytes_FromStringAndSize(code->co_bytecode, _PyCode_GET_SIZE(code)); } static PyGetSetDef code_getsetlist[] = { @@ -1511,7 +1510,7 @@ static PyGetSetDef code_getsetlist[] = { {"co_varnames", (getter)code_getvarnames, NULL, NULL}, {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, {"co_freevars", (getter)code_getfreevars, NULL, NULL}, - {"_co_quickened", (getter)code_getquickened, NULL, NULL}, + {"_co_bytecode", (getter)code_getbytecode, NULL, NULL}, {"co_code", (getter)code_getcode, NULL, NULL}, {0} }; @@ -1701,7 +1700,7 @@ static struct PyMethodDef code_methods[] = { PyTypeObject PyCode_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "code", - offsetof(PyCodeObject, co_quickened), + offsetof(PyCodeObject, co_bytecode), sizeof(_Py_CODEUNIT), (destructor)code_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 33b0a8c3222ab5..ad3945c26eb552 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -31,7 +31,7 @@ unsigned char M_test_frozenmain[] = { 10,11,41,21,24,11,41,11,41,28,34,35,38,28,39,28, 39,28,39,28,39,28,39,11,41,11,41,5,42,5,42,5, 42,5,42,5,42,5,42,5,42,5,42,5,42,1,42,1, - 42,114,9,0,0,0,88,0,0,0,151,0,100,0,100,1, + 42,114,9,0,0,0,176,0,0,0,151,0,100,0,100,1, 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, 100,2,166,1,0,0,171,1,0,0,0,0,0,0,0,0, 1,0,2,0,101,2,100,3,101,0,106,3,0,0,0,0, diff --git a/Python/marshal.c b/Python/marshal.c index 0d178ef633c843..c7c5d75f5fa3c8 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -563,7 +563,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_endlinetable, p); w_object(co->co_columntable, p); w_object(co->co_exceptiontable, p); - w_pstring(co->co_quickened, _PyCode_GET_SIZE(co), p); + w_pstring(co->co_bytecode, _PyCode_GET_SIZE(co), p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ @@ -1418,10 +1418,10 @@ r_object(RFILE *p) if (exceptiontable == NULL) goto code_error; n = r_long(p); - if (size == -1 && PyErr_Occurred()) { + if (n == -1 && PyErr_Occurred()) { break; } - const char *quickened = r_pstring(n, p); + const char *quickened = r_string(n, p); if (quickened == NULL) { break; } diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index ca12bf7d92c88b..2015259173a961 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -274,7 +274,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write("int co_nfreevars;") self.write("PyObject *co_weakreflist;") self.write("void *co_extra;") - self.write(f"char co_quickened[{len(code.co_code)}];") + self.write(f"char co_bytecode[{len(code.co_code)}];") with self.block(f"{name} =", ";"): self.object_var_head("PyCode_Type", len(code.co_code) // 2) # But the ordering here must match that in cpython/code.h @@ -303,7 +303,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_nplaincellvars = {nplaincellvars},") self.write(f".co_ncellvars = {ncellvars},") self.write(f".co_nfreevars = {nfreevars},") - self.write(f".co_quickened = {make_string_literal(code.co_code)},") + self.write(f".co_bytecode = {make_string_literal(code.co_code)},") self.deallocs.append(f"_PyStaticCode_Dealloc(&{name});") self.interns.append(f"_PyStaticCode_InternStrings(&{name})") return f"& {name}.ob_base.ob_base" diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py index 4495de651dd0ef..2bfeab1b065c8b 100644 --- a/Tools/scripts/umarshal.py +++ b/Tools/scripts/umarshal.py @@ -292,7 +292,7 @@ def R_REF(obj: Any) -> Any: retval.co_columntable = self.r_object() retval.co_exceptiontable = self.r_object() n = self.r_long() - retval.co_code = self.r_string(n * 2) + retval.co_code = self.r_string(n) return retval elif type == Type.REF: n = self.r_long() From 975b8d12150120b3b371f17ab41ae139a189a5e2 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 10 Mar 2022 08:59:26 -0800 Subject: [PATCH 03/34] Bump the magic number --- Lib/importlib/_bootstrap_external.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index a6f0a1b3c4c7d8..99992958281a53 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -395,6 +395,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) # Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) # Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) +# Python 3.11a6 3488 (Change the marshal format for code objects) # Python 3.12 will start with magic number 3500 @@ -409,7 +410,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3487).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3488).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' From 40ddf3991bb7b3fce68a2f3c3b8935d63974fb2c Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 10 Mar 2022 14:39:45 -0800 Subject: [PATCH 04/34] co_bytecode -> _co_code --- Include/cpython/code.h | 4 ++-- Objects/codeobject.c | 10 +++++----- Python/marshal.c | 2 +- Tools/scripts/deepfreeze.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index f9bdab9a65ea3d..b4d6219817fe44 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -95,10 +95,10 @@ struct PyCodeObject { /* Quickened instructions and cache, or NULL This should be treated as opaque by all code except the specializer and interpreter. */ - char co_bytecode[1]; + char _co_code[1]; }; -#define _PyCode_GET_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_bytecode) +#define _PyCode_GET_CODE(CO) ((_Py_CODEUNIT *)(CO)->_co_code) #define _PyCode_GET_SIZE(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) /* Masks for co_flags above */ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 680b4a2f1014bb..2ae4ff4b48a42b 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1490,9 +1490,9 @@ code_getfreevars(PyCodeObject *code, void *closure) } static PyObject * -code_getbytecode(PyCodeObject *code, void *closure) +code_getundercode(PyCodeObject *code, void *closure) { - return PyMemoryView_FromMemory(code->co_bytecode, _PyCode_GET_SIZE(code), + return PyMemoryView_FromMemory(code->_co_code, _PyCode_GET_SIZE(code), PyBUF_READ); } @@ -1501,7 +1501,7 @@ code_getcode(PyCodeObject *code, void *closure) { // XXX _Py_SetCountAndUnquicken(code); - return PyBytes_FromStringAndSize(code->co_bytecode, _PyCode_GET_SIZE(code)); + return PyBytes_FromStringAndSize(code->_co_code, _PyCode_GET_SIZE(code)); } static PyGetSetDef code_getsetlist[] = { @@ -1510,7 +1510,7 @@ static PyGetSetDef code_getsetlist[] = { {"co_varnames", (getter)code_getvarnames, NULL, NULL}, {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, {"co_freevars", (getter)code_getfreevars, NULL, NULL}, - {"_co_bytecode", (getter)code_getbytecode, NULL, NULL}, + {"_co_code", (getter)code_getundercode, NULL, NULL}, {"co_code", (getter)code_getcode, NULL, NULL}, {0} }; @@ -1700,7 +1700,7 @@ static struct PyMethodDef code_methods[] = { PyTypeObject PyCode_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "code", - offsetof(PyCodeObject, co_bytecode), + offsetof(PyCodeObject, _co_code), sizeof(_Py_CODEUNIT), (destructor)code_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ diff --git a/Python/marshal.c b/Python/marshal.c index c7c5d75f5fa3c8..1c430b23b4241a 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -563,7 +563,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_endlinetable, p); w_object(co->co_columntable, p); w_object(co->co_exceptiontable, p); - w_pstring(co->co_bytecode, _PyCode_GET_SIZE(co), p); + w_pstring(co->_co_code, _PyCode_GET_SIZE(co), p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 2015259173a961..98496ac8b3cabb 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -274,7 +274,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write("int co_nfreevars;") self.write("PyObject *co_weakreflist;") self.write("void *co_extra;") - self.write(f"char co_bytecode[{len(code.co_code)}];") + self.write(f"char _co_code[{len(code.co_code)}];") with self.block(f"{name} =", ";"): self.object_var_head("PyCode_Type", len(code.co_code) // 2) # But the ordering here must match that in cpython/code.h @@ -303,7 +303,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_nplaincellvars = {nplaincellvars},") self.write(f".co_ncellvars = {ncellvars},") self.write(f".co_nfreevars = {nfreevars},") - self.write(f".co_bytecode = {make_string_literal(code.co_code)},") + self.write(f"._co_code = {make_string_literal(code.co_code)},") self.deallocs.append(f"_PyStaticCode_Dealloc(&{name});") self.interns.append(f"_PyStaticCode_InternStrings(&{name})") return f"& {name}.ob_base.ob_base" From bfcba6d51a3ff40d979a08d5c9526ba1b5d99dcd Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 10 Mar 2022 14:39:57 -0800 Subject: [PATCH 05/34] Generate specialization table --- Include/opcode.h | 164 +++++++++++++++++++-------- Lib/opcode.py | 175 +++++++++++++++++------------ Lib/test/test__opcode.py | 3 +- Python/makeopcodetargets.py | 3 +- Python/opcode_targets.h | 38 +++---- Python/specialize.c | 134 ++-------------------- Tools/scripts/generate_opcode_h.py | 9 +- Tools/scripts/summarize_stats.py | 6 +- 8 files changed, 265 insertions(+), 267 deletions(-) diff --git a/Include/opcode.h b/Include/opcode.h index 7bf0ba70fd7de0..c6e3a3427d3b13 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -138,55 +138,57 @@ extern "C" { #define CALL_ADAPTIVE 34 #define CALL_PY_EXACT_ARGS 36 #define CALL_PY_WITH_DEFAULTS 37 -#define JUMP_ABSOLUTE_QUICK 38 -#define LOAD_ATTR_ADAPTIVE 39 -#define LOAD_ATTR_INSTANCE_VALUE 40 -#define LOAD_ATTR_WITH_HINT 41 -#define LOAD_ATTR_SLOT 42 -#define LOAD_ATTR_MODULE 43 -#define LOAD_GLOBAL_ADAPTIVE 44 -#define LOAD_GLOBAL_MODULE 45 -#define LOAD_GLOBAL_BUILTIN 46 -#define LOAD_METHOD_ADAPTIVE 47 -#define LOAD_METHOD_CLASS 48 -#define LOAD_METHOD_MODULE 55 -#define LOAD_METHOD_NO_DICT 56 -#define LOAD_METHOD_WITH_DICT 57 -#define LOAD_METHOD_WITH_VALUES 58 -#define PRECALL_ADAPTIVE 59 -#define PRECALL_BUILTIN_CLASS 62 -#define PRECALL_NO_KW_BUILTIN_O 63 -#define PRECALL_NO_KW_BUILTIN_FAST 64 -#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 65 -#define PRECALL_NO_KW_LEN 66 -#define PRECALL_NO_KW_ISINSTANCE 67 -#define PRECALL_NO_KW_LIST_APPEND 72 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 73 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 76 -#define PRECALL_NO_KW_STR_1 77 -#define PRECALL_NO_KW_TUPLE_1 78 -#define PRECALL_NO_KW_TYPE_1 79 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 80 -#define PRECALL_BOUND_METHOD 81 -#define PRECALL_PYFUNC 140 -#define RESUME_QUICK 141 -#define STORE_ATTR_ADAPTIVE 143 -#define STORE_ATTR_INSTANCE_VALUE 150 -#define STORE_ATTR_SLOT 153 -#define STORE_ATTR_WITH_HINT 154 -#define UNPACK_SEQUENCE_ADAPTIVE 158 -#define UNPACK_SEQUENCE_LIST 159 -#define UNPACK_SEQUENCE_TUPLE 161 -#define UNPACK_SEQUENCE_TWO_TUPLE 167 -#define LOAD_FAST__LOAD_FAST 168 -#define STORE_FAST__LOAD_FAST 169 -#define LOAD_FAST__LOAD_CONST 170 -#define LOAD_CONST__LOAD_FAST 173 -#define STORE_FAST__STORE_FAST 174 +#define LOAD_ATTR_ADAPTIVE 38 +#define LOAD_ATTR_INSTANCE_VALUE 39 +#define LOAD_ATTR_WITH_HINT 40 +#define LOAD_ATTR_SLOT 41 +#define LOAD_ATTR_MODULE 42 +#define LOAD_GLOBAL_ADAPTIVE 43 +#define LOAD_GLOBAL_MODULE 44 +#define LOAD_GLOBAL_BUILTIN 45 +#define LOAD_METHOD_ADAPTIVE 46 +#define LOAD_METHOD_CLASS 47 +#define LOAD_METHOD_MODULE 48 +#define LOAD_METHOD_NO_DICT 55 +#define LOAD_METHOD_WITH_DICT 56 +#define LOAD_METHOD_WITH_VALUES 57 +#define PRECALL_ADAPTIVE 58 +#define PRECALL_BUILTIN_CLASS 59 +#define PRECALL_NO_KW_BUILTIN_O 62 +#define PRECALL_NO_KW_BUILTIN_FAST 63 +#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 64 +#define PRECALL_NO_KW_LEN 65 +#define PRECALL_NO_KW_ISINSTANCE 66 +#define PRECALL_NO_KW_LIST_APPEND 67 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 72 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 73 +#define PRECALL_NO_KW_STR_1 76 +#define PRECALL_NO_KW_TUPLE_1 77 +#define PRECALL_NO_KW_TYPE_1 78 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 79 +#define PRECALL_BOUND_METHOD 80 +#define PRECALL_PYFUNC 81 +#define STORE_ATTR_ADAPTIVE 140 +#define STORE_ATTR_INSTANCE_VALUE 141 +#define STORE_ATTR_SLOT 143 +#define STORE_ATTR_WITH_HINT 150 +#define UNPACK_SEQUENCE_ADAPTIVE 153 +#define UNPACK_SEQUENCE_LIST 154 +#define UNPACK_SEQUENCE_TUPLE 158 +#define UNPACK_SEQUENCE_TWO_TUPLE 159 +#define LOAD_FAST__LOAD_FAST 161 +#define LOAD_FAST__LOAD_CONST 167 +#define STORE_FAST__LOAD_FAST 168 +#define STORE_FAST__STORE_FAST 169 +#define LOAD_CONST__LOAD_FAST 170 +#define JUMP_ABSOLUTE_QUICK 173 +#define RESUME_QUICK 174 #define DO_TRACING 255 extern const uint8_t _PyOpcode_InlineCacheEntries[256]; +extern const uint8_t _PyOpcode_Specializations[256]; + #ifdef NEED_OPCODE_TABLES static const uint32_t _PyOpcode_RelativeJump[8] = { 0U, @@ -222,6 +224,78 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = { [PRECALL] = 1, [CALL] = 4, }; + +const uint8_t _PyOpcode_Specializations[256] = { + [BINARY_OP_ADAPTIVE] = BINARY_OP, + [BINARY_OP_ADD_INT] = BINARY_OP, + [BINARY_OP_ADD_FLOAT] = BINARY_OP, + [BINARY_OP_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_MULTIPLY_INT] = BINARY_OP, + [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, + [BINARY_OP_SUBTRACT_INT] = BINARY_OP, + [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, + [COMPARE_OP_ADAPTIVE] = COMPARE_OP, + [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP, + [COMPARE_OP_INT_JUMP] = COMPARE_OP, + [COMPARE_OP_STR_JUMP] = COMPARE_OP, + [BINARY_SUBSCR_ADAPTIVE] = BINARY_SUBSCR, + [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, + [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, + [STORE_SUBSCR_ADAPTIVE] = STORE_SUBSCR, + [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, + [STORE_SUBSCR_DICT] = STORE_SUBSCR, + [CALL_ADAPTIVE] = CALL, + [CALL_PY_EXACT_ARGS] = CALL, + [CALL_PY_WITH_DEFAULTS] = CALL, + [LOAD_ATTR_ADAPTIVE] = LOAD_ATTR, + [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, + [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, + [LOAD_ATTR_SLOT] = LOAD_ATTR, + [LOAD_ATTR_MODULE] = LOAD_ATTR, + [LOAD_GLOBAL_ADAPTIVE] = LOAD_GLOBAL, + [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, + [LOAD_METHOD_ADAPTIVE] = LOAD_METHOD, + [LOAD_METHOD_CLASS] = LOAD_METHOD, + [LOAD_METHOD_MODULE] = LOAD_METHOD, + [LOAD_METHOD_NO_DICT] = LOAD_METHOD, + [LOAD_METHOD_WITH_DICT] = LOAD_METHOD, + [LOAD_METHOD_WITH_VALUES] = LOAD_METHOD, + [PRECALL_ADAPTIVE] = PRECALL, + [PRECALL_BUILTIN_CLASS] = PRECALL, + [PRECALL_NO_KW_BUILTIN_O] = PRECALL, + [PRECALL_NO_KW_BUILTIN_FAST] = PRECALL, + [PRECALL_BUILTIN_FAST_WITH_KEYWORDS] = PRECALL, + [PRECALL_NO_KW_LEN] = PRECALL, + [PRECALL_NO_KW_ISINSTANCE] = PRECALL, + [PRECALL_NO_KW_LIST_APPEND] = PRECALL, + [PRECALL_NO_KW_METHOD_DESCRIPTOR_O] = PRECALL, + [PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = PRECALL, + [PRECALL_NO_KW_STR_1] = PRECALL, + [PRECALL_NO_KW_TUPLE_1] = PRECALL, + [PRECALL_NO_KW_TYPE_1] = PRECALL, + [PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST] = PRECALL, + [PRECALL_BOUND_METHOD] = PRECALL, + [PRECALL_PYFUNC] = PRECALL, + [STORE_ATTR_ADAPTIVE] = STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, + [STORE_ATTR_SLOT] = STORE_ATTR, + [STORE_ATTR_WITH_HINT] = STORE_ATTR, + [UNPACK_SEQUENCE_ADAPTIVE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, + [LOAD_FAST__LOAD_FAST] = LOAD_FAST, + [LOAD_FAST__LOAD_CONST] = LOAD_FAST, + [STORE_FAST__LOAD_FAST] = STORE_FAST, + [STORE_FAST__STORE_FAST] = STORE_FAST, + [LOAD_CONST__LOAD_FAST] = LOAD_CONST, + [JUMP_ABSOLUTE_QUICK] = JUMP_ABSOLUTE, + [RESUME_QUICK] = RESUME, +}; #endif /* OPCODE_TABLES */ #define HAS_CONST(op) (false\ diff --git a/Lib/opcode.py b/Lib/opcode.py index eb9dd35fabf8d9..8cf021b566f84a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -229,78 +229,111 @@ def jabs_op(name, op, entries=0): ("NB_INPLACE_XOR", "^="), ] -_specialized_instructions = [ - "BINARY_OP_ADAPTIVE", - "BINARY_OP_ADD_INT", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_INPLACE_ADD_UNICODE", - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_SUBTRACT_INT", - "BINARY_OP_SUBTRACT_FLOAT", - "COMPARE_OP_ADAPTIVE", - "COMPARE_OP_FLOAT_JUMP", - "COMPARE_OP_INT_JUMP", - "COMPARE_OP_STR_JUMP", - "BINARY_SUBSCR_ADAPTIVE", - "BINARY_SUBSCR_GETITEM", - "BINARY_SUBSCR_LIST_INT", - "BINARY_SUBSCR_TUPLE_INT", - "BINARY_SUBSCR_DICT", - "STORE_SUBSCR_ADAPTIVE", - "STORE_SUBSCR_LIST_INT", - "STORE_SUBSCR_DICT", - "CALL_ADAPTIVE", - "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", - "JUMP_ABSOLUTE_QUICK", - "LOAD_ATTR_ADAPTIVE", - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_WITH_HINT", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_MODULE", - "LOAD_GLOBAL_ADAPTIVE", - "LOAD_GLOBAL_MODULE", - "LOAD_GLOBAL_BUILTIN", - "LOAD_METHOD_ADAPTIVE", - "LOAD_METHOD_CLASS", - "LOAD_METHOD_MODULE", - "LOAD_METHOD_NO_DICT", - "LOAD_METHOD_WITH_DICT", - "LOAD_METHOD_WITH_VALUES", - "PRECALL_ADAPTIVE", - "PRECALL_BUILTIN_CLASS", - "PRECALL_NO_KW_BUILTIN_O", - "PRECALL_NO_KW_BUILTIN_FAST", - "PRECALL_BUILTIN_FAST_WITH_KEYWORDS", - "PRECALL_NO_KW_LEN", - "PRECALL_NO_KW_ISINSTANCE", - "PRECALL_NO_KW_LIST_APPEND", - "PRECALL_NO_KW_METHOD_DESCRIPTOR_O", - "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - "PRECALL_NO_KW_STR_1", - "PRECALL_NO_KW_TUPLE_1", - "PRECALL_NO_KW_TYPE_1", - "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST", - "PRECALL_BOUND_METHOD", - "PRECALL_PYFUNC", - "RESUME_QUICK", - "STORE_ATTR_ADAPTIVE", - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - "UNPACK_SEQUENCE_ADAPTIVE", - "UNPACK_SEQUENCE_LIST", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_TWO_TUPLE", +_specializations = { + "BINARY_OP": [ + "BINARY_OP_ADAPTIVE", + "BINARY_OP_ADD_INT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_INPLACE_ADD_UNICODE", + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_SUBTRACT_FLOAT", + ], + "COMPARE_OP": [ + "COMPARE_OP_ADAPTIVE", + "COMPARE_OP_FLOAT_JUMP", + "COMPARE_OP_INT_JUMP", + "COMPARE_OP_STR_JUMP", + ], + "BINARY_SUBSCR": [ + "BINARY_SUBSCR_ADAPTIVE", + "BINARY_SUBSCR_GETITEM", + "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_TUPLE_INT", + "BINARY_SUBSCR_DICT", + ], + "STORE_SUBSCR": [ + "STORE_SUBSCR_ADAPTIVE", + "STORE_SUBSCR_LIST_INT", + "STORE_SUBSCR_DICT", + ], + "CALL": [ + "CALL_ADAPTIVE", + "CALL_PY_EXACT_ARGS", + "CALL_PY_WITH_DEFAULTS", + ], + "LOAD_ATTR": [ + "LOAD_ATTR_ADAPTIVE", + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_MODULE", + ], + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_ADAPTIVE", + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ], + "LOAD_METHOD": [ + "LOAD_METHOD_ADAPTIVE", + "LOAD_METHOD_CLASS", + "LOAD_METHOD_MODULE", + "LOAD_METHOD_NO_DICT", + "LOAD_METHOD_WITH_DICT", + "LOAD_METHOD_WITH_VALUES", + ], + "PRECALL": [ + "PRECALL_ADAPTIVE", + "PRECALL_BUILTIN_CLASS", + "PRECALL_NO_KW_BUILTIN_O", + "PRECALL_NO_KW_BUILTIN_FAST", + "PRECALL_BUILTIN_FAST_WITH_KEYWORDS", + "PRECALL_NO_KW_LEN", + "PRECALL_NO_KW_ISINSTANCE", + "PRECALL_NO_KW_LIST_APPEND", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_O", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + "PRECALL_NO_KW_STR_1", + "PRECALL_NO_KW_TUPLE_1", + "PRECALL_NO_KW_TYPE_1", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST", + "PRECALL_BOUND_METHOD", + "PRECALL_PYFUNC", + ], + "STORE_ATTR": [ + "STORE_ATTR_ADAPTIVE", + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ], + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_ADAPTIVE", + "UNPACK_SEQUENCE_LIST", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_TWO_TUPLE", + ], # Super instructions - "LOAD_FAST__LOAD_FAST", - "STORE_FAST__LOAD_FAST", - "LOAD_FAST__LOAD_CONST", - "LOAD_CONST__LOAD_FAST", - "STORE_FAST__STORE_FAST", -] + "LOAD_FAST": [ + "LOAD_FAST__LOAD_FAST", + "LOAD_FAST__LOAD_CONST", + ], + "STORE_FAST": [ + "STORE_FAST__LOAD_FAST", + "STORE_FAST__STORE_FAST", + ], + "LOAD_CONST": [ + "LOAD_CONST__LOAD_FAST", + ], + # Other instructions + "JUMP_ABSOLUTE": [ + "JUMP_ABSOLUTE_QUICK", + ], + "RESUME": [ + "RESUME_QUICK", + ], +} _specialization_stats = [ "success", "failure", diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 7c1c0cfdb069b8..b4ee149049e7ee 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -1,4 +1,5 @@ import dis +from itertools import chain from test.support.import_helper import import_module import unittest import opcode @@ -70,7 +71,7 @@ def test_specialization_stats(self): specialized_opcodes = [ op[:-len("_ADAPTIVE")].lower() for - op in opcode._specialized_instructions + op in chain.from_iterable(opcode._specializations.values()) if op.endswith("_ADAPTIVE")] self.assertIn('load_attr', specialized_opcodes) self.assertIn('binary_subscr', specialized_opcodes) diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 3bf2e35ccb6dab..ec12478bebac33 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -3,6 +3,7 @@ (for compilers supporting computed gotos or "labels-as-values", such as gcc). """ +from itertools import chain import os import sys @@ -36,7 +37,7 @@ def write_contents(f): for opname, op in opcode.opmap.items(): targets[op] = "TARGET_%s" % opname next_op = 1 - for opname in opcode._specialized_instructions: + for opname in chain.from_iterable(opcode._specializations.values()): while targets[next_op] != '_unknown_opcode': next_op += 1 targets[next_op] = "TARGET_%s" % opname diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 7c94999bfdfe42..5629126e6a7932 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -37,7 +37,6 @@ static void *opcode_targets[256] = { &&TARGET_PUSH_EXC_INFO, &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_JUMP_ABSOLUTE_QUICK, &&TARGET_LOAD_ATTR_ADAPTIVE, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_WITH_HINT, @@ -48,39 +47,40 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_LOAD_METHOD_CLASS, + &&TARGET_LOAD_METHOD_MODULE, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_BEFORE_ASYNC_WITH, &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, - &&TARGET_LOAD_METHOD_MODULE, &&TARGET_LOAD_METHOD_NO_DICT, &&TARGET_LOAD_METHOD_WITH_DICT, &&TARGET_LOAD_METHOD_WITH_VALUES, &&TARGET_PRECALL_ADAPTIVE, + &&TARGET_PRECALL_BUILTIN_CLASS, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_PRECALL_BUILTIN_CLASS, &&TARGET_PRECALL_NO_KW_BUILTIN_O, &&TARGET_PRECALL_NO_KW_BUILTIN_FAST, &&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_PRECALL_NO_KW_LEN, &&TARGET_PRECALL_NO_KW_ISINSTANCE, + &&TARGET_PRECALL_NO_KW_LIST_APPEND, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_PRECALL_NO_KW_LIST_APPEND, &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O, + &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, - &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, &&TARGET_PRECALL_NO_KW_STR_1, &&TARGET_PRECALL_NO_KW_TUPLE_1, &&TARGET_PRECALL_NO_KW_TYPE_1, &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_PRECALL_BOUND_METHOD, + &&TARGET_PRECALL_PYFUNC, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, @@ -139,41 +139,41 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, - &&TARGET_PRECALL_PYFUNC, - &&TARGET_RESUME_QUICK, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_CALL_FUNCTION_EX, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, &&TARGET_COPY_FREE_VARS, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, - &&TARGET_UNPACK_SEQUENCE_LIST, - &&TARGET_LOAD_METHOD, &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_LOAD_METHOD, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_PRECALL, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&TARGET_LOAD_CONST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_JUMP_ABSOLUTE_QUICK, + &&TARGET_RESUME_QUICK, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index d0ef41e71cfa4e..f00dc8e3944e57 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -351,132 +351,14 @@ _Py_SetCountAndUnquicken(PyCodeObject *code) _Py_CODEUNIT *instructions = _PyCode_GET_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _Py_OPCODE(instructions[i]); - // TODO: Generate a table for this: - switch (opcode) { - case BINARY_OP: - case BINARY_OP_ADAPTIVE: - case BINARY_OP_ADD_FLOAT: - case BINARY_OP_ADD_INT: - case BINARY_OP_ADD_UNICODE: - case BINARY_OP_INPLACE_ADD_UNICODE: - case BINARY_OP_MULTIPLY_FLOAT: - case BINARY_OP_MULTIPLY_INT: - case BINARY_OP_SUBTRACT_FLOAT: - case BINARY_OP_SUBTRACT_INT: - opcode = BINARY_OP; - break; - case BINARY_SUBSCR: - case BINARY_SUBSCR_ADAPTIVE: - case BINARY_SUBSCR_DICT: - case BINARY_SUBSCR_GETITEM: - case BINARY_SUBSCR_LIST_INT: - case BINARY_SUBSCR_TUPLE_INT: - opcode = BINARY_SUBSCR; - break; - case CALL: - case CALL_ADAPTIVE: - case CALL_PY_EXACT_ARGS: - case CALL_PY_WITH_DEFAULTS: - opcode = CALL; - break; - case COMPARE_OP: - case COMPARE_OP_ADAPTIVE: - case COMPARE_OP_FLOAT_JUMP: - case COMPARE_OP_INT_JUMP: - case COMPARE_OP_STR_JUMP: - opcode = COMPARE_OP; - break; - case JUMP_ABSOLUTE: - case JUMP_ABSOLUTE_QUICK: - opcode = JUMP_ABSOLUTE; - break; - case LOAD_ATTR: - case LOAD_ATTR_ADAPTIVE: - case LOAD_ATTR_INSTANCE_VALUE: - case LOAD_ATTR_MODULE: - case LOAD_ATTR_SLOT: - case LOAD_ATTR_WITH_HINT: - opcode = LOAD_ATTR; - break; - case LOAD_CONST: - case LOAD_CONST__LOAD_FAST: - opcode = LOAD_CONST; - break; - case LOAD_FAST: - case LOAD_FAST__LOAD_CONST: - case LOAD_FAST__LOAD_FAST: - opcode = LOAD_FAST; - break; - case LOAD_GLOBAL: - case LOAD_GLOBAL_ADAPTIVE: - case LOAD_GLOBAL_BUILTIN: - case LOAD_GLOBAL_MODULE: - opcode = LOAD_GLOBAL; - break; - case LOAD_METHOD: - case LOAD_METHOD_ADAPTIVE: - case LOAD_METHOD_CLASS: - case LOAD_METHOD_MODULE: - case LOAD_METHOD_NO_DICT: - case LOAD_METHOD_WITH_DICT: - case LOAD_METHOD_WITH_VALUES: - opcode = LOAD_METHOD; - break; - case PRECALL: - case PRECALL_ADAPTIVE: - case PRECALL_BOUND_METHOD: - case PRECALL_BUILTIN_CLASS: - case PRECALL_BUILTIN_FAST_WITH_KEYWORDS: - case PRECALL_NO_KW_BUILTIN_FAST: - case PRECALL_NO_KW_BUILTIN_O: - case PRECALL_NO_KW_ISINSTANCE: - case PRECALL_NO_KW_LEN: - case PRECALL_NO_KW_LIST_APPEND: - case PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST: - case PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: - case PRECALL_NO_KW_METHOD_DESCRIPTOR_O: - case PRECALL_NO_KW_STR_1: - case PRECALL_NO_KW_TUPLE_1: - case PRECALL_NO_KW_TYPE_1: - case PRECALL_PYFUNC: - opcode = PRECALL; - break; - case RESUME: - case RESUME_QUICK: - opcode = RESUME; - break; - case STORE_ATTR: - case STORE_ATTR_ADAPTIVE: - case STORE_ATTR_INSTANCE_VALUE: - case STORE_ATTR_SLOT: - case STORE_ATTR_WITH_HINT: - opcode = STORE_ATTR; - break; - case STORE_FAST: - case STORE_FAST__LOAD_FAST: - case STORE_FAST__STORE_FAST: - opcode = STORE_FAST; - break; - case STORE_SUBSCR: - case STORE_SUBSCR_ADAPTIVE: - case STORE_SUBSCR_DICT: - case STORE_SUBSCR_LIST_INT: - opcode = STORE_SUBSCR; - break; - case UNPACK_SEQUENCE: - case UNPACK_SEQUENCE_ADAPTIVE: - case UNPACK_SEQUENCE_LIST: - case UNPACK_SEQUENCE_TUPLE: - case UNPACK_SEQUENCE_TWO_TUPLE: - opcode = UNPACK_SEQUENCE; - break; - default: - assert(!_PyOpcode_InlineCacheEntries[opcode]); - continue; - } - instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instructions[i])); - for (int j = 0; j < _PyOpcode_InlineCacheEntries[opcode]; j++) { - instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); + int deopt = _PyOpcode_Specializations[opcode]; + if (deopt) { + int oparg = _Py_OPARG(instructions[i]); + instructions[i] = _Py_MAKECODEUNIT(deopt, oparg); + int cache_entries = _PyOpcode_InlineCacheEntries[deopt]; + for (int j = 0; j < cache_entries; j++) { + instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); + } } } } diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index 75a9c3f3bfadbc..009b725606cf4e 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -1,5 +1,6 @@ # This script generates the opcode.h header file. +from itertools import chain import sys import tokenize @@ -69,13 +70,14 @@ def main(opcode_py, outfile='Include/opcode.h'): if name == 'POP_EXCEPT': # Special entry for HAVE_ARGUMENT fobj.write(DEFINE.format("HAVE_ARGUMENT", opcode["HAVE_ARGUMENT"])) - for name in opcode['_specialized_instructions']: + for name in chain.from_iterable(opcode['_specializations'].values()): while used[next_op]: next_op += 1 fobj.write(DEFINE.format(name, next_op)) used[next_op] = True fobj.write(DEFINE.format('DO_TRACING', 255)) fobj.write("\nextern const uint8_t _PyOpcode_InlineCacheEntries[256];\n") + fobj.write("\nextern const uint8_t _PyOpcode_Specializations[256];\n") fobj.write("\n#ifdef NEED_OPCODE_TABLES\n") write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj) write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj) @@ -85,6 +87,11 @@ def main(opcode_py, outfile='Include/opcode.h'): if entries: fobj.write(f" [{opname[i]}] = {entries},\n") fobj.write("};\n") + fobj.write("\nconst uint8_t _PyOpcode_Specializations[256] = {\n") + for basic, specializations in opcode["_specializations"].items(): + for specialization in specializations: + fobj.write(f" [{specialization}] = {basic},\n") + fobj.write("};\n") fobj.write("#endif /* OPCODE_TABLES */\n") fobj.write("\n") diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 69babfd2ddafc6..44374c94bd3c22 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -15,7 +15,7 @@ DEFAULT_DIR = "/tmp/py_stats/" #Create list of all instruction names -specialized = iter(opcode._specialized_instructions) +specialized = itertools.chain.from_iterable(opcode._specializations.values()) opname = ["<0>"] for name in opcode.opname[1:]: if name.startswith("<"): @@ -131,10 +131,10 @@ def categorized_counts(opcode_stats): specialized = 0 not_specialized = 0 specialized_instructions = { - op for op in opcode._specialized_instructions + op for op in itertools.chain.from_iterable(opcode._specializations.values()) if "__" not in op and "ADAPTIVE" not in op} adaptive_instructions = { - op for op in opcode._specialized_instructions + op for op in itertools.chain.from_iterable(opcode._specializations.values()) if "ADAPTIVE" in op} for i, opcode_stat in enumerate(opcode_stats): if "execution_count" not in opcode_stat: From 03768229354782cead5ccba7039eb8fb8dd91487 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 10 Mar 2022 14:57:28 -0800 Subject: [PATCH 06/34] Clean things up a bit --- Include/cpython/code.h | 3 --- Include/internal/pycore_code.h | 12 ++++++++++-- Objects/codeobject.c | 1 - Python/ceval.c | 11 +++++++---- Python/specialize.c | 9 ++++----- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index b4d6219817fe44..414737c42eead5 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -92,9 +92,6 @@ struct PyCodeObject { Type is a void* to keep the format private in codeobject.c to force people to go through the proper APIs. */ void *co_extra; - /* Quickened instructions and cache, or NULL - This should be treated as opaque by all code except the specializer and - interpreter. */ char _co_code[1]; }; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index dd7a7309a374c3..24b347c6ed9fea 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -98,19 +98,27 @@ typedef struct { #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) void _Py_Quicken(PyCodeObject *code); +void _Py_Unquicken(PyCodeObject *code); static inline void _Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) { if (code->co_warmup != 0) { code->co_warmup++; - if (code->co_warmup == 0) { + if (++code->co_warmup == 0) { _Py_Quicken(code); } } } -void _Py_SetCountAndUnquicken(PyCodeObject *code); +static inline void +_Py_SetCountAndUnquicken(PyCodeObject *code) +{ + if (code->co_warmup == 0) { + code->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; + _Py_Unquicken(code); + } +} extern Py_ssize_t _Py_QuickenedCount; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 2ae4ff4b48a42b..4bf45e906cfbc5 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1499,7 +1499,6 @@ code_getundercode(PyCodeObject *code, void *closure) static PyObject * code_getcode(PyCodeObject *code, void *closure) { - // XXX _Py_SetCountAndUnquicken(code); return PyBytes_FromStringAndSize(code->_co_code, _PyCode_GET_SIZE(code)); } diff --git a/Python/ceval.c b/Python/ceval.c index fb90fbe8b20bae..9e10ffcd1c87eb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1325,6 +1325,10 @@ eval_frame_handle_pending(PyThreadState *tstate) #define SKIP_CALL() \ JUMPBY(INLINE_CACHE_ENTRIES_PRECALL + 1 + INLINE_CACHE_ENTRIES_CALL) +#define TRACING_NEXTOPARG() do { \ + _Py_SetCountAndUnquicken(frame->f_code); \ + NEXTOPARG(); \ + } while (0) /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to @@ -1562,6 +1566,7 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject static int skip_backwards_over_extended_args(PyCodeObject *code, int offset) { + _Py_SetCountAndUnquicken(code); _Py_CODEUNIT *instrs = _PyCode_GET_CODE(code); while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) { offset--; @@ -5419,11 +5424,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #else case DO_TRACING: { #endif - // Un-quicken in-place: - _Py_SetCountAndUnquicken(frame->f_code); int instr_prev = skip_backwards_over_extended_args(frame->f_code, frame->f_lasti); frame->f_lasti = INSTR_OFFSET(); - NEXTOPARG(); + TRACING_NEXTOPARG(); if (opcode == RESUME) { if (oparg < 2) { CHECK_EVAL_BREAKER(); @@ -5460,7 +5463,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->stacktop = -1; } } - NEXTOPARG(); + TRACING_NEXTOPARG(); PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } diff --git a/Python/specialize.c b/Python/specialize.c index f00dc8e3944e57..7194900d683be6 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -281,6 +281,7 @@ _Py_PrintSpecializationStats(int to_file) void _Py_Quicken(PyCodeObject *code) { + assert(code->co_warmup == 0); _Py_QuickenedCount++; int previous_opcode = -1; int previous_oparg = -1; @@ -342,12 +343,10 @@ _Py_Quicken(PyCodeObject *code) } void -_Py_SetCountAndUnquicken(PyCodeObject *code) +_Py_Unquicken(PyCodeObject *code) { - code->co_warmup = QUICKENING_WARMUP_DELAY; - // if (!code->co_quickened) { - // return; - // } + assert(code->co_warmup == QUICKENING_INITIAL_WARMUP_VALUE); + _Py_QuickenedCount--; _Py_CODEUNIT *instructions = _PyCode_GET_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _Py_OPCODE(instructions[i]); From 3e77b8d969693e3c30f915210c2a688a464f7f0d Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 10 Mar 2022 15:14:48 -0800 Subject: [PATCH 07/34] Pack code objects more efficiently --- Include/cpython/code.h | 17 ++++++++--------- Tools/scripts/deepfreeze.py | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 414737c42eead5..e2d905e2f6a43c 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -63,6 +63,14 @@ struct PyCodeObject { int co_kwonlyargcount; /* #keyword only arguments */ int co_stacksize; /* #entries needed for evaluation stack */ int co_firstlineno; /* first source line number */ + + // redundant values (derived from co_localsplusnames and co_localspluskinds) + int co_nlocalsplus; /* number of local + cell + free variables */ + int co_nlocals; /* number of local variables */ + int co_nplaincellvars; /* number of non-arg cell variables */ + int co_ncellvars; /* total number of cell variables */ + int co_nfreevars; /* number of free variables */ + PyObject *co_localsplusnames; /* tuple mapping offsets to names */ PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */ PyObject *co_filename; /* unicode (where it was loaded from) */ @@ -76,15 +84,6 @@ struct PyCodeObject { PyObject *co_columntable; /* bytes object that holds start/end column offset each instruction */ - /* These fields are set with computed values on new code objects. */ - - // redundant values (derived from co_localsplusnames and co_localspluskinds) - int co_nlocalsplus; /* number of local + cell + free variables */ - int co_nlocals; /* number of local variables */ - int co_nplaincellvars; /* number of non-arg cell variables */ - int co_ncellvars; /* total number of cell variables */ - int co_nfreevars; /* number of free variables */ - /* The remaining fields are zeroed out on new code objects. */ PyObject *co_weakreflist; /* to support weakrefs to code objects */ diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 98496ac8b3cabb..f313d74f57461c 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -259,6 +259,11 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write("int co_kwonlyargcount;") self.write("int co_stacksize;") self.write("int co_firstlineno;") + self.write("int co_nlocalsplus;") + self.write("int co_nlocals;") + self.write("int co_nplaincellvars;") + self.write("int co_ncellvars;") + self.write("int co_nfreevars;") self.write("PyObject *co_localsplusnames;") self.write("PyObject *co_localspluskinds;") self.write("PyObject *co_filename;") @@ -267,11 +272,6 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write("PyObject *co_linetable;") self.write("PyObject *co_endlinetable;") self.write("PyObject *co_columntable;") - self.write("int co_nlocalsplus;") - self.write("int co_nlocals;") - self.write("int co_nplaincellvars;") - self.write("int co_ncellvars;") - self.write("int co_nfreevars;") self.write("PyObject *co_weakreflist;") self.write("void *co_extra;") self.write(f"char _co_code[{len(code.co_code)}];") @@ -290,6 +290,11 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.field(code, "co_kwonlyargcount") self.field(code, "co_stacksize") self.field(code, "co_firstlineno") + self.write(f".co_nlocalsplus = {len(localsplusnames)},") + self.field(code, "co_nlocals") + self.write(f".co_nplaincellvars = {nplaincellvars},") + self.write(f".co_ncellvars = {ncellvars},") + self.write(f".co_nfreevars = {nfreevars},") self.write(f".co_localsplusnames = {co_localsplusnames},") self.write(f".co_localspluskinds = {co_localspluskinds},") self.write(f".co_filename = {co_filename},") @@ -298,11 +303,6 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_linetable = {co_linetable},") self.write(f".co_endlinetable = {co_endlinetable},") self.write(f".co_columntable = {co_columntable},") - self.write(f".co_nlocalsplus = {len(localsplusnames)},") - self.field(code, "co_nlocals") - self.write(f".co_nplaincellvars = {nplaincellvars},") - self.write(f".co_ncellvars = {ncellvars},") - self.write(f".co_nfreevars = {nfreevars},") self.write(f"._co_code = {make_string_literal(code.co_code)},") self.deallocs.append(f"_PyStaticCode_Dealloc(&{name});") self.interns.append(f"_PyStaticCode_InternStrings(&{name})") From 0a598a76a035556298812ebe63e7d8aae1794532 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 10 Mar 2022 15:16:14 -0800 Subject: [PATCH 08/34] Fix typo --- Include/internal/pycore_code.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 24b347c6ed9fea..6536707975ea06 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -105,7 +105,7 @@ _Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) { if (code->co_warmup != 0) { code->co_warmup++; - if (++code->co_warmup == 0) { + if (code->co_warmup == 0) { _Py_Quicken(code); } } From 2fda3b8af4f6bd46428b4c016bb1c720cefe30dc Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 10 Mar 2022 16:04:49 -0800 Subject: [PATCH 09/34] More cleanup --- Include/internal/pycore_code.h | 4 +- Include/opcode.h | 290 ++++++++++++++++++++--------- Lib/opcode.py | 175 ++++++++--------- Lib/test/test__opcode.py | 2 +- Python/makeopcodetargets.py | 2 +- Python/opcode_targets.h | 90 ++++----- Python/specialize.c | 21 +-- Tools/scripts/generate_opcode_h.py | 17 +- Tools/scripts/summarize_stats.py | 6 +- 9 files changed, 356 insertions(+), 251 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 6536707975ea06..db7b125d84caf4 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -98,7 +98,7 @@ typedef struct { #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) void _Py_Quicken(PyCodeObject *code); -void _Py_Unquicken(PyCodeObject *code); +void _Py_Unquicken(_Py_CODEUNIT *instructions, Py_ssize_t size); static inline void _Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) @@ -116,7 +116,7 @@ _Py_SetCountAndUnquicken(PyCodeObject *code) { if (code->co_warmup == 0) { code->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; - _Py_Unquicken(code); + _Py_Unquicken(_PyCode_GET_CODE(code), Py_SIZE(code)); } } diff --git a/Include/opcode.h b/Include/opcode.h index c6e3a3427d3b13..ea041f0fc64344 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -115,79 +115,79 @@ extern "C" { #define CALL 171 #define KW_NAMES 172 #define BINARY_OP_ADAPTIVE 3 -#define BINARY_OP_ADD_INT 4 -#define BINARY_OP_ADD_FLOAT 5 +#define BINARY_OP_ADD_FLOAT 4 +#define BINARY_OP_ADD_INT 5 #define BINARY_OP_ADD_UNICODE 6 #define BINARY_OP_INPLACE_ADD_UNICODE 7 -#define BINARY_OP_MULTIPLY_INT 8 -#define BINARY_OP_MULTIPLY_FLOAT 13 -#define BINARY_OP_SUBTRACT_INT 14 -#define BINARY_OP_SUBTRACT_FLOAT 16 -#define COMPARE_OP_ADAPTIVE 17 -#define COMPARE_OP_FLOAT_JUMP 18 -#define COMPARE_OP_INT_JUMP 19 -#define COMPARE_OP_STR_JUMP 20 -#define BINARY_SUBSCR_ADAPTIVE 21 -#define BINARY_SUBSCR_GETITEM 22 -#define BINARY_SUBSCR_LIST_INT 23 -#define BINARY_SUBSCR_TUPLE_INT 24 -#define BINARY_SUBSCR_DICT 26 -#define STORE_SUBSCR_ADAPTIVE 27 -#define STORE_SUBSCR_LIST_INT 28 -#define STORE_SUBSCR_DICT 29 -#define CALL_ADAPTIVE 34 -#define CALL_PY_EXACT_ARGS 36 -#define CALL_PY_WITH_DEFAULTS 37 -#define LOAD_ATTR_ADAPTIVE 38 -#define LOAD_ATTR_INSTANCE_VALUE 39 +#define BINARY_OP_MULTIPLY_FLOAT 8 +#define BINARY_OP_MULTIPLY_INT 13 +#define BINARY_OP_SUBTRACT_FLOAT 14 +#define BINARY_OP_SUBTRACT_INT 16 +#define BINARY_SUBSCR_ADAPTIVE 17 +#define BINARY_SUBSCR_DICT 18 +#define BINARY_SUBSCR_GETITEM 19 +#define BINARY_SUBSCR_LIST_INT 20 +#define BINARY_SUBSCR_TUPLE_INT 21 +#define CALL_ADAPTIVE 22 +#define CALL_PY_EXACT_ARGS 23 +#define CALL_PY_WITH_DEFAULTS 24 +#define COMPARE_OP_ADAPTIVE 26 +#define COMPARE_OP_FLOAT_JUMP 27 +#define COMPARE_OP_INT_JUMP 28 +#define COMPARE_OP_STR_JUMP 29 +#define JUMP_ABSOLUTE_QUICK 34 +#define LOAD_ATTR_ADAPTIVE 36 +#define LOAD_ATTR_INSTANCE_VALUE 37 +#define LOAD_ATTR_MODULE 38 +#define LOAD_ATTR_SLOT 39 #define LOAD_ATTR_WITH_HINT 40 -#define LOAD_ATTR_SLOT 41 -#define LOAD_ATTR_MODULE 42 -#define LOAD_GLOBAL_ADAPTIVE 43 -#define LOAD_GLOBAL_MODULE 44 +#define LOAD_CONST__LOAD_FAST 41 +#define LOAD_FAST__LOAD_CONST 42 +#define LOAD_FAST__LOAD_FAST 43 +#define LOAD_GLOBAL_ADAPTIVE 44 #define LOAD_GLOBAL_BUILTIN 45 -#define LOAD_METHOD_ADAPTIVE 46 -#define LOAD_METHOD_CLASS 47 -#define LOAD_METHOD_MODULE 48 -#define LOAD_METHOD_NO_DICT 55 -#define LOAD_METHOD_WITH_DICT 56 -#define LOAD_METHOD_WITH_VALUES 57 -#define PRECALL_ADAPTIVE 58 -#define PRECALL_BUILTIN_CLASS 59 -#define PRECALL_NO_KW_BUILTIN_O 62 -#define PRECALL_NO_KW_BUILTIN_FAST 63 +#define LOAD_GLOBAL_MODULE 46 +#define LOAD_METHOD_ADAPTIVE 47 +#define LOAD_METHOD_CLASS 48 +#define LOAD_METHOD_MODULE 55 +#define LOAD_METHOD_NO_DICT 56 +#define LOAD_METHOD_WITH_DICT 57 +#define LOAD_METHOD_WITH_VALUES 58 +#define PRECALL_ADAPTIVE 59 +#define PRECALL_BOUND_METHOD 62 +#define PRECALL_BUILTIN_CLASS 63 #define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 64 -#define PRECALL_NO_KW_LEN 65 -#define PRECALL_NO_KW_ISINSTANCE 66 -#define PRECALL_NO_KW_LIST_APPEND 67 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 72 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 73 -#define PRECALL_NO_KW_STR_1 76 -#define PRECALL_NO_KW_TUPLE_1 77 -#define PRECALL_NO_KW_TYPE_1 78 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 79 -#define PRECALL_BOUND_METHOD 80 -#define PRECALL_PYFUNC 81 -#define STORE_ATTR_ADAPTIVE 140 -#define STORE_ATTR_INSTANCE_VALUE 141 -#define STORE_ATTR_SLOT 143 -#define STORE_ATTR_WITH_HINT 150 -#define UNPACK_SEQUENCE_ADAPTIVE 153 -#define UNPACK_SEQUENCE_LIST 154 -#define UNPACK_SEQUENCE_TUPLE 158 -#define UNPACK_SEQUENCE_TWO_TUPLE 159 -#define LOAD_FAST__LOAD_FAST 161 -#define LOAD_FAST__LOAD_CONST 167 -#define STORE_FAST__LOAD_FAST 168 -#define STORE_FAST__STORE_FAST 169 -#define LOAD_CONST__LOAD_FAST 170 -#define JUMP_ABSOLUTE_QUICK 173 -#define RESUME_QUICK 174 +#define PRECALL_NO_KW_BUILTIN_FAST 65 +#define PRECALL_NO_KW_BUILTIN_O 66 +#define PRECALL_NO_KW_ISINSTANCE 67 +#define PRECALL_NO_KW_LEN 72 +#define PRECALL_NO_KW_LIST_APPEND 73 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 76 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 77 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 78 +#define PRECALL_NO_KW_STR_1 79 +#define PRECALL_NO_KW_TUPLE_1 80 +#define PRECALL_NO_KW_TYPE_1 81 +#define PRECALL_PYFUNC 140 +#define RESUME_QUICK 141 +#define STORE_ATTR_ADAPTIVE 143 +#define STORE_ATTR_INSTANCE_VALUE 150 +#define STORE_ATTR_SLOT 153 +#define STORE_ATTR_WITH_HINT 154 +#define STORE_FAST__LOAD_FAST 158 +#define STORE_FAST__STORE_FAST 159 +#define STORE_SUBSCR_ADAPTIVE 161 +#define STORE_SUBSCR_DICT 167 +#define STORE_SUBSCR_LIST_INT 168 +#define UNPACK_SEQUENCE_ADAPTIVE 169 +#define UNPACK_SEQUENCE_LIST 170 +#define UNPACK_SEQUENCE_TUPLE 173 +#define UNPACK_SEQUENCE_TWO_TUPLE 174 #define DO_TRACING 255 extern const uint8_t _PyOpcode_InlineCacheEntries[256]; -extern const uint8_t _PyOpcode_Specializations[256]; +extern const uint8_t _PyOpcode_Deoptimizations[256]; #ifdef NEED_OPCODE_TABLES static const uint32_t _PyOpcode_RelativeJump[8] = { @@ -225,76 +225,182 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = { [CALL] = 4, }; -const uint8_t _PyOpcode_Specializations[256] = { +const uint8_t _PyOpcode_Deoptimizations[256] = { + [ASYNC_GEN_WRAP] = ASYNC_GEN_WRAP, + [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, + [BEFORE_WITH] = BEFORE_WITH, + [BINARY_OP] = BINARY_OP, [BINARY_OP_ADAPTIVE] = BINARY_OP, - [BINARY_OP_ADD_INT] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, + [BINARY_OP_ADD_INT] = BINARY_OP, [BINARY_OP_ADD_UNICODE] = BINARY_OP, [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, - [BINARY_OP_MULTIPLY_INT] = BINARY_OP, [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, - [BINARY_OP_SUBTRACT_INT] = BINARY_OP, + [BINARY_OP_MULTIPLY_INT] = BINARY_OP, [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, - [COMPARE_OP_ADAPTIVE] = COMPARE_OP, - [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP, - [COMPARE_OP_INT_JUMP] = COMPARE_OP, - [COMPARE_OP_STR_JUMP] = COMPARE_OP, + [BINARY_OP_SUBTRACT_INT] = BINARY_OP, + [BINARY_SUBSCR] = BINARY_SUBSCR, [BINARY_SUBSCR_ADAPTIVE] = BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, - [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, - [STORE_SUBSCR_ADAPTIVE] = STORE_SUBSCR, - [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, - [STORE_SUBSCR_DICT] = STORE_SUBSCR, + [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, + [BUILD_LIST] = BUILD_LIST, + [BUILD_MAP] = BUILD_MAP, + [BUILD_SET] = BUILD_SET, + [BUILD_SLICE] = BUILD_SLICE, + [BUILD_STRING] = BUILD_STRING, + [BUILD_TUPLE] = BUILD_TUPLE, + [CACHE] = CACHE, + [CALL] = CALL, [CALL_ADAPTIVE] = CALL, + [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_PY_EXACT_ARGS] = CALL, [CALL_PY_WITH_DEFAULTS] = CALL, + [COMPARE_OP] = COMPARE_OP, + [COMPARE_OP_ADAPTIVE] = COMPARE_OP, + [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP, + [COMPARE_OP_INT_JUMP] = COMPARE_OP, + [COMPARE_OP_STR_JUMP] = COMPARE_OP, + [CONTAINS_OP] = CONTAINS_OP, + [COPY] = COPY, + [COPY_FREE_VARS] = COPY_FREE_VARS, + [DELETE_ATTR] = DELETE_ATTR, + [DELETE_DEREF] = DELETE_DEREF, + [DELETE_FAST] = DELETE_FAST, + [DELETE_GLOBAL] = DELETE_GLOBAL, + [DELETE_NAME] = DELETE_NAME, + [DELETE_SUBSCR] = DELETE_SUBSCR, + [DICT_MERGE] = DICT_MERGE, + [DICT_UPDATE] = DICT_UPDATE, + [END_ASYNC_FOR] = END_ASYNC_FOR, + [EXTENDED_ARG] = EXTENDED_ARG, + [FORMAT_VALUE] = FORMAT_VALUE, + [FOR_ITER] = FOR_ITER, + [GET_AITER] = GET_AITER, + [GET_ANEXT] = GET_ANEXT, + [GET_AWAITABLE] = GET_AWAITABLE, + [GET_ITER] = GET_ITER, + [GET_LEN] = GET_LEN, + [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, + [IMPORT_FROM] = IMPORT_FROM, + [IMPORT_NAME] = IMPORT_NAME, + [IMPORT_STAR] = IMPORT_STAR, + [IS_OP] = IS_OP, + [JUMP_ABSOLUTE] = JUMP_ABSOLUTE, + [JUMP_ABSOLUTE_QUICK] = JUMP_ABSOLUTE, + [JUMP_FORWARD] = JUMP_FORWARD, + [JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP, + [JUMP_IF_NOT_EG_MATCH] = JUMP_IF_NOT_EG_MATCH, + [JUMP_IF_NOT_EXC_MATCH] = JUMP_IF_NOT_EXC_MATCH, + [JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP, + [JUMP_NO_INTERRUPT] = JUMP_NO_INTERRUPT, + [KW_NAMES] = KW_NAMES, + [LIST_APPEND] = LIST_APPEND, + [LIST_EXTEND] = LIST_EXTEND, + [LIST_TO_TUPLE] = LIST_TO_TUPLE, + [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, + [LOAD_ATTR] = LOAD_ATTR, [LOAD_ATTR_ADAPTIVE] = LOAD_ATTR, [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, - [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, - [LOAD_ATTR_SLOT] = LOAD_ATTR, [LOAD_ATTR_MODULE] = LOAD_ATTR, + [LOAD_ATTR_SLOT] = LOAD_ATTR, + [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, + [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, + [LOAD_CLASSDEREF] = LOAD_CLASSDEREF, + [LOAD_CLOSURE] = LOAD_CLOSURE, + [LOAD_CONST] = LOAD_CONST, + [LOAD_CONST__LOAD_FAST] = LOAD_CONST, + [LOAD_DEREF] = LOAD_DEREF, + [LOAD_FAST] = LOAD_FAST, + [LOAD_FAST__LOAD_CONST] = LOAD_FAST, + [LOAD_FAST__LOAD_FAST] = LOAD_FAST, + [LOAD_GLOBAL] = LOAD_GLOBAL, [LOAD_GLOBAL_ADAPTIVE] = LOAD_GLOBAL, - [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, + [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, + [LOAD_METHOD] = LOAD_METHOD, [LOAD_METHOD_ADAPTIVE] = LOAD_METHOD, [LOAD_METHOD_CLASS] = LOAD_METHOD, [LOAD_METHOD_MODULE] = LOAD_METHOD, [LOAD_METHOD_NO_DICT] = LOAD_METHOD, [LOAD_METHOD_WITH_DICT] = LOAD_METHOD, [LOAD_METHOD_WITH_VALUES] = LOAD_METHOD, + [LOAD_NAME] = LOAD_NAME, + [MAKE_CELL] = MAKE_CELL, + [MAKE_FUNCTION] = MAKE_FUNCTION, + [MAP_ADD] = MAP_ADD, + [MATCH_CLASS] = MATCH_CLASS, + [MATCH_KEYS] = MATCH_KEYS, + [MATCH_MAPPING] = MATCH_MAPPING, + [MATCH_SEQUENCE] = MATCH_SEQUENCE, + [NOP] = NOP, + [POP_EXCEPT] = POP_EXCEPT, + [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, + [POP_TOP] = POP_TOP, + [PRECALL] = PRECALL, [PRECALL_ADAPTIVE] = PRECALL, + [PRECALL_BOUND_METHOD] = PRECALL, [PRECALL_BUILTIN_CLASS] = PRECALL, - [PRECALL_NO_KW_BUILTIN_O] = PRECALL, - [PRECALL_NO_KW_BUILTIN_FAST] = PRECALL, [PRECALL_BUILTIN_FAST_WITH_KEYWORDS] = PRECALL, - [PRECALL_NO_KW_LEN] = PRECALL, + [PRECALL_NO_KW_BUILTIN_FAST] = PRECALL, + [PRECALL_NO_KW_BUILTIN_O] = PRECALL, [PRECALL_NO_KW_ISINSTANCE] = PRECALL, + [PRECALL_NO_KW_LEN] = PRECALL, [PRECALL_NO_KW_LIST_APPEND] = PRECALL, - [PRECALL_NO_KW_METHOD_DESCRIPTOR_O] = PRECALL, + [PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST] = PRECALL, [PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = PRECALL, + [PRECALL_NO_KW_METHOD_DESCRIPTOR_O] = PRECALL, [PRECALL_NO_KW_STR_1] = PRECALL, [PRECALL_NO_KW_TUPLE_1] = PRECALL, [PRECALL_NO_KW_TYPE_1] = PRECALL, - [PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST] = PRECALL, - [PRECALL_BOUND_METHOD] = PRECALL, [PRECALL_PYFUNC] = PRECALL, + [PREP_RERAISE_STAR] = PREP_RERAISE_STAR, + [PRINT_EXPR] = PRINT_EXPR, + [PUSH_EXC_INFO] = PUSH_EXC_INFO, + [PUSH_NULL] = PUSH_NULL, + [RAISE_VARARGS] = RAISE_VARARGS, + [RERAISE] = RERAISE, + [RESUME] = RESUME, + [RESUME_QUICK] = RESUME, + [RETURN_GENERATOR] = RETURN_GENERATOR, + [RETURN_VALUE] = RETURN_VALUE, + [SEND] = SEND, + [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, + [SET_ADD] = SET_ADD, + [SET_UPDATE] = SET_UPDATE, + [STORE_ATTR] = STORE_ATTR, [STORE_ATTR_ADAPTIVE] = STORE_ATTR, [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, [STORE_ATTR_SLOT] = STORE_ATTR, [STORE_ATTR_WITH_HINT] = STORE_ATTR, + [STORE_DEREF] = STORE_DEREF, + [STORE_FAST] = STORE_FAST, + [STORE_FAST__LOAD_FAST] = STORE_FAST, + [STORE_FAST__STORE_FAST] = STORE_FAST, + [STORE_GLOBAL] = STORE_GLOBAL, + [STORE_NAME] = STORE_NAME, + [STORE_SUBSCR] = STORE_SUBSCR, + [STORE_SUBSCR_ADAPTIVE] = STORE_SUBSCR, + [STORE_SUBSCR_DICT] = STORE_SUBSCR, + [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, + [SWAP] = SWAP, + [UNARY_INVERT] = UNARY_INVERT, + [UNARY_NEGATIVE] = UNARY_NEGATIVE, + [UNARY_NOT] = UNARY_NOT, + [UNARY_POSITIVE] = UNARY_POSITIVE, + [UNPACK_EX] = UNPACK_EX, + [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_ADAPTIVE] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, - [LOAD_FAST__LOAD_FAST] = LOAD_FAST, - [LOAD_FAST__LOAD_CONST] = LOAD_FAST, - [STORE_FAST__LOAD_FAST] = STORE_FAST, - [STORE_FAST__STORE_FAST] = STORE_FAST, - [LOAD_CONST__LOAD_FAST] = LOAD_CONST, - [JUMP_ABSOLUTE_QUICK] = JUMP_ABSOLUTE, - [RESUME_QUICK] = RESUME, + [WITH_EXCEPT_START] = WITH_EXCEPT_START, + [YIELD_VALUE] = YIELD_VALUE, }; #endif /* OPCODE_TABLES */ diff --git a/Lib/opcode.py b/Lib/opcode.py index 8cf021b566f84a..e71fbc3273ca30 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -230,110 +230,111 @@ def jabs_op(name, op, entries=0): ] _specializations = { - "BINARY_OP": [ - "BINARY_OP_ADAPTIVE", - "BINARY_OP_ADD_INT", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_INPLACE_ADD_UNICODE", - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_SUBTRACT_INT", - "BINARY_OP_SUBTRACT_FLOAT", + 'BINARY_OP': [ + 'BINARY_OP_ADAPTIVE', + 'BINARY_OP_ADD_FLOAT', + 'BINARY_OP_ADD_INT', + 'BINARY_OP_ADD_UNICODE', + 'BINARY_OP_INPLACE_ADD_UNICODE', + 'BINARY_OP_MULTIPLY_FLOAT', + 'BINARY_OP_MULTIPLY_INT', + 'BINARY_OP_SUBTRACT_FLOAT', + 'BINARY_OP_SUBTRACT_INT', ], - "COMPARE_OP": [ - "COMPARE_OP_ADAPTIVE", - "COMPARE_OP_FLOAT_JUMP", - "COMPARE_OP_INT_JUMP", - "COMPARE_OP_STR_JUMP", + 'BINARY_SUBSCR': [ + 'BINARY_SUBSCR_ADAPTIVE', + 'BINARY_SUBSCR_DICT', + 'BINARY_SUBSCR_GETITEM', + 'BINARY_SUBSCR_LIST_INT', + 'BINARY_SUBSCR_TUPLE_INT', ], - "BINARY_SUBSCR": [ - "BINARY_SUBSCR_ADAPTIVE", - "BINARY_SUBSCR_GETITEM", - "BINARY_SUBSCR_LIST_INT", - "BINARY_SUBSCR_TUPLE_INT", - "BINARY_SUBSCR_DICT", + 'CALL': [ + 'CALL_ADAPTIVE', + 'CALL_PY_EXACT_ARGS', + 'CALL_PY_WITH_DEFAULTS', ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_ADAPTIVE", - "STORE_SUBSCR_LIST_INT", - "STORE_SUBSCR_DICT", + 'COMPARE_OP': [ + 'COMPARE_OP_ADAPTIVE', + 'COMPARE_OP_FLOAT_JUMP', + 'COMPARE_OP_INT_JUMP', + 'COMPARE_OP_STR_JUMP', ], - "CALL": [ - "CALL_ADAPTIVE", - "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", + 'JUMP_ABSOLUTE': [ + 'JUMP_ABSOLUTE_QUICK', ], - "LOAD_ATTR": [ - "LOAD_ATTR_ADAPTIVE", - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_WITH_HINT", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_MODULE", + 'LOAD_ATTR': [ + 'LOAD_ATTR_ADAPTIVE', + 'LOAD_ATTR_INSTANCE_VALUE', + 'LOAD_ATTR_MODULE', + 'LOAD_ATTR_SLOT', + 'LOAD_ATTR_WITH_HINT', ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_ADAPTIVE", - "LOAD_GLOBAL_MODULE", - "LOAD_GLOBAL_BUILTIN", + 'LOAD_CONST': [ + 'LOAD_CONST__LOAD_FAST', ], - "LOAD_METHOD": [ - "LOAD_METHOD_ADAPTIVE", - "LOAD_METHOD_CLASS", - "LOAD_METHOD_MODULE", - "LOAD_METHOD_NO_DICT", - "LOAD_METHOD_WITH_DICT", - "LOAD_METHOD_WITH_VALUES", + 'LOAD_FAST': [ + 'LOAD_FAST__LOAD_CONST', + 'LOAD_FAST__LOAD_FAST', ], - "PRECALL": [ - "PRECALL_ADAPTIVE", - "PRECALL_BUILTIN_CLASS", - "PRECALL_NO_KW_BUILTIN_O", - "PRECALL_NO_KW_BUILTIN_FAST", - "PRECALL_BUILTIN_FAST_WITH_KEYWORDS", - "PRECALL_NO_KW_LEN", - "PRECALL_NO_KW_ISINSTANCE", - "PRECALL_NO_KW_LIST_APPEND", - "PRECALL_NO_KW_METHOD_DESCRIPTOR_O", - "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - "PRECALL_NO_KW_STR_1", - "PRECALL_NO_KW_TUPLE_1", - "PRECALL_NO_KW_TYPE_1", - "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST", - "PRECALL_BOUND_METHOD", - "PRECALL_PYFUNC", + 'LOAD_GLOBAL': [ + 'LOAD_GLOBAL_ADAPTIVE', + 'LOAD_GLOBAL_BUILTIN', + 'LOAD_GLOBAL_MODULE', ], - "STORE_ATTR": [ - "STORE_ATTR_ADAPTIVE", - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", + 'LOAD_METHOD': [ + 'LOAD_METHOD_ADAPTIVE', + 'LOAD_METHOD_CLASS', + 'LOAD_METHOD_MODULE', + 'LOAD_METHOD_NO_DICT', + 'LOAD_METHOD_WITH_DICT', + 'LOAD_METHOD_WITH_VALUES', ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_ADAPTIVE", - "UNPACK_SEQUENCE_LIST", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_TWO_TUPLE", + 'PRECALL': [ + 'PRECALL_ADAPTIVE', + 'PRECALL_BOUND_METHOD', + 'PRECALL_BUILTIN_CLASS', + 'PRECALL_BUILTIN_FAST_WITH_KEYWORDS', + 'PRECALL_NO_KW_BUILTIN_FAST', + 'PRECALL_NO_KW_BUILTIN_O', + 'PRECALL_NO_KW_ISINSTANCE', + 'PRECALL_NO_KW_LEN', + 'PRECALL_NO_KW_LIST_APPEND', + 'PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST', + 'PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS', + 'PRECALL_NO_KW_METHOD_DESCRIPTOR_O', + 'PRECALL_NO_KW_STR_1', + 'PRECALL_NO_KW_TUPLE_1', + 'PRECALL_NO_KW_TYPE_1', + 'PRECALL_PYFUNC', ], - # Super instructions - "LOAD_FAST": [ - "LOAD_FAST__LOAD_FAST", - "LOAD_FAST__LOAD_CONST", + 'RESUME': [ + 'RESUME_QUICK', ], - "STORE_FAST": [ - "STORE_FAST__LOAD_FAST", - "STORE_FAST__STORE_FAST", + 'STORE_ATTR': [ + 'STORE_ATTR_ADAPTIVE', + 'STORE_ATTR_INSTANCE_VALUE', + 'STORE_ATTR_SLOT', + 'STORE_ATTR_WITH_HINT', ], - "LOAD_CONST": [ - "LOAD_CONST__LOAD_FAST", + 'STORE_FAST': [ + 'STORE_FAST__LOAD_FAST', + 'STORE_FAST__STORE_FAST', ], - # Other instructions - "JUMP_ABSOLUTE": [ - "JUMP_ABSOLUTE_QUICK", + 'STORE_SUBSCR': [ + 'STORE_SUBSCR_ADAPTIVE', + 'STORE_SUBSCR_DICT', + 'STORE_SUBSCR_LIST_INT', ], - "RESUME": [ - "RESUME_QUICK", + 'UNPACK_SEQUENCE': [ + 'UNPACK_SEQUENCE_ADAPTIVE', + 'UNPACK_SEQUENCE_LIST', + 'UNPACK_SEQUENCE_TUPLE', + 'UNPACK_SEQUENCE_TWO_TUPLE', ], } +_specialized_instructions = [ + opcode for family in _specializations.values() for opcode in family +] _specialization_stats = [ "success", "failure", diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index b4ee149049e7ee..7a09974d4329ff 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -71,7 +71,7 @@ def test_specialization_stats(self): specialized_opcodes = [ op[:-len("_ADAPTIVE")].lower() for - op in chain.from_iterable(opcode._specializations.values()) + op in opcode._specialized_instructions if op.endswith("_ADAPTIVE")] self.assertIn('load_attr', specialized_opcodes) self.assertIn('binary_subscr', specialized_opcodes) diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index ec12478bebac33..defe7627cabbf8 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -37,7 +37,7 @@ def write_contents(f): for opname, op in opcode.opmap.items(): targets[op] = "TARGET_%s" % opname next_op = 1 - for opname in chain.from_iterable(opcode._specializations.values()): + for opname in opcode._specialized_instructions: while targets[next_op] != '_unknown_opcode': next_op += 1 targets[next_op] = "TARGET_%s" % opname diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 5629126e6a7932..2aa6471abf99a1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -3,84 +3,84 @@ static void *opcode_targets[256] = { &&TARGET_POP_TOP, &&TARGET_PUSH_NULL, &&TARGET_BINARY_OP_ADAPTIVE, - &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_FLOAT, + &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_BINARY_OP_MULTIPLY_INT, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_NOP, &&TARGET_UNARY_POSITIVE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, - &&TARGET_BINARY_OP_SUBTRACT_INT, - &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, - &&TARGET_COMPARE_OP_ADAPTIVE, - &&TARGET_COMPARE_OP_FLOAT_JUMP, - &&TARGET_COMPARE_OP_INT_JUMP, - &&TARGET_COMPARE_OP_STR_JUMP, + &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_SUBSCR_ADAPTIVE, + &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, + &&TARGET_CALL_ADAPTIVE, + &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_BINARY_SUBSCR, - &&TARGET_BINARY_SUBSCR_DICT, - &&TARGET_STORE_SUBSCR_ADAPTIVE, - &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_COMPARE_OP_ADAPTIVE, + &&TARGET_COMPARE_OP_FLOAT_JUMP, + &&TARGET_COMPARE_OP_INT_JUMP, + &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_ADAPTIVE, + &&TARGET_JUMP_ABSOLUTE_QUICK, &&TARGET_PUSH_EXC_INFO, - &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_LOAD_ATTR_ADAPTIVE, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_LOAD_METHOD_CLASS, - &&TARGET_LOAD_METHOD_MODULE, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_BEFORE_ASYNC_WITH, &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, + &&TARGET_LOAD_METHOD_MODULE, &&TARGET_LOAD_METHOD_NO_DICT, &&TARGET_LOAD_METHOD_WITH_DICT, &&TARGET_LOAD_METHOD_WITH_VALUES, &&TARGET_PRECALL_ADAPTIVE, - &&TARGET_PRECALL_BUILTIN_CLASS, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_PRECALL_NO_KW_BUILTIN_O, - &&TARGET_PRECALL_NO_KW_BUILTIN_FAST, + &&TARGET_PRECALL_BOUND_METHOD, + &&TARGET_PRECALL_BUILTIN_CLASS, &&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_PRECALL_NO_KW_LEN, + &&TARGET_PRECALL_NO_KW_BUILTIN_FAST, + &&TARGET_PRECALL_NO_KW_BUILTIN_O, &&TARGET_PRECALL_NO_KW_ISINSTANCE, - &&TARGET_PRECALL_NO_KW_LIST_APPEND, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O, - &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_PRECALL_NO_KW_LEN, + &&TARGET_PRECALL_NO_KW_LIST_APPEND, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST, + &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_PRECALL_NO_KW_STR_1, &&TARGET_PRECALL_NO_KW_TUPLE_1, &&TARGET_PRECALL_NO_KW_TYPE_1, - &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST, - &&TARGET_PRECALL_BOUND_METHOD, - &&TARGET_PRECALL_PYFUNC, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, @@ -139,41 +139,41 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, - &&TARGET_STORE_ATTR_ADAPTIVE, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_PRECALL_PYFUNC, + &&TARGET_RESUME_QUICK, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_ADAPTIVE, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, &&TARGET_COPY_FREE_VARS, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, - &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_LOAD_METHOD, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_PRECALL, - &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&TARGET_JUMP_ABSOLUTE_QUICK, - &&TARGET_RESUME_QUICK, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 7194900d683be6..de42fde994496a 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -343,21 +343,14 @@ _Py_Quicken(PyCodeObject *code) } void -_Py_Unquicken(PyCodeObject *code) +_Py_Unquicken(_Py_CODEUNIT *instructions, Py_ssize_t size) { - assert(code->co_warmup == QUICKENING_INITIAL_WARMUP_VALUE); - _Py_QuickenedCount--; - _Py_CODEUNIT *instructions = _PyCode_GET_CODE(code); - for (int i = 0; i < Py_SIZE(code); i++) { - int opcode = _Py_OPCODE(instructions[i]); - int deopt = _PyOpcode_Specializations[opcode]; - if (deopt) { - int oparg = _Py_OPARG(instructions[i]); - instructions[i] = _Py_MAKECODEUNIT(deopt, oparg); - int cache_entries = _PyOpcode_InlineCacheEntries[deopt]; - for (int j = 0; j < cache_entries; j++) { - instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); - } + for (Py_ssize_t i = 0; i < size; i++) { + int opcode = _PyOpcode_Deoptimizations[_Py_OPCODE(instructions[i])]; + instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instructions[i])); + int cache_entries = _PyOpcode_InlineCacheEntries[opcode]; + while (cache_entries--) { + instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); } } } diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index 009b725606cf4e..036c576f40d2eb 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -70,14 +70,14 @@ def main(opcode_py, outfile='Include/opcode.h'): if name == 'POP_EXCEPT': # Special entry for HAVE_ARGUMENT fobj.write(DEFINE.format("HAVE_ARGUMENT", opcode["HAVE_ARGUMENT"])) - for name in chain.from_iterable(opcode['_specializations'].values()): + for name in opcode['_specialized_instructions']: while used[next_op]: next_op += 1 fobj.write(DEFINE.format(name, next_op)) used[next_op] = True fobj.write(DEFINE.format('DO_TRACING', 255)) fobj.write("\nextern const uint8_t _PyOpcode_InlineCacheEntries[256];\n") - fobj.write("\nextern const uint8_t _PyOpcode_Specializations[256];\n") + fobj.write("\nextern const uint8_t _PyOpcode_Deoptimizations[256];\n") fobj.write("\n#ifdef NEED_OPCODE_TABLES\n") write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj) write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj) @@ -87,10 +87,15 @@ def main(opcode_py, outfile='Include/opcode.h'): if entries: fobj.write(f" [{opname[i]}] = {entries},\n") fobj.write("};\n") - fobj.write("\nconst uint8_t _PyOpcode_Specializations[256] = {\n") - for basic, specializations in opcode["_specializations"].items(): - for specialization in specializations: - fobj.write(f" [{specialization}] = {basic},\n") + deoptcodes = {} + for basic in opmap: + deoptcodes[basic] = basic + for basic, family in opcode["_specializations"].items(): + for specialized in family: + deoptcodes[specialized] = basic + fobj.write("\nconst uint8_t _PyOpcode_Deoptimizations[256] = {\n") + for opt, deopt in sorted(deoptcodes.items()): + fobj.write(f" [{opt}] = {deopt},\n") fobj.write("};\n") fobj.write("#endif /* OPCODE_TABLES */\n") diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 44374c94bd3c22..69babfd2ddafc6 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -15,7 +15,7 @@ DEFAULT_DIR = "/tmp/py_stats/" #Create list of all instruction names -specialized = itertools.chain.from_iterable(opcode._specializations.values()) +specialized = iter(opcode._specialized_instructions) opname = ["<0>"] for name in opcode.opname[1:]: if name.startswith("<"): @@ -131,10 +131,10 @@ def categorized_counts(opcode_stats): specialized = 0 not_specialized = 0 specialized_instructions = { - op for op in itertools.chain.from_iterable(opcode._specializations.values()) + op for op in opcode._specialized_instructions if "__" not in op and "ADAPTIVE" not in op} adaptive_instructions = { - op for op in itertools.chain.from_iterable(opcode._specializations.values()) + op for op in opcode._specialized_instructions if "ADAPTIVE" in op} for i, opcode_stat in enumerate(opcode_stats): if "execution_count" not in opcode_stat: From 42810dd8989d7cd81a0db3e55828ef53aef5416f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 10 Mar 2022 17:32:47 -0800 Subject: [PATCH 10/34] Try a different approach --- Include/internal/pycore_code.h | 11 +---------- Lib/test/test__opcode.py | 1 - Objects/codeobject.c | 7 ++----- Python/ceval.c | 10 +++++----- Python/specialize.c | 23 +++++++++++++---------- Tools/scripts/generate_opcode_h.py | 1 - 6 files changed, 21 insertions(+), 32 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index db7b125d84caf4..d1521b34b3acf1 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -98,7 +98,7 @@ typedef struct { #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) void _Py_Quicken(PyCodeObject *code); -void _Py_Unquicken(_Py_CODEUNIT *instructions, Py_ssize_t size); +_Py_CODEUNIT *_Py_Unquickened(PyCodeObject *code); static inline void _Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) @@ -111,15 +111,6 @@ _Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) } } -static inline void -_Py_SetCountAndUnquicken(PyCodeObject *code) -{ - if (code->co_warmup == 0) { - code->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; - _Py_Unquicken(_PyCode_GET_CODE(code), Py_SIZE(code)); - } -} - extern Py_ssize_t _Py_QuickenedCount; // Borrowed references to common callables: diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 7a09974d4329ff..7c1c0cfdb069b8 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -1,5 +1,4 @@ import dis -from itertools import chain from test.support.import_helper import import_module import unittest import opcode diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 4bf45e906cfbc5..35037b5aaebfc4 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1425,9 +1425,6 @@ code_hash(PyCodeObject *co) Py_hash_t h, h0, h1, h2, h3; h0 = PyObject_Hash(co->co_name); if (h0 == -1) return -1; - // XXX - // h1 = PyObject_Hash(co->co_code); - // if (h1 == -1) return -1; h1 = PyObject_Hash(co->co_consts); if (h1 == -1) return -1; h2 = PyObject_Hash(co->co_names); @@ -1499,8 +1496,8 @@ code_getundercode(PyCodeObject *code, void *closure) static PyObject * code_getcode(PyCodeObject *code, void *closure) { - _Py_SetCountAndUnquicken(code); - return PyBytes_FromStringAndSize(code->_co_code, _PyCode_GET_SIZE(code)); + return PyBytes_FromStringAndSize((char *)_Py_Unquickened(code), + _PyCode_GET_SIZE(code)); } static PyGetSetDef code_getsetlist[] = { diff --git a/Python/ceval.c b/Python/ceval.c index 9e10ffcd1c87eb..285dd549f7afcd 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1326,8 +1326,9 @@ eval_frame_handle_pending(PyThreadState *tstate) JUMPBY(INLINE_CACHE_ENTRIES_PRECALL + 1 + INLINE_CACHE_ENTRIES_CALL) #define TRACING_NEXTOPARG() do { \ - _Py_SetCountAndUnquicken(frame->f_code); \ - NEXTOPARG(); \ + _Py_CODEUNIT word = _Py_Unquickened(frame->f_code)[INSTR_OFFSET()]; \ + opcode = _Py_OPCODE(word); \ + oparg = _Py_OPARG(word); \ } while (0) /* OpCode prediction macros @@ -1566,8 +1567,7 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject static int skip_backwards_over_extended_args(PyCodeObject *code, int offset) { - _Py_SetCountAndUnquicken(code); - _Py_CODEUNIT *instrs = _PyCode_GET_CODE(code); + _Py_CODEUNIT *instrs = _Py_Unquickened(code); while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) { offset--; } @@ -6706,7 +6706,7 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, then call the trace function if we're tracing source lines. */ initialize_trace_info(&tstate->trace_info, frame); - _Py_CODEUNIT prev = _PyCode_GET_CODE(frame->f_code)[instr_prev]; + _Py_CODEUNIT prev = _Py_Unquickened(frame->f_code)[instr_prev]; int lastline; if (_Py_OPCODE(prev) == RESUME && _Py_OPARG(prev) == 0) { lastline = -1; diff --git a/Python/specialize.c b/Python/specialize.c index de42fde994496a..ca9ebcb8945601 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -277,11 +277,9 @@ _Py_PrintSpecializationStats(int to_file) // Insert adaptive instructions and superinstructions. - void _Py_Quicken(PyCodeObject *code) { - assert(code->co_warmup == 0); _Py_QuickenedCount++; int previous_opcode = -1; int previous_oparg = -1; @@ -342,17 +340,22 @@ _Py_Quicken(PyCodeObject *code) } } -void -_Py_Unquicken(_Py_CODEUNIT *instructions, Py_ssize_t size) +_Py_CODEUNIT * +_Py_Unquickened(PyCodeObject *code) { - for (Py_ssize_t i = 0; i < size; i++) { - int opcode = _PyOpcode_Deoptimizations[_Py_OPCODE(instructions[i])]; - instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instructions[i])); - int cache_entries = _PyOpcode_InlineCacheEntries[opcode]; - while (cache_entries--) { - instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); + _Py_CODEUNIT *instructions = _PyCode_GET_CODE(code); + if (code->co_warmup == 0) { + _Py_QuickenedCount--; + for (Py_ssize_t i = 0; i < Py_SIZE(code); i++) { + int opcode = _PyOpcode_Deoptimizations[_Py_OPCODE(instructions[i])]; + instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instructions[i])); + int cache_entries = _PyOpcode_InlineCacheEntries[opcode]; + while (cache_entries--) { + instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); + } } } + return instructions; } static inline int diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index 036c576f40d2eb..1c1ab6a63682b0 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -1,6 +1,5 @@ # This script generates the opcode.h header file. -from itertools import chain import sys import tokenize From 7df49342d5ad5ce921c3f8d17f8f2330823b9cbf Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 10 Mar 2022 17:40:26 -0800 Subject: [PATCH 11/34] Clean up the diff --- Include/cpython/code.h | 3 +-- Python/ceval.c | 3 ++- Python/makeopcodetargets.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index e2d905e2f6a43c..cf28c47fdcabec 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -38,7 +38,6 @@ struct PyCodeObject { * - co_stacksize * - co_flags * - co_firstlineno - * - co_code * - co_consts * - co_names * - co_localsplusnames @@ -55,7 +54,7 @@ struct PyCodeObject { PyObject *co_names; /* list of strings (names used) */ PyObject *co_exceptiontable; /* Byte string encoding exception handling table */ int co_flags; /* CO_..., see below */ - int co_warmup; // XXX + int co_warmup; /* Warmup counter for quickening */ // The rest are not so impactful on performance. int co_argcount; /* #arguments, except *args */ diff --git a/Python/ceval.c b/Python/ceval.c index 285dd549f7afcd..93f180aae30f34 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1325,7 +1325,8 @@ eval_frame_handle_pending(PyThreadState *tstate) #define SKIP_CALL() \ JUMPBY(INLINE_CACHE_ENTRIES_PRECALL + 1 + INLINE_CACHE_ENTRIES_CALL) -#define TRACING_NEXTOPARG() do { \ +/* Get opcode and oparg from original instructions, not quickened form. */ +#define TRACING_NEXTOPARG() do { \ _Py_CODEUNIT word = _Py_Unquickened(frame->f_code)[INSTR_OFFSET()]; \ opcode = _Py_OPCODE(word); \ oparg = _Py_OPARG(word); \ diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index defe7627cabbf8..3bf2e35ccb6dab 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -3,7 +3,6 @@ (for compilers supporting computed gotos or "labels-as-values", such as gcc). """ -from itertools import chain import os import sys From 5fa0ca2a3fac0463c853ce42d958f912539d5293 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 11 Mar 2022 09:37:10 -0800 Subject: [PATCH 12/34] Support equality comparisons again --- Objects/codeobject.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 35037b5aaebfc4..0d0f62140585e8 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1376,9 +1376,23 @@ code_richcompare(PyObject *self, PyObject *other, int op) if (!eq) goto unequal; eq = co->co_firstlineno == cp->co_firstlineno; if (!eq) goto unequal; - // XXX: Compare code! - // eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ); - // if (eq <= 0) goto unequal; + eq = Py_SIZE(co) == Py_SIZE(cp); + if (!eq) { + goto unequal; + } + _Py_CODEUNIT *co_code = _PyCode_GET_CODE(co); + _Py_CODEUNIT *cp_code = _PyCode_GET_CODE(cp); + for (int i = 0; i < Py_SIZE(co); i++) { + int opcode = _PyOpcode_Deoptimizations[_Py_OPCODE(co_code[i])]; + eq = opcode == _PyOpcode_Deoptimizations[_Py_OPCODE(cp_code[i])]; + if (HAS_ARG(opcode)) { + eq &= _Py_OPARG(co_code[i]) == _Py_OPARG(cp_code[i]); + } + if (!eq) { + goto unequal; + } + i += _PyOpcode_InlineCacheEntries[opcode]; + } /* compare constants */ consts1 = _PyCode_ConstantKey(co->co_consts); From 1fc22823a1b1fa9d06b3d5d1ecf8383928687efa Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 11 Mar 2022 13:33:15 -0800 Subject: [PATCH 13/34] Never un-quicken! --- Include/cpython/code.h | 3 --- Include/internal/pycore_code.h | 6 +++++- Objects/codeobject.c | 22 ++++++++++++---------- Objects/frameobject.c | 17 ++++++++++++++--- Objects/genobject.c | 10 ++++------ Python/ceval.c | 7 +++---- Python/specialize.c | 20 +++++--------------- 7 files changed, 43 insertions(+), 42 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index cf28c47fdcabec..4c05274487c686 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -93,9 +93,6 @@ struct PyCodeObject { char _co_code[1]; }; -#define _PyCode_GET_CODE(CO) ((_Py_CODEUNIT *)(CO)->_co_code) -#define _PyCode_GET_SIZE(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) - /* Masks for co_flags above */ #define CO_OPTIMIZED 0x0001 #define CO_NEWLOCALS 0x0002 diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index d1521b34b3acf1..e90d314002af2b 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -97,8 +97,12 @@ typedef struct { /* We want to compare to zero for efficiency, so we offset values accordingly */ #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) +#define _PyCode_GET_CODE(CO) ((_Py_CODEUNIT *)(CO)->_co_code) +#define _PyCode_GET_SIZE(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) + + void _Py_Quicken(PyCodeObject *code); -_Py_CODEUNIT *_Py_Unquickened(PyCodeObject *code); +_Py_CODEUNIT _PyCode_GetUnquickened(PyCodeObject *code, int i); static inline void _Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d0f62140585e8..e3926d0915dbd0 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1380,18 +1380,13 @@ code_richcompare(PyObject *self, PyObject *other, int op) if (!eq) { goto unequal; } - _Py_CODEUNIT *co_code = _PyCode_GET_CODE(co); - _Py_CODEUNIT *cp_code = _PyCode_GET_CODE(cp); for (int i = 0; i < Py_SIZE(co); i++) { - int opcode = _PyOpcode_Deoptimizations[_Py_OPCODE(co_code[i])]; - eq = opcode == _PyOpcode_Deoptimizations[_Py_OPCODE(cp_code[i])]; - if (HAS_ARG(opcode)) { - eq &= _Py_OPARG(co_code[i]) == _Py_OPARG(cp_code[i]); - } + _Py_CODEUNIT instruction = _PyCode_GetUnquickened(co, i); + eq = _PyCode_GetUnquickened(co, i) == _PyCode_GetUnquickened(cp, i); if (!eq) { goto unequal; } - i += _PyOpcode_InlineCacheEntries[opcode]; + i += _PyOpcode_InlineCacheEntries[_Py_OPCODE(instruction)]; } /* compare constants */ @@ -1510,8 +1505,15 @@ code_getundercode(PyCodeObject *code, void *closure) static PyObject * code_getcode(PyCodeObject *code, void *closure) { - return PyBytes_FromStringAndSize((char *)_Py_Unquickened(code), - _PyCode_GET_SIZE(code)); + PyObject *co_code = PyBytes_FromStringAndSize(NULL, _PyCode_GET_SIZE(code)); + if (co_code == NULL) { + return NULL; + } + _Py_CODEUNIT *instructions = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code); + for (int i = 0; i < Py_SIZE(code); i++) { + instructions[i] = _PyCode_GetUnquickened(code, i); + } + return co_code; } static PyGetSetDef code_getsetlist[] = { diff --git a/Objects/frameobject.c b/Objects/frameobject.c index cd18da2e78a481..64c9b5d5583627 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -170,12 +170,21 @@ top_of_stack(int64_t stack) static int64_t * mark_stacks(PyCodeObject *code_obj, int len) { - const _Py_CODEUNIT *code = _PyCode_GET_CODE(code_obj); + // XXX: this is one big TODO!!! + PyObject *xxx = PyBytes_FromStringAndSize(NULL, _PyCode_GET_SIZE(code_obj)); + if (xxx == NULL) { + return NULL; + } + _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(xxx); + for (int i = 0; i < Py_SIZE(code_obj); i++) { + code[i] = _PyCode_GetUnquickened(code_obj, i); + } int64_t *stacks = PyMem_New(int64_t, len+1); int i, j, opcode; if (stacks == NULL) { PyErr_NoMemory(); + Py_DECREF(xxx); return NULL; } for (int i = 1; i <= len; i++) { @@ -303,6 +312,7 @@ mark_stacks(PyCodeObject *code_obj, int len) } } } + Py_DECREF(xxx); return stacks; } @@ -837,9 +847,10 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, static int _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) { - const _Py_CODEUNIT *code = _PyCode_GET_CODE(frame->f_code); + // XXX: Does this handle EXTENDED_ARGs? for (int i = 0; i < frame->f_lasti; i++) { - if (_Py_OPCODE(code[i]) == opcode && _Py_OPARG(code[i]) == oparg) { + _Py_CODEUNIT instruction = _PyCode_GetUnquickened(frame->f_code, i); + if (instruction == _Py_MAKECODEUNIT(opcode, oparg)) { return 1; } } diff --git a/Objects/genobject.c b/Objects/genobject.c index 39520b5a683101..abc30075f3c7f2 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -349,17 +349,16 @@ _PyGen_yf(PyGenObject *gen) if (gen->gi_frame_valid) { _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - _Py_CODEUNIT *code = _PyCode_GET_CODE(gen->gi_code); if (frame->f_lasti < 1) { /* Return immediately if the frame didn't start yet. SEND always come after LOAD_CONST: a code object should not start with SEND */ - assert(_Py_OPCODE(code[0]) != SEND); + assert(_Py_OPCODE(_PyCode_GetUnquickened(gen->gi_code, 0)) != SEND); return NULL; } - if (_Py_OPCODE(code[frame->f_lasti - 1]) != SEND || frame->stacktop < 0) + if (_Py_OPCODE(_PyCode_GetUnquickened(gen->gi_code, frame->f_lasti - 1)) != SEND || frame->stacktop < 0) { return NULL; } @@ -485,12 +484,11 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, Py_DECREF(ret); /* Termination repetition of SEND loop */ assert(frame->f_lasti >= 0); - _Py_CODEUNIT *code = _PyCode_GET_CODE(gen->gi_code); /* Backup to SEND */ frame->f_lasti--; - assert(_Py_OPCODE(code[frame->f_lasti]) == SEND); + assert(_Py_OPCODE(_PyCode_GetUnquickened(gen->gi_code, frame->f_lasti)) == SEND); // XXX: This doesn't seem to handle EXTENDED_ARGs: - int jump = _Py_OPARG(code[frame->f_lasti]); + int jump = _Py_OPARG(_PyCode_GetUnquickened(gen->gi_code, frame->f_lasti)); frame->f_lasti += jump; if (_PyGen_FetchStopIterationValue(&val) == 0) { ret = gen_send(gen, val); diff --git a/Python/ceval.c b/Python/ceval.c index 93f180aae30f34..f479b6a6ba750b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1327,7 +1327,7 @@ eval_frame_handle_pending(PyThreadState *tstate) /* Get opcode and oparg from original instructions, not quickened form. */ #define TRACING_NEXTOPARG() do { \ - _Py_CODEUNIT word = _Py_Unquickened(frame->f_code)[INSTR_OFFSET()]; \ + _Py_CODEUNIT word = _PyCode_GetUnquickened(frame->f_code, INSTR_OFFSET()); \ opcode = _Py_OPCODE(word); \ oparg = _Py_OPARG(word); \ } while (0) @@ -1568,8 +1568,7 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject static int skip_backwards_over_extended_args(PyCodeObject *code, int offset) { - _Py_CODEUNIT *instrs = _Py_Unquickened(code); - while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) { + while (offset > 0 && _Py_OPCODE(_PyCode_GetUnquickened(code, offset - 1)) == EXTENDED_ARG) { offset--; } return offset; @@ -6707,7 +6706,7 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, then call the trace function if we're tracing source lines. */ initialize_trace_info(&tstate->trace_info, frame); - _Py_CODEUNIT prev = _Py_Unquickened(frame->f_code)[instr_prev]; + _Py_CODEUNIT prev = _PyCode_GetUnquickened(frame->f_code, instr_prev); int lastline; if (_Py_OPCODE(prev) == RESUME && _Py_OPARG(prev) == 0) { lastline = -1; diff --git a/Python/specialize.c b/Python/specialize.c index ca9ebcb8945601..2d96bd28234acc 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -340,22 +340,12 @@ _Py_Quicken(PyCodeObject *code) } } -_Py_CODEUNIT * -_Py_Unquickened(PyCodeObject *code) +_Py_CODEUNIT +_PyCode_GetUnquickened(PyCodeObject *code, int i) { - _Py_CODEUNIT *instructions = _PyCode_GET_CODE(code); - if (code->co_warmup == 0) { - _Py_QuickenedCount--; - for (Py_ssize_t i = 0; i < Py_SIZE(code); i++) { - int opcode = _PyOpcode_Deoptimizations[_Py_OPCODE(instructions[i])]; - instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instructions[i])); - int cache_entries = _PyOpcode_InlineCacheEntries[opcode]; - while (cache_entries--) { - instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); - } - } - } - return instructions; + _Py_CODEUNIT instruction = _PyCode_GET_CODE(code)[i]; + int opcode = _PyOpcode_Deoptimizations[_Py_OPCODE(instruction)]; + return _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction)); } static inline int From b40e300cd6e98ca8709a508fa7e1b17018cb04e3 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 11 Mar 2022 14:31:31 -0800 Subject: [PATCH 14/34] More renaming and cleanup --- Include/internal/pycore_code.h | 6 +-- Include/opcode.h | 8 ++-- Objects/codeobject.c | 68 ++++++++++++++++++------------ Objects/frameobject.c | 15 ++++--- Objects/genobject.c | 9 ++-- Objects/typeobject.c | 4 +- Python/ceval.c | 17 ++++---- Python/compile.c | 4 +- Python/marshal.c | 2 +- Python/specialize.c | 41 ++++++------------ Tools/scripts/generate_opcode_h.py | 8 ++-- 11 files changed, 93 insertions(+), 89 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index e90d314002af2b..83f882818f7de4 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -97,12 +97,11 @@ typedef struct { /* We want to compare to zero for efficiency, so we offset values accordingly */ #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) -#define _PyCode_GET_CODE(CO) ((_Py_CODEUNIT *)(CO)->_co_code) -#define _PyCode_GET_SIZE(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) +#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->_co_code) +#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) void _Py_Quicken(PyCodeObject *code); -_Py_CODEUNIT _PyCode_GetUnquickened(PyCodeObject *code, int i); static inline void _Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) @@ -222,6 +221,7 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *); extern PyObject* _PyCode_GetVarnames(PyCodeObject *); extern PyObject* _PyCode_GetCellvars(PyCodeObject *); extern PyObject* _PyCode_GetFreevars(PyCodeObject *); +extern PyObject* _PyCode_GetCode(PyCodeObject *); /* Return the ending source code line number from a bytecode index. */ extern int _PyCode_Addr2EndLine(PyCodeObject *, int); diff --git a/Include/opcode.h b/Include/opcode.h index ea041f0fc64344..ca080af18ebd9f 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -185,9 +185,9 @@ extern "C" { #define UNPACK_SEQUENCE_TWO_TUPLE 174 #define DO_TRACING 255 -extern const uint8_t _PyOpcode_InlineCacheEntries[256]; +extern const uint8_t _PyOpcode_Caches[256]; -extern const uint8_t _PyOpcode_Deoptimizations[256]; +extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_TABLES static const uint32_t _PyOpcode_RelativeJump[8] = { @@ -211,7 +211,7 @@ static const uint32_t _PyOpcode_Jump[8] = { 0U, }; -const uint8_t _PyOpcode_InlineCacheEntries[256] = { +const uint8_t _PyOpcode_Caches[256] = { [BINARY_SUBSCR] = 4, [STORE_SUBSCR] = 1, [UNPACK_SEQUENCE] = 1, @@ -225,7 +225,7 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = { [CALL] = 4, }; -const uint8_t _PyOpcode_Deoptimizations[256] = { +const uint8_t _PyOpcode_Deopt[256] = { [ASYNC_GEN_WRAP] = ASYNC_GEN_WRAP, [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, [BEFORE_WITH] = BEFORE_WITH, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index e3926d0915dbd0..baf7a8a8ac4c6a 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -344,7 +344,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_extra = NULL; co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; - memcpy(_PyCode_GET_CODE(co), PyBytes_AS_STRING(con->code), + memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code), PyBytes_GET_SIZE(con->code)); } @@ -381,7 +381,8 @@ _PyCode_New(struct _PyCodeConstructor *con) con->columntable = Py_None; } - PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, PyBytes_GET_SIZE(con->code) / 2); + Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT); + PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); if (co == NULL) { PyErr_NoMemory(); return NULL; @@ -599,7 +600,7 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) if (addrq < 0) { return co->co_firstlineno; } - assert(addrq >= 0 && addrq < _PyCode_GET_SIZE(co)); + assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); PyCodeAddressRange bounds; _PyCode_InitAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -627,7 +628,7 @@ _PyCode_Addr2EndLine(PyCodeObject* co, int addrq) return -1; } - assert(addrq >= 0 && addrq < _PyCode_GET_SIZE(co)); + assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); PyCodeAddressRange bounds; _PyCode_InitEndAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -983,7 +984,7 @@ _source_offset_converter(int* value) { static PyObject* positionsiter_next(positionsiterator* pi) { - if (pi->pi_offset >= _PyCode_GET_SIZE(pi->pi_code)) { + if (pi->pi_offset >= _PyCode_NBYTES(pi->pi_code)) { return NULL; } @@ -1154,6 +1155,26 @@ _PyCode_GetFreevars(PyCodeObject *co) return get_localsplus_names(co, CO_FAST_FREE, co->co_nfreevars); } +PyObject * +_PyCode_GetCode(PyCodeObject *co) +{ + PyObject *code = PyBytes_FromStringAndSize(NULL, _PyCode_NBYTES(co)); + if (code == NULL) { + return NULL; + } + _Py_CODEUNIT *instructions = (_Py_CODEUNIT *)PyBytes_AS_STRING(code); + for (int i = 0; i < Py_SIZE(co); i++) { + _Py_CODEUNIT instruction = _PyCode_CODE(co)[i]; + int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; + int caches = _PyOpcode_Caches[opcode]; + instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction)); + while (caches--) { + instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); + } + } + return code; +} + /****************** * PyCode_Type @@ -1376,17 +1397,20 @@ code_richcompare(PyObject *self, PyObject *other, int op) if (!eq) goto unequal; eq = co->co_firstlineno == cp->co_firstlineno; if (!eq) goto unequal; - eq = Py_SIZE(co) == Py_SIZE(cp); - if (!eq) { - goto unequal; + PyObject *co_code = _PyCode_GetCode(co); + if (co_code == NULL) { + return NULL; } - for (int i = 0; i < Py_SIZE(co); i++) { - _Py_CODEUNIT instruction = _PyCode_GetUnquickened(co, i); - eq = _PyCode_GetUnquickened(co, i) == _PyCode_GetUnquickened(cp, i); - if (!eq) { - goto unequal; - } - i += _PyOpcode_InlineCacheEntries[_Py_OPCODE(instruction)]; + PyObject *cp_code = _PyCode_GetCode(cp); + if (cp_code == NULL) { + Py_DECREF(co_code); + return NULL; + } + eq = PyObject_RichCompareBool(co_code, cp_code, Py_EQ); + Py_DECREF(co_code); + Py_DECREF(cp_code); + if (eq <= 0) { + goto unequal; } /* compare constants */ @@ -1498,22 +1522,14 @@ code_getfreevars(PyCodeObject *code, void *closure) static PyObject * code_getundercode(PyCodeObject *code, void *closure) { - return PyMemoryView_FromMemory(code->_co_code, _PyCode_GET_SIZE(code), + return PyMemoryView_FromMemory(code->_co_code, _PyCode_NBYTES(code), PyBUF_READ); } static PyObject * code_getcode(PyCodeObject *code, void *closure) { - PyObject *co_code = PyBytes_FromStringAndSize(NULL, _PyCode_GET_SIZE(code)); - if (co_code == NULL) { - return NULL; - } - _Py_CODEUNIT *instructions = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code); - for (int i = 0; i < Py_SIZE(code); i++) { - instructions[i] = _PyCode_GetUnquickened(code, i); - } - return co_code; + return _PyCode_GetCode(code); } static PyGetSetDef code_getsetlist[] = { @@ -1608,7 +1624,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, PyObject *code = NULL; if (co_code == NULL) { - code = code_getcode(self, NULL); + code = _PyCode_GetCode(self); if (code == NULL) { return NULL; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 64c9b5d5583627..a541c551264f4a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -107,6 +107,7 @@ frame_getback(PyFrameObject *f, void *closure) /* Given the index of the effective opcode, scan back to construct the oparg with EXTENDED_ARG */ +// XXX This is broken! static unsigned int get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i) { @@ -171,14 +172,11 @@ static int64_t * mark_stacks(PyCodeObject *code_obj, int len) { // XXX: this is one big TODO!!! - PyObject *xxx = PyBytes_FromStringAndSize(NULL, _PyCode_GET_SIZE(code_obj)); + PyObject *xxx = _PyCode_GetCode(code_obj); if (xxx == NULL) { return NULL; } _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(xxx); - for (int i = 0; i < Py_SIZE(code_obj); i++) { - code[i] = _PyCode_GetUnquickened(code_obj, i); - } int64_t *stacks = PyMem_New(int64_t, len+1); int i, j, opcode; @@ -847,12 +845,15 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, static int _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) { - // XXX: Does this handle EXTENDED_ARGs? + // XXX: Does this handle EXTENDED_ARGs/CACHEs? for (int i = 0; i < frame->f_lasti; i++) { - _Py_CODEUNIT instruction = _PyCode_GetUnquickened(frame->f_code, i); + _Py_CODEUNIT instruction = _PyCode_CODE(frame->f_code)[i]; + int deopt = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; + instruction = _Py_MAKECODEUNIT(deopt, oparg); if (instruction == _Py_MAKECODEUNIT(opcode, oparg)) { return 1; } + i += _PyOpcode_Caches[deopt]; } return 0; } @@ -871,7 +872,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) { } co = frame->f_code; fast = _PyFrame_GetLocalsArray(frame); - if (frame->f_lasti < 0 && _Py_OPCODE(_PyCode_GET_CODE(co)[0]) == COPY_FREE_VARS) { + if (frame->f_lasti < 0 && _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS) { /* Free vars have not been initialized -- Do that */ PyCodeObject *co = frame->f_code; PyObject *closure = frame->f_func->func_closure; diff --git a/Objects/genobject.c b/Objects/genobject.c index abc30075f3c7f2..015f267855e416 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -354,11 +354,11 @@ _PyGen_yf(PyGenObject *gen) /* Return immediately if the frame didn't start yet. SEND always come after LOAD_CONST: a code object should not start with SEND */ - assert(_Py_OPCODE(_PyCode_GetUnquickened(gen->gi_code, 0)) != SEND); + assert(_Py_OPCODE(_PyCode_CODE(gen->gi_code)[0]) != SEND); return NULL; } - if (_Py_OPCODE(_PyCode_GetUnquickened(gen->gi_code, frame->f_lasti - 1)) != SEND || frame->stacktop < 0) + if (_Py_OPCODE(_PyCode_CODE(gen->gi_code)[frame->f_lasti - 1]) != SEND || frame->stacktop < 0) { return NULL; } @@ -486,9 +486,10 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, assert(frame->f_lasti >= 0); /* Backup to SEND */ frame->f_lasti--; - assert(_Py_OPCODE(_PyCode_GetUnquickened(gen->gi_code, frame->f_lasti)) == SEND); + _Py_CODEUNIT instruction = _PyCode_CODE(gen->gi_code)[frame->f_lasti]; + assert(_Py_OPCODE(instruction) == SEND); // XXX: This doesn't seem to handle EXTENDED_ARGs: - int jump = _Py_OPARG(_PyCode_GetUnquickened(gen->gi_code, frame->f_lasti)); + int jump = _Py_OPARG(instruction); frame->f_lasti += jump; if (_PyGen_FetchStopIterationValue(&val) == 0) { ret = gen_send(gen, val); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 7498012c9de0e9..fd2f80d7ac3abc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8949,8 +8949,8 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, // "firstarg" is a cell here unless (very unlikely) super() // was called from the C-API before the first MAKE_CELL op. if (cframe->f_lasti >= 0) { - assert(_Py_OPCODE(_PyCode_GET_CODE(co)[0]) == MAKE_CELL || - _Py_OPCODE(_PyCode_GET_CODE(co)[0]) == COPY_FREE_VARS); + assert(_Py_OPCODE(_PyCode_CODE(co)[0]) == MAKE_CELL || + _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS); assert(PyCell_Check(firstarg)); firstarg = PyCell_GET(firstarg); } diff --git a/Python/ceval.c b/Python/ceval.c index f479b6a6ba750b..e99339c5068bd0 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1327,9 +1327,8 @@ eval_frame_handle_pending(PyThreadState *tstate) /* Get opcode and oparg from original instructions, not quickened form. */ #define TRACING_NEXTOPARG() do { \ - _Py_CODEUNIT word = _PyCode_GetUnquickened(frame->f_code, INSTR_OFFSET()); \ - opcode = _Py_OPCODE(word); \ - oparg = _Py_OPARG(word); \ + NEXTOPARG(); \ + opcode = _PyOpcode_Deopt[opcode]; \ } while (0) /* OpCode prediction macros @@ -1568,7 +1567,8 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject static int skip_backwards_over_extended_args(PyCodeObject *code, int offset) { - while (offset > 0 && _Py_OPCODE(_PyCode_GetUnquickened(code, offset - 1)) == EXTENDED_ARG) { + // XXX: I think this is broken? + while (offset > 0 && _Py_OPCODE(_PyCode_CODE(code)[offset]) == EXTENDED_ARG) { offset--; } return offset; @@ -1659,7 +1659,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyCodeObject *co = frame->f_code; \ names = co->co_names; \ consts = co->co_consts; \ - first_instr = _PyCode_GET_CODE(co); \ + first_instr = _PyCode_CODE(co); \ } \ assert(frame->f_lasti >= -1); \ next_instr = first_instr + frame->f_lasti + 1; \ @@ -6706,9 +6706,10 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, then call the trace function if we're tracing source lines. */ initialize_trace_info(&tstate->trace_info, frame); - _Py_CODEUNIT prev = _PyCode_GetUnquickened(frame->f_code, instr_prev); + // XXX: Is this broken? + int prev = _PyCode_CODE(frame->f_code)[instr_prev]; int lastline; - if (_Py_OPCODE(prev) == RESUME && _Py_OPARG(prev) == 0) { + if (_PyOpcode_Deopt[_Py_OPCODE(prev)] == RESUME && _Py_OPARG(prev) == 0) { lastline = -1; } else { @@ -6723,7 +6724,7 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, /* Trace backward edges (except in 'yield from') or if line number has changed */ int trace = line != lastline || (frame->f_lasti < instr_prev && - _Py_OPCODE(_PyCode_GET_CODE(frame->f_code)[frame->f_lasti]) != SEND); + _Py_OPCODE(_PyCode_CODE(frame->f_code)[frame->f_lasti]) != SEND); if (trace) { result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); } diff --git a/Python/compile.c b/Python/compile.c index ac9ddbcd79d033..dc0907cf68aef1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -136,7 +136,7 @@ instr_size(struct instr *instruction) int opcode = instruction->i_opcode; int oparg = instruction->i_oparg; int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg); - int caches = _PyOpcode_InlineCacheEntries[opcode]; + int caches = _PyOpcode_Caches[opcode]; return extended_args + 1 + caches; } @@ -145,7 +145,7 @@ write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) { int opcode = instruction->i_opcode; int oparg = instruction->i_oparg; - int caches = _PyOpcode_InlineCacheEntries[opcode]; + int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { case 4: *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 24) & 0xFF); diff --git a/Python/marshal.c b/Python/marshal.c index 1c430b23b4241a..df3b14e71a5731 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -563,7 +563,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_endlinetable, p); w_object(co->co_columntable, p); w_object(co->co_exceptiontable, p); - w_pstring(co->_co_code, _PyCode_GET_SIZE(co), p); + w_pstring(co->_co_code, _PyCode_NBYTES(co), p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ diff --git a/Python/specialize.c b/Python/specialize.c index 2d96bd28234acc..f20661e368bf4b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -283,7 +283,7 @@ _Py_Quicken(PyCodeObject *code) _Py_QuickenedCount++; int previous_opcode = -1; int previous_oparg = -1; - _Py_CODEUNIT *instructions = _PyCode_GET_CODE(code); + _Py_CODEUNIT *instructions = _PyCode_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _Py_OPCODE(instructions[i]); int oparg = _Py_OPARG(instructions[i]); @@ -294,10 +294,10 @@ _Py_Quicken(PyCodeObject *code) assert(instructions[i + 1] == 0); previous_opcode = -1; previous_oparg = -1; - i += _PyOpcode_InlineCacheEntries[opcode]; + i += _PyOpcode_Caches[opcode]; } else { - assert(!_PyOpcode_InlineCacheEntries[opcode]); + assert(!_PyOpcode_Caches[opcode]); switch (opcode) { case JUMP_ABSOLUTE: instructions[i] = _Py_MAKECODEUNIT(JUMP_ABSOLUTE_QUICK, oparg); @@ -340,14 +340,6 @@ _Py_Quicken(PyCodeObject *code) } } -_Py_CODEUNIT -_PyCode_GetUnquickened(PyCodeObject *code, int i) -{ - _Py_CODEUNIT instruction = _PyCode_GET_CODE(code)[i]; - int opcode = _PyOpcode_Deoptimizations[_Py_OPCODE(instruction)]; - return _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction)); -} - static inline int initial_counter_value(void) { /* Starting value for the counter. @@ -684,8 +676,7 @@ specialize_dict_access( int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - assert(_PyOpcode_InlineCacheEntries[LOAD_ATTR] == - INLINE_CACHE_ENTRIES_LOAD_ATTR); + assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); if (PyModule_CheckExact(owner)) { int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR, @@ -783,8 +774,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - assert(_PyOpcode_InlineCacheEntries[STORE_ATTR] == - INLINE_CACHE_ENTRIES_STORE_ATTR); + assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); if (PyModule_CheckExact(owner)) { @@ -944,8 +934,7 @@ typedef enum { int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - assert(_PyOpcode_InlineCacheEntries[LOAD_METHOD] == - INLINE_CACHE_ENTRIES_LOAD_METHOD); + assert(_PyOpcode_Caches[LOAD_METHOD] == INLINE_CACHE_ENTRIES_LOAD_METHOD); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); PyTypeObject *owner_cls = Py_TYPE(owner); @@ -1077,8 +1066,7 @@ _Py_Specialize_LoadGlobal( PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name) { - assert(_PyOpcode_InlineCacheEntries[LOAD_GLOBAL] == - INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL); /* Use inline cache */ _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1); assert(PyUnicode_CheckExact(name)); @@ -1214,7 +1202,7 @@ int _Py_Specialize_BinarySubscr( PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) { - assert(_PyOpcode_InlineCacheEntries[BINARY_SUBSCR] == + assert(_PyOpcode_Caches[BINARY_SUBSCR] == INLINE_CACHE_ENTRIES_BINARY_SUBSCR); _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1); PyTypeObject *container_type = Py_TYPE(container); @@ -1652,8 +1640,7 @@ int _Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames, int oparg) { - assert(_PyOpcode_InlineCacheEntries[PRECALL] == - INLINE_CACHE_ENTRIES_PRECALL); + assert(_PyOpcode_Caches[PRECALL] == INLINE_CACHE_ENTRIES_PRECALL); _PyPrecallCache *cache = (_PyPrecallCache *)(instr + 1); int fail; if (PyCFunction_CheckExact(callable)) { @@ -1699,7 +1686,7 @@ int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { - assert(_PyOpcode_InlineCacheEntries[CALL] == INLINE_CACHE_ENTRIES_CALL); + assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyFunction_Check(callable)) { @@ -1797,8 +1784,7 @@ void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg) { - assert(_PyOpcode_InlineCacheEntries[BINARY_OP] == - INLINE_CACHE_ENTRIES_BINARY_OP); + assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); switch (oparg) { case NB_ADD: @@ -1926,8 +1912,7 @@ void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg) { - assert(_PyOpcode_InlineCacheEntries[COMPARE_OP] == - INLINE_CACHE_ENTRIES_COMPARE_OP); + assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]); if (next_opcode != POP_JUMP_IF_FALSE && next_opcode != POP_JUMP_IF_TRUE) { @@ -2009,7 +1994,7 @@ unpack_sequence_fail_kind(PyObject *seq) void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) { - assert(_PyOpcode_InlineCacheEntries[UNPACK_SEQUENCE] == + assert(_PyOpcode_Caches[UNPACK_SEQUENCE] == INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1); if (PyTuple_CheckExact(seq)) { diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index 1c1ab6a63682b0..45dd0123c16ec5 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -75,13 +75,13 @@ def main(opcode_py, outfile='Include/opcode.h'): fobj.write(DEFINE.format(name, next_op)) used[next_op] = True fobj.write(DEFINE.format('DO_TRACING', 255)) - fobj.write("\nextern const uint8_t _PyOpcode_InlineCacheEntries[256];\n") - fobj.write("\nextern const uint8_t _PyOpcode_Deoptimizations[256];\n") + fobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n") + fobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n") fobj.write("\n#ifdef NEED_OPCODE_TABLES\n") write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj) write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj) - fobj.write("\nconst uint8_t _PyOpcode_InlineCacheEntries[256] = {\n") + fobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n") for i, entries in enumerate(opcode["_inline_cache_entries"]): if entries: fobj.write(f" [{opname[i]}] = {entries},\n") @@ -92,7 +92,7 @@ def main(opcode_py, outfile='Include/opcode.h'): for basic, family in opcode["_specializations"].items(): for specialized in family: deoptcodes[specialized] = basic - fobj.write("\nconst uint8_t _PyOpcode_Deoptimizations[256] = {\n") + fobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n") for opt, deopt in sorted(deoptcodes.items()): fobj.write(f" [{opt}] = {deopt},\n") fobj.write("};\n") From af27670344b64b2c6e7593c4e84f9eb07d830016 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 11 Mar 2022 15:24:41 -0800 Subject: [PATCH 15/34] Revert marshal format changes --- Include/cpython/code.h | 2 + Include/internal/pycore_code.h | 8 +- Include/marshal.h | 2 +- Include/opcode.h | 404 +++++++++++++-------------- Lib/importlib/_bootstrap_external.py | 1 - Objects/codeobject.c | 50 ++-- Programs/test_frozenmain.h | 64 ++--- Python/compile.c | 2 +- Python/marshal.c | 23 +- Python/specialize.c | 2 +- Tools/scripts/deepfreeze.py | 4 +- Tools/scripts/generate_opcode_h.py | 2 +- Tools/scripts/umarshal.py | 3 +- 13 files changed, 279 insertions(+), 288 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 4c05274487c686..1a8f5f23d020c4 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -132,6 +132,8 @@ PyAPI_DATA(PyTypeObject) PyCode_Type; #define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type) #define PyCode_GetNumFree(op) ((op)->co_nfreevars) +#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->_co_code) +#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) /* Public interface */ PyAPI_FUNC(PyCodeObject *) PyCode_New( diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 83f882818f7de4..e4ac5c93c8e3fd 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -97,10 +97,6 @@ typedef struct { /* We want to compare to zero for efficiency, so we offset values accordingly */ #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) -#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->_co_code) -#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) - - void _Py_Quicken(PyCodeObject *code); static inline void @@ -274,9 +270,9 @@ extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg); /* Deallocator function for static codeobjects used in deepfreeze.py */ -extern void _PyStaticCode_Dealloc(void *code); +extern void _PyStaticCode_Dealloc(PyCodeObject *co); /* Function to intern strings of codeobjects */ -extern int _PyStaticCode_InternStrings(void *code); +extern int _PyStaticCode_InternStrings(PyCodeObject *co); #ifdef Py_STATS diff --git a/Include/marshal.h b/Include/marshal.h index f773587bdd0429..f8b0de80cfc38d 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -13,7 +13,7 @@ PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); -#define Py_MARSHAL_VERSION 5 +#define Py_MARSHAL_VERSION 4 PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); diff --git a/Include/opcode.h b/Include/opcode.h index ca080af18ebd9f..dfc7b72e3cdc68 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -7,183 +7,183 @@ extern "C" { /* Instruction opcodes for compiled code */ -#define CACHE 0 -#define POP_TOP 1 -#define PUSH_NULL 2 -#define NOP 9 -#define UNARY_POSITIVE 10 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 -#define UNARY_INVERT 15 -#define BINARY_SUBSCR 25 -#define GET_LEN 30 -#define MATCH_MAPPING 31 -#define MATCH_SEQUENCE 32 -#define MATCH_KEYS 33 -#define PUSH_EXC_INFO 35 -#define WITH_EXCEPT_START 49 -#define GET_AITER 50 -#define GET_ANEXT 51 -#define BEFORE_ASYNC_WITH 52 -#define BEFORE_WITH 53 -#define END_ASYNC_FOR 54 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 -#define GET_ITER 68 -#define GET_YIELD_FROM_ITER 69 -#define PRINT_EXPR 70 -#define LOAD_BUILD_CLASS 71 -#define LOAD_ASSERTION_ERROR 74 -#define RETURN_GENERATOR 75 -#define LIST_TO_TUPLE 82 -#define RETURN_VALUE 83 -#define IMPORT_STAR 84 -#define SETUP_ANNOTATIONS 85 -#define YIELD_VALUE 86 -#define ASYNC_GEN_WRAP 87 -#define PREP_RERAISE_STAR 88 -#define POP_EXCEPT 89 -#define HAVE_ARGUMENT 90 -#define STORE_NAME 90 -#define DELETE_NAME 91 -#define UNPACK_SEQUENCE 92 -#define FOR_ITER 93 -#define UNPACK_EX 94 -#define STORE_ATTR 95 -#define DELETE_ATTR 96 -#define STORE_GLOBAL 97 -#define DELETE_GLOBAL 98 -#define SWAP 99 -#define LOAD_CONST 100 -#define LOAD_NAME 101 -#define BUILD_TUPLE 102 -#define BUILD_LIST 103 -#define BUILD_SET 104 -#define BUILD_MAP 105 -#define LOAD_ATTR 106 -#define COMPARE_OP 107 -#define IMPORT_NAME 108 -#define IMPORT_FROM 109 -#define JUMP_FORWARD 110 -#define JUMP_IF_FALSE_OR_POP 111 -#define JUMP_IF_TRUE_OR_POP 112 -#define JUMP_ABSOLUTE 113 -#define POP_JUMP_IF_FALSE 114 -#define POP_JUMP_IF_TRUE 115 -#define LOAD_GLOBAL 116 -#define IS_OP 117 -#define CONTAINS_OP 118 -#define RERAISE 119 -#define COPY 120 -#define JUMP_IF_NOT_EXC_MATCH 121 -#define BINARY_OP 122 -#define SEND 123 -#define LOAD_FAST 124 -#define STORE_FAST 125 -#define DELETE_FAST 126 -#define JUMP_IF_NOT_EG_MATCH 127 -#define POP_JUMP_IF_NOT_NONE 128 -#define POP_JUMP_IF_NONE 129 -#define RAISE_VARARGS 130 -#define GET_AWAITABLE 131 -#define MAKE_FUNCTION 132 -#define BUILD_SLICE 133 -#define JUMP_NO_INTERRUPT 134 -#define MAKE_CELL 135 -#define LOAD_CLOSURE 136 -#define LOAD_DEREF 137 -#define STORE_DEREF 138 -#define DELETE_DEREF 139 -#define CALL_FUNCTION_EX 142 -#define EXTENDED_ARG 144 -#define LIST_APPEND 145 -#define SET_ADD 146 -#define MAP_ADD 147 -#define LOAD_CLASSDEREF 148 -#define COPY_FREE_VARS 149 -#define RESUME 151 -#define MATCH_CLASS 152 -#define FORMAT_VALUE 155 -#define BUILD_CONST_KEY_MAP 156 -#define BUILD_STRING 157 -#define LOAD_METHOD 160 -#define LIST_EXTEND 162 -#define SET_UPDATE 163 -#define DICT_MERGE 164 -#define DICT_UPDATE 165 -#define PRECALL 166 -#define CALL 171 -#define KW_NAMES 172 -#define BINARY_OP_ADAPTIVE 3 -#define BINARY_OP_ADD_FLOAT 4 -#define BINARY_OP_ADD_INT 5 -#define BINARY_OP_ADD_UNICODE 6 -#define BINARY_OP_INPLACE_ADD_UNICODE 7 -#define BINARY_OP_MULTIPLY_FLOAT 8 -#define BINARY_OP_MULTIPLY_INT 13 -#define BINARY_OP_SUBTRACT_FLOAT 14 -#define BINARY_OP_SUBTRACT_INT 16 -#define BINARY_SUBSCR_ADAPTIVE 17 -#define BINARY_SUBSCR_DICT 18 -#define BINARY_SUBSCR_GETITEM 19 -#define BINARY_SUBSCR_LIST_INT 20 -#define BINARY_SUBSCR_TUPLE_INT 21 -#define CALL_ADAPTIVE 22 -#define CALL_PY_EXACT_ARGS 23 -#define CALL_PY_WITH_DEFAULTS 24 -#define COMPARE_OP_ADAPTIVE 26 -#define COMPARE_OP_FLOAT_JUMP 27 -#define COMPARE_OP_INT_JUMP 28 -#define COMPARE_OP_STR_JUMP 29 -#define JUMP_ABSOLUTE_QUICK 34 -#define LOAD_ATTR_ADAPTIVE 36 -#define LOAD_ATTR_INSTANCE_VALUE 37 -#define LOAD_ATTR_MODULE 38 -#define LOAD_ATTR_SLOT 39 -#define LOAD_ATTR_WITH_HINT 40 -#define LOAD_CONST__LOAD_FAST 41 -#define LOAD_FAST__LOAD_CONST 42 -#define LOAD_FAST__LOAD_FAST 43 -#define LOAD_GLOBAL_ADAPTIVE 44 -#define LOAD_GLOBAL_BUILTIN 45 -#define LOAD_GLOBAL_MODULE 46 -#define LOAD_METHOD_ADAPTIVE 47 -#define LOAD_METHOD_CLASS 48 -#define LOAD_METHOD_MODULE 55 -#define LOAD_METHOD_NO_DICT 56 -#define LOAD_METHOD_WITH_DICT 57 -#define LOAD_METHOD_WITH_VALUES 58 -#define PRECALL_ADAPTIVE 59 -#define PRECALL_BOUND_METHOD 62 -#define PRECALL_BUILTIN_CLASS 63 -#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 64 -#define PRECALL_NO_KW_BUILTIN_FAST 65 -#define PRECALL_NO_KW_BUILTIN_O 66 -#define PRECALL_NO_KW_ISINSTANCE 67 -#define PRECALL_NO_KW_LEN 72 -#define PRECALL_NO_KW_LIST_APPEND 73 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 76 +#define CACHE 0 +#define POP_TOP 1 +#define PUSH_NULL 2 +#define NOP 9 +#define UNARY_POSITIVE 10 +#define UNARY_NEGATIVE 11 +#define UNARY_NOT 12 +#define UNARY_INVERT 15 +#define BINARY_SUBSCR 25 +#define GET_LEN 30 +#define MATCH_MAPPING 31 +#define MATCH_SEQUENCE 32 +#define MATCH_KEYS 33 +#define PUSH_EXC_INFO 35 +#define WITH_EXCEPT_START 49 +#define GET_AITER 50 +#define GET_ANEXT 51 +#define BEFORE_ASYNC_WITH 52 +#define BEFORE_WITH 53 +#define END_ASYNC_FOR 54 +#define STORE_SUBSCR 60 +#define DELETE_SUBSCR 61 +#define GET_ITER 68 +#define GET_YIELD_FROM_ITER 69 +#define PRINT_EXPR 70 +#define LOAD_BUILD_CLASS 71 +#define LOAD_ASSERTION_ERROR 74 +#define RETURN_GENERATOR 75 +#define LIST_TO_TUPLE 82 +#define RETURN_VALUE 83 +#define IMPORT_STAR 84 +#define SETUP_ANNOTATIONS 85 +#define YIELD_VALUE 86 +#define ASYNC_GEN_WRAP 87 +#define PREP_RERAISE_STAR 88 +#define POP_EXCEPT 89 +#define HAVE_ARGUMENT 90 +#define STORE_NAME 90 +#define DELETE_NAME 91 +#define UNPACK_SEQUENCE 92 +#define FOR_ITER 93 +#define UNPACK_EX 94 +#define STORE_ATTR 95 +#define DELETE_ATTR 96 +#define STORE_GLOBAL 97 +#define DELETE_GLOBAL 98 +#define SWAP 99 +#define LOAD_CONST 100 +#define LOAD_NAME 101 +#define BUILD_TUPLE 102 +#define BUILD_LIST 103 +#define BUILD_SET 104 +#define BUILD_MAP 105 +#define LOAD_ATTR 106 +#define COMPARE_OP 107 +#define IMPORT_NAME 108 +#define IMPORT_FROM 109 +#define JUMP_FORWARD 110 +#define JUMP_IF_FALSE_OR_POP 111 +#define JUMP_IF_TRUE_OR_POP 112 +#define JUMP_ABSOLUTE 113 +#define POP_JUMP_IF_FALSE 114 +#define POP_JUMP_IF_TRUE 115 +#define LOAD_GLOBAL 116 +#define IS_OP 117 +#define CONTAINS_OP 118 +#define RERAISE 119 +#define COPY 120 +#define JUMP_IF_NOT_EXC_MATCH 121 +#define BINARY_OP 122 +#define SEND 123 +#define LOAD_FAST 124 +#define STORE_FAST 125 +#define DELETE_FAST 126 +#define JUMP_IF_NOT_EG_MATCH 127 +#define POP_JUMP_IF_NOT_NONE 128 +#define POP_JUMP_IF_NONE 129 +#define RAISE_VARARGS 130 +#define GET_AWAITABLE 131 +#define MAKE_FUNCTION 132 +#define BUILD_SLICE 133 +#define JUMP_NO_INTERRUPT 134 +#define MAKE_CELL 135 +#define LOAD_CLOSURE 136 +#define LOAD_DEREF 137 +#define STORE_DEREF 138 +#define DELETE_DEREF 139 +#define CALL_FUNCTION_EX 142 +#define EXTENDED_ARG 144 +#define LIST_APPEND 145 +#define SET_ADD 146 +#define MAP_ADD 147 +#define LOAD_CLASSDEREF 148 +#define COPY_FREE_VARS 149 +#define RESUME 151 +#define MATCH_CLASS 152 +#define FORMAT_VALUE 155 +#define BUILD_CONST_KEY_MAP 156 +#define BUILD_STRING 157 +#define LOAD_METHOD 160 +#define LIST_EXTEND 162 +#define SET_UPDATE 163 +#define DICT_MERGE 164 +#define DICT_UPDATE 165 +#define PRECALL 166 +#define CALL 171 +#define KW_NAMES 172 +#define BINARY_OP_ADAPTIVE 3 +#define BINARY_OP_ADD_FLOAT 4 +#define BINARY_OP_ADD_INT 5 +#define BINARY_OP_ADD_UNICODE 6 +#define BINARY_OP_INPLACE_ADD_UNICODE 7 +#define BINARY_OP_MULTIPLY_FLOAT 8 +#define BINARY_OP_MULTIPLY_INT 13 +#define BINARY_OP_SUBTRACT_FLOAT 14 +#define BINARY_OP_SUBTRACT_INT 16 +#define BINARY_SUBSCR_ADAPTIVE 17 +#define BINARY_SUBSCR_DICT 18 +#define BINARY_SUBSCR_GETITEM 19 +#define BINARY_SUBSCR_LIST_INT 20 +#define BINARY_SUBSCR_TUPLE_INT 21 +#define CALL_ADAPTIVE 22 +#define CALL_PY_EXACT_ARGS 23 +#define CALL_PY_WITH_DEFAULTS 24 +#define COMPARE_OP_ADAPTIVE 26 +#define COMPARE_OP_FLOAT_JUMP 27 +#define COMPARE_OP_INT_JUMP 28 +#define COMPARE_OP_STR_JUMP 29 +#define JUMP_ABSOLUTE_QUICK 34 +#define LOAD_ATTR_ADAPTIVE 36 +#define LOAD_ATTR_INSTANCE_VALUE 37 +#define LOAD_ATTR_MODULE 38 +#define LOAD_ATTR_SLOT 39 +#define LOAD_ATTR_WITH_HINT 40 +#define LOAD_CONST__LOAD_FAST 41 +#define LOAD_FAST__LOAD_CONST 42 +#define LOAD_FAST__LOAD_FAST 43 +#define LOAD_GLOBAL_ADAPTIVE 44 +#define LOAD_GLOBAL_BUILTIN 45 +#define LOAD_GLOBAL_MODULE 46 +#define LOAD_METHOD_ADAPTIVE 47 +#define LOAD_METHOD_CLASS 48 +#define LOAD_METHOD_MODULE 55 +#define LOAD_METHOD_NO_DICT 56 +#define LOAD_METHOD_WITH_DICT 57 +#define LOAD_METHOD_WITH_VALUES 58 +#define PRECALL_ADAPTIVE 59 +#define PRECALL_BOUND_METHOD 62 +#define PRECALL_BUILTIN_CLASS 63 +#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 64 +#define PRECALL_NO_KW_BUILTIN_FAST 65 +#define PRECALL_NO_KW_BUILTIN_O 66 +#define PRECALL_NO_KW_ISINSTANCE 67 +#define PRECALL_NO_KW_LEN 72 +#define PRECALL_NO_KW_LIST_APPEND 73 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 76 #define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 77 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 78 -#define PRECALL_NO_KW_STR_1 79 -#define PRECALL_NO_KW_TUPLE_1 80 -#define PRECALL_NO_KW_TYPE_1 81 -#define PRECALL_PYFUNC 140 -#define RESUME_QUICK 141 -#define STORE_ATTR_ADAPTIVE 143 -#define STORE_ATTR_INSTANCE_VALUE 150 -#define STORE_ATTR_SLOT 153 -#define STORE_ATTR_WITH_HINT 154 -#define STORE_FAST__LOAD_FAST 158 -#define STORE_FAST__STORE_FAST 159 -#define STORE_SUBSCR_ADAPTIVE 161 -#define STORE_SUBSCR_DICT 167 -#define STORE_SUBSCR_LIST_INT 168 -#define UNPACK_SEQUENCE_ADAPTIVE 169 -#define UNPACK_SEQUENCE_LIST 170 -#define UNPACK_SEQUENCE_TUPLE 173 -#define UNPACK_SEQUENCE_TWO_TUPLE 174 -#define DO_TRACING 255 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 78 +#define PRECALL_NO_KW_STR_1 79 +#define PRECALL_NO_KW_TUPLE_1 80 +#define PRECALL_NO_KW_TYPE_1 81 +#define PRECALL_PYFUNC 140 +#define RESUME_QUICK 141 +#define STORE_ATTR_ADAPTIVE 143 +#define STORE_ATTR_INSTANCE_VALUE 150 +#define STORE_ATTR_SLOT 153 +#define STORE_ATTR_WITH_HINT 154 +#define STORE_FAST__LOAD_FAST 158 +#define STORE_FAST__STORE_FAST 159 +#define STORE_SUBSCR_ADAPTIVE 161 +#define STORE_SUBSCR_DICT 167 +#define STORE_SUBSCR_LIST_INT 168 +#define UNPACK_SEQUENCE_ADAPTIVE 169 +#define UNPACK_SEQUENCE_LIST 170 +#define UNPACK_SEQUENCE_TUPLE 173 +#define UNPACK_SEQUENCE_TWO_TUPLE 174 +#define DO_TRACING 255 extern const uint8_t _PyOpcode_Caches[256]; @@ -409,32 +409,32 @@ const uint8_t _PyOpcode_Deopt[256] = { || ((op) == 172) \ ) -#define NB_ADD 0 -#define NB_AND 1 -#define NB_FLOOR_DIVIDE 2 -#define NB_LSHIFT 3 -#define NB_MATRIX_MULTIPLY 4 -#define NB_MULTIPLY 5 -#define NB_REMAINDER 6 -#define NB_OR 7 -#define NB_POWER 8 -#define NB_RSHIFT 9 -#define NB_SUBTRACT 10 -#define NB_TRUE_DIVIDE 11 -#define NB_XOR 12 -#define NB_INPLACE_ADD 13 -#define NB_INPLACE_AND 14 -#define NB_INPLACE_FLOOR_DIVIDE 15 -#define NB_INPLACE_LSHIFT 16 -#define NB_INPLACE_MATRIX_MULTIPLY 17 -#define NB_INPLACE_MULTIPLY 18 -#define NB_INPLACE_REMAINDER 19 -#define NB_INPLACE_OR 20 -#define NB_INPLACE_POWER 21 -#define NB_INPLACE_RSHIFT 22 -#define NB_INPLACE_SUBTRACT 23 -#define NB_INPLACE_TRUE_DIVIDE 24 -#define NB_INPLACE_XOR 25 +#define NB_ADD 0 +#define NB_AND 1 +#define NB_FLOOR_DIVIDE 2 +#define NB_LSHIFT 3 +#define NB_MATRIX_MULTIPLY 4 +#define NB_MULTIPLY 5 +#define NB_REMAINDER 6 +#define NB_OR 7 +#define NB_POWER 8 +#define NB_RSHIFT 9 +#define NB_SUBTRACT 10 +#define NB_TRUE_DIVIDE 11 +#define NB_XOR 12 +#define NB_INPLACE_ADD 13 +#define NB_INPLACE_AND 14 +#define NB_INPLACE_FLOOR_DIVIDE 15 +#define NB_INPLACE_LSHIFT 16 +#define NB_INPLACE_MATRIX_MULTIPLY 17 +#define NB_INPLACE_MULTIPLY 18 +#define NB_INPLACE_REMAINDER 19 +#define NB_INPLACE_OR 20 +#define NB_INPLACE_POWER 21 +#define NB_INPLACE_RSHIFT 22 +#define NB_INPLACE_SUBTRACT 23 +#define NB_INPLACE_TRUE_DIVIDE 24 +#define NB_INPLACE_XOR 25 #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 99992958281a53..a93e695aa9378e 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -395,7 +395,6 @@ def _write_atomic(path, data, mode=0o666): # Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) # Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) # Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) -# Python 3.11a6 3488 (Change the marshal format for code objects) # Python 3.12 will start with magic number 3500 diff --git a/Objects/codeobject.c b/Objects/codeobject.c index baf7a8a8ac4c6a..bc9452c3e396da 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1475,22 +1475,22 @@ code_hash(PyCodeObject *co) #define OFF(x) offsetof(PyCodeObject, x) static PyMemberDef code_memberlist[] = { - {"co_argcount", T_INT, OFF(co_argcount), READONLY}, - {"co_posonlyargcount", T_INT, OFF(co_posonlyargcount), READONLY}, - {"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY}, - {"co_stacksize",T_INT, OFF(co_stacksize), READONLY}, - {"co_flags", T_INT, OFF(co_flags), READONLY}, - {"co_consts", T_OBJECT, OFF(co_consts), READONLY}, - {"co_names", T_OBJECT, OFF(co_names), READONLY}, - {"co_filename", T_OBJECT, OFF(co_filename), READONLY}, - {"co_name", T_OBJECT, OFF(co_name), READONLY}, - {"co_qualname", T_OBJECT, OFF(co_qualname), READONLY}, - {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY}, - {"co_linetable", T_OBJECT, OFF(co_linetable), READONLY}, - {"co_endlinetable", T_OBJECT, OFF(co_endlinetable), READONLY}, - {"co_columntable", T_OBJECT, OFF(co_columntable), READONLY}, - {"co_exceptiontable", T_OBJECT, OFF(co_exceptiontable), READONLY}, - {"co_nlocals", T_INT, OFF(co_nlocals), READONLY}, + {"co_argcount", T_INT, OFF(co_argcount), READONLY}, + {"co_posonlyargcount", T_INT, OFF(co_posonlyargcount), READONLY}, + {"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY}, + {"co_stacksize", T_INT, OFF(co_stacksize), READONLY}, + {"co_flags", T_INT, OFF(co_flags), READONLY}, + {"co_consts", T_OBJECT, OFF(co_consts), READONLY}, + {"co_names", T_OBJECT, OFF(co_names), READONLY}, + {"co_filename", T_OBJECT, OFF(co_filename), READONLY}, + {"co_name", T_OBJECT, OFF(co_name), READONLY}, + {"co_qualname", T_OBJECT, OFF(co_qualname), READONLY}, + {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY}, + {"co_linetable", T_OBJECT, OFF(co_linetable), READONLY}, + {"co_endlinetable", T_OBJECT, OFF(co_endlinetable), READONLY}, + {"co_columntable", T_OBJECT, OFF(co_columntable), READONLY}, + {"co_exceptiontable", T_OBJECT, OFF(co_exceptiontable), READONLY}, + {"co_nlocals", T_INT, OFF(co_nlocals), READONLY}, {NULL} /* Sentinel */ }; @@ -1533,13 +1533,13 @@ code_getcode(PyCodeObject *code, void *closure) } static PyGetSetDef code_getsetlist[] = { - {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, + {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, + {"_co_code", (getter)code_getundercode, NULL, NULL}, // The following old names are kept for backward compatibility. - {"co_varnames", (getter)code_getvarnames, NULL, NULL}, - {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, - {"co_freevars", (getter)code_getfreevars, NULL, NULL}, - {"_co_code", (getter)code_getundercode, NULL, NULL}, - {"co_code", (getter)code_getcode, NULL, NULL}, + {"co_varnames", (getter)code_getvarnames, NULL, NULL}, + {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, + {"co_freevars", (getter)code_getfreevars, NULL, NULL}, + {"co_code", (getter)code_getcode, NULL, NULL}, {0} }; @@ -1902,9 +1902,8 @@ _PyCode_ConstantKey(PyObject *op) } void -_PyStaticCode_Dealloc(void *code) +_PyStaticCode_Dealloc(PyCodeObject *co) { - PyCodeObject *co = (PyCodeObject *)code; if (co->co_warmup == 0) { _Py_QuickenedCount--; } @@ -1918,9 +1917,8 @@ _PyStaticCode_Dealloc(void *code) } int -_PyStaticCode_InternStrings(void *code) +_PyStaticCode_InternStrings(PyCodeObject *co) { - PyCodeObject *co = (PyCodeObject *)code; int res = intern_strings(co->co_names); if (res < 0) { return -1; diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index ad3945c26eb552..8cae77a4899f12 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,37 +1,7 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,41,8,233,0,0,0,0,78,122,18,70, - 114,111,122,101,110,32,72,101,108,108,111,32,87,111,114,108, - 100,122,8,115,121,115,46,97,114,103,118,218,6,99,111,110, - 102,105,103,41,5,90,12,112,114,111,103,114,97,109,95,110, - 97,109,101,218,10,101,120,101,99,117,116,97,98,108,101,90, - 15,117,115,101,95,101,110,118,105,114,111,110,109,101,110,116, - 90,17,99,111,110,102,105,103,117,114,101,95,99,95,115,116, - 100,105,111,90,14,98,117,102,102,101,114,101,100,95,115,116, - 100,105,111,122,7,99,111,110,102,105,103,32,122,2,58,32, - 41,7,218,3,115,121,115,90,17,95,116,101,115,116,105,110, - 116,101,114,110,97,108,99,97,112,105,218,5,112,114,105,110, - 116,218,4,97,114,103,118,90,11,103,101,116,95,99,111,110, - 102,105,103,115,114,2,0,0,0,218,3,107,101,121,169,0, - 243,0,0,0,0,250,18,116,101,115,116,95,102,114,111,122, - 101,110,109,97,105,110,46,112,121,250,8,60,109,111,100,117, - 108,101,62,114,11,0,0,0,1,0,0,0,115,18,0,0, - 0,2,128,8,3,8,1,22,2,34,1,42,1,8,1,48, - 7,4,249,115,20,0,0,0,2,128,8,3,8,1,22,2, - 34,1,42,1,2,7,4,1,2,249,52,7,115,176,0,0, - 0,0,0,1,11,1,11,1,11,1,11,1,25,1,25,1, - 25,1,25,1,6,1,6,7,27,1,28,1,28,1,28,1, - 28,1,28,1,28,1,28,1,28,1,6,1,6,7,17,19, - 22,19,27,19,27,19,27,19,27,19,27,1,28,1,28,1, - 28,1,28,1,28,1,28,1,28,1,28,10,39,10,27,10, - 39,10,39,10,39,10,39,10,39,10,41,10,41,10,41,10, - 41,10,41,10,41,10,41,42,50,10,51,10,51,10,51,10, - 51,10,51,1,7,12,2,1,42,1,42,5,8,5,10,5, - 10,11,41,21,24,11,41,11,41,28,34,35,38,28,39,28, - 39,28,39,28,39,28,39,11,41,11,41,5,42,5,42,5, - 42,5,42,5,42,5,42,5,42,5,42,5,42,1,42,1, - 42,114,9,0,0,0,176,0,0,0,151,0,100,0,100,1, + 0,0,0,0,0,115,176,0,0,0,151,0,100,0,100,1, 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, 100,2,166,1,0,0,171,1,0,0,0,0,0,0,0,0, 1,0,2,0,101,2,100,3,101,0,106,3,0,0,0,0, @@ -42,5 +12,35 @@ unsigned char M_test_frozenmain[] = { 68,0,93,25,90,6,2,0,101,2,100,6,101,6,155,0, 100,7,101,5,101,6,25,0,0,0,0,0,0,0,0,0, 155,0,157,4,166,1,0,0,171,1,0,0,0,0,0,0, - 0,0,1,0,113,60,100,1,83,0, + 0,0,1,0,113,60,100,1,83,0,41,8,233,0,0,0, + 0,78,122,18,70,114,111,122,101,110,32,72,101,108,108,111, + 32,87,111,114,108,100,122,8,115,121,115,46,97,114,103,118, + 218,6,99,111,110,102,105,103,41,5,90,12,112,114,111,103, + 114,97,109,95,110,97,109,101,218,10,101,120,101,99,117,116, + 97,98,108,101,90,15,117,115,101,95,101,110,118,105,114,111, + 110,109,101,110,116,90,17,99,111,110,102,105,103,117,114,101, + 95,99,95,115,116,100,105,111,90,14,98,117,102,102,101,114, + 101,100,95,115,116,100,105,111,122,7,99,111,110,102,105,103, + 32,122,2,58,32,41,7,218,3,115,121,115,90,17,95,116, + 101,115,116,105,110,116,101,114,110,97,108,99,97,112,105,218, + 5,112,114,105,110,116,218,4,97,114,103,118,90,11,103,101, + 116,95,99,111,110,102,105,103,115,114,2,0,0,0,218,3, + 107,101,121,169,0,243,0,0,0,0,250,18,116,101,115,116, + 95,102,114,111,122,101,110,109,97,105,110,46,112,121,250,8, + 60,109,111,100,117,108,101,62,114,11,0,0,0,1,0,0, + 0,115,18,0,0,0,2,128,8,3,8,1,22,2,34,1, + 42,1,8,1,48,7,4,249,115,20,0,0,0,2,128,8, + 3,8,1,22,2,34,1,42,1,2,7,4,1,2,249,52, + 7,115,176,0,0,0,0,0,1,11,1,11,1,11,1,11, + 1,25,1,25,1,25,1,25,1,6,1,6,7,27,1,28, + 1,28,1,28,1,28,1,28,1,28,1,28,1,28,1,6, + 1,6,7,17,19,22,19,27,19,27,19,27,19,27,19,27, + 1,28,1,28,1,28,1,28,1,28,1,28,1,28,1,28, + 10,39,10,27,10,39,10,39,10,39,10,39,10,39,10,41, + 10,41,10,41,10,41,10,41,10,41,10,41,42,50,10,51, + 10,51,10,51,10,51,10,51,1,7,12,2,1,42,1,42, + 5,8,5,10,5,10,11,41,21,24,11,41,11,41,28,34, + 35,38,28,39,28,39,28,39,28,39,28,39,11,41,11,41, + 5,42,5,42,5,42,5,42,5,42,5,42,5,42,5,42, + 5,42,1,42,1,42,114,9,0,0,0, }; diff --git a/Python/compile.c b/Python/compile.c index dc0907cf68aef1..542770e8e186ba 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -144,7 +144,7 @@ static void write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) { int opcode = instruction->i_opcode; - int oparg = instruction->i_oparg; + int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { case 4: diff --git a/Python/marshal.c b/Python/marshal.c index df3b14e71a5731..c571cd79b97fee 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -545,12 +545,18 @@ w_complex_object(PyObject *v, char flag, WFILE *p) } else if (PyCode_Check(v)) { PyCodeObject *co = (PyCodeObject *)v; + PyObject *co_code = _PyCode_GetCode(co); + if (co_code == NULL) { + p->error = WFERR_NOMEMORY; + return; + } W_TYPE(TYPE_CODE, p); w_long(co->co_argcount, p); w_long(co->co_posonlyargcount, p); w_long(co->co_kwonlyargcount, p); w_long(co->co_stacksize, p); w_long(co->co_flags, p); + w_object(co_code, p); w_object(co->co_consts, p); w_object(co->co_names, p); w_object(co->co_localsplusnames, p); @@ -563,7 +569,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_endlinetable, p); w_object(co->co_columntable, p); w_object(co->co_exceptiontable, p); - w_pstring(co->_co_code, _PyCode_NBYTES(co), p); + Py_DECREF(co_code); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ @@ -1381,6 +1387,9 @@ r_object(RFILE *p) flags = (int)r_long(p); if (PyErr_Occurred()) goto code_error; + code = r_object(p); + if (code == NULL) + goto code_error; consts = r_object(p); if (consts == NULL) goto code_error; @@ -1417,18 +1426,6 @@ r_object(RFILE *p) exceptiontable = r_object(p); if (exceptiontable == NULL) goto code_error; - n = r_long(p); - if (n == -1 && PyErr_Occurred()) { - break; - } - const char *quickened = r_string(n, p); - if (quickened == NULL) { - break; - } - code = PyBytes_FromStringAndSize(quickened, n); - if (code == NULL) { - goto code_error; - } struct _PyCodeConstructor con = { .filename = filename, diff --git a/Python/specialize.c b/Python/specialize.c index f20661e368bf4b..d37c666ba02395 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -276,7 +276,7 @@ _Py_PrintSpecializationStats(int to_file) #endif -// Insert adaptive instructions and superinstructions. +// Insert adaptive instructions and superinstructions. This cannot fail. void _Py_Quicken(PyCodeObject *code) { diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index f313d74f57461c..5c57bf969ba079 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -304,8 +304,8 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_endlinetable = {co_endlinetable},") self.write(f".co_columntable = {co_columntable},") self.write(f"._co_code = {make_string_literal(code.co_code)},") - self.deallocs.append(f"_PyStaticCode_Dealloc(&{name});") - self.interns.append(f"_PyStaticCode_InternStrings(&{name})") + self.deallocs.append(f"_PyStaticCode_Dealloc((PyCodeObject *)&{name});") + self.interns.append(f"_PyStaticCode_InternStrings((PyCodeObject *)&{name})") return f"& {name}.ob_base.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index 45dd0123c16ec5..3b79dc6b7359f6 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -28,7 +28,7 @@ #endif /* !Py_OPCODE_H */ """ -DEFINE = "#define {:<31} {:>3}\n" +DEFINE = "#define {:<38} {:>3}\n" UINT32_MASK = (1<<32)-1 diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py index 2bfeab1b065c8b..2eaaa7ce2d95bc 100644 --- a/Tools/scripts/umarshal.py +++ b/Tools/scripts/umarshal.py @@ -279,6 +279,7 @@ def R_REF(obj: Any) -> Any: retval.co_kwonlyargcount = self.r_long() retval.co_stacksize = self.r_long() retval.co_flags = self.r_long() + retval.co_code = self.r_object() retval.co_consts = self.r_object() retval.co_names = self.r_object() retval.co_localsplusnames = self.r_object() @@ -291,8 +292,6 @@ def R_REF(obj: Any) -> Any: retval.co_endlinetable = self.r_object() retval.co_columntable = self.r_object() retval.co_exceptiontable = self.r_object() - n = self.r_long() - retval.co_code = self.r_string(n) return retval elif type == Type.REF: n = self.r_long() From 629bf8b3ab3b8ca54084e8b095074c9c02606792 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 11 Mar 2022 15:47:03 -0800 Subject: [PATCH 16/34] More cleanup --- Include/internal/pycore_code.h | 2 +- Lib/importlib/_bootstrap_external.py | 2 +- Objects/codeobject.c | 3 ++- Python/ceval.c | 4 ++-- Python/specialize.c | 1 - 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index e4ac5c93c8e3fd..ce6611a5a30c78 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -100,7 +100,7 @@ typedef struct { void _Py_Quicken(PyCodeObject *code); static inline void -_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) +_PyCode_Warmup(PyCodeObject *code) { if (code->co_warmup != 0) { code->co_warmup++; diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index a93e695aa9378e..a6f0a1b3c4c7d8 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -409,7 +409,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3488).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3487).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Objects/codeobject.c b/Objects/codeobject.c index bc9452c3e396da..7cd2c08f4b30cb 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1341,8 +1341,9 @@ code_dealloc(PyCodeObject *co) Py_XDECREF(co->co_endlinetable); Py_XDECREF(co->co_columntable); Py_XDECREF(co->co_exceptiontable); - if (co->co_weakreflist != NULL) + if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); + } if (co->co_warmup == 0) { _Py_QuickenedCount--; } diff --git a/Python/ceval.c b/Python/ceval.c index e99339c5068bd0..03ad7c0d918717 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1731,7 +1731,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } TARGET(RESUME) { - _Py_IncrementCountAndMaybeQuicken(frame->f_code); + _PyCode_Warmup(frame->f_code); JUMP_TO_INSTRUCTION(RESUME_QUICK); } @@ -4059,7 +4059,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(JUMP_ABSOLUTE) { PREDICTED(JUMP_ABSOLUTE); - _Py_IncrementCountAndMaybeQuicken(frame->f_code); + _PyCode_Warmup(frame->f_code); JUMP_TO_INSTRUCTION(JUMP_ABSOLUTE_QUICK); } diff --git a/Python/specialize.c b/Python/specialize.c index d37c666ba02395..4c05cbb9a5788f 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -275,7 +275,6 @@ _Py_PrintSpecializationStats(int to_file) #define SPECIALIZATION_FAIL(opcode, kind) ((void)0) #endif - // Insert adaptive instructions and superinstructions. This cannot fail. void _Py_Quicken(PyCodeObject *code) From 59cda59035902945fc0ccfc7f48a312154fb1344 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 11 Mar 2022 15:50:11 -0800 Subject: [PATCH 17/34] Clean up the diff --- Objects/codeobject.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 7cd2c08f4b30cb..3651e77f665fbc 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1343,7 +1343,6 @@ code_dealloc(PyCodeObject *co) Py_XDECREF(co->co_exceptiontable); if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); - } if (co->co_warmup == 0) { _Py_QuickenedCount--; } @@ -1481,6 +1480,7 @@ static PyMemberDef code_memberlist[] = { {"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY}, {"co_stacksize", T_INT, OFF(co_stacksize), READONLY}, {"co_flags", T_INT, OFF(co_flags), READONLY}, + {"co_nlocals", T_INT, OFF(co_nlocals), READONLY}, {"co_consts", T_OBJECT, OFF(co_consts), READONLY}, {"co_names", T_OBJECT, OFF(co_names), READONLY}, {"co_filename", T_OBJECT, OFF(co_filename), READONLY}, @@ -1491,7 +1491,6 @@ static PyMemberDef code_memberlist[] = { {"co_endlinetable", T_OBJECT, OFF(co_endlinetable), READONLY}, {"co_columntable", T_OBJECT, OFF(co_columntable), READONLY}, {"co_exceptiontable", T_OBJECT, OFF(co_exceptiontable), READONLY}, - {"co_nlocals", T_INT, OFF(co_nlocals), READONLY}, {NULL} /* Sentinel */ }; From ecfb193e11b6fb8408cc7ed132c2984aac8810b2 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 14 Mar 2022 12:40:46 -0700 Subject: [PATCH 18/34] Miscellaneous cleanup --- Objects/frameobject.c | 40 +++++++++++++++++++++++-------------- Objects/typeobject.c | 2 ++ Python/ceval.c | 3 ++- Tools/scripts/deepfreeze.py | 5 +++-- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index a541c551264f4a..0f7225b568b446 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -105,9 +105,9 @@ frame_getback(PyFrameObject *f, void *closure) return res; } -/* Given the index of the effective opcode, - scan back to construct the oparg with EXTENDED_ARG */ -// XXX This is broken! +// Given the index of the effective opcode, scan back to construct the oparg +// with EXTENDED_ARG. This only works correctly with *unquickened* code, +// obtained via a call to _PyCode_GetCode! static unsigned int get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i) { @@ -171,18 +171,17 @@ top_of_stack(int64_t stack) static int64_t * mark_stacks(PyCodeObject *code_obj, int len) { - // XXX: this is one big TODO!!! - PyObject *xxx = _PyCode_GetCode(code_obj); - if (xxx == NULL) { + PyObject *co_code = _PyCode_GetCode(code_obj); + if (co_code == NULL) { return NULL; } - _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(xxx); + _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code); int64_t *stacks = PyMem_New(int64_t, len+1); int i, j, opcode; if (stacks == NULL) { PyErr_NoMemory(); - Py_DECREF(xxx); + Py_DECREF(co_code); return NULL; } for (int i = 1; i <= len; i++) { @@ -310,7 +309,7 @@ mark_stacks(PyCodeObject *code_obj, int len) } } } - Py_DECREF(xxx); + Py_DECREF(co_code); return stacks; } @@ -845,15 +844,23 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, static int _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) { - // XXX: Does this handle EXTENDED_ARGs/CACHEs? + // This only works when opcode is a non-quickened form: + assert(_PyOpcode_Deopt[opcode] == opcode); + int check_oparg = 0; for (int i = 0; i < frame->f_lasti; i++) { _Py_CODEUNIT instruction = _PyCode_CODE(frame->f_code)[i]; - int deopt = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; - instruction = _Py_MAKECODEUNIT(deopt, oparg); - if (instruction == _Py_MAKECODEUNIT(opcode, oparg)) { + int check_opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; + check_oparg |= _Py_OPARG(instruction); + if (check_opcode == opcode && check_oparg == oparg) { return 1; } - i += _PyOpcode_Caches[deopt]; + if (check_opcode == EXTENDED_ARG) { + check_oparg <<= 8; + } + else { + check_oparg = 0; + } + i += _PyOpcode_Caches[check_opcode]; } return 0; } @@ -872,7 +879,10 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) { } co = frame->f_code; fast = _PyFrame_GetLocalsArray(frame); - if (frame->f_lasti < 0 && _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS) { + // COPY_FREE_VARS has no quickened forms, so no need to use _PyOpcode_Deopt + // here: + if (frame->f_lasti < 0 && _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS) + { /* Free vars have not been initialized -- Do that */ PyCodeObject *co = frame->f_code; PyObject *closure = frame->f_func->func_closure; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index fd2f80d7ac3abc..4bed3ef49289ae 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8949,6 +8949,8 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, // "firstarg" is a cell here unless (very unlikely) super() // was called from the C-API before the first MAKE_CELL op. if (cframe->f_lasti >= 0) { + // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need + // to use _PyOpcode_Deopt here: assert(_Py_OPCODE(_PyCode_CODE(co)[0]) == MAKE_CELL || _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS); assert(PyCell_Check(firstarg)); diff --git a/Python/ceval.c b/Python/ceval.c index 160549f3a3a1ab..02e4340f0bdbff 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -6706,7 +6706,6 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, then call the trace function if we're tracing source lines. */ initialize_trace_info(&tstate->trace_info, frame); - // XXX: Is this broken? int prev = _PyCode_CODE(frame->f_code)[instr_prev]; int lastline; if (_PyOpcode_Deopt[_Py_OPCODE(prev)] == RESUME && _Py_OPARG(prev) == 0) { @@ -6724,6 +6723,8 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, /* Trace backward edges (except in 'yield from') or if line number has changed */ int trace = line != lastline || (frame->f_lasti < instr_prev && + // SEND has no quickened forms, so no need to use _PyOpcode_Deopt + // here: _Py_OPCODE(_PyCode_CODE(frame->f_code)[frame->f_lasti]) != SEND); if (trace) { result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 5c57bf969ba079..1dfc0ea7062adf 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -304,8 +304,9 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_endlinetable = {co_endlinetable},") self.write(f".co_columntable = {co_columntable},") self.write(f"._co_code = {make_string_literal(code.co_code)},") - self.deallocs.append(f"_PyStaticCode_Dealloc((PyCodeObject *)&{name});") - self.interns.append(f"_PyStaticCode_InternStrings((PyCodeObject *)&{name})") + cast = f"(PyCodeObject *)&{name}" + self.deallocs.append(f"_PyStaticCode_Dealloc({cast});") + self.interns.append(f"_PyStaticCode_InternStrings({cast})") return f"& {name}.ob_base.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: From 824b2daef449e235d274c1ede12c4c71f4352e31 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 14 Mar 2022 12:58:24 -0700 Subject: [PATCH 19/34] Remove outdated comment --- Python/specialize.c | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index 4a20770bfb4329..c3d36f942bd24b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -15,31 +15,6 @@ * ./adaptive.md */ - -/* We layout the quickened data as a bi-directional array: - * Instructions upwards, cache entries downwards. - * first_instr is aligned to a SpecializedCacheEntry. - * The nth instruction is located at first_instr[n] - * The nth cache is located at ((SpecializedCacheEntry *)first_instr)[-1-n] - * The first (index 0) cache entry is reserved for the count, to enable finding - * the first instruction from the base pointer. - * The cache_count argument must include space for the count. - * We use the SpecializedCacheOrInstruction union to refer to the data - * to avoid type punning. - - Layout of quickened data, each line 8 bytes for M cache entries and N instructions: - - <---- co->co_quickened - - - ... - - <--- co->co_first_instr - - ... - -*/ - /* Map from opcode to adaptive opcode. Values of zero are ignored. */ static uint8_t adaptive_opcodes[256] = { From 8164f413028a499de352b9edc4094d242c9111d7 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 14 Mar 2022 12:58:41 -0700 Subject: [PATCH 20/34] Properly skip over EXTENDED_ARG instructions --- Python/ceval.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 02e4340f0bdbff..ca81b719d47418 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1565,10 +1565,17 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject } static int -skip_backwards_over_extended_args(PyCodeObject *code, int offset) +skip_backwards_over_extended_args(PyCodeObject *code, int offset, int oparg) { - // XXX: I think this is broken? - while (offset > 0 && _Py_OPCODE(_PyCode_CODE(code)[offset]) == EXTENDED_ARG) { + // You typically cannot scan backwards like this over quickened code, since + // inline cache entries might *appear* to be valid instructions. However, + // our check for oparg makes this particular case safe: the instruction at + // the current offset can *only* be an EXTENDED_ARG iff oparg is still + // nonzero. Also, EXTENDED_ARG has no quickened forms, so no need to use + // _PyOpcode_Deopt here: + while (oparg && _Py_OPCODE(_PyCode_CODE(code)[offset]) == EXTENDED_ARG) { + assert(0 < offset); + oparg >>= 8; offset--; } return offset; @@ -5409,6 +5416,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(EXTENDED_ARG) { int oldoparg = oparg; + assert(oldoparg); NEXTOPARG(); oparg |= oldoparg << 8; PRE_DISPATCH_GOTO(); @@ -5424,7 +5432,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #else case DO_TRACING: { #endif - int instr_prev = skip_backwards_over_extended_args(frame->f_code, frame->f_lasti); + int instr_prev = skip_backwards_over_extended_args( + frame->f_code, frame->f_lasti, oparg); frame->f_lasti = INSTR_OFFSET(); TRACING_NEXTOPARG(); if (opcode == RESUME) { From 932a3f27b5957202eb1900c939e0223719f671ea Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 14 Mar 2022 13:36:24 -0700 Subject: [PATCH 21/34] Make sure that f_lasti is always valid --- Python/ceval.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index ca81b719d47418..d1fa6a8a7e1673 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1669,7 +1669,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int first_instr = _PyCode_CODE(co); \ } \ assert(frame->f_lasti >= -1); \ - next_instr = first_instr + frame->f_lasti + 1; \ + /* Jump back to the last instruction executed... */ \ + next_instr = first_instr + frame->f_lasti; \ + /* ...past any inline cache entries... */ \ + next_instr += _PyOpcode_Caches[_PyOpcode_Deopt[_Py_OPCODE(*next_instr)]]; \ + /* ...and onto the true next instruction: */ \ + next_instr++; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ /* Set stackdepth to -1. \ Update when returning or calling trace function. \ @@ -2221,7 +2226,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int new_frame->localsplus[i] = NULL; } _PyFrame_SetStackPointer(frame, stack_pointer); - frame->f_lasti += INLINE_CACHE_ENTRIES_BINARY_SUBSCR; new_frame->previous = frame; frame = cframe.current_frame = new_frame; CALL_STAT_INC(inlined_py_calls); @@ -4631,7 +4635,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); - frame->f_lasti += INLINE_CACHE_ENTRIES_CALL; new_frame->previous = frame; cframe.current_frame = frame = new_frame; CALL_STAT_INC(inlined_py_calls); @@ -4736,7 +4739,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } STACK_SHRINK(2-is_meth); _PyFrame_SetStackPointer(frame, stack_pointer); - frame->f_lasti += INLINE_CACHE_ENTRIES_CALL; new_frame->previous = frame; frame = cframe.current_frame = new_frame; goto start_frame; @@ -4776,7 +4778,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } STACK_SHRINK(2-is_meth); _PyFrame_SetStackPointer(frame, stack_pointer); - frame->f_lasti += INLINE_CACHE_ENTRIES_CALL; new_frame->previous = frame; frame = cframe.current_frame = new_frame; goto start_frame; From c0c54981fed604b54c2fc1e907b2b1fac6509d22 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 14 Mar 2022 13:36:44 -0700 Subject: [PATCH 22/34] Add some comments --- Objects/frameobject.c | 1 + Objects/genobject.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 0f7225b568b446..ee60ea91bc08a5 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -892,6 +892,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) { Py_INCREF(o); frame->localsplus[offset + i] = o; } + // COPY_FREE_VARS doesn't have inline CACHEs, either: frame->f_lasti = 0; } for (int i = 0; i < co->co_nlocalsplus; i++) { diff --git a/Objects/genobject.c b/Objects/genobject.c index ef980dedb23abb..2e6eeb287052cb 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -357,7 +357,7 @@ _PyGen_yf(PyGenObject *gen) assert(_Py_OPCODE(_PyCode_CODE(gen->gi_code)[0]) != SEND); return NULL; } - + // XXX: Bad use of f_lasti? if (_Py_OPCODE(_PyCode_CODE(gen->gi_code)[frame->f_lasti - 1]) != SEND || frame->stacktop < 0) { return NULL; @@ -485,6 +485,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, /* Termination repetition of SEND loop */ assert(frame->f_lasti >= 0); /* Backup to SEND */ + // XXX: Bad use of f_lasti? frame->f_lasti--; _Py_CODEUNIT instruction = _PyCode_CODE(gen->gi_code)[frame->f_lasti]; assert(_Py_OPCODE(instruction) == SEND); From e7464a37b197fb96884ccbf9f355272d6f341a51 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 14 Mar 2022 14:08:33 -0700 Subject: [PATCH 23/34] Check opargs during size calculations --- Python/ceval.c | 2 +- Python/compile.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index d2775d9f89246f..4bc4e8d5a26949 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5416,8 +5416,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } TARGET(EXTENDED_ARG) { + assert(oparg); int oldoparg = oparg; - assert(oldoparg); NEXTOPARG(); oparg |= oldoparg << 8; PRE_DISPATCH_GOTO(); diff --git a/Python/compile.c b/Python/compile.c index 542770e8e186ba..7bd9a078184f88 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -134,7 +134,7 @@ static int instr_size(struct instr *instruction) { int opcode = instruction->i_opcode; - int oparg = instruction->i_oparg; + int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg); int caches = _PyOpcode_Caches[opcode]; return extended_args + 1 + caches; From 4f51fdd22726cbaf45ea32b567a52becc23037fd Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 14 Mar 2022 16:33:01 -0700 Subject: [PATCH 24/34] Add another TODO --- Python/ceval.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/ceval.c b/Python/ceval.c index 4bc4e8d5a26949..d4a3bc2b6fdb27 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -6716,6 +6716,7 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, then call the trace function if we're tracing source lines. */ initialize_trace_info(&tstate->trace_info, frame); + // XXX! int prev = _PyCode_CODE(frame->f_code)[instr_prev]; int lastline; if (_PyOpcode_Deopt[_Py_OPCODE(prev)] == RESUME && _Py_OPARG(prev) == 0) { From 75bd37546db01be600b115df93c67dec5b6eb285 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 15 Mar 2022 10:50:30 -0700 Subject: [PATCH 25/34] Clean up formatting --- Lib/opcode.py | 170 +++++++++++++++++++++++++------------------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index e71fbc3273ca30..7a52c13579af7c 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -230,106 +230,106 @@ def jabs_op(name, op, entries=0): ] _specializations = { - 'BINARY_OP': [ - 'BINARY_OP_ADAPTIVE', - 'BINARY_OP_ADD_FLOAT', - 'BINARY_OP_ADD_INT', - 'BINARY_OP_ADD_UNICODE', - 'BINARY_OP_INPLACE_ADD_UNICODE', - 'BINARY_OP_MULTIPLY_FLOAT', - 'BINARY_OP_MULTIPLY_INT', - 'BINARY_OP_SUBTRACT_FLOAT', - 'BINARY_OP_SUBTRACT_INT', + "BINARY_OP": [ + "BINARY_OP_ADAPTIVE", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_ADD_INT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_INPLACE_ADD_UNICODE", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_SUBTRACT_INT", ], - 'BINARY_SUBSCR': [ - 'BINARY_SUBSCR_ADAPTIVE', - 'BINARY_SUBSCR_DICT', - 'BINARY_SUBSCR_GETITEM', - 'BINARY_SUBSCR_LIST_INT', - 'BINARY_SUBSCR_TUPLE_INT', + "BINARY_SUBSCR": [ + "BINARY_SUBSCR_ADAPTIVE", + "BINARY_SUBSCR_DICT", + "BINARY_SUBSCR_GETITEM", + "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_TUPLE_INT", ], - 'CALL': [ - 'CALL_ADAPTIVE', - 'CALL_PY_EXACT_ARGS', - 'CALL_PY_WITH_DEFAULTS', + "CALL": [ + "CALL_ADAPTIVE", + "CALL_PY_EXACT_ARGS", + "CALL_PY_WITH_DEFAULTS", ], - 'COMPARE_OP': [ - 'COMPARE_OP_ADAPTIVE', - 'COMPARE_OP_FLOAT_JUMP', - 'COMPARE_OP_INT_JUMP', - 'COMPARE_OP_STR_JUMP', + "COMPARE_OP": [ + "COMPARE_OP_ADAPTIVE", + "COMPARE_OP_FLOAT_JUMP", + "COMPARE_OP_INT_JUMP", + "COMPARE_OP_STR_JUMP", ], - 'JUMP_ABSOLUTE': [ - 'JUMP_ABSOLUTE_QUICK', + "JUMP_ABSOLUTE": [ + "JUMP_ABSOLUTE_QUICK", ], - 'LOAD_ATTR': [ - 'LOAD_ATTR_ADAPTIVE', - 'LOAD_ATTR_INSTANCE_VALUE', - 'LOAD_ATTR_MODULE', - 'LOAD_ATTR_SLOT', - 'LOAD_ATTR_WITH_HINT', + "LOAD_ATTR": [ + "LOAD_ATTR_ADAPTIVE", + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_WITH_HINT", ], - 'LOAD_CONST': [ - 'LOAD_CONST__LOAD_FAST', + "LOAD_CONST": [ + "LOAD_CONST__LOAD_FAST", ], - 'LOAD_FAST': [ - 'LOAD_FAST__LOAD_CONST', - 'LOAD_FAST__LOAD_FAST', + "LOAD_FAST": [ + "LOAD_FAST__LOAD_CONST", + "LOAD_FAST__LOAD_FAST", ], - 'LOAD_GLOBAL': [ - 'LOAD_GLOBAL_ADAPTIVE', - 'LOAD_GLOBAL_BUILTIN', - 'LOAD_GLOBAL_MODULE', + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_ADAPTIVE", + "LOAD_GLOBAL_BUILTIN", + "LOAD_GLOBAL_MODULE", ], - 'LOAD_METHOD': [ - 'LOAD_METHOD_ADAPTIVE', - 'LOAD_METHOD_CLASS', - 'LOAD_METHOD_MODULE', - 'LOAD_METHOD_NO_DICT', - 'LOAD_METHOD_WITH_DICT', - 'LOAD_METHOD_WITH_VALUES', + "LOAD_METHOD": [ + "LOAD_METHOD_ADAPTIVE", + "LOAD_METHOD_CLASS", + "LOAD_METHOD_MODULE", + "LOAD_METHOD_NO_DICT", + "LOAD_METHOD_WITH_DICT", + "LOAD_METHOD_WITH_VALUES", ], - 'PRECALL': [ - 'PRECALL_ADAPTIVE', - 'PRECALL_BOUND_METHOD', - 'PRECALL_BUILTIN_CLASS', - 'PRECALL_BUILTIN_FAST_WITH_KEYWORDS', - 'PRECALL_NO_KW_BUILTIN_FAST', - 'PRECALL_NO_KW_BUILTIN_O', - 'PRECALL_NO_KW_ISINSTANCE', - 'PRECALL_NO_KW_LEN', - 'PRECALL_NO_KW_LIST_APPEND', - 'PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST', - 'PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS', - 'PRECALL_NO_KW_METHOD_DESCRIPTOR_O', - 'PRECALL_NO_KW_STR_1', - 'PRECALL_NO_KW_TUPLE_1', - 'PRECALL_NO_KW_TYPE_1', - 'PRECALL_PYFUNC', + "PRECALL": [ + "PRECALL_ADAPTIVE", + "PRECALL_BOUND_METHOD", + "PRECALL_BUILTIN_CLASS", + "PRECALL_BUILTIN_FAST_WITH_KEYWORDS", + "PRECALL_NO_KW_BUILTIN_FAST", + "PRECALL_NO_KW_BUILTIN_O", + "PRECALL_NO_KW_ISINSTANCE", + "PRECALL_NO_KW_LEN", + "PRECALL_NO_KW_LIST_APPEND", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_O", + "PRECALL_NO_KW_STR_1", + "PRECALL_NO_KW_TUPLE_1", + "PRECALL_NO_KW_TYPE_1", + "PRECALL_PYFUNC", ], - 'RESUME': [ - 'RESUME_QUICK', + "RESUME": [ + "RESUME_QUICK", ], - 'STORE_ATTR': [ - 'STORE_ATTR_ADAPTIVE', - 'STORE_ATTR_INSTANCE_VALUE', - 'STORE_ATTR_SLOT', - 'STORE_ATTR_WITH_HINT', + "STORE_ATTR": [ + "STORE_ATTR_ADAPTIVE", + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", ], - 'STORE_FAST': [ - 'STORE_FAST__LOAD_FAST', - 'STORE_FAST__STORE_FAST', + "STORE_FAST": [ + "STORE_FAST__LOAD_FAST", + "STORE_FAST__STORE_FAST", ], - 'STORE_SUBSCR': [ - 'STORE_SUBSCR_ADAPTIVE', - 'STORE_SUBSCR_DICT', - 'STORE_SUBSCR_LIST_INT', + "STORE_SUBSCR": [ + "STORE_SUBSCR_ADAPTIVE", + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", ], - 'UNPACK_SEQUENCE': [ - 'UNPACK_SEQUENCE_ADAPTIVE', - 'UNPACK_SEQUENCE_LIST', - 'UNPACK_SEQUENCE_TUPLE', - 'UNPACK_SEQUENCE_TWO_TUPLE', + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_ADAPTIVE", + "UNPACK_SEQUENCE_LIST", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_TWO_TUPLE", ], } _specialized_instructions = [ From d6d51289a68dcc076f93800b9d895177aaee7991 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 15 Mar 2022 10:50:59 -0700 Subject: [PATCH 26/34] Fix compiler warning --- Objects/frameobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ee60ea91bc08a5..5d1a50c7c2a7be 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -499,7 +499,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore /* PyCode_NewWithPosOnlyArgs limits co_code to be under INT_MAX so this * should never overflow. */ - int len = Py_SIZE(f->f_frame->f_code); + int len = (int)Py_SIZE(f->f_frame->f_code); int *lines = marklines(f->f_frame->f_code, len); if (lines == NULL) { return -1; From 82145c18b5078d6b10fa4697c91757e90df86416 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 15 Mar 2022 10:51:28 -0700 Subject: [PATCH 27/34] Simplify calculation of instr_prev --- Python/ceval.c | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index d4a3bc2b6fdb27..f5c92fce95ba8f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1564,23 +1564,6 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject return 0; } -static int -skip_backwards_over_extended_args(PyCodeObject *code, int offset, int oparg) -{ - // You typically cannot scan backwards like this over quickened code, since - // inline cache entries might *appear* to be valid instructions. However, - // our check for oparg makes this particular case safe: the instruction at - // the current offset can *only* be an EXTENDED_ARG iff oparg is still - // nonzero. Also, EXTENDED_ARG has no quickened forms, so no need to use - // _PyOpcode_Deopt here: - while (oparg && _Py_OPCODE(_PyCode_CODE(code)[offset]) == EXTENDED_ARG) { - assert(0 < offset); - oparg >>= 8; - offset--; - } - return offset; -} - static _PyInterpreterFrame * pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) { @@ -5433,8 +5416,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #else case DO_TRACING: { #endif - int instr_prev = skip_backwards_over_extended_args( - frame->f_code, frame->f_lasti, oparg); + int instr_prev = frame->f_lasti; frame->f_lasti = INSTR_OFFSET(); TRACING_NEXTOPARG(); if (opcode == RESUME) { From ca176ac9c8b155276a7936a211abacae20a13bf4 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 15 Mar 2022 10:57:32 -0700 Subject: [PATCH 28/34] _Py_Quicken -> _PyCode_Quicken --- Include/internal/pycore_code.h | 4 ++-- Python/specialize.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index ce6611a5a30c78..82dc9e4bdc62f8 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -97,7 +97,7 @@ typedef struct { /* We want to compare to zero for efficiency, so we offset values accordingly */ #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) -void _Py_Quicken(PyCodeObject *code); +void _PyCode_Quicken(PyCodeObject *code); static inline void _PyCode_Warmup(PyCodeObject *code) @@ -105,7 +105,7 @@ _PyCode_Warmup(PyCodeObject *code) if (code->co_warmup != 0) { code->co_warmup++; if (code->co_warmup == 0) { - _Py_Quicken(code); + _PyCode_Quicken(code); } } } diff --git a/Python/specialize.c b/Python/specialize.c index c3d36f942bd24b..ce091a2c5f0784 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -252,7 +252,7 @@ _Py_PrintSpecializationStats(int to_file) // Insert adaptive instructions and superinstructions. This cannot fail. void -_Py_Quicken(PyCodeObject *code) +_PyCode_Quicken(PyCodeObject *code) { _Py_QuickenedCount++; int previous_opcode = -1; From 1e06bb5e80092d3f72449aa768b4767d715f73f9 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 15 Mar 2022 11:11:16 -0700 Subject: [PATCH 29/34] Revert expensive f_lasti changes --- Python/ceval.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index f5c92fce95ba8f..6ba6bbd731d48d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1653,11 +1653,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } \ assert(frame->f_lasti >= -1); \ /* Jump back to the last instruction executed... */ \ - next_instr = first_instr + frame->f_lasti; \ - /* ...past any inline cache entries... */ \ - next_instr += _PyOpcode_Caches[_PyOpcode_Deopt[_Py_OPCODE(*next_instr)]]; \ - /* ...and onto the true next instruction: */ \ - next_instr++; \ + next_instr = first_instr + frame->f_lasti + 1; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ /* Set stackdepth to -1. \ Update when returning or calling trace function. \ @@ -2209,6 +2205,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int new_frame->localsplus[i] = NULL; } _PyFrame_SetStackPointer(frame, stack_pointer); + frame->f_lasti += INLINE_CACHE_ENTRIES_BINARY_SUBSCR; new_frame->previous = frame; frame = cframe.current_frame = new_frame; CALL_STAT_INC(inlined_py_calls); @@ -4618,6 +4615,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); + frame->f_lasti += INLINE_CACHE_ENTRIES_CALL; new_frame->previous = frame; cframe.current_frame = frame = new_frame; CALL_STAT_INC(inlined_py_calls); @@ -4722,6 +4720,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } STACK_SHRINK(2-is_meth); _PyFrame_SetStackPointer(frame, stack_pointer); + frame->f_lasti += INLINE_CACHE_ENTRIES_CALL; new_frame->previous = frame; frame = cframe.current_frame = new_frame; goto start_frame; @@ -4761,6 +4760,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } STACK_SHRINK(2-is_meth); _PyFrame_SetStackPointer(frame, stack_pointer); + frame->f_lasti += INLINE_CACHE_ENTRIES_CALL; new_frame->previous = frame; frame = cframe.current_frame = new_frame; goto start_frame; From e70819f8caaa20cf4eed033b2ea4dd80a50ad384 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 15 Mar 2022 17:16:41 -0700 Subject: [PATCH 30/34] Naming is hard --- Include/cpython/code.h | 4 ++-- Objects/codeobject.c | 18 +++++++++--------- Tools/scripts/deepfreeze.py | 11 ++++++----- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index b538ac04e42498..d9b7aefefb5938 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -93,7 +93,7 @@ struct PyCodeObject { Type is a void* to keep the format private in codeobject.c to force people to go through the proper APIs. */ void *co_extra; - char _co_code[1]; + char co_code_adaptive[1]; }; /* Masks for co_flags above */ @@ -135,7 +135,7 @@ PyAPI_DATA(PyTypeObject) PyCode_Type; #define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type) #define PyCode_GetNumFree(op) ((op)->co_nfreevars) -#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->_co_code) +#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive) #define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) /* Public interface */ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0b6e70d471e469..224493edb19ea3 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1521,9 +1521,9 @@ code_getfreevars(PyCodeObject *code, void *closure) } static PyObject * -code_getundercode(PyCodeObject *code, void *closure) +code_getcodeadaptive(PyCodeObject *code, void *closure) { - return PyMemoryView_FromMemory(code->_co_code, _PyCode_NBYTES(code), + return PyMemoryView_FromMemory(code->co_code_adaptive, _PyCode_NBYTES(code), PyBUF_READ); } @@ -1534,13 +1534,13 @@ code_getcode(PyCodeObject *code, void *closure) } static PyGetSetDef code_getsetlist[] = { - {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, - {"_co_code", (getter)code_getundercode, NULL, NULL}, + {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, + {"_co_code_adaptive", (getter)code_getcodeadaptive, NULL, NULL}, // The following old names are kept for backward compatibility. - {"co_varnames", (getter)code_getvarnames, NULL, NULL}, - {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, - {"co_freevars", (getter)code_getfreevars, NULL, NULL}, - {"co_code", (getter)code_getcode, NULL, NULL}, + {"co_varnames", (getter)code_getvarnames, NULL, NULL}, + {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, + {"co_freevars", (getter)code_getfreevars, NULL, NULL}, + {"co_code", (getter)code_getcode, NULL, NULL}, {0} }; @@ -1729,7 +1729,7 @@ static struct PyMethodDef code_methods[] = { PyTypeObject PyCode_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "code", - offsetof(PyCodeObject, _co_code), + offsetof(PyCodeObject, co_code_adaptive), sizeof(_Py_CODEUNIT), (destructor)code_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 1dfc0ea7062adf..593fe1b8f3534b 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -245,6 +245,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: # Derived values nlocals, nplaincellvars, ncellvars, nfreevars = \ get_localsplus_counts(code, localsplusnames, localspluskinds) + co_code_adaptive = make_string_literal(code.co_code) self.write("static") with self.indent(): with self.block("struct"): @@ -274,7 +275,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write("PyObject *co_columntable;") self.write("PyObject *co_weakreflist;") self.write("void *co_extra;") - self.write(f"char _co_code[{len(code.co_code)}];") + self.write(f"char co_code_adaptive[{len(code.co_code)}];") with self.block(f"{name} =", ";"): self.object_var_head("PyCode_Type", len(code.co_code) // 2) # But the ordering here must match that in cpython/code.h @@ -303,10 +304,10 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_linetable = {co_linetable},") self.write(f".co_endlinetable = {co_endlinetable},") self.write(f".co_columntable = {co_columntable},") - self.write(f"._co_code = {make_string_literal(code.co_code)},") - cast = f"(PyCodeObject *)&{name}" - self.deallocs.append(f"_PyStaticCode_Dealloc({cast});") - self.interns.append(f"_PyStaticCode_InternStrings({cast})") + self.write(f".co_code_adaptive = {co_code_adaptive},") + name_as_code = f"(PyCodeObject *)&{name}" + self.deallocs.append(f"_PyStaticCode_Dealloc({name_as_code});") + self.interns.append(f"_PyStaticCode_InternStrings({name_as_code})") return f"& {name}.ob_base.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: From 6b96204ee39b1789c714f4d438612e5aba2ab472 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 16 Mar 2022 11:01:09 -0700 Subject: [PATCH 31/34] make patchcheck --- Tools/scripts/deepfreeze.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 593fe1b8f3534b..f03b9a47f6970a 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -474,13 +474,13 @@ def generate(args: list[str], output: TextIO) -> None: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"): - for p in printer.deallocs: - printer.write(p) + for p in printer.deallocs: + printer.write(p) with printer.block(f"int\n_Py_Deepfreeze_Init(void)"): - for p in printer.interns: - with printer.block(f"if ({p} < 0)"): - printer.write("return -1;") - printer.write("return 0;") + for p in printer.interns: + with printer.block(f"if ({p} < 0)"): + printer.write("return -1;") + printer.write("return 0;") if verbose: print(f"Cache hits: {printer.hits}, misses: {printer.misses}") From 3087025c936d2ce13572d9b6d6bb7b5bba8b2eda Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 16 Mar 2022 11:06:01 -0700 Subject: [PATCH 32/34] blurb add --- .../Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst new file mode 100644 index 00000000000000..99fad382d13bb8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst @@ -0,0 +1,2 @@ +Quicken bytecode in-place by storing it as part of the corresponding +``PyCodeObject``. From 6f3bc38364762c80d5a1931b7ccc5a213e7ea90d Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 16 Mar 2022 16:52:34 -0700 Subject: [PATCH 33/34] Reuse the PyCodeObject definition for deepfreeze --- Include/cpython/code.h | 139 +++++++++++++++++++----------------- Tools/scripts/deepfreeze.py | 29 +------- 2 files changed, 73 insertions(+), 95 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index d9b7aefefb5938..157678317931ec 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -26,75 +26,80 @@ typedef uint16_t _Py_CODEUNIT; // Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing: #define _Py_SET_OPCODE(word, opcode) (((unsigned char *)&(word))[0] = (opcode)) +// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are +// defined in this macro: +#define _PyCode_DEF(SIZE) { \ + PyObject_VAR_HEAD \ + \ + /* Note only the following fields are used in hash and/or comparisons \ + * \ + * - co_name \ + * - co_argcount \ + * - co_posonlyargcount \ + * - co_kwonlyargcount \ + * - co_nlocals \ + * - co_stacksize \ + * - co_flags \ + * - co_firstlineno \ + * - co_consts \ + * - co_names \ + * - co_localsplusnames \ + * This is done to preserve the name and line number for tracebacks \ + * and debuggers; otherwise, constant de-duplication would collapse \ + * identical functions/lambdas defined on different lines. \ + */ \ + \ + /* These fields are set with provided values on new code objects. */ \ + \ + /* The hottest fields (in the eval loop) are grouped here at the top. */ \ + PyObject *co_consts; /* list (constants used) */ \ + PyObject *co_names; /* list of strings (names used) */ \ + PyObject *co_exceptiontable; /* Byte string encoding exception handling \ + table */ \ + int co_flags; /* CO_..., see below */ \ + int co_warmup; /* Warmup counter for quickening */ \ + \ + /* The rest are not so impactful on performance. */ \ + int co_argcount; /* #arguments, except *args */ \ + int co_posonlyargcount; /* #positional only arguments */ \ + int co_kwonlyargcount; /* #keyword only arguments */ \ + int co_stacksize; /* #entries needed for evaluation stack */ \ + int co_firstlineno; /* first source line number */ \ + \ + /* redundant values (derived from co_localsplusnames and \ + co_localspluskinds) */ \ + int co_nlocalsplus; /* number of local + cell + free variables \ + */ \ + int co_nlocals; /* number of local variables */ \ + int co_nplaincellvars; /* number of non-arg cell variables */ \ + int co_ncellvars; /* total number of cell variables */ \ + int co_nfreevars; /* number of free variables */ \ + \ + PyObject *co_localsplusnames; /* tuple mapping offsets to names */ \ + PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte \ + per variable) */ \ + PyObject *co_filename; /* unicode (where it was loaded from) */ \ + PyObject *co_name; /* unicode (name, for reference) */ \ + PyObject *co_qualname; /* unicode (qualname, for reference) */ \ + PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) \ + See Objects/lnotab_notes.txt for details. \ + */ \ + PyObject *co_endlinetable; /* bytes object that holds end lineno for \ + instructions separated across different \ + lines */ \ + PyObject *co_columntable; /* bytes object that holds start/end column \ + offset each instruction */ \ + \ + PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ + /* Scratch space for extra data relating to the code object. \ + Type is a void* to keep the format private in codeobject.c to force \ + people to go through the proper APIs. */ \ + void *co_extra; \ + char co_code_adaptive[(SIZE)]; \ +} /* Bytecode object */ -struct PyCodeObject { - PyObject_VAR_HEAD - - /* Note only the following fields are used in hash and/or comparisons - * - * - co_name - * - co_argcount - * - co_posonlyargcount - * - co_kwonlyargcount - * - co_nlocals - * - co_stacksize - * - co_flags - * - co_firstlineno - * - co_consts - * - co_names - * - co_localsplusnames - * - * This is done to preserve the name and line number for tracebacks - * and debuggers; otherwise, constant de-duplication would collapse - * identical functions/lambdas defined on different lines. - */ - - /* These fields are set with provided values on new code objects. */ - - // The hottest fields (in the eval loop) are grouped here at the top. - PyObject *co_consts; /* list (constants used) */ - PyObject *co_names; /* list of strings (names used) */ - PyObject *co_exceptiontable; /* Byte string encoding exception handling table */ - int co_flags; /* CO_..., see below */ - int co_warmup; /* Warmup counter for quickening */ - - // The rest are not so impactful on performance. - int co_argcount; /* #arguments, except *args */ - int co_posonlyargcount; /* #positional only arguments */ - int co_kwonlyargcount; /* #keyword only arguments */ - int co_stacksize; /* #entries needed for evaluation stack */ - int co_firstlineno; /* first source line number */ - - // redundant values (derived from co_localsplusnames and co_localspluskinds) - int co_nlocalsplus; /* number of local + cell + free variables */ - int co_nlocals; /* number of local variables */ - int co_nplaincellvars; /* number of non-arg cell variables */ - int co_ncellvars; /* total number of cell variables */ - int co_nfreevars; /* number of free variables */ - - PyObject *co_localsplusnames; /* tuple mapping offsets to names */ - PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */ - PyObject *co_filename; /* unicode (where it was loaded from) */ - PyObject *co_name; /* unicode (name, for reference) */ - PyObject *co_qualname; /* unicode (qualname, for reference) */ - PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) See - Objects/lnotab_notes.txt for details. */ - PyObject *co_endlinetable; /* bytes object that holds end lineno for - instructions separated across different - lines */ - PyObject *co_columntable; /* bytes object that holds start/end column - offset each instruction */ - - /* The remaining fields are zeroed out on new code objects. */ - - PyObject *co_weakreflist; /* to support weakrefs to code objects */ - /* Scratch space for extra data relating to the code object. - Type is a void* to keep the format private in codeobject.c to force - people to go through the proper APIs. */ - void *co_extra; - char co_code_adaptive[1]; -}; +struct PyCodeObject _PyCode_DEF(1); /* Masks for co_flags above */ #define CO_OPTIMIZED 0x0001 diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index f03b9a47f6970a..d208258dbc54c4 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -248,34 +248,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: co_code_adaptive = make_string_literal(code.co_code) self.write("static") with self.indent(): - with self.block("struct"): - self.write("PyObject_VAR_HEAD") - self.write("PyObject *co_consts;") - self.write("PyObject *co_names;") - self.write("PyObject *co_exceptiontable;") - self.write("int co_flags;") - self.write("int co_warmup;") - self.write("int co_argcount;") - self.write("int co_posonlyargcount;") - self.write("int co_kwonlyargcount;") - self.write("int co_stacksize;") - self.write("int co_firstlineno;") - self.write("int co_nlocalsplus;") - self.write("int co_nlocals;") - self.write("int co_nplaincellvars;") - self.write("int co_ncellvars;") - self.write("int co_nfreevars;") - self.write("PyObject *co_localsplusnames;") - self.write("PyObject *co_localspluskinds;") - self.write("PyObject *co_filename;") - self.write("PyObject *co_name;") - self.write("PyObject *co_qualname;") - self.write("PyObject *co_linetable;") - self.write("PyObject *co_endlinetable;") - self.write("PyObject *co_columntable;") - self.write("PyObject *co_weakreflist;") - self.write("void *co_extra;") - self.write(f"char co_code_adaptive[{len(code.co_code)}];") + self.write(f"struct _PyCode_DEF({len(code.co_code)})") with self.block(f"{name} =", ";"): self.object_var_head("PyCode_Type", len(code.co_code) // 2) # But the ordering here must match that in cpython/code.h From c8054b9fec736611b68b3703924110dc1c7dd91f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 18 Mar 2022 13:07:40 -0700 Subject: [PATCH 34/34] Clean up TODO --- Objects/genobject.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index acb8c07d2bc859..3ad8dc1c459420 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -483,15 +483,15 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, ret = _PyFrame_StackPop((_PyInterpreterFrame *)gen->gi_iframe); assert(ret == yf); Py_DECREF(ret); + // XXX: Performing this jump ourselves is awkward and problematic. + // See https://github.com/python/cpython/pull/31968. /* Termination repetition of SEND loop */ assert(frame->f_lasti >= 0); + _Py_CODEUNIT *code = _PyCode_CODE(gen->gi_code); /* Backup to SEND */ - // XXX: Bad use of f_lasti? frame->f_lasti--; - _Py_CODEUNIT instruction = _PyCode_CODE(gen->gi_code)[frame->f_lasti]; - assert(_Py_OPCODE(instruction) == SEND); - // XXX: This doesn't seem to handle EXTENDED_ARGs: - int jump = _Py_OPARG(instruction); + assert(_Py_OPCODE(code[frame->f_lasti]) == SEND); + int jump = _Py_OPARG(code[frame->f_lasti]); frame->f_lasti += jump; if (_PyGen_FetchStopIterationValue(&val) == 0) { ret = gen_send(gen, val);