fixup

Python/ceval.c

@@ -866,13 +866,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #endif
 
 #ifdef Py_TAIL_CALL_INTERP
-    return _TAIL_CALL_start_frame(frame, stack_pointer, tstate, next_instr, 0, 0);
+    return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0, 0);
+#   include "generated_tail_call_labels.c.h"
 #else
     goto start_frame;
+#   include "generated_cases.c.h"
 #endif
 
-#include "generated_cases.c.h"
-
 #ifdef _Py_TIER2
 
 // Tier 2 is also here!

Python/generated_tail_call_labels.c.h

@@ -0,0 +1,107 @@
+// This file is generated by Tools/cases_generator/tier1_tail_call_generator.py
+// from:
+//   Python/bytecodes.c
+// Do not edit!
+
+#ifndef Py_TAIL_CALL_INTERP
+    #error "This file is for tail-calling interpreter only."
+#endif
+#define TIER_ONE 1
+
+        error:
+        {
+            /* Double-check exception status. */
+            #ifdef NDEBUG
+            if (!_PyErr_Occurred(tstate)) {
+                _PyErr_SetString(tstate, PyExc_SystemError,
+                             "error return without exception set");
+            }
+            #else
+            assert(_PyErr_Occurred(tstate));
+            #endif
+
+            /* Log traceback info. */
+            assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
+            if (!_PyFrame_IsIncomplete(frame)) {
+                PyFrameObject *f = _PyFrame_GetFrameObject(frame);
+                if (f != NULL) {
+                    PyTraceBack_Here(f);
+                }
+            }
+            _PyEval_MonitorRaise(tstate, frame, next_instr-1);
+            goto exception_unwind;
+        }
+
+        exception_unwind:
+        {
+            /* We can't use frame->instr_ptr here, as RERAISE may have set it */
+            int offset = INSTR_OFFSET()-1;
+            int level, handler, lasti;
+            if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) {
+                // No handlers, so exit.
+                assert(_PyErr_Occurred(tstate));
+                /* Pop remaining stack entries. */
+                _PyStackRef *stackbase = _PyFrame_Stackbase(frame);
+                while (stack_pointer > stackbase) {
+                    PyStackRef_XCLOSE(POP());
+                }
+                assert(STACK_LEVEL() == 0);
+                _PyFrame_SetStackPointer(frame, stack_pointer);
+                monitor_unwind(tstate, frame, next_instr-1);
+                goto exit_unwind;
+            }
+            assert(STACK_LEVEL() >= level);
+            _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level;
+            while (stack_pointer > new_top) {
+                PyStackRef_XCLOSE(POP());
+            }
+            if (lasti) {
+                int frame_lasti = _PyInterpreterFrame_LASTI(frame);
+                PyObject *lasti = PyLong_FromLong(frame_lasti);
+                if (lasti == NULL) {
+                    goto exception_unwind;
+                }
+                PUSH(PyStackRef_FromPyObjectSteal(lasti));
+            }
+            /* Make the raw exception data
+               available to the handler,
+               so a program can emulate the
+               Python main loop. */
+            PyObject *exc = _PyErr_GetRaisedException(tstate);
+            PUSH(PyStackRef_FromPyObjectSteal(exc));
+            next_instr = _PyFrame_GetBytecode(frame) + handler;
+            if (monitor_handled(tstate, frame, next_instr, exc) < 0) {
+                goto exception_unwind;
+            }
+            /* Resume normal execution */
+            #ifdef LLTRACE
+            if (frame->lltrace >= 5) {
+                lltrace_resume_frame(frame);
+            }
+            #endif
+            _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0);
+            ;
+        }
+
+        exit_unwind:
+        {
+            assert(_PyErr_Occurred(tstate));
+            _Py_LeaveRecursiveCallPy(tstate);
+            assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
+            // GH-99729: We need to unlink the frame *before* clearing it:
+            _PyInterpreterFrame *dying = frame;
+            frame = tstate->current_frame = dying->previous;
+            _PyEval_FrameClearAndPop(tstate, dying);
+            frame->return_offset = 0;
+            if (frame->owner == FRAME_OWNED_BY_INTERPRETER) {
+                /* Restore previous frame and exit */
+                tstate->current_frame = frame->previous;
+                tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;
+                return NULL;
+            }
+            next_instr = frame->instr_ptr;
+            stack_pointer = _PyFrame_GetStackPointer(frame);
+            goto error;
+        }
+
+#undef TIER_ONE

Tools/cases_generator/tier1_generator.py

@@ -15,6 +15,7 @@
     Flush,
     analysis_error,
     StackItem,
+    Label,
 )
 from generators_common import (
     DEFAULT_INPUT,
@@ -221,21 +222,21 @@ def generate_tier1(
 #endif /* Py_TAIL_CALL_INTERP */
         {LABEL_START_MARKER}
 """)
-    generate_tier1_labels(analysis, outfile, lines)
+    out = CWriter(outfile, 2, lines)
+    emitter = Emitter(out)
+    generate_tier1_labels(analysis.labels, emitter)
     outfile.write(f"{LABEL_END_MARKER}\n")
     outfile.write(FOOTER)
 
 def generate_tier1_labels(
-    analysis: Analysis, outfile: TextIO, lines: bool
+    labels: dict[str, Label], emitter: Emitter
 ) -> None:
-    out = CWriter(outfile, 2, lines)
-    out.emit("\n")
-    for name, label in analysis.labels.items():
-        out.emit(f"{name}:\n")
-        for tkn in label.body:
-            out.emit(tkn)
-        out.emit("\n")
-        out.emit("\n")
+    emitter.emit("\n")
+    for name, label in labels.items():
+        emitter.emit(f"{name}:\n")
+        emitter.emit_label(label)
+        emitter.emit("\n")
+        emitter.emit("\n")
 
 def generate_tier1_cases(
     analysis: Analysis, outfile: TextIO, lines: bool

Tools/cases_generator/tier1_tail_call_generator.py

@@ -26,7 +26,8 @@
 )
 
 from tier1_generator import (
-    write_single_inst
+    write_single_inst,
+    generate_tier1_labels,
 )
 
 from lexer import Token
@@ -35,9 +36,22 @@
 
 DEFAULT_INPUT = ROOT / "Python/bytecodes.c"
 DEFAULT_OUTPUT = ROOT / "Python/generated_tail_call_handlers.c.h"
+DEFAULT_LABELS_OUTPUT = ROOT / "Python/generated_tail_call_labels.c.h"
 
+PRELUDE = """
+#ifndef Py_TAIL_CALL_INTERP
+    #error "This file is for tail-calling interpreter only."
+#endif
+#define TIER_ONE 1
+"""
 FOOTER = "#undef TIER_ONE\n"
 
+NEEDED_LABELS = {
+    "error",
+    "exception_unwind",
+    "exit_unwind",
+}
+
 class TailCallEmitter(Emitter):
 
     def __init__(self, out: CWriter, analysis: Analysis):
@@ -110,11 +124,32 @@ def goto(
         name = next(tkn_iter)
         next(tkn_iter)
         assert name.kind == "IDENTIFIER"
-        self.out.emit("\n")
+        self.out.start_line()
         self.emit(f"TAIL_CALL({name.text});\n")
         return True
 
 
+class TailCallCevalLabelsEmitter(Emitter):
+    def __init__(self, out: CWriter):
+        super().__init__(out)
+        self._replacers = {
+            'DISPATCH': self.dispatch,
+        }
+
+    def dispatch(
+        self,
+        tkn: Token,
+        tkn_iter: TokenIterator,
+        uop: Uop | Label,
+        storage: Storage,
+        inst: Instruction | None,
+    ) -> bool:
+        # Replace DISPATCH with _TAIL_CALL_entry(...)
+        next(tkn_iter)
+        next(tkn_iter)
+        self.emit("_TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0);\n")
+        return True
+
 
 def function_proto(name: str) -> str:
     return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)"
@@ -152,17 +187,17 @@ def uses_this(inst: Instruction) -> bool:
     return False
 
 def generate_tier1(
-    filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool
+    filenames: list[str], analysis: Analysis, outfile: TextIO, labels_outfile: TextIO,  lines: bool
 ) -> None:
+    # Write labels required for main ceval
+    write_header(__file__, filenames, labels_outfile)
+    labels_outfile.write(PRELUDE)
+    out = CWriter(labels_outfile, 2, lines)
+    emitter = TailCallCevalLabelsEmitter(out)
+    generate_tier1_labels({label: analysis.labels[label] for label in NEEDED_LABELS}, emitter)
+    labels_outfile.write(FOOTER)
     write_header(__file__, filenames, outfile)
-    outfile.write(
-        """
-#ifndef Py_TAIL_CALL_INTERP
-    #error "This file is for tail-calling interpreter only."
-#endif
-#define TIER_ONE 1
-"""
-    )
+    outfile.write(PRELUDE)
     out = CWriter(outfile, 0, lines)
     out.emit("static inline PyObject *_TAIL_CALL_entry(TAIL_CALL_PARAMS);\n")
     out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n");
@@ -232,6 +267,10 @@ def generate_tier1(
     "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT
 )
 
+arg_parser.add_argument(
+    "-lo", "--labels-output", type=str, help="Generated labels", default=DEFAULT_LABELS_OUTPUT
+)
+
 arg_parser.add_argument(
     "-l", "--emit-line-directives", help="Emit #line directives", action="store_true"
 )
@@ -246,5 +285,5 @@ def generate_tier1(
     if len(args.input) == 0:
         args.input.append(DEFAULT_INPUT)
     data = analyze_files(args.input)
-    with open(args.output, "w") as outfile:
-        generate_tier1(args.input, data, outfile, args.emit_line_directives)
+    with open(args.output, "w") as outfile, open(args.labels_output, 'w') as labels_outfile:
+        generate_tier1(args.input, data, outfile, labels_outfile, args.emit_line_directives)