diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index cb9c0aa27a1785..fb5c14db32e633 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -556,7 +556,7 @@ _PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) // Return a pointer to the thread-local bytecode for the current thread, // creating it if necessary. -extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co); +PyAPI_FUNC(_Py_CODEUNIT *) _PyCode_GetTLBC(PyCodeObject *co); // Reserve an index for the current thread into thread-local bytecode // arrays diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index ebc5622912f0cb..3775b074ecf54c 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -67,8 +67,8 @@ _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, extern int _Py_Instrumentation_GetLine(PyCodeObject *code, int index); -extern PyObject _PyInstrumentation_MISSING; -extern PyObject _PyInstrumentation_DISABLE; +PyAPI_DATA(PyObject) _PyInstrumentation_MISSING; +PyAPI_DATA(PyObject) _PyInstrumentation_DISABLE; /* Total tool ids available */ diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 8949d6cc2fc4bb..19dc6df32e7107 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -279,7 +279,7 @@ _PyThreadState_GetFrame(PyThreadState *tstate) /* For use by _PyFrame_GetFrameObject Do not call directly. */ -PyFrameObject * +PyAPI_FUNC(PyFrameObject *) _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); /* Gets the PyFrameObject for this frame, lazily diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index ffbcebdb7dfb50..b39639a9063260 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -17,12 +17,12 @@ PyAPI_FUNC(PyObject) *_PyList_SliceSubscript(PyObject*, PyObject*); extern void _PyList_DebugMallocStats(FILE *out); // _PyList_GetItemRef should be used only when the object is known as a list // because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does. -extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i); +PyAPI_FUNC(PyObject *) _PyList_GetItemRef(PyListObject *, Py_ssize_t i); #ifdef Py_GIL_DISABLED // Returns -1 in case of races with other threads. -extern int _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *); +PyAPI_FUNC(int) _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *); #endif #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 906ea0db0a5fe0..13b19abdeaa7ac 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1075,7 +1075,7 @@ struct opcode_metadata { uint32_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; +PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1516,7 +1516,7 @@ _PyOpcode_macro_expansion[256] = { }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[267]; +PyAPI_DATA(const char) *_PyOpcode_OpName[267]; #ifdef NEED_OPCODE_METADATA const char *_PyOpcode_OpName[267] = { [ANNOTATIONS_PLACEHOLDER] = "ANNOTATIONS_PLACEHOLDER", @@ -1759,7 +1759,7 @@ const char *_PyOpcode_OpName[267] = { }; #endif -extern const uint8_t _PyOpcode_Caches[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { [TO_BOOL] = 3, @@ -1784,7 +1784,7 @@ const uint8_t _PyOpcode_Caches[256] = { }; #endif -extern const uint8_t _PyOpcode_Deopt[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { [121] = 121, diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index f80808fcc8c4d7..1023dbc3395b2f 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -11,8 +11,8 @@ extern "C" { /* Error handling definitions */ -extern _PyErr_StackItem* _PyErr_GetTopmostException(PyThreadState *tstate); -extern PyObject* _PyErr_GetHandledException(PyThreadState *); +PyAPI_FUNC(_PyErr_StackItem*) _PyErr_GetTopmostException(PyThreadState *tstate); +PyAPI_FUNC(PyObject*) _PyErr_GetHandledException(PyThreadState *); extern void _PyErr_SetHandledException(PyThreadState *, PyObject *); extern void _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); @@ -108,7 +108,7 @@ extern void _PyErr_Restore( PyObject *value, PyObject *traceback); -extern void _PyErr_SetObject( +PyAPI_FUNC(void) _PyErr_SetObject( PyThreadState *tstate, PyObject *type, PyObject *value); diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3997acbdf84695..c9c757857a8a5d 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2857,6 +2857,24 @@ def func(): names = ["func", "outer", "outer", "inner", "inner", "outer", "inner"] self.do_test(func, names) + def test_replaced_interpreter(self): + def inner(): + yield 'abc' + def outer(): + yield from inner() + def func(): + list(outer()) + _testinternalcapi.set_eval_frame_interp() + try: + func() + finally: + _testinternalcapi.set_eval_frame_default() + + stats = _testinternalcapi.get_eval_frame_stats() + + self.assertEqual(stats["resumes"], 5) + self.assertEqual(stats["loads"], 5) + @unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED') class TestPyThreadId(unittest.TestCase): diff --git a/Makefile.pre.in b/Makefile.pre.in index a6beb96d12a3f2..52a91ee6bb6abd 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2150,7 +2150,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ regen-generated-cases regen-executor-cases regen-optimizer-cases \ - regen-opcode-metadata regen-uop-metadata + regen-opcode-metadata regen-uop-metadata regen-test-cases regen-test-opcode-targets .PHONY: regen-opcode-ids regen-opcode-ids: @@ -2182,6 +2182,20 @@ regen-generated-cases: -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new +.PHONY: regen-test-cases +regen-test-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_generator.py \ + -o $(srcdir)/Modules/_testinternalcapi/test_cases.c.h.new $(srcdir)/Python/bytecodes.c \ + $(srcdir)/Modules/_testinternalcapi/testbytecodes.c + $(UPDATE_FILE) $(srcdir)/Modules/_testinternalcapi/test_cases.c.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h.new + +.PHONY: regen-test-opcode-targets +regen-test-opcode-targets: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/target_generator.py \ + -o $(srcdir)/Modules/_testinternalcapi/test_targets.h.new $(srcdir)/Python/bytecodes.c \ + $(srcdir)/Modules/_testinternalcapi/testbytecodes.c + $(UPDATE_FILE) $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_targets.h.new + .PHONY: regen-executor-cases regen-executor-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ @@ -3428,7 +3442,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo. MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h -MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index acb08400e24e2e..e6e0e937e6eb70 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -174,7 +174,7 @@ @MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c -@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c +@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index a7fbb0f87b6e9c..3f772799409acf 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -693,6 +693,39 @@ set_eval_frame_record(PyObject *self, PyObject *list) Py_RETURN_NONE; } +// Defined in interpreter.c +extern PyObject* +Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag); +extern int Test_EvalFrame_Resumes, Test_EvalFrame_Loads; + +static PyObject * +get_eval_frame_stats(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *res = PyDict_New(); + if (res == NULL) { + return NULL; + } + PyObject *resumes = PyLong_FromLong(Test_EvalFrame_Resumes); + if (resumes == NULL || PyDict_SetItemString(res, "resumes", resumes) < 0) { + Py_XDECREF(resumes); + return NULL; + } + PyObject *loads = PyLong_FromLong(Test_EvalFrame_Loads); + if (loads == NULL || PyDict_SetItemString(res, "loads", loads) < 0) { + Py_XDECREF(loads); + return NULL; + } + Test_EvalFrame_Resumes = Test_EvalFrame_Loads = 0; + return res; +} + +static PyObject * +set_eval_frame_interp(PyObject *self, PyObject *Py_UNUSED(args)) +{ + _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), Test_EvalFrame); + Py_RETURN_NONE; +} + /*[clinic input] _testinternalcapi.compiler_cleandoc -> object @@ -2507,6 +2540,7 @@ test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args)) static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, + {"get_eval_frame_stats", get_eval_frame_stats, METH_NOARGS, NULL}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS}, {"get_stack_pointer", get_stack_pointer, METH_NOARGS}, @@ -2523,6 +2557,7 @@ static PyMethodDef module_functions[] = { {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS}, {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS}, {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, + {"set_eval_frame_interp", set_eval_frame_interp, METH_NOARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c new file mode 100644 index 00000000000000..c3191ac20a3199 --- /dev/null +++ b/Modules/_testinternalcapi/interpreter.c @@ -0,0 +1,254 @@ +#ifndef Py_BUILD_CORE +#define Py_BUILD_CORE +#endif +#ifndef Py_BUILD_CORE_MODULE +#define Py_BUILD_CORE_MODULE +#endif + +#include "../../Python/ceval.h" + +#include "../../Python/ceval_macros.h" + +int Test_EvalFrame_Resumes, Test_EvalFrame_Loads; + +#if _Py_TAIL_CALL_INTERP +#include "test_targets.h" +#include "test_cases.c.h" +#endif + +PyObject* _Py_HOT_FUNCTION +Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) +{ + _Py_EnsureTstateNotNULL(tstate); + check_invalid_reentrancy(); + CALL_STAT_INC(pyeval_calls); + +#if USE_COMPUTED_GOTOS && !_Py_TAIL_CALL_INTERP +/* Import the static jump table */ +#include "test_targets.h" + void **opcode_targets = opcode_targets_table; +#endif + +#ifdef Py_STATS + int lastopcode = 0; +#endif +#if !_Py_TAIL_CALL_INTERP + uint8_t opcode; /* Current opcode */ + int oparg; /* Current opcode argument, if any */ + assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); +#if !USE_COMPUTED_GOTOS + uint8_t tracing_mode = 0; + uint8_t dispatch_code; +#endif +#endif + _PyEntryFrame entry; + + if (_Py_EnterRecursiveCallTstate(tstate, "")) { + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyEval_FrameClearAndPop(tstate, frame); + return NULL; + } + + /* Local "register" variables. + * These are cached values from the frame and code object. */ + _Py_CODEUNIT *next_instr; + _PyStackRef *stack_pointer; + entry.stack[0] = PyStackRef_NULL; +#ifdef Py_STACKREF_DEBUG + entry.frame.f_funcobj = PyStackRef_None; +#elif defined(Py_DEBUG) + /* Set these to invalid but identifiable values for debugging. */ + entry.frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; + entry.frame.f_locals = (PyObject*)0xaaa1; + entry.frame.frame_obj = (PyFrameObject*)0xaaa2; + entry.frame.f_globals = (PyObject*)0xaaa3; + entry.frame.f_builtins = (PyObject*)0xaaa4; +#endif + entry.frame.f_executable = PyStackRef_None; + entry.frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR + 1; + entry.frame.stackpointer = entry.stack; + entry.frame.owner = FRAME_OWNED_BY_INTERPRETER; + entry.frame.visited = 0; + entry.frame.return_offset = 0; +#ifdef Py_DEBUG + entry.frame.lltrace = 0; +#endif + /* Push frame */ + entry.frame.previous = tstate->current_frame; + frame->previous = &entry.frame; + tstate->current_frame = frame; + entry.frame.localsplus[0] = PyStackRef_NULL; +#ifdef _Py_TIER2 + if (tstate->current_executor != NULL) { + entry.frame.localsplus[0] = PyStackRef_FromPyObjectNew(tstate->current_executor); + tstate->current_executor = NULL; + } +#endif + + /* support for generator.throw() */ + if (throwflag) { + if (_Py_EnterRecursivePy(tstate)) { + goto early_exit; + } +#ifdef Py_GIL_DISABLED + /* Load thread-local bytecode */ + if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + if (bytecode == NULL) { + goto early_exit; + } + ptrdiff_t off = frame->instr_ptr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + } +#endif + /* Because this avoids the RESUME, we need to update instrumentation */ + _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + next_instr = frame->instr_ptr; + monitor_throw(tstate, frame, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); +#if _Py_TAIL_CALL_INTERP +# if Py_STATS + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); +# else + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0); +# endif +#else + goto error; +#endif + } + + #if defined(_Py_TIER2) && !defined(_Py_JIT) + /* Tier 2 interpreter state */ + _PyExecutorObject *current_executor = NULL; + const _PyUOpInstruction *next_uop = NULL; +#endif +#if _Py_TAIL_CALL_INTERP +# if Py_STATS + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode); +# else + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0); +# endif +#else + goto start_frame; +#include "test_cases.c.h" +#endif + + +#ifdef _Py_TIER2 + +// Tier 2 is also here! +enter_tier_two: + +#ifdef _Py_JIT + assert(0); +#else + +#undef LOAD_IP +#define LOAD_IP(UNUSED) (void)0 + +#ifdef Py_STATS +// Disable these macros that apply to Tier 1 stats when we are in Tier 2 +#undef STAT_INC +#define STAT_INC(opname, name) ((void)0) +#undef STAT_DEC +#define STAT_DEC(opname, name) ((void)0) +#endif + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION 0 +#undef ENABLE_SPECIALIZATION_FT +#define ENABLE_SPECIALIZATION_FT 0 + + ; // dummy statement after a label, before a declaration + uint16_t uopcode; +#ifdef Py_STATS + int lastuop = 0; + uint64_t trace_uop_execution_counter = 0; +#endif + + assert(next_uop->opcode == _START_EXECUTOR); +tier2_dispatch: + for (;;) { + uopcode = next_uop->opcode; +#ifdef Py_DEBUG + if (frame->lltrace >= 3) { + dump_stack(frame, stack_pointer); + if (next_uop->opcode == _START_EXECUTOR) { + printf("%4d uop: ", 0); + } + else { + printf("%4d uop: ", (int)(next_uop - current_executor->trace)); + } + _PyUOpPrint(next_uop); + printf("\n"); + } +#endif + next_uop++; + OPT_STAT_INC(uops_executed); + UOP_STAT_INC(uopcode, execution_count); + UOP_PAIR_INC(uopcode, lastuop); +#ifdef Py_STATS + trace_uop_execution_counter++; + ((_PyUOpInstruction *)next_uop)[-1].execution_count++; +#endif + + switch (uopcode) { + +#include "executor_cases.c.h" + + default: +#ifdef Py_DEBUG + { + printf("Unknown uop: "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d\n", (int)(next_uop - current_executor->trace - 1)); + Py_FatalError("Unknown uop"); + } +#else + Py_UNREACHABLE(); +#endif + + } + } + +jump_to_error_target: +#ifdef Py_DEBUG + if (frame->lltrace >= 2) { + printf("Error: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d -> %s]\n", + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif + assert(next_uop[-1].format == UOP_FORMAT_JUMP); + uint16_t target = uop_get_error_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +jump_to_jump_target: + assert(next_uop[-1].format == UOP_FORMAT_JUMP); + target = uop_get_jump_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +#endif // _Py_JIT + +#endif // _Py_TIER2 + +early_exit: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + return NULL; +} diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h new file mode 100644 index 00000000000000..133dcb2062d5a7 --- /dev/null +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -0,0 +1,12250 @@ +// This file is generated by Tools/cases_generator/tier1_generator.py +// from: +// Python/bytecodes.c, Modules/_testinternalcapi/testbytecodes.c +// Do not edit! + +#ifdef TIER_TWO + #error "This file is for Tier 1 only" +#endif +#define TIER_ONE 1 + +#if !_Py_TAIL_CALL_INTERP +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (dispatch_code) +#endif + { +#endif /* _Py_TAIL_CALL_INTERP */ + /* BEGIN INSTRUCTIONS */ + + + TARGET(BINARY_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED_BINARY_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_OPARG_LAST); + } + /* Skip 4 cache entries */ + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = lhs; + lhs = res; + stack_pointer[-2] = lhs; + PyStackRef_CLOSE(tmp); + tmp = rhs; + rhs = PyStackRef_NULL; + stack_pointer[-1] = rhs; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + } + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + JUMP_TO_LABEL(error); + } + l = left; + r = right; + } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_UNICODE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + res = PyStackRef_FromPyObjectSteal(res_o); + if (PyStackRef_IsNull(res)) { + JUMP_TO_LABEL(error); + } + l = left; + r = right; + } + // _POP_TOP_UNICODE + { + value = r; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + // _POP_TOP_UNICODE + { + value = l; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_EXTEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_INPLACE_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = (int)CURRENT_OPERAND0_16(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyUnicode_Append(&temp, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*target_local)) { + JUMP_TO_LABEL(error); + } + #if TIER_ONE + + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + } + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + JUMP_TO_LABEL(error); + } + l = left; + r = right; + } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_DICT + { + sub_st = stack_pointer[-1]; + dict_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (rc <= 0) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_GETITEM) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_GETITEM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef new_frame; + /* Skip 5 cache entries */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _BINARY_OP_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_OP, hit); + } + // _BINARY_OP_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_LIST_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = list_st; + list_st = res; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + } + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_SLICE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_SLICE + { + sub_st = tos; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PySlice_Check(sub)); + assert(PyList_CheckExact(list)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_SliceSubscript(list, sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = list_st; + list_st = PyStackRef_NULL; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_STR_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_STR_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_STR_INT + { + sub_st = value; + str_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyUnicode_IS_COMPACT_ASCII(str)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + uint8_t c = PyUnicode_1BYTE_DATA(str)[index]; + assert(c < 128); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectBorrow(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_TUPLE_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_TUPLE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_TUPLE_INT + { + sub_st = value; + tuple_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = tuple_st; + tuple_st = res; + stack_pointer[-1] = tuple_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + JUMP_TO_LABEL(error); + } + l = left; + r = right; + } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_SLICE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(container); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_INTERPOLATION) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_INTERPOLATION; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_INTERPOLATION); + _PyStackRef value; + _PyStackRef str; + _PyStackRef *format; + _PyStackRef interpolation; + format = &stack_pointer[-(oparg & 1)]; + str = stack_pointer[-1 - (oparg & 1)]; + value = stack_pointer[-2 - (oparg & 1)]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); + int conversion = oparg >> 2; + PyObject *format_o; + if (oparg & 1) { + format_o = PyStackRef_AsPyObjectBorrow(format[0]); + } + else { + format_o = &_Py_STR(empty); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (oparg & 1) { + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(format[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer += -(oparg & 1); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (interpolation_o == NULL) { + JUMP_TO_LABEL(error); + } + interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); + stack_pointer[0] = interpolation; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_LIST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (list_o == NULL) { + JUMP_TO_LABEL(error); + } + list = PyStackRef_FromPyObjectStealMortal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_MAP) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_MAP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (map_o == NULL) { + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + map = PyStackRef_FromPyObjectStealMortal(map_o); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_SET) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SET; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + int err = 0; + for (Py_ssize_t i = 0; i < oparg; i++) { + _PyStackRef value = values[i]; + values[i] = PyStackRef_NULL; + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + if (err) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + set = PyStackRef_FromPyObjectStealMortal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SLICE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + _PyStackRef *args; + _PyStackRef slice; + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (slice_o == NULL) { + JUMP_TO_LABEL(error); + } + slice = PyStackRef_FromPyObjectStealMortal(slice_o); + stack_pointer[0] = slice; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_STRING) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_STRING; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (str_o == NULL) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_TEMPLATE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TEMPLATE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TEMPLATE); + _PyStackRef strings; + _PyStackRef interpolations; + _PyStackRef template; + interpolations = stack_pointer[-1]; + strings = stack_pointer[-2]; + PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); + PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(interpolations); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(strings); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (template_o == NULL) { + JUMP_TO_LABEL(error); + } + template = PyStackRef_FromPyObjectSteal(template_o); + stack_pointer[0] = template; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TUPLE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + JUMP_TO_LABEL(error); + } + tup = PyStackRef_FromPyObjectStealMortal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CACHE) { + #if _Py_TAIL_CALL_INTERP + int opcode = CACHE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + DISPATCH(); + } + + TARGET(CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_ALLOC_AND_ENTER_INIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ALLOC_AND_ENTER_INIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef init; + _PyStackRef self; + _PyStackRef *args; + _PyStackRef init_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_AND_ALLOCATE_OBJECT + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + JUMP_TO_LABEL(error); + } + self_or_null = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = self_or_null; + init = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + shim->localsplus[0] = PyStackRef_DUP(self); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init, NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); + } + // _PUSH_FRAME + { + new_frame = init_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_BOUND_METHOD_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_BUILTIN_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_BUILTIN_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_BUILTIN_O) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + _PyStackRef a; + _PyStackRef c; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + a = arg; + c = callable; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = c; + stack_pointer[-2 - oparg] = res; + stack_pointer[-1 - oparg] = a; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = a; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + opcode = CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_INTRINSIC_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CALL_INTRINSIC_2) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_2; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; + PyStackRef_CLOSE(tmp); + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CALL_ISINSTANCE) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ISINSTANCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef instance; + _PyStackRef cls; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_THIRD_NULL + { + null = stack_pointer[-3]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_ISINSTANCE + { + callable = stack_pointer[-4]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_ISINSTANCE + { + cls = stack_pointer[-1]; + instance = stack_pointer[-2]; + STAT_INC(CALL, hit); + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(inst_o, cls_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + JUMP_TO_LABEL(error); + } + (void)null; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(cls); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(instance); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CALL_KW) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_KW_NON_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_NON_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + opcode = CALL_KW_NON_PY; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_KW_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_LEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef c; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_LEN + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_LEN + { + arg = stack_pointer[-1]; + STAT_INC(CALL, hit); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + JUMP_TO_LABEL(error); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + a = arg; + c = callable; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = c; + stack_pointer[-3] = res; + stack_pointer[-2] = a; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = a; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(CALL_LIST_APPEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef nos; + _PyStackRef self; + _PyStackRef arg; + _PyStackRef c; + _PyStackRef s; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_LIST_APPEND + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_NOS_LIST + { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_LIST_APPEND + { + arg = stack_pointer[-1]; + self = nos; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + if (err) { + JUMP_TO_LABEL(error); + } + c = callable; + s = self; + #if TIER_ONE + + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + } + // _POP_TOP + { + value = s; + stack_pointer[-3] = c; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = c; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_O) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_NON_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_NON_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + opcode = CALL_NON_PY_GENERAL; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_STR_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_STR_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_STR_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + a = arg; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = a; + stack_pointer[-3] = res; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_TUPLE_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TUPLE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_TUPLE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + a = arg; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = a; + stack_pointer[-3] = res; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_TYPE_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TYPE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_TYPE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_TYPE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + (void)callable; + (void)null; + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + stack_pointer[-3] = res; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(CHECK_EG_MATCH) { + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EG_MATCH; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res < 0) { + JUMP_TO_LABEL(error); + } + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) { + JUMP_TO_LABEL(error); + } + if (!Py_IsNone(match_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CHECK_EXC_MATCH) { + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EXC_MATCH; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CLEANUP_THROW) { + #if _Py_TAIL_CALL_INTERP + int opcode = CLEANUP_THROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + _PyStackRef sub_iter; + _PyStackRef last_sent_val; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #if !_Py_TAIL_CALL_INTERP + assert(throwflag); + #endif + assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_iter; + sub_iter = value; + stack_pointer[-3] = sub_iter; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-1] = exc_value_st; + PyStackRef_CLOSE(tmp); + tmp = last_sent_val; + last_sent_val = PyStackRef_NULL; + stack_pointer[-2] = last_sent_val; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + none = PyStackRef_None; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + stack_pointer[0] = none; + stack_pointer[1] = value; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COMPARE_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED_COMPARE_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + if (oparg & 16) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) { + JUMP_TO_LABEL(error); + } + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COMPARE_OP_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COMPARE_OP_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int sign_ish = COMPARISON_BIT(ileft, iright); + l = left; + r = right; + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COMPARE_OP_STR) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CONTAINS_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED_CONTAINS_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CONTAINS_OP_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_DICT + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_DICT + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CONTAINS_OP_SET) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_ANY_SET + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_SET + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CONVERT_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONVERT_VALUE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[0] = result; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COPY) { + #if _Py_TAIL_CALL_INTERP + int opcode = COPY; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COPY_FREE_VARS) { + #if _Py_TAIL_CALL_INTERP + int opcode = COPY_FREE_VARS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + DISPATCH(); + } + + TARGET(DELETE_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(DELETE_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(DELETE_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(DELETE_GLOBAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_GLOBAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(DELETE_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_NAME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(DELETE_SUBSCR) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_SUBSCR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(DICT_MERGE) { + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_MERGE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(DICT_UPDATE) { + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_UPDATE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(END_ASYNC_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + DISPATCH(); + } + + TARGET(END_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_FOR; + (void)(opcode); + #endif + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(END_SEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_SEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(ENTER_EXECUTOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = ENTER_EXECUTOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + opcode = ENTER_EXECUTOR; + #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + next_instr = this_instr; + JUMP_TO_LABEL(stop_tracing); + } + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->current_executor == NULL); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; + TIER1_TO_TIER2(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + } + + TARGET(EXIT_INIT_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = EXIT_INIT_CHECK; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(EXTENDED_ARG) { + #if _Py_TAIL_CALL_INTERP + int opcode = EXTENDED_ARG; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + opcode = EXTENDED_ARG; + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TARGET(FORMAT_SIMPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_SIMPLE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + else { + res = value; + stack_pointer += -1; + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FORMAT_WITH_SPEC) { + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_WITH_SPEC; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED_FOR_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _FOR_ITER + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + next = item; + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(FOR_ITER_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif + } + // _ITER_JUMP_LIST + { + #ifdef Py_GIL_DISABLED + + #else + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + #endif + } + // _ITER_NEXT_LIST + { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); + #ifdef Py_GIL_DISABLED + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); + STAT_INC(FOR_ITER, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result < 0) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + if (result == 0) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + #else + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); + #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER_RANGE) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_RANGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-2]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + JUMP_TO_LABEL(error); + } + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + } + // _ITER_JUMP_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_AITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_ANEXT) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ANEXT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + JUMP_TO_LABEL(error); + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_AWAITABLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AWAITABLE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef index_or_null; + iterable = stack_pointer[-1]; + #ifdef Py_STATS + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_GatherStats_GetIter(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_LEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_LEN; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + JUMP_TO_LABEL(error); + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + JUMP_TO_LABEL(error); + } + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_YIELD_FROM_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_YIELD_FROM_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + iter = iterable; + } + else if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(IMPORT_FROM) { + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_FROM; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(IMPORT_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_NAME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + opcode = INSTRUMENTED_CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef *args; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = self_or_null; + func = callable; + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL_KW) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_KW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + opcode = INSTRUMENTED_CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null); + } + else if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } + PyObject *function = PyStackRef_AsPyObjectBorrow(callable); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_ASYNC_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + // _MONITOR_END_ASYNC_FOR + { + assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); + INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + // _END_ASYNC_FOR + { + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-3]; + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_SEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(INSTRUMENTED_FOR_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_FOR_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_INSTRUCTION) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_INSTRUCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + opcode = INSTRUMENTED_INSTRUCTION; + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_BACKWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_FORWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); + } + + TARGET(INSTRUMENTED_LINE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LINE; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + opcode = INSTRUMENTED_LINE; + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + JUMP_TO_LABEL(error); + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (super == NULL) { + JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_NOT_TAKEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + (void)this_instr; + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = !PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RESUME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + DISPATCH(); + } + } + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + Test_EvalFrame_Resumes++; + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_RETURN_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _RETURN_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_YIELD_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } + } + // _YIELD_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INTERPRETER_EXIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = INTERPRETER_EXIT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(frame)); + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(executor); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + #endif + LLTRACE_RESUME_FRAME(); + return result; + } + + TARGET(IS_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = IS_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(JUMP_BACKWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + /* Skip 1 cache entry */ + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == JUMP_BACKWARD) { + uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); + next_instr = this_instr; + DISPATCH_SAME_OPARG(); + } + #endif + } + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_JIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + // _JIT + { + #ifdef _Py_TIER2 + _Py_BackoffCounter counter = this_instr[1].counter; + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); + } + else { + this_instr[1].counter = restart_backoff_counter(counter); + } + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif + } + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_INTERRUPT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_NO_JIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_JIT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + DISPATCH(); + } + + TARGET(JUMP_FORWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_FORWARD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + DISPATCH(); + } + + TARGET(LIST_APPEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_APPEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LIST_EXTEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_EXTEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + assert(Py_IsNone(none_val)); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(LOAD_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED_LOAD_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *self_or_null; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + self_or_null = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + _PyCStackRef method; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyThreadState_PushCStackRef(tstate, &method); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, &method.ref); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + assert(!PyStackRef_IsNull(method.ref)); + self_or_null[0] = owner; + attr = _PyThreadState_PopCStackRefSteal(tstate, &method); + } + else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null[0] = PyStackRef_NULL; + attr = _PyThreadState_PopCStackRefSteal(tstate, &method); + if (PyStackRef_IsNull(attr)) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1; + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer += 1; + } + } + stack_pointer[-1] = attr; + stack_pointer += (oparg&1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + new_frame->localsplus[0] = owner; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10u ; + DISPATCH_INLINED(new_frame); + } + + TARGET(LOAD_ATTR_INSTANCE_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef value; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + } + // _POP_TOP + { + value = o; + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_LAZY_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_MODULE) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_ATTR_MODULE + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_PROPERTY) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_PROPERTY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(LOAD_ATTR_SLOT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_WITH_HINT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); + assert(PyDict_CheckExact((PyObject *)dict)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #endif + PyObject *attr_o; + if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (dk->dk_kind != DICT_KEYS_UNICODE) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; + if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); + if (attr_o == NULL) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_BUILD_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_BUILD_CLASS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_COMMON_CONSTANT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_COMMON_CONSTANT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_CONST) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_CONST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + _PyStackRef value; + Test_EvalFrame_Loads++; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = _PyCell_GetStackRef(cell); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(value)) { + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_AND_CLEAR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_AND_CLEAR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + _PyStackRef value; + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_BORROW) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); + value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_CHECK; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_LOAD_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_LOAD_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FROM_DICT_OR_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_GLOBALS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(mod_or_class_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + } + } + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED_LOAD_GLOBAL:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef *res; + _PyStackRef *null; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) { + JUMP_TO_LABEL(error); + } + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_BUILTIN) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_BUILTIN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t version = read_u16(&this_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_MODULE) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _NOP + { + } + // _LOAD_GLOBAL_MODULE + { + uint16_t version = read_u16(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_LOCALS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_LOCALS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_NAME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + JUMP_TO_LABEL(error); + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_SMALL_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SMALL_INT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_SPECIAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SPECIAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + _PyStackRef self; + _PyStackRef *method_and_self; + // _INSERT_NULL + { + self = stack_pointer[-1]; + method_and_self = &stack_pointer[-1]; + method_and_self[1] = self; + method_and_self[0] = PyStackRef_NULL; + } + // _LOAD_SPECIAL + { + method_and_self = &stack_pointer[-1]; + PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err <= 0) { + if (err == 0) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) + ? _Py_SpecialMethods[oparg].error_suggestion + : _Py_SpecialMethods[oparg].error; + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); + assert(errfmt != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED_LOAD_SUPER_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (super == NULL) { + JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (attr == NULL) { + JUMP_TO_LABEL(error); + } + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[0] = attr_st; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(MAKE_CELL) { + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_CELL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(MAKE_FUNCTION) { + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_FUNCTION; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(codeobj_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + JUMP_TO_LABEL(error); + } + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[0] = func; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(MAP_ADD) { + #if _Py_TAIL_CALL_INTERP + int opcode = MAP_ADD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(pop_2_error); + } + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(MATCH_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_CLASS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + JUMP_TO_LABEL(error); + } + attrs = PyStackRef_None; + } + stack_pointer[0] = attrs; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(MATCH_KEYS) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_KEYS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) { + JUMP_TO_LABEL(error); + } + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(MATCH_MAPPING) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_MAPPING; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(MATCH_SEQUENCE) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_SEQUENCE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(NOP) { + #if _Py_TAIL_CALL_INTERP + int opcode = NOP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + DISPATCH(); + } + + TARGET(NOT_TAKEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = NOT_TAKEN; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + DISPATCH(); + } + + TARGET(POP_EXCEPT) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_EXCEPT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_FALSE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NOT_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_TRUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_TOP) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_TOP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(PUSH_EXC_INFO) { + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_EXC_INFO; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(PUSH_NULL) { + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_NULL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(RAISE_VARARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = RAISE_VARARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + JUMP_TO_LABEL(error); + } + + TARGET(RERAISE) { + #if _Py_TAIL_CALL_INTERP + int opcode = RERAISE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); + } + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + + TARGET(RESERVED) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESERVED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + DISPATCH(); + } + + TARGET(RESUME) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED_RESUME:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + DISPATCH(); + } + } + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + Test_EvalFrame_Resumes++; + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + } + DISPATCH(); + } + + TARGET(RESUME_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #endif + DISPATCH(); + } + + TARGET(RETURN_GENERATOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_GENERATOR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(RETURN_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_VALUE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(SEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED_SEND:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + retval = PyStackRef_FromPyObjectSteal(retval_o); + } + stack_pointer[0] = retval; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(SEND_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(SETUP_ANNOTATIONS) { + #if _Py_TAIL_CALL_INTERP + int opcode = SETUP_ANNOTATIONS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(SET_ADD) { + #if _Py_TAIL_CALL_INTERP + int opcode = SET_ADD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(SET_FUNCTION_ATTRIBUTE) { + #if _Py_TAIL_CALL_INTERP + int opcode = SET_FUNCTION_ATTRIBUTE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(SET_UPDATE) { + #if _Py_TAIL_CALL_INTERP + int opcode = SET_UPDATE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(STORE_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED_STORE_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(STORE_ATTR_INSTANCE_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef o; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + } + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + o = owner; + stack_pointer[-2] = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(STORE_ATTR_SLOT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef o; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + o = owner; + stack_pointer[-2] = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(STORE_ATTR_WITH_HINT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef o; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + STAT_INC(STORE_ATTR, hit); + o = owner; + stack_pointer[-2] = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(STORE_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(STORE_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + _PyStackRef value; + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(STORE_FAST_LOAD_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_LOAD_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(STORE_FAST_STORE_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_STORE_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + tmp = GETLOCAL(oparg2); + GETLOCAL(oparg2) = value2; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(STORE_GLOBAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_GLOBAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(STORE_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_NAME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(STORE_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SLICE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-4] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -4; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(STORE_SUBSCR) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED_STORE_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(STORE_SUBSCR_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + _PyStackRef st; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_DICT + { + sub = stack_pointer[-1]; + dict_st = nos; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + st = dict_st; + } + // _POP_TOP + { + value = st; + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(STORE_SUBSCR_LIST_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef ls; + _PyStackRef ss; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + ls = list_st; + ss = sub_st; + stack_pointer[-3] = ls; + stack_pointer[-2] = ss; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP_INT + { + value = ss; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP + { + value = ls; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(SWAP) { + #if _Py_TAIL_CALL_INTERP + int opcode = SWAP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + _PyStackRef bottom; + _PyStackRef top; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-2 - (oparg-2)] = bottom; + stack_pointer[-1] = top; + DISPATCH(); + } + + TARGET(TO_BOOL) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED_TO_BOOL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(TO_BOOL_ALWAYS_TRUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_ALWAYS_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + // _REPLACE_WITH_TRUE + { + value = owner; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(TO_BOOL_BOOL) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_BOOL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + DISPATCH(); + } + + TARGET(TO_BOOL_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_LIST + { + value = tos; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(TO_BOOL_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_STR) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_STR + { + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + assert(Py_SIZE(value_o)); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TRACE_RECORD) { + #if _Py_TAIL_CALL_INTERP + int opcode = TRACE_RECORD; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TRACE_RECORD); + opcode = TRACE_RECORD; + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); + #else + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); + #endif + } + + TARGET(UNARY_INVERT) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_INVERT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(UNARY_NEGATIVE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NEGATIVE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(UNARY_NOT) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NOT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNPACK_EX) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_EX; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + _PyStackRef seq; + _PyStackRef *top; + seq = stack_pointer[-1]; + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED_UNPACK_SEQUENCE:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *top; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + JUMP_TO_LABEL(error); + } + } + stack_pointer += oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_LIST + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyList_CheckExact(seq_o)); + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + stack_pointer += -1 + oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TUPLE + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + stack_pointer += -1 + oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TWO_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TWO_TUPLE + { + seq = tos; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + DISPATCH(); + } + + TARGET(WITH_EXCEPT_START) { + #if _Py_TAIL_CALL_INTERP + int opcode = WITH_EXCEPT_START; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + PyObject *original_tb = tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + assert(PyStackRef_IsTaggedInt(lasti)); + (void)lasti; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + Py_XDECREF(original_tb); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(YIELD_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = YIELD_VALUE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + /* END INSTRUCTIONS */ +#if !_Py_TAIL_CALL_INTERP +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); + + + } + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); +#endif /* _Py_TAIL_CALL_INTERP */ + /* BEGIN LABELS */ + + LABEL(pop_2_error) + { + stack_pointer -= 2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + + LABEL(pop_1_error) + { + stack_pointer -= 1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + + LABEL(error) + { + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (f != NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyTraceBack_Here(f); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + JUMP_TO_LABEL(exception_unwind); + } + + LABEL(exception_unwind) + { + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); + if (handled == 0) { + assert(_PyErr_Occurred(tstate)); + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (frame->stackpointer > stackbase) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); + } + monitor_unwind(tstate, frame, next_instr-1); + JUMP_TO_LABEL(exit_unwind); + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + assert(frame->stackpointer >= new_top); + while (frame->stackpointer > new_top) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + _PyStackRef lasti = PyStackRef_TagInt(frame_lasti); + _PyFrame_StackPush(frame, lasti); + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + int err = monitor_handled(tstate, frame, next_instr, exc); + if (err < 0) { + JUMP_TO_LABEL(exception_unwind); + } + #ifdef Py_DEBUG + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + + LABEL(exit_unwind) + { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + tstate->current_frame = frame->previous; + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + PyStackRef_CLOSE(executor); + } + #endif + return NULL; + } + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + + LABEL(start_frame) + { + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + JUMP_TO_LABEL(exit_unwind); + } + next_instr = frame->instr_ptr; + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; + assert(!_PyErr_Occurred(tstate)); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + + LABEL(stop_tracing) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); + stack_pointer = _PyFrame_GetStackPointer(frame); + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + +/* END LABELS */ +#undef TIER_ONE diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h new file mode 100644 index 00000000000000..b2fa7d01e8f6c2 --- /dev/null +++ b/Modules/_testinternalcapi/test_targets.h @@ -0,0 +1,1286 @@ +#if !_Py_TAIL_CALL_INTERP +static void *opcode_targets_table[256] = { + &&TARGET_CACHE, + &&TARGET_BINARY_SLICE, + &&TARGET_BUILD_TEMPLATE, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, + &&TARGET_CALL_FUNCTION_EX, + &&TARGET_CHECK_EG_MATCH, + &&TARGET_CHECK_EXC_MATCH, + &&TARGET_CLEANUP_THROW, + &&TARGET_DELETE_SUBSCR, + &&TARGET_END_FOR, + &&TARGET_END_SEND, + &&TARGET_EXIT_INIT_CHECK, + &&TARGET_FORMAT_SIMPLE, + &&TARGET_FORMAT_WITH_SPEC, + &&TARGET_GET_AITER, + &&TARGET_GET_ANEXT, + &&TARGET_GET_ITER, + &&TARGET_RESERVED, + &&TARGET_GET_LEN, + &&TARGET_GET_YIELD_FROM_ITER, + &&TARGET_INTERPRETER_EXIT, + &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_LOCALS, + &&TARGET_MAKE_FUNCTION, + &&TARGET_MATCH_KEYS, + &&TARGET_MATCH_MAPPING, + &&TARGET_MATCH_SEQUENCE, + &&TARGET_NOP, + &&TARGET_NOT_TAKEN, + &&TARGET_POP_EXCEPT, + &&TARGET_POP_ITER, + &&TARGET_POP_TOP, + &&TARGET_PUSH_EXC_INFO, + &&TARGET_PUSH_NULL, + &&TARGET_RETURN_GENERATOR, + &&TARGET_RETURN_VALUE, + &&TARGET_SETUP_ANNOTATIONS, + &&TARGET_STORE_SLICE, + &&TARGET_STORE_SUBSCR, + &&TARGET_TO_BOOL, + &&TARGET_UNARY_INVERT, + &&TARGET_UNARY_NEGATIVE, + &&TARGET_UNARY_NOT, + &&TARGET_WITH_EXCEPT_START, + &&TARGET_BINARY_OP, + &&TARGET_BUILD_INTERPOLATION, + &&TARGET_BUILD_LIST, + &&TARGET_BUILD_MAP, + &&TARGET_BUILD_SET, + &&TARGET_BUILD_SLICE, + &&TARGET_BUILD_STRING, + &&TARGET_BUILD_TUPLE, + &&TARGET_CALL, + &&TARGET_CALL_INTRINSIC_1, + &&TARGET_CALL_INTRINSIC_2, + &&TARGET_CALL_KW, + &&TARGET_COMPARE_OP, + &&TARGET_CONTAINS_OP, + &&TARGET_CONVERT_VALUE, + &&TARGET_COPY, + &&TARGET_COPY_FREE_VARS, + &&TARGET_DELETE_ATTR, + &&TARGET_DELETE_DEREF, + &&TARGET_DELETE_FAST, + &&TARGET_DELETE_GLOBAL, + &&TARGET_DELETE_NAME, + &&TARGET_DICT_MERGE, + &&TARGET_DICT_UPDATE, + &&TARGET_END_ASYNC_FOR, + &&TARGET_EXTENDED_ARG, + &&TARGET_FOR_ITER, + &&TARGET_GET_AWAITABLE, + &&TARGET_IMPORT_FROM, + &&TARGET_IMPORT_NAME, + &&TARGET_IS_OP, + &&TARGET_JUMP_BACKWARD, + &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, + &&TARGET_JUMP_FORWARD, + &&TARGET_LIST_APPEND, + &&TARGET_LIST_EXTEND, + &&TARGET_LOAD_ATTR, + &&TARGET_LOAD_COMMON_CONSTANT, + &&TARGET_LOAD_CONST, + &&TARGET_LOAD_DEREF, + &&TARGET_LOAD_FAST, + &&TARGET_LOAD_FAST_AND_CLEAR, + &&TARGET_LOAD_FAST_BORROW, + &&TARGET_LOAD_FAST_BORROW_LOAD_FAST_BORROW, + &&TARGET_LOAD_FAST_CHECK, + &&TARGET_LOAD_FAST_LOAD_FAST, + &&TARGET_LOAD_FROM_DICT_OR_DEREF, + &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, + &&TARGET_LOAD_GLOBAL, + &&TARGET_LOAD_NAME, + &&TARGET_LOAD_SMALL_INT, + &&TARGET_LOAD_SPECIAL, + &&TARGET_LOAD_SUPER_ATTR, + &&TARGET_MAKE_CELL, + &&TARGET_MAP_ADD, + &&TARGET_MATCH_CLASS, + &&TARGET_POP_JUMP_IF_FALSE, + &&TARGET_POP_JUMP_IF_NONE, + &&TARGET_POP_JUMP_IF_NOT_NONE, + &&TARGET_POP_JUMP_IF_TRUE, + &&TARGET_RAISE_VARARGS, + &&TARGET_RERAISE, + &&TARGET_SEND, + &&TARGET_SET_ADD, + &&TARGET_SET_FUNCTION_ATTRIBUTE, + &&TARGET_SET_UPDATE, + &&TARGET_STORE_ATTR, + &&TARGET_STORE_DEREF, + &&TARGET_STORE_FAST, + &&TARGET_STORE_FAST_LOAD_FAST, + &&TARGET_STORE_FAST_STORE_FAST, + &&TARGET_STORE_GLOBAL, + &&TARGET_STORE_NAME, + &&TARGET_SWAP, + &&TARGET_UNPACK_EX, + &&TARGET_UNPACK_SEQUENCE, + &&TARGET_YIELD_VALUE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_RESUME, + &&TARGET_BINARY_OP_ADD_FLOAT, + &&TARGET_BINARY_OP_ADD_INT, + &&TARGET_BINARY_OP_ADD_UNICODE, + &&TARGET_BINARY_OP_EXTEND, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BINARY_OP_MULTIPLY_INT, + &&TARGET_BINARY_OP_SUBSCR_DICT, + &&TARGET_BINARY_OP_SUBSCR_GETITEM, + &&TARGET_BINARY_OP_SUBSCR_LIST_INT, + &&TARGET_BINARY_OP_SUBSCR_LIST_SLICE, + &&TARGET_BINARY_OP_SUBSCR_STR_INT, + &&TARGET_BINARY_OP_SUBSCR_TUPLE_INT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_BINARY_OP_SUBTRACT_INT, + &&TARGET_CALL_ALLOC_AND_ENTER_INIT, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_GENERAL, + &&TARGET_CALL_BUILTIN_CLASS, + &&TARGET_CALL_BUILTIN_FAST, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_BUILTIN_O, + &&TARGET_CALL_ISINSTANCE, + &&TARGET_CALL_KW_BOUND_METHOD, + &&TARGET_CALL_KW_NON_PY, + &&TARGET_CALL_KW_PY, + &&TARGET_CALL_LEN, + &&TARGET_CALL_LIST_APPEND, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_CALL_METHOD_DESCRIPTOR_O, + &&TARGET_CALL_NON_PY_GENERAL, + &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_GENERAL, + &&TARGET_CALL_STR_1, + &&TARGET_CALL_TUPLE_1, + &&TARGET_CALL_TYPE_1, + &&TARGET_COMPARE_OP_FLOAT, + &&TARGET_COMPARE_OP_INT, + &&TARGET_COMPARE_OP_STR, + &&TARGET_CONTAINS_OP_DICT, + &&TARGET_CONTAINS_OP_SET, + &&TARGET_FOR_ITER_GEN, + &&TARGET_FOR_ITER_LIST, + &&TARGET_FOR_ITER_RANGE, + &&TARGET_FOR_ITER_TUPLE, + &&TARGET_JUMP_BACKWARD_JIT, + &&TARGET_JUMP_BACKWARD_NO_JIT, + &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_SUPER_ATTR_ATTR, + &&TARGET_LOAD_SUPER_ATTR_METHOD, + &&TARGET_RESUME_CHECK, + &&TARGET_SEND_GEN, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_TO_BOOL_ALWAYS_TRUE, + &&TARGET_TO_BOOL_BOOL, + &&TARGET_TO_BOOL_INT, + &&TARGET_TO_BOOL_LIST, + &&TARGET_TO_BOOL_NONE, + &&TARGET_TO_BOOL_STR, + &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_INSTRUMENTED_END_FOR, + &&TARGET_INSTRUMENTED_POP_ITER, + &&TARGET_INSTRUMENTED_END_SEND, + &&TARGET_INSTRUMENTED_FOR_ITER, + &&TARGET_INSTRUMENTED_INSTRUCTION, + &&TARGET_INSTRUMENTED_JUMP_FORWARD, + &&TARGET_INSTRUMENTED_NOT_TAKEN, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_INSTRUMENTED_RESUME, + &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_END_ASYNC_FOR, + &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, + &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_CALL_KW, + &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, + &&TARGET_INSTRUMENTED_JUMP_BACKWARD, + &&TARGET_INSTRUMENTED_LINE, + &&TARGET_ENTER_EXECUTOR, + &&TARGET_TRACE_RECORD, +}; +#if _Py_TIER2 +static void *opcode_tracing_targets_table[256] = { + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, +}; +#endif +#else /* _Py_TAIL_CALL_INTERP */ +static py_tail_call_funcptr instruction_funcptr_handler_table[256]; + +static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); +} + +static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { + [BINARY_OP] = _TAIL_CALL_BINARY_OP, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, + [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_ADD_UNICODE, + [BINARY_OP_EXTEND] = _TAIL_CALL_BINARY_OP_EXTEND, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_BINARY_OP_SUBSCR_DICT, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_BINARY_OP_SUBTRACT_INT, + [BINARY_SLICE] = _TAIL_CALL_BINARY_SLICE, + [BUILD_INTERPOLATION] = _TAIL_CALL_BUILD_INTERPOLATION, + [BUILD_LIST] = _TAIL_CALL_BUILD_LIST, + [BUILD_MAP] = _TAIL_CALL_BUILD_MAP, + [BUILD_SET] = _TAIL_CALL_BUILD_SET, + [BUILD_SLICE] = _TAIL_CALL_BUILD_SLICE, + [BUILD_STRING] = _TAIL_CALL_BUILD_STRING, + [BUILD_TEMPLATE] = _TAIL_CALL_BUILD_TEMPLATE, + [BUILD_TUPLE] = _TAIL_CALL_BUILD_TUPLE, + [CACHE] = _TAIL_CALL_CACHE, + [CALL] = _TAIL_CALL_CALL, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, + [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, + [CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O, + [CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2, + [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, + [CALL_KW] = _TAIL_CALL_CALL_KW, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, + [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, + [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, + [CALL_LEN] = _TAIL_CALL_CALL_LEN, + [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, + [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, + [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, + [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, + [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, + [CHECK_EG_MATCH] = _TAIL_CALL_CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = _TAIL_CALL_CHECK_EXC_MATCH, + [CLEANUP_THROW] = _TAIL_CALL_CLEANUP_THROW, + [COMPARE_OP] = _TAIL_CALL_COMPARE_OP, + [COMPARE_OP_FLOAT] = _TAIL_CALL_COMPARE_OP_FLOAT, + [COMPARE_OP_INT] = _TAIL_CALL_COMPARE_OP_INT, + [COMPARE_OP_STR] = _TAIL_CALL_COMPARE_OP_STR, + [CONTAINS_OP] = _TAIL_CALL_CONTAINS_OP, + [CONTAINS_OP_DICT] = _TAIL_CALL_CONTAINS_OP_DICT, + [CONTAINS_OP_SET] = _TAIL_CALL_CONTAINS_OP_SET, + [CONVERT_VALUE] = _TAIL_CALL_CONVERT_VALUE, + [COPY] = _TAIL_CALL_COPY, + [COPY_FREE_VARS] = _TAIL_CALL_COPY_FREE_VARS, + [DELETE_ATTR] = _TAIL_CALL_DELETE_ATTR, + [DELETE_DEREF] = _TAIL_CALL_DELETE_DEREF, + [DELETE_FAST] = _TAIL_CALL_DELETE_FAST, + [DELETE_GLOBAL] = _TAIL_CALL_DELETE_GLOBAL, + [DELETE_NAME] = _TAIL_CALL_DELETE_NAME, + [DELETE_SUBSCR] = _TAIL_CALL_DELETE_SUBSCR, + [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, + [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, + [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, + [END_FOR] = _TAIL_CALL_END_FOR, + [END_SEND] = _TAIL_CALL_END_SEND, + [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, + [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, + [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, + [FOR_ITER] = _TAIL_CALL_FOR_ITER, + [FOR_ITER_GEN] = _TAIL_CALL_FOR_ITER_GEN, + [FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST, + [FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE, + [FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE, + [GET_AITER] = _TAIL_CALL_GET_AITER, + [GET_ANEXT] = _TAIL_CALL_GET_ANEXT, + [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, + [GET_ITER] = _TAIL_CALL_GET_ITER, + [GET_LEN] = _TAIL_CALL_GET_LEN, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, + [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, + [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, + [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = _TAIL_CALL_INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, + [IS_OP] = _TAIL_CALL_IS_OP, + [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_JUMP_BACKWARD_JIT, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, + [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, + [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, + [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, + [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, + [LOAD_ATTR_CLASS] = _TAIL_CALL_LOAD_ATTR_CLASS, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, + [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_LOAD_ATTR_PROPERTY, + [LOAD_ATTR_SLOT] = _TAIL_CALL_LOAD_ATTR_SLOT, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_LOAD_ATTR_WITH_HINT, + [LOAD_BUILD_CLASS] = _TAIL_CALL_LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, + [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, + [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, + [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, + [LOAD_FAST_BORROW] = _TAIL_CALL_LOAD_FAST_BORROW, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW, + [LOAD_FAST_CHECK] = _TAIL_CALL_LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = _TAIL_CALL_LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, + [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, + [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, + [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, + [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, + [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, + [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, + [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, + [MAP_ADD] = _TAIL_CALL_MAP_ADD, + [MATCH_CLASS] = _TAIL_CALL_MATCH_CLASS, + [MATCH_KEYS] = _TAIL_CALL_MATCH_KEYS, + [MATCH_MAPPING] = _TAIL_CALL_MATCH_MAPPING, + [MATCH_SEQUENCE] = _TAIL_CALL_MATCH_SEQUENCE, + [NOP] = _TAIL_CALL_NOP, + [NOT_TAKEN] = _TAIL_CALL_NOT_TAKEN, + [POP_EXCEPT] = _TAIL_CALL_POP_EXCEPT, + [POP_ITER] = _TAIL_CALL_POP_ITER, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = _TAIL_CALL_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_POP_JUMP_IF_TRUE, + [POP_TOP] = _TAIL_CALL_POP_TOP, + [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, + [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, + [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, + [RERAISE] = _TAIL_CALL_RERAISE, + [RESERVED] = _TAIL_CALL_RESERVED, + [RESUME] = _TAIL_CALL_RESUME, + [RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK, + [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, + [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, + [SEND] = _TAIL_CALL_SEND, + [SEND_GEN] = _TAIL_CALL_SEND_GEN, + [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, + [SET_ADD] = _TAIL_CALL_SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = _TAIL_CALL_SET_UPDATE, + [STORE_ATTR] = _TAIL_CALL_STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE, + [STORE_ATTR_SLOT] = _TAIL_CALL_STORE_ATTR_SLOT, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_STORE_ATTR_WITH_HINT, + [STORE_DEREF] = _TAIL_CALL_STORE_DEREF, + [STORE_FAST] = _TAIL_CALL_STORE_FAST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = _TAIL_CALL_STORE_GLOBAL, + [STORE_NAME] = _TAIL_CALL_STORE_NAME, + [STORE_SLICE] = _TAIL_CALL_STORE_SLICE, + [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, + [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [SWAP] = _TAIL_CALL_SWAP, + [TO_BOOL] = _TAIL_CALL_TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, + [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, + [TO_BOOL_INT] = _TAIL_CALL_TO_BOOL_INT, + [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, + [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, + [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, + [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, + [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, + [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, + [UNPACK_EX] = _TAIL_CALL_UNPACK_EX, + [UNPACK_SEQUENCE] = _TAIL_CALL_UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_UNPACK_SEQUENCE_LIST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TUPLE, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, + [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, + [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [210] = _TAIL_CALL_UNKNOWN_OPCODE, + [211] = _TAIL_CALL_UNKNOWN_OPCODE, + [212] = _TAIL_CALL_UNKNOWN_OPCODE, + [213] = _TAIL_CALL_UNKNOWN_OPCODE, + [214] = _TAIL_CALL_UNKNOWN_OPCODE, + [215] = _TAIL_CALL_UNKNOWN_OPCODE, + [216] = _TAIL_CALL_UNKNOWN_OPCODE, + [217] = _TAIL_CALL_UNKNOWN_OPCODE, + [218] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, +}; +static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_INTERPOLATION] = _TAIL_CALL_TRACE_RECORD, + [BUILD_LIST] = _TAIL_CALL_TRACE_RECORD, + [BUILD_MAP] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SET] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_STRING] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TEMPLATE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [CACHE] = _TAIL_CALL_TRACE_RECORD, + [CALL] = _TAIL_CALL_TRACE_RECORD, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_2] = _TAIL_CALL_TRACE_RECORD, + [CALL_ISINSTANCE] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_NON_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_LEN] = _TAIL_CALL_TRACE_RECORD, + [CALL_LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_STR_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TUPLE_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TYPE_1] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EG_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EXC_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CLEANUP_THROW] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_INT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_STR] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_DICT] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_SET] = _TAIL_CALL_TRACE_RECORD, + [CONVERT_VALUE] = _TAIL_CALL_TRACE_RECORD, + [COPY] = _TAIL_CALL_TRACE_RECORD, + [COPY_FREE_VARS] = _TAIL_CALL_TRACE_RECORD, + [DELETE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [DELETE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [DELETE_FAST] = _TAIL_CALL_TRACE_RECORD, + [DELETE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [DELETE_NAME] = _TAIL_CALL_TRACE_RECORD, + [DELETE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [DICT_MERGE] = _TAIL_CALL_TRACE_RECORD, + [DICT_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_SEND] = _TAIL_CALL_TRACE_RECORD, + [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, + [EXIT_INIT_CHECK] = _TAIL_CALL_TRACE_RECORD, + [EXTENDED_ARG] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_SIMPLE] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_GEN] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_LIST] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_RANGE] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [GET_AITER] = _TAIL_CALL_TRACE_RECORD, + [GET_ANEXT] = _TAIL_CALL_TRACE_RECORD, + [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER] = _TAIL_CALL_TRACE_RECORD, + [GET_LEN] = _TAIL_CALL_TRACE_RECORD, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LINE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INTERPRETER_EXIT] = _TAIL_CALL_TRACE_RECORD, + [IS_OP] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [LIST_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_CONST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_LOCALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_NAME] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SMALL_INT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SPECIAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACE_RECORD, + [MAKE_CELL] = _TAIL_CALL_TRACE_RECORD, + [MAKE_FUNCTION] = _TAIL_CALL_TRACE_RECORD, + [MAP_ADD] = _TAIL_CALL_TRACE_RECORD, + [MATCH_CLASS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_KEYS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_MAPPING] = _TAIL_CALL_TRACE_RECORD, + [MATCH_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [NOP] = _TAIL_CALL_TRACE_RECORD, + [NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [POP_EXCEPT] = _TAIL_CALL_TRACE_RECORD, + [POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [POP_TOP] = _TAIL_CALL_TRACE_RECORD, + [PUSH_EXC_INFO] = _TAIL_CALL_TRACE_RECORD, + [PUSH_NULL] = _TAIL_CALL_TRACE_RECORD, + [RAISE_VARARGS] = _TAIL_CALL_TRACE_RECORD, + [RERAISE] = _TAIL_CALL_TRACE_RECORD, + [RESERVED] = _TAIL_CALL_TRACE_RECORD, + [RESUME] = _TAIL_CALL_TRACE_RECORD, + [RESUME_CHECK] = _TAIL_CALL_TRACE_RECORD, + [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, + [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, + [SET_ADD] = _TAIL_CALL_TRACE_RECORD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, + [SET_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [STORE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [STORE_NAME] = _TAIL_CALL_TRACE_RECORD, + [STORE_SLICE] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [SWAP] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_INT] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_LIST] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_NONE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_STR] = _TAIL_CALL_TRACE_RECORD, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, + [UNARY_INVERT] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NEGATIVE] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NOT] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_EX] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, + [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [210] = _TAIL_CALL_UNKNOWN_OPCODE, + [211] = _TAIL_CALL_UNKNOWN_OPCODE, + [212] = _TAIL_CALL_UNKNOWN_OPCODE, + [213] = _TAIL_CALL_UNKNOWN_OPCODE, + [214] = _TAIL_CALL_UNKNOWN_OPCODE, + [215] = _TAIL_CALL_UNKNOWN_OPCODE, + [216] = _TAIL_CALL_UNKNOWN_OPCODE, + [217] = _TAIL_CALL_UNKNOWN_OPCODE, + [218] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, +}; +#endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Modules/_testinternalcapi/testbytecodes.c b/Modules/_testinternalcapi/testbytecodes.c new file mode 100644 index 00000000000000..da3025d1f4fdca --- /dev/null +++ b/Modules/_testinternalcapi/testbytecodes.c @@ -0,0 +1,177 @@ +// This file contains instruction definitions. +// It is read by generators stored in Tools/cases_generator/ +// to generate Python/generated_cases.c.h and others. +// Note that there is some dummy C code at the top and bottom of the file +// to fool text editors like VS Code into believing this is valid C code. +// The actual instruction definitions start at // BEGIN BYTECODES //. +// See Tools/cases_generator/README.md for more information. + +#include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _PySys_Audit() +#include "pycore_backoff.h" +#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_code.h" +#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS +#include "pycore_function.h" +#include "pycore_instruments.h" +#include "pycore_interpolation.h" // _PyInterpolation_Build() +#include "pycore_intrinsics.h" +#include "pycore_long.h" // _PyLong_ExactDealloc(), _PyLong_GetZero() +#include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_opcode_metadata.h" // uop names +#include "pycore_opcode_utils.h" // MAKE_FUNCTION_* +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_range.h" // _PyRangeIterObject +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs +#include "pycore_stackref.h" +#include "pycore_template.h" // _PyTemplate_Build() +#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_typeobject.h" // _PySuper_Lookup() + +#include "pycore_dict.h" +#include "dictobject.h" +#include "pycore_frame.h" +#include "opcode.h" +#include "optimizer.h" +#include "pydtrace.h" +#include "setobject.h" + + +#define USE_COMPUTED_GOTOS 0 +#include "ceval_macros.h" + +/* Flow control macros */ + +#define inst(name, ...) case name: +#define op(name, ...) /* NAME is ignored */ +#define macro(name) static int MACRO_##name +#define super(name) static int SUPER_##name +#define family(name, ...) static int family_##name +#define pseudo(name) static int pseudo_##name +#define label(name) name: + +/* Annotations */ +#define guard +#define override +#define specializing +#define replicate(TIMES) +#define tier1 +#define no_save_ip + +// Dummy variables for stack effects. +static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; +static PyObject *container, *start, *stop, *v, *lhs, *rhs, *res2; +static PyObject *list, *tuple, *dict, *owner, *set, *str, *tup, *map, *keys; +static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter, *exhausted; +static PyObject *aiter, *awaitable, *iterable, *w, *exc_value, *bc, *locals; +static PyObject *orig, *excs, *update, *b, *fromlist, *level, *from; +static PyObject **pieces, **values; +static size_t jump; +// Dummy variables for cache effects +static uint16_t invert, counter, index, hint; +#define unused 0 // Used in a macro def, can't be static +static uint32_t type_version; +static _PyExecutorObject *current_executor; + +static PyObject * +dummy_func( + PyThreadState *tstate, + _PyInterpreterFrame *frame, + unsigned char opcode, + unsigned int oparg, + _Py_CODEUNIT *next_instr, + PyObject **stack_pointer, + int throwflag, + PyObject *args[] +) +{ + // Dummy labels. + pop_1_error: + // Dummy locals. + PyObject *dummy; + _Py_CODEUNIT *this_instr; + PyObject *attr; + PyObject *attrs; + PyObject *bottom; + PyObject *callable; + PyObject *callargs; + PyObject *codeobj; + PyObject *cond; + PyObject *descr; + PyObject *exc; + PyObject *exit; + PyObject *fget; + PyObject *fmt_spec; + PyObject *func; + uint32_t func_version; + PyObject *getattribute; + PyObject *kwargs; + PyObject *kwdefaults; + PyObject *len_o; + PyObject *match; + PyObject *match_type; + PyObject *method; + PyObject *mgr; + Py_ssize_t min_args; + PyObject *names; + PyObject *new_exc; + PyObject *next; + PyObject *none; + PyObject *null; + PyObject *prev_exc; + PyObject *receiver; + PyObject *rest; + int result; + PyObject *self; + PyObject *seq; + PyObject *slice; + PyObject *step; + PyObject *subject; + PyObject *top; + PyObject *type; + PyObject *typevars; + PyObject *val0; + PyObject *val1; + int values_or_none; + + switch (opcode) { + +// BEGIN BYTECODES // + + // Override an op + override op(_CHECK_PERIODIC_IF_NOT_YIELD_FROM, (--)) { + Test_EvalFrame_Resumes++; + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + int err = check_periodics(tstate); + ERROR_IF(err != 0); + } + } + + // Override an inst + override inst(LOAD_CONST, (-- value)) { + Test_EvalFrame_Loads++; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + } + + + // END BYTECODES // + + } + dispatch_opcode: + error: + exception_unwind: + exit_unwind: + handle_eval_breaker: + resume_frame: + start_frame: + unbound_local_error: + ; +} + +// Future families go below this point // diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 5186ff4f6f0cf5..104ea0ad0525dd 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -19,6 +19,7 @@ this type and there is exactly one in existence. #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #define _PySlice_CAST(op) _Py_CAST(PySliceObject*, (op)) diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index f7e050b7c74180..3818e6d3f7bbd2 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -99,6 +99,7 @@ + diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 742597f5cb5ebd..41af9cacfb912b 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -112,6 +112,10 @@ WorkingDirectory="$(PySourcePath)" /> + + // bool - -#if !defined(Py_BUILD_CORE) -# error "ceval.c must be build with Py_BUILD_CORE define for best performance" -#endif - -#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) -// GH-89279: The MSVC compiler does not inline these static inline functions -// in PGO build in _PyEval_EvalFrameDefault(), because this function is over -// the limit of PGO, and that limit cannot be configured. -// Define them as macros to make sure that they are always inlined by the -// preprocessor. - -#undef Py_IS_TYPE -#define Py_IS_TYPE(ob, type) \ - (_PyObject_CAST(ob)->ob_type == (type)) - -#undef Py_XDECREF -#define Py_XDECREF(arg) \ - do { \ - PyObject *xop = _PyObject_CAST(arg); \ - if (xop != NULL) { \ - Py_DECREF(xop); \ - } \ - } while (0) - -#ifndef Py_GIL_DISABLED - -#undef Py_DECREF -#define Py_DECREF(arg) \ - do { \ - PyObject *op = _PyObject_CAST(arg); \ - if (_Py_IsImmortal(op)) { \ - _Py_DECREF_IMMORTAL_STAT_INC(); \ - break; \ - } \ - _Py_DECREF_STAT_INC(); \ - if (--op->ob_refcnt == 0) { \ - _PyReftracerTrack(op, PyRefTracer_DESTROY); \ - destructor dealloc = Py_TYPE(op)->tp_dealloc; \ - (*dealloc)(op); \ - } \ - } while (0) - -#undef _Py_DECREF_SPECIALIZED -#define _Py_DECREF_SPECIALIZED(arg, dealloc) \ - do { \ - PyObject *op = _PyObject_CAST(arg); \ - if (_Py_IsImmortal(op)) { \ - _Py_DECREF_IMMORTAL_STAT_INC(); \ - break; \ - } \ - _Py_DECREF_STAT_INC(); \ - if (--op->ob_refcnt == 0) { \ - _PyReftracerTrack(op, PyRefTracer_DESTROY); \ - destructor d = (destructor)(dealloc); \ - d(op); \ - } \ - } while (0) - -#else // Py_GIL_DISABLED - -#undef Py_DECREF -#define Py_DECREF(arg) \ - do { \ - PyObject *op = _PyObject_CAST(arg); \ - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \ - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \ - _Py_DECREF_IMMORTAL_STAT_INC(); \ - break; \ - } \ - _Py_DECREF_STAT_INC(); \ - if (_Py_IsOwnedByCurrentThread(op)) { \ - local--; \ - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \ - if (local == 0) { \ - _Py_MergeZeroLocalRefcount(op); \ - } \ - } \ - else { \ - _Py_DecRefShared(op); \ - } \ - } while (0) - -#undef _Py_DECREF_SPECIALIZED -#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg) - -#endif -#endif - - -static void -check_invalid_reentrancy(void) -{ -#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) - // In the free-threaded build, the interpreter must not be re-entered if - // the world-is-stopped. If so, that's a bug somewhere (quite likely in - // the painfully complex typeobject code). - PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(!interp->stoptheworld.world_stopped); -#endif -} - - -#ifdef Py_DEBUG -static void -dump_item(_PyStackRef item) -{ - if (PyStackRef_IsNull(item)) { - printf(""); - return; - } - if (PyStackRef_IsMalformed(item)) { - printf(""); - return; - } - if (PyStackRef_IsTaggedInt(item)) { - printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item)); - return; - } - PyObject *obj = PyStackRef_AsPyObjectBorrow(item); - if (obj == NULL) { - printf(""); - return; - } - // Don't call __repr__(), it might recurse into the interpreter. - printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); -} - -static void -dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) -{ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame); - _PyStackRef *stack_base = _PyFrame_Stackbase(frame); - PyObject *exc = PyErr_GetRaisedException(); - printf(" locals=["); - for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) { - if (ptr != locals_base) { - printf(", "); - } - dump_item(*ptr); - } - printf("]\n"); - if (stack_pointer < stack_base) { - printf(" stack=%d\n", (int)(stack_pointer-stack_base)); - } - else { - printf(" stack=["); - for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) { - if (ptr != stack_base) { - printf(", "); - } - dump_item(*ptr); - } - printf("]\n"); - } - fflush(stdout); - PyErr_SetRaisedException(exc); - _PyFrame_GetStackPointer(frame); -} - -#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG) -static void -dump_cache_item(_PyStackRef cache, int position, int depth) -{ - if (position < depth) { - dump_item(cache); - } - else { - printf("---"); - } -} -#endif - -static void -lltrace_instruction(_PyInterpreterFrame *frame, - _PyStackRef *stack_pointer, - _Py_CODEUNIT *next_instr, - int opcode, - int oparg) -{ - int offset = 0; - if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { - dump_stack(frame, stack_pointer); - offset = (int)(next_instr - _PyFrame_GetBytecode(frame)); - } - const char *opname = _PyOpcode_OpName[opcode]; - assert(opname != NULL); - if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) { - printf("%d: %s %d\n", offset * 2, opname, oparg); - } - else { - printf("%d: %s\n", offset * 2, opname); - } - fflush(stdout); -} - -static void -lltrace_resume_frame(_PyInterpreterFrame *frame) -{ - PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - if (!PyStackRef_CodeCheck(frame->f_executable) || - fobj == NULL || - !PyFunction_Check(fobj) - ) { - printf("\nResuming frame.\n"); - return; - } - PyFunctionObject *f = (PyFunctionObject *)fobj; - PyObject *exc = PyErr_GetRaisedException(); - PyObject *name = f->func_qualname; - if (name == NULL) { - name = f->func_name; - } - printf("\nResuming frame"); - if (name) { - printf(" for "); - if (PyObject_Print(name, stdout, 0) < 0) { - PyErr_Clear(); - } - } - if (f->func_module) { - printf(" in module "); - if (PyObject_Print(f->func_module, stdout, 0) < 0) { - PyErr_Clear(); - } - } - printf("\n"); - fflush(stdout); - PyErr_SetRaisedException(exc); -} - -static int -maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) -{ - if (globals == NULL) { - return 0; - } - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - return 0; - } - int r = PyDict_Contains(globals, &_Py_ID(__lltrace__)); - if (r < 0) { - PyErr_Clear(); - return 0; - } - int lltrace = r * 5; // Levels 1-4 only trace uops - if (!lltrace) { - // Can also be controlled by environment variable - char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); - if (python_lltrace != NULL && *python_lltrace >= '0') { - lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that - } - } - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } - return lltrace; -} - -#endif - -static void monitor_reraise(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr); -static int monitor_stop_iteration(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, - PyObject *value); -static void monitor_unwind(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr); -static int monitor_handled(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, PyObject *exc); -static void monitor_throw(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr); - -static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); -static _PyInterpreterFrame * -_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); - -#ifdef HAVE_ERRNO_H -#include -#endif +#include "ceval.h" int Py_GetRecursionLimit(void) @@ -1345,19 +1010,6 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif -/* Disable unused label warnings. They are handy for debugging, even - if computed gotos aren't used. */ - -/* TBD - what about other compilers? */ -#if defined(__GNUC__) || defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-label" -#elif defined(_MSC_VER) /* MS_WINDOWS */ -# pragma warning(push) -# pragma warning(disable:4102) -#endif - - PyObject ** _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch) { @@ -1388,12 +1040,6 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } -#ifdef Py_DEBUG -#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) -#else -#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 -#endif - #if _Py_TIER2 // 0 for success, -1 for error. static int @@ -1465,11 +1111,6 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) #define DONT_SLP_VECTORIZE #endif -typedef struct { - _PyInterpreterFrame frame; - _PyStackRef stack[1]; -} _PyEntryFrame; - PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -1968,74 +1609,6 @@ positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, } - -static inline unsigned char * -scan_back_to_entry_start(unsigned char *p) { - for (; (p[0]&128) == 0; p--); - return p; -} - -static inline unsigned char * -skip_to_next_entry(unsigned char *p, unsigned char *end) { - while (p < end && ((p[0] & 128) == 0)) { - p++; - } - return p; -} - - -#define MAX_LINEAR_SEARCH 40 - -static Py_NO_INLINE int -get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti) -{ - unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); - unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); - /* Invariants: - * start_table == end_table OR - * start_table points to a legal entry and end_table points - * beyond the table or to a legal entry that is after index. - */ - if (end - start > MAX_LINEAR_SEARCH) { - int offset; - parse_varint(start, &offset); - if (offset > index) { - return 0; - } - do { - unsigned char * mid = start + ((end-start)>>1); - mid = scan_back_to_entry_start(mid); - parse_varint(mid, &offset); - if (offset > index) { - end = mid; - } - else { - start = mid; - } - - } while (end - start > MAX_LINEAR_SEARCH); - } - unsigned char *scan = start; - while (scan < end) { - int start_offset, size; - scan = parse_varint(scan, &start_offset); - if (start_offset > index) { - break; - } - scan = parse_varint(scan, &size); - if (start_offset + size > index) { - scan = parse_varint(scan, handler); - int depth_and_lasti; - parse_varint(scan, &depth_and_lasti); - *level = depth_and_lasti >> 1; - *lasti = depth_and_lasti & 1; - return 1; - } - scan = skip_to_next_entry(scan, end); - } - return 0; -} - static int initialize_locals(PyThreadState *tstate, PyFunctionObject *func, _PyStackRef *localsplus, _PyStackRef const *args, @@ -2372,70 +1945,6 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, return NULL; } -/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. - Steals references to func, callargs and kwargs. -*/ -static _PyInterpreterFrame * -_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous) -{ - bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); - PyObject *kwnames = NULL; - _PyStackRef *newargs; - PyObject *const *object_array = NULL; - _PyStackRef stack_array[8] = {0}; - if (has_dict) { - object_array = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); - if (object_array == NULL) { - PyStackRef_CLOSE(func); - goto error; - } - size_t total_args = nargs + PyDict_GET_SIZE(kwargs); - assert(sizeof(PyObject *) == sizeof(_PyStackRef)); - newargs = (_PyStackRef *)object_array; - for (size_t i = 0; i < total_args; i++) { - newargs[i] = PyStackRef_FromPyObjectSteal(object_array[i]); - } - } - else { - if (nargs <= 8) { - newargs = stack_array; - } - else { - newargs = PyMem_Malloc(sizeof(_PyStackRef) *nargs); - if (newargs == NULL) { - PyErr_NoMemory(); - PyStackRef_CLOSE(func); - goto error; - } - } - /* We need to create a new reference for all our args since the new frame steals them. */ - for (Py_ssize_t i = 0; i < nargs; i++) { - newargs[i] = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(callargs, i)); - } - } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, func, locals, - newargs, nargs, kwnames, previous - ); - if (has_dict) { - _PyStack_UnpackDict_FreeNoDecRef(object_array, kwnames); - } - else if (nargs > 8) { - PyMem_Free((void *)newargs); - } - /* No need to decref func here because the reference has been stolen by - _PyEvalFramePushAndInit. - */ - Py_DECREF(callargs); - Py_XDECREF(kwargs); - return new_frame; -error: - Py_DECREF(callargs); - Py_XDECREF(kwargs); - return NULL; -} - PyObject * _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, @@ -2556,109 +2065,6 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, return res; } - -/* Logic for the raise statement (too complicated for inlining). - This *consumes* a reference count to each of its arguments. */ -static int -do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) -{ - PyObject *type = NULL, *value = NULL; - - if (exc == NULL) { - /* Reraise */ - _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); - exc = exc_info->exc_value; - if (Py_IsNone(exc) || exc == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "No active exception to reraise"); - return 0; - } - Py_INCREF(exc); - assert(PyExceptionInstance_Check(exc)); - _PyErr_SetRaisedException(tstate, exc); - return 1; - } - - /* We support the following forms of raise: - raise - raise - raise */ - - if (PyExceptionClass_Check(exc)) { - type = exc; - value = _PyObject_CallNoArgs(exc); - if (value == NULL) - goto raise_error; - if (!PyExceptionInstance_Check(value)) { - _PyErr_Format(tstate, PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto raise_error; - } - } - else if (PyExceptionInstance_Check(exc)) { - value = exc; - type = PyExceptionInstance_Class(exc); - Py_INCREF(type); - } - else { - /* Not something you can raise. You get an exception - anyway, just not what you specified :-) */ - Py_DECREF(exc); - _PyErr_SetString(tstate, PyExc_TypeError, - "exceptions must derive from BaseException"); - goto raise_error; - } - - assert(type != NULL); - assert(value != NULL); - - if (cause) { - PyObject *fixed_cause; - if (PyExceptionClass_Check(cause)) { - fixed_cause = _PyObject_CallNoArgs(cause); - if (fixed_cause == NULL) - goto raise_error; - if (!PyExceptionInstance_Check(fixed_cause)) { - _PyErr_Format(tstate, PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - cause, Py_TYPE(fixed_cause)); - Py_DECREF(fixed_cause); - goto raise_error; - } - Py_DECREF(cause); - } - else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - } - else if (Py_IsNone(cause)) { - Py_DECREF(cause); - fixed_cause = NULL; - } - else { - _PyErr_SetString(tstate, PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto raise_error; - } - PyException_SetCause(value, fixed_cause); - } - - _PyErr_SetObject(tstate, type, value); - /* _PyErr_SetObject incref's its arguments */ - Py_DECREF(value); - Py_DECREF(type); - return 0; - -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(cause); - return 0; -} - /* Logic for matching an exception in an except* clause (too complicated for inlining). */ @@ -2856,45 +2262,7 @@ _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, return 0; } -static int -do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, int event) -{ - assert(event < _PY_MONITORING_UNGROUPED_EVENTS); - if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { - return 0; - } - PyObject *exc = PyErr_GetRaisedException(); - assert(exc != NULL); - int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); - if (err == 0) { - PyErr_SetRaisedException(exc); - } - else { - assert(PyErr_Occurred()); - Py_DECREF(exc); - } - return err; -} -static inline bool -no_tools_for_global_event(PyThreadState *tstate, int event) -{ - return tstate->interp->monitors.tools[event] == 0; -} - -static inline bool -no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) -{ - assert(event < _PY_MONITORING_LOCAL_EVENTS); - _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; - if (data) { - return data->active_monitors.tools[event] == 0; - } - else { - return no_tools_for_global_event(tstate, event); - } -} void _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, @@ -2906,70 +2274,11 @@ _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); } -static void -monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) -{ - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { - return; - } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); -} - -static int -monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, PyObject *value) -{ - if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { - return 0; - } - assert(!PyErr_Occurred()); - PyErr_SetObject(PyExc_StopIteration, value); - int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); - if (res < 0) { - return res; - } - PyErr_SetRaisedException(NULL); - return 0; -} - -static void -monitor_unwind(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) -{ - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { - return; - } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); -} - bool _PyEval_NoToolsForUnwind(PyThreadState *tstate) { return no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND); } -static int -monitor_handled(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, PyObject *exc) -{ - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { - return 0; - } - return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); -} - -static void -monitor_throw(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) -{ - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { - return; - } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); -} void PyThreadState_EnterTracing(PyThreadState *tstate) diff --git a/Python/ceval.h b/Python/ceval.h new file mode 100644 index 00000000000000..c5f116f23085d6 --- /dev/null +++ b/Python/ceval.h @@ -0,0 +1,693 @@ +#define _PY_INTERPRETER + +#include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _PySys_Audit() +#include "pycore_backoff.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_ceval.h" // SPECIAL___ENTER__ +#include "pycore_code.h" +#include "pycore_dict.h" +#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS +#include "pycore_floatobject.h" // _PyFloat_ExactDealloc() +#include "pycore_frame.h" +#include "pycore_function.h" +#include "pycore_genobject.h" // _PyCoro_GetAwaitableIter() +#include "pycore_import.h" // _PyImport_IsDefaultImportFunc() +#include "pycore_instruments.h" +#include "pycore_interpframe.h" // _PyFrame_SetStackPointer() +#include "pycore_interpolation.h" // _PyInterpolation_Build() +#include "pycore_intrinsics.h" +#include "pycore_jit.h" +#include "pycore_list.h" // _PyList_GetItemRef() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_opcode_metadata.h" // EXTRA_CASES +#include "pycore_opcode_utils.h" // MAKE_FUNCTION_* +#include "pycore_optimizer.h" // _PyUOpExecutor_Type +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_range.h" // _PyRangeIterObject +#include "pycore_setobject.h" // _PySet_Update() +#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs +#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() +#include "pycore_template.h" // _PyTemplate_Build() +#include "pycore_traceback.h" // _PyTraceBack_FromFrame +#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_uop_ids.h" // Uops + +#include "dictobject.h" +#include "frameobject.h" // _PyInterpreterFrame_GetLine +#include "opcode.h" +#include "pydtrace.h" +#include "setobject.h" +#include "pycore_stackref.h" + +#include // bool + +#if !defined(Py_BUILD_CORE) +# error "ceval.c must be build with Py_BUILD_CORE define for best performance" +#endif + +#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) +// GH-89279: The MSVC compiler does not inline these static inline functions +// in PGO build in _PyEval_EvalFrameDefault(), because this function is over +// the limit of PGO, and that limit cannot be configured. +// Define them as macros to make sure that they are always inlined by the +// preprocessor. + +#undef Py_IS_TYPE +#define Py_IS_TYPE(ob, type) \ + (_PyObject_CAST(ob)->ob_type == (type)) + +#undef Py_XDECREF +#define Py_XDECREF(arg) \ + do { \ + PyObject *xop = _PyObject_CAST(arg); \ + if (xop != NULL) { \ + Py_DECREF(xop); \ + } \ + } while (0) + +#ifndef Py_GIL_DISABLED + +#undef Py_DECREF +#define Py_DECREF(arg) \ + do { \ + PyObject *op = _PyObject_CAST(arg); \ + if (_Py_IsImmortal(op)) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ + if (--op->ob_refcnt == 0) { \ + _PyReftracerTrack(op, PyRefTracer_DESTROY); \ + destructor dealloc = Py_TYPE(op)->tp_dealloc; \ + (*dealloc)(op); \ + } \ + } while (0) + +#undef _Py_DECREF_SPECIALIZED +#define _Py_DECREF_SPECIALIZED(arg, dealloc) \ + do { \ + PyObject *op = _PyObject_CAST(arg); \ + if (_Py_IsImmortal(op)) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ + if (--op->ob_refcnt == 0) { \ + _PyReftracerTrack(op, PyRefTracer_DESTROY); \ + destructor d = (destructor)(dealloc); \ + d(op); \ + } \ + } while (0) + +#else // Py_GIL_DISABLED + +#undef Py_DECREF +#define Py_DECREF(arg) \ + do { \ + PyObject *op = _PyObject_CAST(arg); \ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \ + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ + if (_Py_IsOwnedByCurrentThread(op)) { \ + local--; \ + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \ + if (local == 0) { \ + _Py_MergeZeroLocalRefcount(op); \ + } \ + } \ + else { \ + _Py_DecRefShared(op); \ + } \ + } while (0) + +#undef _Py_DECREF_SPECIALIZED +#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg) + +#endif +#endif + +typedef struct { + _PyInterpreterFrame frame; + _PyStackRef stack[1]; +} _PyEntryFrame; + +static void +check_invalid_reentrancy(void) +{ +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) + // In the free-threaded build, the interpreter must not be re-entered if + // the world-is-stopped. If so, that's a bug somewhere (quite likely in + // the painfully complex typeobject code). + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!interp->stoptheworld.world_stopped); +#endif +} + + +#ifdef Py_DEBUG +static void +dump_item(_PyStackRef item) +{ + if (PyStackRef_IsNull(item)) { + printf(""); + return; + } + if (PyStackRef_IsMalformed(item)) { + printf(""); + return; + } + if (PyStackRef_IsTaggedInt(item)) { + printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item)); + return; + } + PyObject *obj = PyStackRef_AsPyObjectBorrow(item); + if (obj == NULL) { + printf(""); + return; + } + // Don't call __repr__(), it might recurse into the interpreter. + printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); +} + +static void +dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) +{ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame); + _PyStackRef *stack_base = _PyFrame_Stackbase(frame); + PyObject *exc = PyErr_GetRaisedException(); + printf(" locals=["); + for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) { + if (ptr != locals_base) { + printf(", "); + } + dump_item(*ptr); + } + printf("]\n"); + if (stack_pointer < stack_base) { + printf(" stack=%d\n", (int)(stack_pointer-stack_base)); + } + else { + printf(" stack=["); + for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) { + if (ptr != stack_base) { + printf(", "); + } + dump_item(*ptr); + } + printf("]\n"); + } + fflush(stdout); + PyErr_SetRaisedException(exc); + _PyFrame_GetStackPointer(frame); +} + +#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG) +static void +dump_cache_item(_PyStackRef cache, int position, int depth) +{ + if (position < depth) { + dump_item(cache); + } + else { + printf("---"); + } +} +#endif + +static void +lltrace_instruction(_PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, + _Py_CODEUNIT *next_instr, + int opcode, + int oparg) +{ + int offset = 0; + if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + dump_stack(frame, stack_pointer); + offset = (int)(next_instr - _PyFrame_GetBytecode(frame)); + } + const char *opname = _PyOpcode_OpName[opcode]; + assert(opname != NULL); + if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) { + printf("%d: %s %d\n", offset * 2, opname, oparg); + } + else { + printf("%d: %s\n", offset * 2, opname); + } + fflush(stdout); +} + +static void +lltrace_resume_frame(_PyInterpreterFrame *frame) +{ + PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (!PyStackRef_CodeCheck(frame->f_executable) || + fobj == NULL || + !PyFunction_Check(fobj) + ) { + printf("\nResuming frame.\n"); + return; + } + PyFunctionObject *f = (PyFunctionObject *)fobj; + PyObject *exc = PyErr_GetRaisedException(); + PyObject *name = f->func_qualname; + if (name == NULL) { + name = f->func_name; + } + printf("\nResuming frame"); + if (name) { + printf(" for "); + if (PyObject_Print(name, stdout, 0) < 0) { + PyErr_Clear(); + } + } + if (f->func_module) { + printf(" in module "); + if (PyObject_Print(f->func_module, stdout, 0) < 0) { + PyErr_Clear(); + } + } + printf("\n"); + fflush(stdout); + PyErr_SetRaisedException(exc); +} + +static int +maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) +{ + if (globals == NULL) { + return 0; + } + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + return 0; + } + int r = PyDict_Contains(globals, &_Py_ID(__lltrace__)); + if (r < 0) { + PyErr_Clear(); + return 0; + } + int lltrace = r * 5; // Levels 1-4 only trace uops + if (!lltrace) { + // Can also be controlled by environment variable + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + } + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } + return lltrace; +} + +#endif + +static void monitor_reraise(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static int monitor_stop_iteration(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, + PyObject *value); +static void monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static int monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc); +static void monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); + +static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); + +#ifdef HAVE_ERRNO_H +#include +#endif + +static int +do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event) +{ + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); + if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { + return 0; + } + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); + if (err == 0) { + PyErr_SetRaisedException(exc); + } + else { + assert(PyErr_Occurred()); + Py_DECREF(exc); + } + return err; +} + +static inline bool +no_tools_for_global_event(PyThreadState *tstate, int event) +{ + return tstate->interp->monitors.tools[event] == 0; +} + +static inline bool +no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) +{ + assert(event < _PY_MONITORING_LOCAL_EVENTS); + _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; + if (data) { + return data->active_monitors.tools[event] == 0; + } + else { + return no_tools_for_global_event(tstate, event); + } +} + +static int +monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) +{ + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + return 0; + } + return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); +} + +static void +monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); +} + +static void +monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); +} + +static int +monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *value) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { + return 0; + } + assert(!PyErr_Occurred()); + PyErr_SetObject(PyExc_StopIteration, value); + int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); + if (res < 0) { + return res; + } + PyErr_SetRaisedException(NULL); + return 0; +} + +static void +monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); +} + +static inline unsigned char * +scan_back_to_entry_start(unsigned char *p) { + for (; (p[0]&128) == 0; p--); + return p; +} + +static inline unsigned char * +skip_to_next_entry(unsigned char *p, unsigned char *end) { + while (p < end && ((p[0] & 128) == 0)) { + p++; + } + return p; +} + + +#define MAX_LINEAR_SEARCH 40 + +static Py_NO_INLINE int +get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti) +{ + unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); + unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); + /* Invariants: + * start_table == end_table OR + * start_table points to a legal entry and end_table points + * beyond the table or to a legal entry that is after index. + */ + if (end - start > MAX_LINEAR_SEARCH) { + int offset; + parse_varint(start, &offset); + if (offset > index) { + return 0; + } + do { + unsigned char * mid = start + ((end-start)>>1); + mid = scan_back_to_entry_start(mid); + parse_varint(mid, &offset); + if (offset > index) { + end = mid; + } + else { + start = mid; + } + + } while (end - start > MAX_LINEAR_SEARCH); + } + unsigned char *scan = start; + while (scan < end) { + int start_offset, size; + scan = parse_varint(scan, &start_offset); + if (start_offset > index) { + break; + } + scan = parse_varint(scan, &size); + if (start_offset + size > index) { + scan = parse_varint(scan, handler); + int depth_and_lasti; + parse_varint(scan, &depth_and_lasti); + *level = depth_and_lasti >> 1; + *lasti = depth_and_lasti & 1; + return 1; + } + scan = skip_to_next_entry(scan, end); + } + return 0; +} + + +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + +/* Logic for the raise statement (too complicated for inlining). + This *consumes* a reference count to each of its arguments. */ +static int +do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) +{ + PyObject *type = NULL, *value = NULL; + + if (exc == NULL) { + /* Reraise */ + _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); + exc = exc_info->exc_value; + if (Py_IsNone(exc) || exc == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "No active exception to reraise"); + return 0; + } + Py_INCREF(exc); + assert(PyExceptionInstance_Check(exc)); + _PyErr_SetRaisedException(tstate, exc); + return 1; + } + + /* We support the following forms of raise: + raise + raise + raise */ + + if (PyExceptionClass_Check(exc)) { + type = exc; + value = _PyObject_CallNoArgs(exc); + if (value == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(value)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto raise_error; + } + } + else if (PyExceptionInstance_Check(exc)) { + value = exc; + type = PyExceptionInstance_Class(exc); + Py_INCREF(type); + } + else { + /* Not something you can raise. You get an exception + anyway, just not what you specified :-) */ + Py_DECREF(exc); + _PyErr_SetString(tstate, PyExc_TypeError, + "exceptions must derive from BaseException"); + goto raise_error; + } + + assert(type != NULL); + assert(value != NULL); + + if (cause) { + PyObject *fixed_cause; + if (PyExceptionClass_Check(cause)) { + fixed_cause = _PyObject_CallNoArgs(cause); + if (fixed_cause == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(fixed_cause)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + cause, Py_TYPE(fixed_cause)); + Py_DECREF(fixed_cause); + goto raise_error; + } + Py_DECREF(cause); + } + else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + } + else if (Py_IsNone(cause)) { + Py_DECREF(cause); + fixed_cause = NULL; + } + else { + _PyErr_SetString(tstate, PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto raise_error; + } + PyException_SetCause(value, fixed_cause); + } + + _PyErr_SetObject(tstate, type, value); + /* _PyErr_SetObject incref's its arguments */ + Py_DECREF(value); + Py_DECREF(type); + return 0; + +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(cause); + return 0; +} + +/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. + Steals references to func, callargs and kwargs. +*/ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous) +{ + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + _PyStackRef *newargs; + PyObject *const *object_array = NULL; + _PyStackRef stack_array[8] = {0}; + if (has_dict) { + object_array = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); + if (object_array == NULL) { + PyStackRef_CLOSE(func); + goto error; + } + size_t total_args = nargs + PyDict_GET_SIZE(kwargs); + assert(sizeof(PyObject *) == sizeof(_PyStackRef)); + newargs = (_PyStackRef *)object_array; + for (size_t i = 0; i < total_args; i++) { + newargs[i] = PyStackRef_FromPyObjectSteal(object_array[i]); + } + } + else { + if (nargs <= 8) { + newargs = stack_array; + } + else { + newargs = PyMem_Malloc(sizeof(_PyStackRef) *nargs); + if (newargs == NULL) { + PyErr_NoMemory(); + PyStackRef_CLOSE(func); + goto error; + } + } + /* We need to create a new reference for all our args since the new frame steals them. */ + for (Py_ssize_t i = 0; i < nargs; i++) { + newargs[i] = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, func, locals, + newargs, nargs, kwnames, previous + ); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(object_array, kwnames); + } + else if (nargs > 8) { + PyMem_Free((void *)newargs); + } + /* No need to decref func here because the reference has been stolen by + _PyEvalFramePushAndInit. + */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return new_frame; +error: + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return NULL; +} + + +/* Disable unused label warnings. They are handy for debugging, even + if computed gotos aren't used. */ + +/* TBD - what about other compilers? */ +#if defined(__GNUC__) || defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-label" +#elif defined(_MSC_VER) /* MS_WINDOWS */ +# pragma warning(push) +# pragma warning(disable:4102) +#endif diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 6332dec3aefe4f..322c9f232f2796 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -81,12 +81,15 @@ def format_tsv_lines(lines): 'Python/executor_cases.c.h', 'Python/optimizer_cases.c.h', 'Python/opcode_targets.h', + 'Modules/_testinternalcapi/test_targets.h', + 'Modules/_testinternalcapi/test_cases.c.h', # XXX: Throws errors if PY_VERSION_HEX is not mocked out 'Modules/clinic/_testclinic_depr.c.h', # not actually source 'Python/bytecodes.c', 'Python/optimizer_bytecodes.c', + 'Modules/_testinternalcapi/testbytecodes.c', # mimalloc 'Objects/mimalloc/*.c', diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index adb183000deeff..6ad3fc5f76e57a 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -567,6 +567,10 @@ Modules/_testimportmultiple.c - _barmodule - Modules/_testimportmultiple.c - _foomodule - Modules/_testimportmultiple.c - _testimportmultiple - Modules/_testinternalcapi.c - pending_identify_result - +Modules/_testinternalcapi.c - Test_EvalFrame_Resumes - +Modules/_testinternalcapi.c - Test_EvalFrame_Loads - +Modules/_testinternalcapi/interpreter.c - Test_EvalFrame_Resumes - +Modules/_testinternalcapi/interpreter.c - Test_EvalFrame_Loads - Modules/_testmultiphase.c - Example_Type_slots - Modules/_testmultiphase.c - Example_Type_spec - Modules/_testmultiphase.c - Example_methods - diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index a0664cdfde19b1..a31f57850f6a0f 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -150,7 +150,7 @@ def generate_instruction_formats(analysis: Analysis, out: CWriter) -> None: def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint8_t _PyOpcode_Deopt[256];\n") + out.emit("PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256];\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint8_t _PyOpcode_Deopt[256] = {\n") deopts: list[tuple[str, str]] = [] @@ -173,7 +173,7 @@ def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: def generate_cache_table(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint8_t _PyOpcode_Caches[256];\n") + out.emit("PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256];\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint8_t _PyOpcode_Caches[256] = {\n") for inst in analysis.instructions.values(): @@ -189,7 +189,7 @@ def generate_cache_table(analysis: Analysis, out: CWriter) -> None: def generate_name_table(analysis: Analysis, out: CWriter) -> None: table_size = 256 + len(analysis.pseudos) - out.emit(f"extern const char *_PyOpcode_OpName[{table_size}];\n") + out.emit(f"PyAPI_DATA(const char) *_PyOpcode_OpName[{table_size}];\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit(f"const char *_PyOpcode_OpName[{table_size}] = {{\n") names = list(analysis.instructions) + list(analysis.pseudos) @@ -207,7 +207,7 @@ def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: out.emit("uint32_t flags;\n") out.emit("};\n\n") out.emit( - f"extern const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}];\n" + f"PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[{table_size}];\n" ) out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit(