GH-148960: Reduce the size of the debug stencils to less than half. (GH-150551)

For AArch64 linux, reduces the total bytes in the code bodies from 489kb to 218kb.
Reduces the size of the stencils files from 394k lines to 167k lines.

Author: markshannon

Include/cpython/pystats.h

@@ -163,6 +163,7 @@ typedef struct _optimization_stats {
     uint64_t jit_code_size;
     uint64_t jit_trampoline_size;
     uint64_t jit_data_size;
+    uint64_t jit_got_size;
     uint64_t jit_padding_size;
     uint64_t jit_freed_memory_size;
     uint64_t trace_total_memory_hist[_Py_UOP_HIST_SIZE];

Include/internal/pycore_interpframe.h

@@ -230,7 +230,9 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
 static inline _PyStackRef*
 _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 {
+#ifndef _Py_JIT
     assert(frame->stackpointer != NULL);
+#endif
     _PyStackRef *sp = frame->stackpointer;
 #ifndef NDEBUG
     frame->stackpointer = NULL;
@@ -241,7 +243,10 @@ _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 static inline void
 _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
 {
+/* Avoid bloating the JIT code */
+#ifndef _Py_JIT
     assert(frame->stackpointer == NULL);
+#endif
     frame->stackpointer = stack_pointer;
 }
 

Include/internal/pycore_jit.h

@@ -31,6 +31,8 @@ _Py_CODEUNIT *_PyJIT_Entry(
 int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length);
 void _PyJIT_Free(_PyExecutorObject *executor);
 PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr);
+PyAPI_FUNC(void) _Py_jit_assert_within_stack_bounds(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno);
+PyAPI_FUNC(int) _Py_jit_assertion_failure(int line);
 
 #endif  // _Py_JIT
 

Include/internal/pycore_uop.h

@@ -38,12 +38,7 @@ typedef struct _PyUOpInstruction{
 
 // Fitness is the target length of the trace we translate initially. The uop
 // buffer has a small amount of extra space for entry/loop-closing overhead.
-#if defined(Py_DEBUG) && defined(_Py_JIT)
-    // With asserts, the stencils are a lot larger
-#define FITNESS_INITIAL 1000
-#else
 #define FITNESS_INITIAL 2500
-#endif
 
 #define UOP_TRACE_BUFFER_OVERHEAD 10
 #define UOP_MAX_TRACE_LENGTH (FITNESS_INITIAL + UOP_TRACE_BUFFER_OVERHEAD)

Python/ceval.c

@@ -997,6 +997,14 @@ _Py_assert_within_stack_bounds(
         abort();
     }
 }
+#ifdef _Py_JIT
+void
+_Py_jit_assert_within_stack_bounds(
+    _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno
+) {
+    _Py_assert_within_stack_bounds(frame, stack_pointer, "executor_cases.c.h", lineno);
+}
+#endif
 #endif
 
 int _Py_CheckRecursiveCallPy(

Python/ceval_macros.h

@@ -268,10 +268,10 @@ GETITEM(PyObject *v, Py_ssize_t i) {
 
 #if defined(Py_DEBUG) && !defined(_Py_JIT)
 // This allows temporary stack "overflows", provided it's all in the cache at any point of time.
-#define WITHIN_STACK_BOUNDS_IGNORING_CACHE() \
-   (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE()))
+#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(F, L) \
+   assert(frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE()))
 #else
-#define WITHIN_STACK_BOUNDS_IGNORING_CACHE WITHIN_STACK_BOUNDS
+#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE ASSERT_WITHIN_STACK_BOUNDS
 #endif
 
 /* Data access macros */