| /* |
| * Python JIT - DWARF .eh_frame builder |
| * |
| * This file contains the DWARF CFI generator used to build .eh_frame |
| * data for JIT code (perf jitdump and other unwinders). |
| */ |
| |
| #include "Python.h" |
| #include "pycore_jit_unwind.h" |
| #include "pycore_lock.h" |
| |
| #if defined(PY_HAVE_JIT_GDB_UNWIND) |
| # include "jit_unwind_info.h" |
| # if !JIT_UNWIND_INFO_SUPPORTED |
| # error "JIT unwind info was not generated for this target" |
| # endif |
| #endif |
| |
| #if defined(PY_HAVE_PERF_TRAMPOLINE) \ |
| || defined(PY_HAVE_JIT_GDB_UNWIND) \ |
| || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) |
| |
| #if defined(PY_HAVE_JIT_GDB_UNWIND) |
| # include <elf.h> |
| #endif |
| #if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) |
| /* |
| * libgcc exposes frame registration entry points, but GCC's public headers |
| * on some distributions do not declare them even though the symbols are |
| * available in libgcc_s. |
| */ |
| void __register_frame(const void *); |
| void __deregister_frame(const void *); |
| #endif |
| #include <stdio.h> |
| #include <string.h> |
| |
| // ============================================================================= |
| // DWARF CONSTANTS |
| // ============================================================================= |
| |
| /* |
| * DWARF (Debug With Arbitrary Record Formats) constants |
| * |
| * DWARF is a debugging data format used to provide stack unwinding information. |
| * These constants define the various encoding types and opcodes used in |
| * DWARF Call Frame Information (CFI) records. |
| */ |
| |
| /* DWARF Call Frame Information version */ |
| #define DWRF_CIE_VERSION 1 |
| |
| /* DWARF CFA (Call Frame Address) opcodes */ |
| enum { |
| DWRF_CFA_nop = 0x0, // No operation |
| DWRF_CFA_offset_extended = 0x5, // Extended offset instruction |
| DWRF_CFA_def_cfa = 0xc, // Define CFA rule |
| DWRF_CFA_def_cfa_register = 0xd, // Define CFA register |
| DWRF_CFA_def_cfa_offset = 0xe, // Define CFA offset |
| DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset |
| DWRF_CFA_advance_loc = 0x40, // Advance location counter |
| DWRF_CFA_offset = 0x80, // Simple offset instruction |
| DWRF_CFA_restore = 0xc0 // Restore register |
| }; |
| |
| /* |
| * Architecture-specific DWARF register numbers |
| * |
| * These constants define the register numbering scheme used by DWARF |
| * for each supported architecture. The numbers must match the ABI |
| * specification for proper stack unwinding. |
| */ |
| enum { |
| #ifdef __x86_64__ |
| /* x86_64 register numbering (note: order is defined by x86_64 ABI) */ |
| DWRF_REG_AX, // RAX |
| DWRF_REG_DX, // RDX |
| DWRF_REG_CX, // RCX |
| DWRF_REG_BX, // RBX |
| DWRF_REG_SI, // RSI |
| DWRF_REG_DI, // RDI |
| DWRF_REG_BP, // RBP |
| DWRF_REG_SP, // RSP |
| DWRF_REG_8, // R8 |
| DWRF_REG_9, // R9 |
| DWRF_REG_10, // R10 |
| DWRF_REG_11, // R11 |
| DWRF_REG_12, // R12 |
| DWRF_REG_13, // R13 |
| DWRF_REG_14, // R14 |
| DWRF_REG_15, // R15 |
| DWRF_REG_RA, // Return address (RIP) |
| #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) |
| /* AArch64 register numbering */ |
| DWRF_REG_FP = 29, // Frame Pointer |
| DWRF_REG_RA = 30, // Link register (return address) |
| DWRF_REG_SP = 31, // Stack pointer |
| #else |
| # error "Unsupported target architecture" |
| #endif |
| }; |
| |
| // ============================================================================= |
| // ELF OBJECT CONTEXT |
| // ============================================================================= |
| |
| /* |
| * Context for building ELF/DWARF structures |
| * |
| * This structure maintains state while constructing DWARF unwind information. |
| * It acts as a simple buffer manager with pointers to track current position |
| * and important landmarks within the buffer. |
| */ |
| typedef struct ELFObjectContext { |
| uint8_t* p; // Current write position in buffer |
| uint8_t* startp; // Start of buffer (for offset calculations) |
| uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) |
| uintptr_t code_addr; // Address of the code section |
| size_t code_size; // Size of the code section |
| } ELFObjectContext; |
| |
| // ============================================================================= |
| // DWARF GENERATION UTILITIES |
| // ============================================================================= |
| |
| /* |
| * Append a null-terminated string to the ELF context buffer. |
| * |
| * Args: |
| * ctx: ELF object context |
| * str: String to append (must be null-terminated) |
| * |
| * Returns: Offset from start of buffer where string was written |
| */ |
| static uint32_t elfctx_append_string(ELFObjectContext* ctx, const char* str) { |
| uint8_t* p = ctx->p; |
| uint32_t ofs = (uint32_t)(p - ctx->startp); |
| |
| /* Copy string including null terminator */ |
| do { |
| *p++ = (uint8_t)*str; |
| } while (*str++); |
| |
| ctx->p = p; |
| return ofs; |
| } |
| |
| /* |
| * Append a SLEB128 (Signed Little Endian Base 128) value |
| * |
| * SLEB128 is a variable-length encoding used extensively in DWARF. |
| * It efficiently encodes small numbers in fewer bytes. |
| * |
| * Args: |
| * ctx: ELF object context |
| * v: Signed value to encode |
| */ |
| static void elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) { |
| uint8_t* p = ctx->p; |
| |
| /* Encode 7 bits at a time, with continuation bit in MSB */ |
| for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { |
| *p++ = (uint8_t)((v & 0x7f) | 0x80); // Set continuation bit |
| } |
| *p++ = (uint8_t)(v & 0x7f); // Final byte without continuation bit |
| |
| ctx->p = p; |
| } |
| |
| /* |
| * Append a ULEB128 (Unsigned Little Endian Base 128) value |
| * |
| * Similar to SLEB128 but for unsigned values. |
| * |
| * Args: |
| * ctx: ELF object context |
| * v: Unsigned value to encode |
| */ |
| static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { |
| uint8_t* p = ctx->p; |
| |
| /* Encode 7 bits at a time, with continuation bit in MSB */ |
| for (; v >= 0x80; v >>= 7) { |
| *p++ = (char)((v & 0x7f) | 0x80); // Set continuation bit |
| } |
| *p++ = (char)v; // Final byte without continuation bit |
| |
| ctx->p = p; |
| } |
| |
| /* |
| * Macros for generating DWARF structures |
| * |
| * These macros provide a convenient way to write various data types |
| * to the DWARF buffer while automatically advancing the pointer. |
| */ |
| #define DWRF_U8(x) (*p++ = (x)) // Write unsigned 8-bit |
| #define DWRF_I8(x) (*(int8_t*)p = (x), p++) // Write signed 8-bit |
| #define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) // Write unsigned 16-bit |
| #define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) // Write unsigned 32-bit |
| #define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) // Write address |
| #define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) // Write ULEB128 |
| #define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) // Write SLEB128 |
| #define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) // Write string |
| |
| /* Align to specified boundary with NOP instructions */ |
| #define DWRF_ALIGNNOP(s) \ |
| while ((uintptr_t)p & ((s)-1)) { \ |
| *p++ = DWRF_CFA_nop; \ |
| } |
| |
| /* Write a DWARF section with automatic size calculation */ |
| #define DWRF_SECTION(name, stmt) \ |
| { \ |
| uint32_t* szp_##name = (uint32_t*)p; \ |
| p += 4; \ |
| stmt; \ |
| *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ |
| } |
| |
| // ============================================================================= |
| // DWARF EH FRAME GENERATION |
| // ============================================================================= |
| |
| static void elf_init_ehframe_perf(ELFObjectContext* ctx); |
| #if defined(PY_HAVE_JIT_GDB_UNWIND) |
| static void elf_init_ehframe_gdb(ELFObjectContext* ctx); |
| #endif |
| |
| static inline void elf_init_ehframe(ELFObjectContext* ctx, int absolute_addr) { |
| if (absolute_addr) { |
| #if defined(PY_HAVE_JIT_GDB_UNWIND) |
| elf_init_ehframe_gdb(ctx); |
| #else |
| Py_UNREACHABLE(); |
| #endif |
| } |
| else { |
| elf_init_ehframe_perf(ctx); |
| } |
| } |
| |
| size_t |
| _PyJitUnwind_EhFrameSize(int absolute_addr) |
| { |
| /* The .eh_frame we emit is small and bounded; keep a generous buffer. */ |
| uint8_t scratch[512]; |
| _Static_assert(sizeof(scratch) >= 256, |
| "scratch buffer may be too small for elf_init_ehframe"); |
| ELFObjectContext ctx; |
| ctx.code_size = 1; |
| ctx.code_addr = 0; |
| ctx.startp = ctx.p = scratch; |
| ctx.fde_p = NULL; |
| /* Generate once into scratch to learn the required size. */ |
| elf_init_ehframe(&ctx, absolute_addr); |
| ptrdiff_t size = ctx.p - ctx.startp; |
| assert(size <= (ptrdiff_t)sizeof(scratch)); |
| return (size_t)size; |
| } |
| |
| size_t |
| _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, |
| const void *code_addr, size_t code_size, |
| int absolute_addr) |
| { |
| if (buffer == NULL || code_addr == NULL || code_size == 0) { |
| return 0; |
| } |
| /* Generate the frame twice: once to size-check, once to write. */ |
| size_t required = _PyJitUnwind_EhFrameSize(absolute_addr); |
| if (required == 0 || required > buffer_size) { |
| return 0; |
| } |
| ELFObjectContext ctx; |
| ctx.code_size = code_size; |
| ctx.code_addr = (uintptr_t)code_addr; |
| ctx.startp = ctx.p = buffer; |
| ctx.fde_p = NULL; |
| elf_init_ehframe(&ctx, absolute_addr); |
| size_t written = (size_t)(ctx.p - ctx.startp); |
| /* The frame size is independent of code_addr/code_size (fixed-width fields). */ |
| assert(written == required); |
| return written; |
| } |
| |
| /* |
| * Generate a minimal .eh_frame for a single JIT code region. |
| * |
| * The .eh_frame section contains Call Frame Information (CFI) that describes |
| * how to unwind the stack at any point in the code. This is essential for |
| * unwinding through JIT-generated code. |
| * |
| * The generated data contains: |
| * 1. A CIE (Common Information Entry) describing the calling convention. |
| * 2. An FDE (Frame Description Entry) describing how to unwind the JIT frame. |
| * |
| * Two flavors are emitted, dispatched on the absolute_addr flag: |
| * |
| * - absolute_addr == 0 (elf_init_ehframe_perf): PC-relative FDE address |
| * encoding for perf's synthesized DSO layout. The CIE describes the |
| * trampoline's entry state and the FDE walks through the prologue and |
| * epilogue with advance_loc instructions. This matches the pre-existing |
| * perf_jit_trampoline behavior byte-for-byte. |
| * |
| * - absolute_addr == 1 (elf_init_ehframe_gdb): absolute FDE address |
| * encoding for the GDB JIT in-memory ELF. The CIE describes the |
| * steady-state frame layout (CFA = %rbp+16 / x29+16, with saved fp and |
| * return-address column at fixed offsets) and the FDE emits no further |
| * CFI. The same rule applies at every PC in the registered region, |
| * which is correct for executor stencils (they pin the frame pointer |
| * across the region). This is the GDB-side fix; see elf_init_ehframe_gdb |
| * for details. |
| */ |
| static void elf_init_ehframe_perf(ELFObjectContext* ctx) { |
| int fde_ptr_enc = DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4; |
| uint8_t* p = ctx->p; |
| uint8_t* framep = p; // Remember start of frame data |
| |
| /* |
| * DWARF Unwind Table for Trampoline Function |
| * |
| * This section defines DWARF Call Frame Information (CFI) using encoded macros |
| * like `DWRF_U8`, `DWRF_UV`, and `DWRF_SECTION` to describe how the trampoline function |
| * preserves and restores registers. This is used by profiling tools (e.g., `perf`) |
| * and debuggers for stack unwinding in JIT-compiled code. |
| * |
| * ------------------------------------------------- |
| * TO REGENERATE THIS TABLE FROM GCC OBJECTS: |
| * ------------------------------------------------- |
| * |
| * 1. Create a trampoline source file (e.g., `trampoline.c`): |
| * |
| * #include <Python.h> |
| * typedef PyObject* (*py_evaluator)(void*, void*, int); |
| * PyObject* trampoline(void *ts, void *f, int throwflag, py_evaluator evaluator) { |
| * return evaluator(ts, f, throwflag); |
| * } |
| * |
| * 2. Compile to an object file with frame pointer preservation: |
| * |
| * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c |
| * |
| * 3. Extract DWARF unwind info from the object file: |
| * |
| * readelf -w trampoline.o |
| * |
| * Example output from `.eh_frame`: |
| * |
| * 00000000 CIE |
| * Version: 1 |
| * Augmentation: "zR" |
| * Code alignment factor: 4 |
| * Data alignment factor: -8 |
| * Return address column: 30 |
| * DW_CFA_def_cfa: r31 (sp) ofs 0 |
| * |
| * 00000014 FDE cie=00000000 pc=0..14 |
| * DW_CFA_advance_loc: 4 |
| * DW_CFA_def_cfa_offset: 16 |
| * DW_CFA_offset: r29 at cfa-16 |
| * DW_CFA_offset: r30 at cfa-8 |
| * DW_CFA_advance_loc: 12 |
| * DW_CFA_restore: r30 |
| * DW_CFA_restore: r29 |
| * DW_CFA_def_cfa_offset: 0 |
| * |
| * -- These values can be verified by comparing with `readelf -w` or `llvm-dwarfdump --eh-frame`. |
| * |
| * ---------------------------------- |
| * HOW TO TRANSLATE TO DWRF_* MACROS: |
| * ---------------------------------- |
| * |
| * After compiling your trampoline with: |
| * |
| * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c |
| * |
| * run: |
| * |
| * readelf -w trampoline.o |
| * |
| * to inspect the generated `.eh_frame` data. You will see two main components: |
| * |
| * 1. A CIE (Common Information Entry): shared configuration used by all FDEs. |
| * 2. An FDE (Frame Description Entry): function-specific unwind instructions. |
| * |
| * --------------------- |
| * Translating the CIE: |
| * --------------------- |
| * From `readelf -w`, you might see: |
| * |
| * 00000000 0000000000000010 00000000 CIE |
| * Version: 1 |
| * Augmentation: "zR" |
| * Code alignment factor: 4 |
| * Data alignment factor: -8 |
| * Return address column: 30 |
| * Augmentation data: 1b |
| * DW_CFA_def_cfa: r31 (sp) ofs 0 |
| * |
| * Map this to: |
| * |
| * DWRF_SECTION(CIE, |
| * DWRF_U32(0); // CIE ID (always 0 for CIEs) |
| * DWRF_U8(DWRF_CIE_VERSION); // Version: 1 |
| * DWRF_STR("zR"); // Augmentation string "zR" |
| * DWRF_UV(4); // Code alignment factor = 4 |
| * DWRF_SV(-8); // Data alignment factor = -8 |
| * DWRF_U8(DWRF_REG_RA); // Return address register (e.g., x30 = 30) |
| * DWRF_UV(1); // Augmentation data length = 1 |
| * DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // Encoding for FDE pointers |
| * |
| * DWRF_U8(DWRF_CFA_def_cfa); // DW_CFA_def_cfa |
| * DWRF_UV(DWRF_REG_SP); // Register: SP (r31) |
| * DWRF_UV(0); // Offset = 0 |
| * |
| * DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer size boundary |
| * ) |
| * |
| * Notes: |
| * - Use `DWRF_UV` for unsigned LEB128, `DWRF_SV` for signed LEB128. |
| * - `DWRF_REG_RA` and `DWRF_REG_SP` are architecture-defined constants. |
| * |
| * --------------------- |
| * Translating the FDE: |
| * --------------------- |
| * From `readelf -w`: |
| * |
| * 00000014 0000000000000020 00000018 FDE cie=00000000 pc=0000000000000000..0000000000000014 |
| * DW_CFA_advance_loc: 4 |
| * DW_CFA_def_cfa_offset: 16 |
| * DW_CFA_offset: r29 at cfa-16 |
| * DW_CFA_offset: r30 at cfa-8 |
| * DW_CFA_advance_loc: 12 |
| * DW_CFA_restore: r30 |
| * DW_CFA_restore: r29 |
| * DW_CFA_def_cfa_offset: 0 |
| * |
| * Map the FDE header and instructions to: |
| * |
| * DWRF_SECTION(FDE, |
| * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) |
| * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) |
| * DWRF_U32(ctx->code_size); // Code range covered by this FDE |
| * DWRF_U8(0); // Augmentation data length (none) |
| * |
| * DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 unit (1 * 4 = 4 bytes) |
| * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 |
| * DWRF_UV(16); |
| * |
| * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Save x29 (frame pointer) |
| * DWRF_UV(2); // At offset 2 * 8 = 16 bytes |
| * |
| * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Save x30 (return address) |
| * DWRF_UV(1); // At offset 1 * 8 = 8 bytes |
| * |
| * DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance location by 3 units (3 * 4 = 12 bytes) |
| * |
| * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore x30 |
| * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore x29 |
| * |
| * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP |
| * DWRF_UV(0); |
| * ) |
| * |
| * To regenerate: |
| * 1. Get the `code alignment factor`, `data alignment factor`, and `RA column` from the CIE. |
| * 2. Note the range of the function from the FDE's `pc=...` line and map it to the JIT code as |
| * the code is in a different address space every time. |
| * 3. For each `DW_CFA_*` entry, use the corresponding `DWRF_*` macro: |
| * - `DW_CFA_def_cfa_offset` → DWRF_U8(DWRF_CFA_def_cfa_offset), DWRF_UV(value) |
| * - `DW_CFA_offset: rX` → DWRF_U8(DWRF_CFA_offset | reg), DWRF_UV(offset) |
| * - `DW_CFA_restore: rX` → DWRF_U8(DWRF_CFA_offset | reg) // restore is same as reusing offset |
| * - `DW_CFA_advance_loc: N` → DWRF_U8(DWRF_CFA_advance_loc | (N / code_alignment_factor)) |
| * 4. Use `DWRF_REG_FP`, `DWRF_REG_RA`, etc., for register numbers. |
| * 5. Use `sizeof(uintptr_t)` (typically 8) for pointer size calculations and alignment. |
| */ |
| |
| /* |
| * Emit DWARF EH CIE (Common Information Entry) |
| * |
| * The CIE describes the calling conventions and basic unwinding rules |
| * that apply to all functions in this compilation unit. |
| */ |
| DWRF_SECTION(CIE, |
| DWRF_U32(0); // CIE ID (0 indicates this is a CIE) |
| DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) |
| DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) |
| #ifdef __x86_64__ |
| DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) |
| #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) |
| DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) |
| #endif |
| DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) |
| DWRF_U8(DWRF_REG_RA); // Return address register number |
| DWRF_UV(1); // Augmentation data length |
| DWRF_U8(fde_ptr_enc); // FDE pointer encoding |
| |
| /* Initial CFI instructions - describe default calling convention */ |
| #ifdef __x86_64__ |
| /* x86_64 initial CFI state */ |
| DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) |
| DWRF_UV(DWRF_REG_SP); // CFA = SP register |
| DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size |
| DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved |
| DWRF_UV(1); // At offset 1 from CFA |
| #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) |
| /* AArch64 initial CFI state */ |
| DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) |
| DWRF_UV(DWRF_REG_SP); // CFA = SP register |
| DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) |
| // No initial register saves in AArch64 CIE |
| #endif |
| DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary |
| ) |
| |
| /* |
| * Emit DWARF EH FDE (Frame Description Entry) |
| * |
| * The FDE describes unwinding information specific to this function. |
| * It references the CIE and provides function-specific CFI instructions. |
| * |
| * The PC-relative offset is calculated after the entire EH frame is built |
| * to ensure accurate positioning relative to the synthesized DSO layout. |
| */ |
| DWRF_SECTION(FDE, |
| DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) |
| /* |
| * In perf jitdump mode the FDE PC field is encoded PC-relative and |
| * points back to code_start. Record where that field lives so we can |
| * patch in the final offset after the rest of the synthetic DSO |
| * layout is known. |
| */ |
| ctx->fde_p = p; // Remember where PC offset field is located for later calculation |
| DWRF_U32(0); // Placeholder for PC-relative offset (calculated below) |
| DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) |
| DWRF_U8(0); // Augmentation data length (none) |
| |
| /* |
| * Architecture-specific CFI instructions |
| * |
| * These instructions describe how registers are saved and restored |
| * during function calls. Each architecture has different calling |
| * conventions and register usage patterns. |
| */ |
| #ifdef __x86_64__ |
| /* x86_64 calling convention unwinding rules */ |
| # if defined(__CET__) && (__CET__ & 1) |
| DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) |
| # endif |
| DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) |
| DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 |
| DWRF_UV(16); // New offset: SP + 16 |
| DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 |
| DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes |
| DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) |
| DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 |
| DWRF_UV(DWRF_REG_BP); // Use base pointer register |
| DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 |
| DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 |
| DWRF_UV(DWRF_REG_SP); // Use stack pointer register |
| DWRF_UV(8); // New offset: SP + 8 |
| #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) |
| /* AArch64 calling convention unwinding rules */ |
| DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) |
| DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 |
| DWRF_UV(16); // Stack pointer moved by 16 bytes |
| DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved |
| DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) |
| DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved |
| DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) |
| DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) |
| DWRF_U8(DWRF_CFA_def_cfa_register); // CFA = FP (x29) + 16 |
| DWRF_UV(DWRF_REG_FP); |
| DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! |
| DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! |
| DWRF_U8(DWRF_CFA_def_cfa); // CFA = SP + 0 (stack restored) |
| DWRF_UV(DWRF_REG_SP); |
| DWRF_UV(0); |
| |
| #else |
| # error "Unsupported target architecture" |
| #endif |
| |
| DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary |
| ) |
| |
| ctx->p = p; // Update context pointer to end of generated data |
| |
| /* Calculate and update the PC-relative offset in the FDE |
| * |
| * When perf processes the jitdump, it creates a synthesized DSO with this layout: |
| * |
| * Synthesized DSO Memory Layout: |
| * ┌─────────────────────────────────────────────────────────────┐ < code_start |
| * │ Code Section │ |
| * │ (round_up(code_size, 8) bytes) │ |
| * ├─────────────────────────────────────────────────────────────┤ < start of EH frame data |
| * │ EH Frame Data │ |
| * │ ┌─────────────────────────────────────────────────────┐ │ |
| * │ │ CIE data │ │ |
| * │ └─────────────────────────────────────────────────────┘ │ |
| * │ ┌─────────────────────────────────────────────────────┐ │ |
| * │ │ FDE Header: │ │ |
| * │ │ - CIE offset (4 bytes) │ │ |
| * │ │ - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start |
| * │ │ - address range (4 bytes) │ │ (this specific field) |
| * │ │ CFI Instructions... │ │ |
| * │ └─────────────────────────────────────────────────────┘ │ |
| * ├─────────────────────────────────────────────────────────────┤ < reference_point |
| * │ EhFrameHeader │ |
| * │ (navigation metadata) │ |
| * └─────────────────────────────────────────────────────────────┘ |
| * |
| * The PC offset field in the FDE must contain the distance from itself to code_start: |
| * |
| * distance = code_start - fde_pc_field |
| * |
| * Where: |
| * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame |
| * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) |
| * |
| * Therefore: |
| * distance = code_start_location - fde_pc_field_location |
| * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) |
| * = -rounded_code_size - fde_offset_in_frame |
| * = -(round_up(code_size, 8) + fde_offset_in_frame) |
| * |
| * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field. |
| * |
| */ |
| int32_t rounded_code_size = |
| (int32_t)_Py_SIZE_ROUND_UP(ctx->code_size, 8); |
| int32_t fde_offset_in_frame = (int32_t)(ctx->fde_p - framep); |
| *(int32_t *)ctx->fde_p = -(rounded_code_size + fde_offset_in_frame); |
| } |
| |
| /* |
| * Build .eh_frame data for the GDB JIT interface. |
| * |
| * The executor runs inside the frame established by _PyJIT_Entry, but the |
| * synthetic executor FDE collapses that state into a single logical JIT frame |
| * that unwinds directly into _PyEval_*. Executor stencils never touch the |
| * frame pointer - enforced by Tools/jit/_optimizers.py _validate() and |
| * -mframe-pointer=reserved - so the steady-state rule is valid at every PC |
| * and the FDE body is empty. Tools/jit/_targets.py derives the initial CFI |
| * rules from the row active at the executor call in the compiled shim object. |
| */ |
| #if defined(PY_HAVE_JIT_GDB_UNWIND) |
| static void elf_init_ehframe_gdb(ELFObjectContext* ctx) { |
| int fde_ptr_enc = DWRF_EH_PE_absptr; |
| uint8_t* p = ctx->p; |
| uint8_t* framep = p; |
| |
| DWRF_SECTION(CIE, |
| DWRF_U32(0); // CIE ID |
| DWRF_U8(DWRF_CIE_VERSION); |
| DWRF_STR("zR"); // aug data length + FDE ptr encoding follow |
| DWRF_UV(JIT_UNWIND_CODE_ALIGNMENT_FACTOR); |
| DWRF_SV(JIT_UNWIND_DATA_ALIGNMENT_FACTOR); |
| DWRF_U8(JIT_UNWIND_RA_REG); |
| DWRF_UV(1); // Augmentation data length |
| DWRF_U8(fde_ptr_enc); // FDE pointer encoding |
| |
| /* Executor steady-state rule (our invariant, not the compiler's). */ |
| DWRF_U8(DWRF_CFA_def_cfa); |
| DWRF_UV(JIT_UNWIND_CFA_REG); |
| DWRF_UV(JIT_UNWIND_CFA_OFFSET); |
| DWRF_U8(DWRF_CFA_offset | JIT_UNWIND_FP_REG); |
| DWRF_UV(JIT_UNWIND_FP_OFFSET); |
| DWRF_U8(DWRF_CFA_offset | JIT_UNWIND_RA_REG); |
| DWRF_UV(JIT_UNWIND_RA_OFFSET); |
| DWRF_ALIGNNOP(sizeof(uintptr_t)); |
| ) |
| |
| DWRF_SECTION(FDE, |
| DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) |
| DWRF_ADDR(ctx->code_addr); // Absolute code start |
| DWRF_ADDR((uintptr_t)ctx->code_size); // Code range covered |
| DWRF_U8(0); // Augmentation data length (none) |
| DWRF_ALIGNNOP(sizeof(uintptr_t)); |
| ) |
| |
| ctx->p = p; |
| } |
| #endif |
| |
| #if defined(PY_HAVE_JIT_GDB_UNWIND) |
| enum { |
| JIT_NOACTION = 0, |
| JIT_REGISTER_FN = 1, |
| JIT_UNREGISTER_FN = 2, |
| }; |
| |
| struct jit_code_entry { |
| struct jit_code_entry *next; |
| struct jit_code_entry *prev; |
| const char *symfile_addr; |
| uint64_t symfile_size; |
| const void *code_addr; |
| }; |
| |
| struct jit_descriptor { |
| uint32_t version; |
| uint32_t action_flag; |
| struct jit_code_entry *relevant_entry; |
| struct jit_code_entry *first_entry; |
| }; |
| |
| PyMutex _Py_jit_debug_mutex = {0}; |
| |
| Py_EXPORTED_SYMBOL volatile struct jit_descriptor __jit_debug_descriptor = { |
| 1, JIT_NOACTION, NULL, NULL |
| }; |
| |
| Py_EXPORTED_SYMBOL void __attribute__((noinline)) |
| __jit_debug_register_code(void) |
| { |
| /* Keep this call visible to debuggers and not optimized away. */ |
| (void)__jit_debug_descriptor.action_flag; |
| #if defined(__GNUC__) || defined(__clang__) |
| __asm__ __volatile__("" ::: "memory"); |
| #endif |
| } |
| |
| static uint16_t |
| gdb_jit_machine_id(void) |
| { |
| /* Map the current target to ELF e_machine; return 0 to skip registration. */ |
| #if defined(__x86_64__) || defined(_M_X64) |
| return EM_X86_64; |
| #elif defined(__aarch64__) && !defined(__ILP32__) |
| return EM_AARCH64; |
| #else |
| return 0; |
| #endif |
| } |
| |
| static struct jit_code_entry * |
| gdb_jit_register_code( |
| const void *code_addr, |
| size_t code_size, |
| const char *symname, |
| const uint8_t *eh_frame, |
| size_t eh_frame_size |
| ) |
| { |
| /* |
| * Build a minimal in-memory ELF for GDB's JIT interface and link it into |
| * __jit_debug_descriptor so debuggers can resolve JIT code. |
| */ |
| if (code_addr == NULL || code_size == 0 || symname == NULL) { |
| return NULL; |
| } |
| |
| const uint16_t machine = gdb_jit_machine_id(); |
| if (machine == 0) { |
| return NULL; |
| } |
| |
| enum { |
| SH_NULL = 0, |
| SH_TEXT, |
| SH_EH_FRAME, |
| SH_SHSTRTAB, |
| SH_STRTAB, |
| SH_SYMTAB, |
| SH_NUM, |
| }; |
| static const char shstrtab[] = |
| "\0.text\0.eh_frame\0.shstrtab\0.strtab\0.symtab"; |
| _Static_assert(sizeof(shstrtab) == |
| 1 + sizeof(".text") + sizeof(".eh_frame") + |
| sizeof(".shstrtab") + sizeof(".strtab") + sizeof(".symtab"), |
| "shstrtab size mismatch"); |
| const size_t shstrtab_size = sizeof(shstrtab); |
| const size_t sh_text = 1; |
| const size_t sh_eh_frame = sh_text + sizeof(".text"); |
| const size_t sh_shstrtab = sh_eh_frame + sizeof(".eh_frame"); |
| const size_t sh_strtab = sh_shstrtab + sizeof(".shstrtab"); |
| const size_t sh_symtab = sh_strtab + sizeof(".strtab"); |
| const size_t text_size = code_size; |
| const size_t text_padded = _Py_SIZE_ROUND_UP(text_size, 8); |
| const size_t strtab_size = 1 + strlen(symname) + 1; |
| const size_t symtab_size = 3 * sizeof(Elf64_Sym); |
| |
| size_t offset = sizeof(Elf64_Ehdr); |
| offset = _Py_SIZE_ROUND_UP(offset, 16); |
| const size_t text_off = offset; |
| const size_t eh_off = text_off + text_padded; |
| offset = eh_off + eh_frame_size; |
| const size_t shstr_off = offset; |
| offset += shstrtab_size; |
| const size_t str_off = offset; |
| offset += strtab_size; |
| /* Elf64_Sym requires 8-byte alignment for st_value/st_size. */ |
| offset = _Py_SIZE_ROUND_UP(offset, 8); |
| const size_t sym_off = offset; |
| offset += symtab_size; |
| offset = _Py_SIZE_ROUND_UP(offset, sizeof(Elf64_Shdr)); |
| const size_t sh_off = offset; |
| |
| const size_t shnum = SH_NUM; |
| const size_t total_size = sh_off + shnum * sizeof(Elf64_Shdr); |
| uint8_t *buf = (uint8_t *)PyMem_RawMalloc(total_size); |
| if (buf == NULL) { |
| return NULL; |
| } |
| memset(buf, 0, total_size); |
| |
| Elf64_Ehdr *ehdr = (Elf64_Ehdr *)buf; |
| memcpy(ehdr->e_ident, ELFMAG, SELFMAG); |
| ehdr->e_ident[EI_CLASS] = ELFCLASS64; |
| ehdr->e_ident[EI_DATA] = ELFDATA2LSB; |
| ehdr->e_ident[EI_VERSION] = EV_CURRENT; |
| ehdr->e_ident[EI_OSABI] = ELFOSABI_NONE; |
| ehdr->e_type = ET_DYN; |
| ehdr->e_machine = machine; |
| ehdr->e_version = EV_CURRENT; |
| ehdr->e_entry = 0; |
| ehdr->e_phoff = 0; |
| ehdr->e_shoff = sh_off; |
| ehdr->e_ehsize = sizeof(Elf64_Ehdr); |
| ehdr->e_shentsize = sizeof(Elf64_Shdr); |
| ehdr->e_shnum = shnum; |
| ehdr->e_shstrndx = SH_SHSTRTAB; |
| |
| memcpy(buf + text_off, code_addr, text_size); |
| memcpy(buf + eh_off, eh_frame, eh_frame_size); |
| |
| char *shstr = (char *)(buf + shstr_off); |
| memcpy(shstr, shstrtab, shstrtab_size); |
| |
| char *strtab = (char *)(buf + str_off); |
| strtab[0] = '\0'; |
| memcpy(strtab + 1, symname, strlen(symname)); |
| strtab[strtab_size - 1] = '\0'; |
| |
| Elf64_Sym *syms = (Elf64_Sym *)(buf + sym_off); |
| memset(syms, 0, symtab_size); |
| /* Section symbol for .text (local) */ |
| syms[1].st_info = ELF64_ST_INFO(STB_LOCAL, STT_SECTION); |
| syms[1].st_shndx = 1; |
| /* Function symbol */ |
| syms[2].st_name = 1; |
| syms[2].st_info = ELF64_ST_INFO(STB_GLOBAL, STT_FUNC); |
| syms[2].st_other = STV_DEFAULT; |
| syms[2].st_shndx = 1; |
| /* For ET_DYN/ET_EXEC, st_value is the absolute virtual address. */ |
| syms[2].st_value = (Elf64_Addr)(uintptr_t)code_addr; |
| syms[2].st_size = code_size; |
| |
| Elf64_Shdr *shdrs = (Elf64_Shdr *)(buf + sh_off); |
| memset(shdrs, 0, shnum * sizeof(Elf64_Shdr)); |
| |
| shdrs[SH_TEXT].sh_name = sh_text; |
| shdrs[SH_TEXT].sh_type = SHT_PROGBITS; |
| shdrs[SH_TEXT].sh_flags = SHF_ALLOC | SHF_EXECINSTR; |
| shdrs[SH_TEXT].sh_addr = (Elf64_Addr)(uintptr_t)code_addr; |
| shdrs[SH_TEXT].sh_offset = text_off; |
| shdrs[SH_TEXT].sh_size = text_size; |
| shdrs[SH_TEXT].sh_addralign = 16; |
| |
| shdrs[SH_EH_FRAME].sh_name = sh_eh_frame; |
| shdrs[SH_EH_FRAME].sh_type = SHT_PROGBITS; |
| shdrs[SH_EH_FRAME].sh_flags = SHF_ALLOC; |
| shdrs[SH_EH_FRAME].sh_addr = |
| (Elf64_Addr)((uintptr_t)code_addr + text_padded); |
| shdrs[SH_EH_FRAME].sh_offset = eh_off; |
| shdrs[SH_EH_FRAME].sh_size = eh_frame_size; |
| shdrs[SH_EH_FRAME].sh_addralign = 8; |
| |
| shdrs[SH_SHSTRTAB].sh_name = sh_shstrtab; |
| shdrs[SH_SHSTRTAB].sh_type = SHT_STRTAB; |
| shdrs[SH_SHSTRTAB].sh_offset = shstr_off; |
| shdrs[SH_SHSTRTAB].sh_size = shstrtab_size; |
| shdrs[SH_SHSTRTAB].sh_addralign = 1; |
| |
| shdrs[SH_STRTAB].sh_name = sh_strtab; |
| shdrs[SH_STRTAB].sh_type = SHT_STRTAB; |
| shdrs[SH_STRTAB].sh_offset = str_off; |
| shdrs[SH_STRTAB].sh_size = strtab_size; |
| shdrs[SH_STRTAB].sh_addralign = 1; |
| |
| shdrs[SH_SYMTAB].sh_name = sh_symtab; |
| shdrs[SH_SYMTAB].sh_type = SHT_SYMTAB; |
| shdrs[SH_SYMTAB].sh_offset = sym_off; |
| shdrs[SH_SYMTAB].sh_size = symtab_size; |
| shdrs[SH_SYMTAB].sh_link = SH_STRTAB; |
| shdrs[SH_SYMTAB].sh_info = 2; |
| shdrs[SH_SYMTAB].sh_addralign = 8; |
| shdrs[SH_SYMTAB].sh_entsize = sizeof(Elf64_Sym); |
| |
| struct jit_code_entry *entry = PyMem_RawMalloc(sizeof(*entry)); |
| if (entry == NULL) { |
| PyMem_RawFree(buf); |
| return NULL; |
| } |
| entry->symfile_addr = (const char *)buf; |
| entry->symfile_size = total_size; |
| entry->code_addr = code_addr; |
| |
| PyMutex_Lock(&_Py_jit_debug_mutex); |
| entry->prev = NULL; |
| entry->next = __jit_debug_descriptor.first_entry; |
| if (entry->next != NULL) { |
| entry->next->prev = entry; |
| } |
| __jit_debug_descriptor.first_entry = entry; |
| __jit_debug_descriptor.relevant_entry = entry; |
| __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; |
| __jit_debug_register_code(); |
| __jit_debug_descriptor.action_flag = JIT_NOACTION; |
| __jit_debug_descriptor.relevant_entry = NULL; |
| PyMutex_Unlock(&_Py_jit_debug_mutex); |
| return entry; |
| } |
| #endif // defined(PY_HAVE_JIT_GDB_UNWIND) |
| |
| void * |
| _PyJitUnwind_GdbRegisterCode(const void *code_addr, |
| size_t code_size, |
| const char *entry, |
| const char *filename) |
| { |
| #if defined(PY_HAVE_JIT_GDB_UNWIND) |
| /* GDB expects a stable symbol name and absolute addresses in .eh_frame. */ |
| if (entry == NULL) { |
| entry = ""; |
| } |
| if (filename == NULL) { |
| filename = ""; |
| } |
| size_t name_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; |
| char *name = (char *)PyMem_RawMalloc(name_size); |
| if (name == NULL) { |
| return NULL; |
| } |
| snprintf(name, name_size, "py::%s:%s", entry, filename); |
| |
| uint8_t buffer[1024]; |
| size_t eh_frame_size = _PyJitUnwind_BuildEhFrame( |
| buffer, sizeof(buffer), code_addr, code_size, 1); |
| if (eh_frame_size == 0) { |
| PyMem_RawFree(name); |
| return NULL; |
| } |
| |
| void *handle = gdb_jit_register_code(code_addr, code_size, name, |
| buffer, eh_frame_size); |
| PyMem_RawFree(name); |
| return handle; |
| #else |
| (void)code_addr; |
| (void)code_size; |
| (void)entry; |
| (void)filename; |
| return NULL; |
| #endif |
| } |
| |
| void |
| _PyJitUnwind_GdbUnregisterCode(void *handle) |
| { |
| #if defined(PY_HAVE_JIT_GDB_UNWIND) |
| struct jit_code_entry *entry = (struct jit_code_entry *)handle; |
| if (entry == NULL) { |
| return; |
| } |
| |
| PyMutex_Lock(&_Py_jit_debug_mutex); |
| if (entry->prev != NULL) { |
| entry->prev->next = entry->next; |
| } |
| else { |
| __jit_debug_descriptor.first_entry = entry->next; |
| } |
| if (entry->next != NULL) { |
| entry->next->prev = entry->prev; |
| } |
| |
| __jit_debug_descriptor.relevant_entry = entry; |
| __jit_debug_descriptor.action_flag = JIT_UNREGISTER_FN; |
| __jit_debug_register_code(); |
| __jit_debug_descriptor.action_flag = JIT_NOACTION; |
| __jit_debug_descriptor.relevant_entry = NULL; |
| |
| PyMutex_Unlock(&_Py_jit_debug_mutex); |
| |
| PyMem_RawFree((void *)entry->symfile_addr); |
| PyMem_RawFree(entry); |
| #else |
| (void)handle; |
| #endif |
| } |
| |
| #if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) |
| void * |
| _PyJitUnwind_GnuBacktraceRegisterCode(const void *code_addr, size_t code_size) |
| { |
| if (code_addr == NULL || code_size == 0) { |
| return NULL; |
| } |
| |
| size_t eh_frame_size = _PyJitUnwind_EhFrameSize(1); |
| if (eh_frame_size == 0) { |
| return NULL; |
| } |
| size_t total_size = eh_frame_size + sizeof(uint32_t); |
| if (total_size < eh_frame_size) { |
| return NULL; |
| } |
| |
| /* |
| * libgcc's __register_frame walks a .eh_frame section until it finds a |
| * zero-length terminator entry, so keep an extra zeroed word after the |
| * generated CIE/FDE pair. |
| * |
| * See GCC's libgcc/unwind-dw2-fde.c (__register_frame) and |
| * libgcc/unwind-dw2-fde.h (last_fde/next_fde): |
| * https://github.com/gcc-mirror/gcc/blob/master/libgcc/unwind-dw2-fde.c |
| * https://github.com/gcc-mirror/gcc/blob/master/libgcc/unwind-dw2-fde.h |
| */ |
| uint8_t *eh_frame = PyMem_RawCalloc(1, total_size); |
| if (eh_frame == NULL) { |
| return NULL; |
| } |
| if (_PyJitUnwind_BuildEhFrame( |
| eh_frame, eh_frame_size, code_addr, code_size, 1) == 0) { |
| PyMem_RawFree(eh_frame); |
| return NULL; |
| } |
| |
| __register_frame(eh_frame); |
| return eh_frame; |
| } |
| |
| void |
| _PyJitUnwind_GnuBacktraceUnregisterCode(void *handle) |
| { |
| if (handle == NULL) { |
| return; |
| } |
| __deregister_frame(handle); |
| PyMem_RawFree(handle); |
| } |
| #endif // defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) |
| |
| #endif // JIT unwind support |
