| /****************************************************************************** | |
| * Python Remote Debugging Module - Binary Writer Implementation | |
| * | |
| * High-performance binary file writer for profiling data with optional zstd | |
| * compression. | |
| ******************************************************************************/ | |
| #ifndef Py_BUILD_CORE_MODULE | |
| # define Py_BUILD_CORE_MODULE | |
| #endif | |
| #include "binary_io.h" | |
| #include "_remote_debugging.h" | |
| #include "pycore_bitutils.h" // _Py_bit_length(), _Py_popcount32() | |
| #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE | |
| #include <math.h> | |
| #include <string.h> | |
| #ifdef HAVE_ZSTD | |
| #include <zstd.h> | |
| #endif | |
| #ifdef _Py_MEMORY_SANITIZER | |
| # include <sanitizer/msan_interface.h> | |
| #endif | |
| /* ============================================================================ | |
| * CONSTANTS FOR BINARY FORMAT SIZES | |
| * ============================================================================ */ | |
| #define MAX_VARINT_SIZE 10 /* Maximum bytes for a varint64 */ | |
| #define MAX_VARINT_SIZE_U32 5 /* Maximum bytes for a varint32 */ | |
| #define MAX_BLOCK_HEADER_SIZE \ | |
| (BLOCK_MAGIC_SIZE + MAX_VARINT_SIZE_U32 + MAX_VARINT_SIZE + \ | |
| 3 * sizeof(uint8_t) + \ | |
| COLUMN_COUNT * (2 * MAX_VARINT_SIZE_U32 + sizeof(uint8_t))) | |
| #define ZSTD_MIN_SIZE 32 | |
| #define STACK_REUSE_MIN_FRAMES 3 | |
| #define MAX_SAMPLE_SIZE \ | |
| (STREAM_ENTRY_SIZE + 5 * MAX_VARINT_SIZE + sizeof(uint8_t) + \ | |
| MAX_STACK_DEPTH * sizeof(uint32_t) + sizeof(Repeat)) | |
| #define MAX_BITMAP_PADDING (CHAR_BIT - 1) | |
| #define BLOCK_FLUSH_SIZE \ | |
| (MAX_BLOCK_SIZE - MAX_SAMPLE_SIZE - MAX_BITMAP_PADDING) | |
| static int | |
| buffer_append(ByteBuffer *buffer, const void *data, size_t size) | |
| { | |
| if (buffer->size > MAX_BLOCK_SIZE || | |
| size > MAX_BLOCK_SIZE - buffer->size) { | |
| PyErr_SetString(PyExc_RuntimeError, | |
| "sample block buffer exceeds the writer safety limit"); | |
| return -1; | |
| } | |
| if (buffer_reserve(buffer, buffer->size + size) < 0) { | |
| return -1; | |
| } | |
| if (size != 0) { | |
| memcpy(buffer->data + buffer->size, data, size); | |
| } | |
| buffer->size += size; | |
| return 0; | |
| } | |
| static int | |
| buffer_append_varint(ByteBuffer *buffer, uint64_t value) | |
| { | |
| uint8_t bytes[MAX_VARINT_SIZE]; | |
| size_t size = encode_varint_u64(bytes, value); | |
| return buffer_append(buffer, bytes, size); | |
| } | |
| static int | |
| column_write(SampleBlock *block, unsigned column, const void *data, | |
| size_t size) | |
| { | |
| if (buffer_append(&block->columns[column], data, size) < 0) { | |
| return -1; | |
| } | |
| block->size += size; | |
| return 0; | |
| } | |
| static int | |
| column_write_varint(SampleBlock *block, unsigned column, uint64_t value) | |
| { | |
| uint8_t bytes[MAX_VARINT_SIZE]; | |
| size_t size = encode_varint_u64(bytes, value); | |
| return column_write(block, column, bytes, size); | |
| } | |
| /* Helper macro: convert PyLong to int32, using default_val if conversion fails */ | |
| #define PYLONG_TO_INT32_OR_DEFAULT(obj, var, default_val) \ | |
| do { \ | |
| (var) = (int32_t)PyLong_AsLong(obj); \ | |
| if (UNLIKELY(PyErr_Occurred() != NULL)) { \ | |
| PyErr_Clear(); \ | |
| (var) = (default_val); \ | |
| } \ | |
| } while (0) | |
| /* ============================================================================ | |
| * WRITER-SPECIFIC UTILITY HELPERS | |
| * ============================================================================ */ | |
| /* Grow two parallel arrays together (e.g., strings and string_lengths). | |
| * Returns 0 on success, -1 on error (sets PyErr). | |
| * On error, original arrays are preserved (truly atomic update). */ | |
| static inline int | |
| grow_parallel_arrays(void **array1, void **array2, size_t *capacity, | |
| size_t elem_size1, size_t elem_size2) | |
| { | |
| size_t old_cap = *capacity; | |
| if (old_cap > SIZE_MAX / 2) { | |
| PyErr_SetString(PyExc_OverflowError, "Array capacity overflow"); | |
| return -1; | |
| } | |
| size_t new_cap = old_cap * 2; | |
| if (new_cap > SIZE_MAX / elem_size1 || new_cap > SIZE_MAX / elem_size2) { | |
| PyErr_SetString(PyExc_OverflowError, "Array allocation size overflow"); | |
| return -1; | |
| } | |
| size_t new_size1 = new_cap * elem_size1; | |
| size_t new_size2 = new_cap * elem_size2; | |
| size_t old_size1 = old_cap * elem_size1; | |
| size_t old_size2 = old_cap * elem_size2; | |
| /* Allocate fresh memory blocks (not realloc) to ensure atomicity. | |
| * If either allocation fails, original arrays are completely unchanged. */ | |
| void *new_array1 = PyMem_Malloc(new_size1); | |
| if (!new_array1) { | |
| PyErr_NoMemory(); | |
| return -1; | |
| } | |
| void *new_array2 = PyMem_Malloc(new_size2); | |
| if (!new_array2) { | |
| /* Second allocation failed - free first and return with no state change */ | |
| PyMem_Free(new_array1); | |
| PyErr_NoMemory(); | |
| return -1; | |
| } | |
| /* Both allocations succeeded - copy data and update pointers atomically */ | |
| memcpy(new_array1, *array1, old_size1); | |
| memcpy(new_array2, *array2, old_size2); | |
| PyMem_Free(*array1); | |
| PyMem_Free(*array2); | |
| *array1 = new_array1; | |
| *array2 = new_array2; | |
| *capacity = new_cap; | |
| return 0; | |
| } | |
| /* Checked fwrite with GIL release - returns 0 on success, -1 on error (sets PyErr). | |
| * This version releases the GIL during the write operation to allow other Python | |
| * threads to run during potentially blocking I/O. */ | |
| static inline int | |
| fwrite_checked_allow_threads(const void *data, size_t size, FILE *fp) | |
| { | |
| size_t written; | |
| Py_BEGIN_ALLOW_THREADS | |
| written = fwrite(data, 1, size, fp); | |
| Py_END_ALLOW_THREADS | |
| if (written != size) { | |
| int err = errno; | |
| if (ferror(fp) && err != 0) { | |
| errno = err; | |
| PyErr_SetFromErrno(PyExc_IOError); | |
| } | |
| else { | |
| PyErr_Format(PyExc_IOError, | |
| "short write: wrote %zu of %zu bytes", written, size); | |
| } | |
| return -1; | |
| } | |
| return 0; | |
| } | |
| /* ============================================================================ | |
| * UTILITY FUNCTIONS | |
| * ============================================================================ */ | |
| int | |
| binary_io_zstd_available(void) | |
| { | |
| #ifdef HAVE_ZSTD | |
| return 1; | |
| #else | |
| return 0; | |
| #endif | |
| } | |
| int | |
| binary_io_get_best_compression(void) | |
| { | |
| #ifdef HAVE_ZSTD | |
| return COMPRESSION_ZSTD; | |
| #else | |
| return COMPRESSION_NONE; | |
| #endif | |
| } | |
| /* ============================================================================ | |
| * BINARY WRITER IMPLEMENTATION | |
| * ============================================================================ */ | |
| static int | |
| writer_init_zstd(BinaryWriter *writer) | |
| { | |
| #ifdef HAVE_ZSTD | |
| writer->zstd.cctx = ZSTD_createCCtx(); | |
| if (!writer->zstd.cctx) { | |
| PyErr_SetString(PyExc_MemoryError, "Failed to create zstd compression context"); | |
| return -1; | |
| } | |
| /* Compression level 3: good ratio without slowing down each block flush */ | |
| size_t result = ZSTD_CCtx_setParameter(writer->zstd.cctx, | |
| ZSTD_c_compressionLevel, 3); | |
| if (ZSTD_isError(result)) { | |
| PyErr_Format(PyExc_RuntimeError, "Failed to set zstd compression level: %s", | |
| ZSTD_getErrorName(result)); | |
| ZSTD_freeCCtx(writer->zstd.cctx); | |
| writer->zstd.cctx = NULL; | |
| return -1; | |
| } | |
| return 0; | |
| #else | |
| PyErr_SetString(PyExc_RuntimeError, | |
| "zstd compression requested but not available (HAVE_ZSTD not defined)"); | |
| return -1; | |
| #endif | |
| } | |
| /* ============================================================================ | |
| * HASH TABLE SUPPORT FUNCTIONS (using _Py_hashtable) | |
| * ============================================================================ */ | |
| /* Hash function for Python strings - uses Python's cached hash */ | |
| static Py_uhash_t | |
| string_hash_func(const void *key) | |
| { | |
| PyObject *str = (PyObject *)key; | |
| Py_hash_t hash = PyObject_Hash(str); | |
| if (hash == -1) { | |
| PyErr_Clear(); | |
| return 0; | |
| } | |
| return (Py_uhash_t)hash; | |
| } | |
| static int | |
| string_compare_func(const void *key1, const void *key2) | |
| { | |
| PyObject *str1 = (PyObject *)key1; | |
| PyObject *str2 = (PyObject *)key2; | |
| if (str1 == str2) { | |
| return 1; | |
| } | |
| int result = PyObject_RichCompareBool(str1, str2, Py_EQ); | |
| if (result == -1) { | |
| PyErr_Clear(); | |
| return 0; | |
| } | |
| return result; | |
| } | |
| static void | |
| string_key_destroy(void *key) | |
| { | |
| Py_XDECREF((PyObject *)key); | |
| } | |
| static Py_uhash_t | |
| frame_key_hash_func(const void *key) | |
| { | |
| const FrameKey *fk = (const FrameKey *)key; | |
| /* FNV-1a style hash combining all fields */ | |
| Py_uhash_t hash = 2166136261u; | |
| hash ^= fk->filename_idx; | |
| hash *= 16777619u; | |
| hash ^= fk->funcname_idx; | |
| hash *= 16777619u; | |
| hash ^= (uint32_t)fk->lineno; | |
| hash *= 16777619u; | |
| hash ^= (uint32_t)fk->end_lineno; | |
| hash *= 16777619u; | |
| hash ^= (uint32_t)fk->column; | |
| hash *= 16777619u; | |
| hash ^= (uint32_t)fk->end_column; | |
| hash *= 16777619u; | |
| hash ^= fk->opcode; | |
| hash *= 16777619u; | |
| return hash; | |
| } | |
| static int | |
| frame_key_compare_func(const void *key1, const void *key2) | |
| { | |
| const FrameKey *fk1 = (const FrameKey *)key1; | |
| const FrameKey *fk2 = (const FrameKey *)key2; | |
| return (fk1->filename_idx == fk2->filename_idx && | |
| fk1->funcname_idx == fk2->funcname_idx && | |
| fk1->lineno == fk2->lineno && | |
| fk1->end_lineno == fk2->end_lineno && | |
| fk1->column == fk2->column && | |
| fk1->end_column == fk2->end_column && | |
| fk1->opcode == fk2->opcode); | |
| } | |
| static void | |
| frame_key_destroy(void *key) | |
| { | |
| PyMem_Free(key); | |
| } | |
| static inline int | |
| writer_intern_string(BinaryWriter *writer, PyObject *string, uint32_t *index) | |
| { | |
| void *existing = _Py_hashtable_get(writer->string_hash, string); | |
| if (existing != NULL) { | |
| *index = (uint32_t)(uintptr_t)existing - 1; /* index+1 stored to distinguish from NULL */ | |
| return 0; | |
| } | |
| if (writer->string_count >= UINT32_MAX) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| PyErr_SetString(PyExc_OverflowError, | |
| "too many strings for binary format"); | |
| return -1; | |
| } | |
| if (writer->string_count >= writer->string_capacity) { | |
| if (grow_parallel_arrays((void **)&writer->strings, | |
| (void **)&writer->string_lengths, | |
| &writer->string_capacity, | |
| sizeof(char *), sizeof(size_t)) < 0) { | |
| if (PyErr_ExceptionMatches(PyExc_OverflowError)) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| } | |
| return -1; | |
| } | |
| } | |
| Py_ssize_t str_len; | |
| const char *str_data = PyUnicode_AsUTF8AndSize(string, &str_len); | |
| if (!str_data) { | |
| return -1; | |
| } | |
| if ((uintmax_t)str_len > UINT32_MAX) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| PyErr_Format(PyExc_OverflowError, | |
| "string length %zd exceeds binary format maximum %u", | |
| str_len, UINT32_MAX); | |
| return -1; | |
| } | |
| char *str_copy = PyMem_Malloc(str_len + 1); | |
| if (!str_copy) { | |
| PyErr_NoMemory(); | |
| return -1; | |
| } | |
| memcpy(str_copy, str_data, str_len + 1); | |
| *index = (uint32_t)writer->string_count; | |
| /* Add to hash table FIRST to ensure atomic rollback on failure */ | |
| Py_INCREF(string); | |
| if (_Py_hashtable_set(writer->string_hash, string, (void *)(uintptr_t)(*index + 1)) < 0) { | |
| Py_DECREF(string); | |
| PyMem_Free(str_copy); | |
| PyErr_NoMemory(); | |
| return -1; | |
| } | |
| writer->strings[writer->string_count] = str_copy; | |
| writer->string_lengths[writer->string_count] = str_len; | |
| writer->string_count++; | |
| return 0; | |
| } | |
| static inline int | |
| writer_intern_frame(BinaryWriter *writer, const FrameEntry *entry, uint32_t *index) | |
| { | |
| FrameKey lookup_key = { | |
| entry->filename_idx, entry->funcname_idx, | |
| entry->lineno, entry->end_lineno, | |
| entry->column, entry->end_column, | |
| entry->opcode | |
| }; | |
| void *existing = _Py_hashtable_get(writer->frame_hash, &lookup_key); | |
| if (existing != NULL) { | |
| *index = (uint32_t)(uintptr_t)existing - 1; /* index+1 stored to distinguish from NULL */ | |
| return 0; | |
| } | |
| if (writer->frame_count >= UINT32_MAX) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| PyErr_SetString(PyExc_OverflowError, | |
| "too many frames for binary format"); | |
| return -1; | |
| } | |
| if (GROW_ARRAY(writer->frame_entries, writer->frame_count, | |
| writer->frame_capacity, FrameEntry) < 0) { | |
| if (PyErr_ExceptionMatches(PyExc_OverflowError)) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| } | |
| return -1; | |
| } | |
| FrameKey *key = PyMem_Malloc(sizeof(FrameKey)); | |
| if (!key) { | |
| PyErr_NoMemory(); | |
| return -1; | |
| } | |
| *key = lookup_key; | |
| *index = (uint32_t)writer->frame_count; | |
| writer->frame_entries[writer->frame_count] = *entry; | |
| if (_Py_hashtable_set(writer->frame_hash, key, (void *)(uintptr_t)(*index + 1)) < 0) { | |
| PyMem_Free(key); | |
| PyErr_NoMemory(); | |
| return -1; | |
| } | |
| writer->frame_count++; | |
| return 0; | |
| } | |
| /* Get or create the entry for a (thread, interpreter) pair. */ | |
| static ThreadEntry * | |
| writer_get_or_create_thread_entry(BinaryWriter *writer, uint64_t thread_id, | |
| uint32_t interpreter_id) | |
| { | |
| /* Linear search is OK for small number of threads. | |
| * Key is (thread_id, interpreter_id) since same thread_id can exist in different interpreters. */ | |
| for (size_t i = 0; i < writer->thread_count; i++) { | |
| if (writer->thread_entries[i].thread_id == thread_id && | |
| writer->thread_entries[i].interpreter_id == interpreter_id) { | |
| return &writer->thread_entries[i]; | |
| } | |
| } | |
| if (writer->thread_count >= UINT32_MAX) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| PyErr_SetString(PyExc_OverflowError, | |
| "too many threads for binary format"); | |
| return NULL; | |
| } | |
| if (GROW_ARRAY(writer->thread_entries, writer->thread_count, | |
| writer->thread_capacity, ThreadEntry) < 0) { | |
| if (PyErr_ExceptionMatches(PyExc_OverflowError)) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| } | |
| return NULL; | |
| } | |
| ThreadEntry *entry = &writer->thread_entries[writer->thread_count]; | |
| memset(entry, 0, sizeof(ThreadEntry)); | |
| entry->thread_id = thread_id; | |
| entry->interpreter_id = interpreter_id; | |
| entry->block_stream_index = UINT32_MAX; | |
| writer->thread_count++; | |
| return entry; | |
| } | |
| static int | |
| encode_column(BinaryWriter *writer, const ByteBuffer *decoded, | |
| ColumnDescriptor *descriptor) | |
| { | |
| SampleBlock *block = writer->block; | |
| const uint8_t *data = decoded->data; | |
| size_t size = decoded->size; | |
| uint8_t encoding = COMPRESSION_NONE; | |
| if (size > 1 && memcmp(data, data + 1, size - 1) == 0) { | |
| encoding = COLUMN_CONSTANT; | |
| size = 1; | |
| } | |
| #ifdef HAVE_ZSTD | |
| if (encoding == COMPRESSION_NONE && size >= ZSTD_MIN_SIZE && | |
| writer->compression_type == COMPRESSION_ZSTD) { | |
| size_t compress_bound = ZSTD_compressBound(size); | |
| if (buffer_reserve(&block->compressed, compress_bound) < 0) { | |
| return -1; | |
| } | |
| size_t compressed_size = ZSTD_compress2( | |
| writer->zstd.cctx, block->compressed.data, compress_bound, | |
| data, size); | |
| if (ZSTD_isError(compressed_size)) { | |
| PyErr_Format(PyExc_IOError, "zstd compression error: %s", | |
| ZSTD_getErrorName(compressed_size)); | |
| return -1; | |
| } | |
| _Py_MSAN_UNPOISON(block->compressed.data, compressed_size); | |
| uint8_t stored_size[MAX_VARINT_SIZE]; | |
| if (compressed_size + encode_varint_u64(stored_size, compressed_size) | |
| < size) { | |
| data = block->compressed.data; | |
| size = compressed_size; | |
| encoding = COMPRESSION_ZSTD; | |
| } | |
| } | |
| #endif | |
| descriptor->decoded_size = (uint32_t)decoded->size; | |
| descriptor->stored_size = (uint32_t)size; | |
| descriptor->encoding = encoding; | |
| if (buffer_reserve(&block->output, block->output.size + size) < 0) { | |
| return -1; | |
| } | |
| if (size != 0) { | |
| memcpy(block->output.data + block->output.size, data, size); | |
| } | |
| block->output.size += size; | |
| return 0; | |
| } | |
| static ByteBuffer * | |
| transform_statuses(SampleBlock *block, uint8_t bitmap_flags) | |
| { | |
| size_t plane_size = status_bitmap_size(block->sample_count); | |
| unsigned plane_count = (unsigned)_Py_popcount32(bitmap_flags); | |
| if (buffer_reserve(&block->transformed, | |
| plane_size * plane_count) < 0) { | |
| return NULL; | |
| } | |
| const uint8_t *statuses = | |
| block->columns[COLUMN_STATUS_BITMAPS].data; | |
| block->transformed.size = 0; | |
| for (unsigned bit = 0; bit < CHAR_BIT; bit++) { | |
| if (!(bitmap_flags & (1U << bit))) { | |
| continue; | |
| } | |
| uint8_t *plane = block->transformed.data + block->transformed.size; | |
| memset(plane, 0, plane_size); | |
| for (uint32_t sample = 0; sample < block->sample_count; sample++) { | |
| plane[sample / CHAR_BIT] |= ((statuses[sample] >> bit) & 1U) | |
| << (sample % CHAR_BIT); | |
| } | |
| block->transformed.size += plane_size; | |
| } | |
| return &block->transformed; | |
| } | |
| static ByteBuffer * | |
| transform_frame_ids(SampleBlock *block, unsigned width) | |
| { | |
| ByteBuffer *frames = &block->columns[COLUMN_FRAME_IDS]; | |
| size_t frame_count = frames->size / sizeof(uint32_t); | |
| if (buffer_reserve(&block->transformed, | |
| frame_count * width) < 0) { | |
| return NULL; | |
| } | |
| const uint32_t *frame_ids = (const uint32_t *)frames->data; | |
| for (unsigned byte = 0; frame_count && byte < width; byte++) { | |
| uint8_t *plane = | |
| block->transformed.data + (size_t)byte * frame_count; | |
| for (uint32_t frame = 0; frame < frame_count; frame++) { | |
| plane[frame] = | |
| (uint8_t)(frame_ids[frame] >> (byte * CHAR_BIT)); | |
| } | |
| } | |
| block->transformed.size = frame_count * width; | |
| return &block->transformed; | |
| } | |
| static ByteBuffer * | |
| transform_stack_instructions(SampleBlock *block) | |
| { | |
| ByteBuffer *instructions = | |
| &block->columns[COLUMN_STACK_INSTRUCTIONS]; | |
| size_t repeat_count = block->repeats.size / sizeof(Repeat); | |
| if (repeat_count == 0) { | |
| return instructions; | |
| } | |
| const Repeat *repeats = (const Repeat *)block->repeats.data; | |
| size_t copied_offset = 0; | |
| block->transformed.size = 0; | |
| for (size_t i = 0; i < repeat_count; i++) { | |
| if (buffer_append(&block->transformed, | |
| instructions->data + copied_offset, | |
| repeats[i].offset - copied_offset) < 0 || | |
| buffer_append_varint( | |
| &block->transformed, STACK_REPEAT) < 0 || | |
| buffer_append_varint( | |
| &block->transformed, repeats[i].count) < 0) { | |
| return NULL; | |
| } | |
| copied_offset = repeats[i].offset; | |
| } | |
| if (buffer_append(&block->transformed, | |
| instructions->data + copied_offset, | |
| instructions->size - copied_offset) < 0) { | |
| return NULL; | |
| } | |
| return &block->transformed; | |
| } | |
| static void | |
| block_reset(BinaryWriter *writer) | |
| { | |
| SampleBlock *block = writer->block; | |
| /* Only threads with a stream in this block need resetting */ | |
| for (uint32_t i = 0; i < block->stream_count; i++) { | |
| size_t thread = block->streams[i].thread_index; | |
| writer->thread_entries[thread].block_stream_index = UINT32_MAX; | |
| } | |
| for (unsigned column = 0; column < COLUMN_COUNT; column++) { | |
| block->columns[column].size = 0; | |
| } | |
| block->sample_count = 0; | |
| block->stream_count = 0; | |
| block->repeats.size = 0; | |
| block->max_frame = 0; | |
| block->size = 0; | |
| memset(block->cache, 0, sizeof(block->cache)); | |
| } | |
| static unsigned | |
| block_frame_width(const SampleBlock *block) | |
| { | |
| if (block->max_frame == 0) { | |
| return 1; | |
| } | |
| return (unsigned)(_Py_bit_length(block->max_frame) + CHAR_BIT - 1) | |
| / CHAR_BIT; | |
| } | |
| static void | |
| block_status_flags(const SampleBlock *block, uint8_t *common_flags, | |
| uint8_t *bitmap_flags) | |
| { | |
| uint8_t common_status = UINT8_MAX; | |
| uint8_t any_status = 0; | |
| const uint8_t *statuses = block->columns[COLUMN_STATUS_BITMAPS].data; | |
| for (uint32_t sample = 0; sample < block->sample_count; sample++) { | |
| common_status &= statuses[sample]; | |
| any_status |= statuses[sample]; | |
| } | |
| *common_flags = common_status; | |
| *bitmap_flags = any_status ^ common_status; | |
| } | |
| static size_t | |
| write_block_header(const BlockHeader *header, uint8_t *bytes) | |
| { | |
| /* Magic "TBLK", sample count and base timestamp (varints), then frame | |
| * ID width and the two status flag masks (1 byte each) */ | |
| size_t size = BLOCK_MAGIC_SIZE; | |
| memcpy(bytes, BLOCK_MAGIC, BLOCK_MAGIC_SIZE); | |
| size += encode_varint_u32(bytes + size, header->sample_count); | |
| size += encode_varint_u64(bytes + size, header->base_timestamp); | |
| bytes[size++] = header->frame_width; | |
| bytes[size++] = header->common_status_flags; | |
| bytes[size++] = header->status_bitmap_flags; | |
| /* Each column: decoded size (varint), encoding (1 byte), and the stored | |
| * size (varint) for zstd only */ | |
| for (unsigned column = 0; column < COLUMN_COUNT; column++) { | |
| const ColumnDescriptor *descriptor = &header->columns[column]; | |
| size += encode_varint_u32(bytes + size, descriptor->decoded_size); | |
| bytes[size++] = descriptor->encoding; | |
| if (descriptor->encoding == COMPRESSION_ZSTD) { | |
| size += encode_varint_u32(bytes + size, descriptor->stored_size); | |
| } | |
| } | |
| return size; | |
| } | |
| static int | |
| block_flush(BinaryWriter *writer) | |
| { | |
| SampleBlock *block = writer->block; | |
| if (block->sample_count == 0) { | |
| return 0; | |
| } | |
| BlockHeader header = {0}; | |
| header.sample_count = block->sample_count; | |
| header.base_timestamp = block->base_timestamp; | |
| header.frame_width = (uint8_t)block_frame_width(block); | |
| block_status_flags(block, &header.common_status_flags, | |
| &header.status_bitmap_flags); | |
| /* Convert each column to its file form, then encode it */ | |
| /* Leave room for the header, which is written once its size is known */ | |
| if (buffer_reserve(&block->output, MAX_BLOCK_HEADER_SIZE) < 0) { | |
| return -1; | |
| } | |
| block->output.size = MAX_BLOCK_HEADER_SIZE; | |
| size_t decoded_payload_size = 0; | |
| for (unsigned column = 0; column < COLUMN_COUNT; column++) { | |
| ByteBuffer *decoded = &block->columns[column]; | |
| switch (column) { | |
| case COLUMN_STATUS_BITMAPS: | |
| decoded = transform_statuses(block, header.status_bitmap_flags); | |
| break; | |
| case COLUMN_STACK_INSTRUCTIONS: | |
| decoded = transform_stack_instructions(block); | |
| break; | |
| case COLUMN_FRAME_IDS: | |
| decoded = transform_frame_ids(block, header.frame_width); | |
| break; | |
| } | |
| if (decoded == NULL) { | |
| return -1; | |
| } | |
| if (decoded->size > | |
| MAX_BLOCK_SIZE - decoded_payload_size) { | |
| PyErr_SetString(PyExc_RuntimeError, | |
| "sample block exceeds the decoded size limit"); | |
| return -1; | |
| } | |
| decoded_payload_size += decoded->size; | |
| if (encode_column(writer, decoded, &header.columns[column]) < 0) { | |
| return -1; | |
| } | |
| } | |
| /* Put the header right before the columns and write the block */ | |
| uint8_t header_bytes[MAX_BLOCK_HEADER_SIZE]; | |
| size_t header_size = write_block_header(&header, header_bytes); | |
| size_t unused = MAX_BLOCK_HEADER_SIZE - header_size; | |
| uint8_t *header_start = block->output.data + unused; | |
| memcpy(header_start, header_bytes, header_size); | |
| if (fwrite_checked_allow_threads(header_start, | |
| block->output.size - unused, | |
| writer->fp) < 0) { | |
| return -1; | |
| } | |
| writer->stats.bytes_written += decoded_payload_size; | |
| block_reset(writer); | |
| return 0; | |
| } | |
| /* FNV-1a over the frame indices, the same in every process */ | |
| static Py_uhash_t | |
| stack_hash(const uint32_t *stack, size_t depth) | |
| { | |
| uint64_t hash = 0xcbf29ce484222325; | |
| for (size_t i = 0; i < depth; i++) { | |
| hash = (hash ^ stack[i]) * 0x100000001b3; | |
| } | |
| /* Mix the high bits into the low bits that select the cache bucket */ | |
| hash ^= hash >> 33; | |
| hash *= 0xff51afd7ed558ccd; | |
| hash ^= hash >> 33; | |
| return (Py_uhash_t)hash; | |
| } | |
| static size_t | |
| common_outer_frames(const BlockStream *stream, const uint32_t *stack, | |
| size_t depth) | |
| { | |
| size_t shared = 0; | |
| while (shared < depth && shared < stream->prev_stack_depth && | |
| stack[depth - 1 - shared] == | |
| stream->prev_stack[stream->prev_stack_depth - 1 - shared]) { | |
| shared++; | |
| } | |
| return shared; | |
| } | |
| static int | |
| cached_stack_matches(const StackCacheEntry *cached, | |
| const ByteBuffer *frames, Py_uhash_t hash, | |
| const uint32_t *stack, size_t depth) | |
| { | |
| if (cached->hash != hash || cached->depth != depth) { | |
| return 0; | |
| } | |
| const uint32_t *frame_ids = (const uint32_t *)frames->data; | |
| for (size_t frame = 0; frame < depth; frame++) { | |
| if (stack[frame] != | |
| frame_ids[cached->offset + depth - 1 - frame]) { | |
| return 0; | |
| } | |
| } | |
| return 1; | |
| } | |
| static StackCacheEntry * | |
| stack_cache_lookup(StackCacheEntry *bucket, const ByteBuffer *frames, | |
| Py_uhash_t hash, const uint32_t *stack, size_t depth) | |
| { | |
| for (size_t i = 0; i < STACK_CACHE_BUCKET_SIZE; i++) { | |
| if (cached_stack_matches(&bucket[i], frames, hash, stack, depth)) { | |
| return &bucket[i]; | |
| } | |
| } | |
| return NULL; | |
| } | |
| static void | |
| stack_cache_insert(StackCacheEntry *bucket, StackCacheEntry entry) | |
| { | |
| memmove(&bucket[1], &bucket[0], | |
| (STACK_CACHE_BUCKET_SIZE - 1) * sizeof(*bucket)); | |
| bucket[0] = entry; | |
| } | |
| static int | |
| write_stack_repeat(BinaryWriter *writer, BlockStream *stream, size_t depth) | |
| { | |
| SampleBlock *block = writer->block; | |
| if (stream->current_repeat == UINT32_MAX) { | |
| Repeat repeat = { | |
| (uint32_t)block->columns[COLUMN_STACK_INSTRUCTIONS].size, 1}; | |
| if (buffer_append(&block->repeats, | |
| &repeat, sizeof(repeat)) < 0) { | |
| return -1; | |
| } | |
| block->size += sizeof(repeat); | |
| stream->current_repeat = | |
| (uint32_t)(block->repeats.size / sizeof(repeat) - 1); | |
| writer->stats.repeat_records++; | |
| } | |
| else { | |
| Repeat *repeats = (Repeat *)block->repeats.data; | |
| repeats[stream->current_repeat].count++; | |
| } | |
| writer->stats.repeat_samples++; | |
| writer->stats.frames_saved += depth; | |
| return 0; | |
| } | |
| static int | |
| write_stack_reuse(BinaryWriter *writer, | |
| const StackCacheEntry *cached, size_t depth) | |
| { | |
| SampleBlock *block = writer->block; | |
| if (column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, | |
| STACK_REUSE) < 0 || | |
| column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, | |
| cached->offset) < 0 || | |
| column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, depth) < 0) { | |
| return -1; | |
| } | |
| writer->stats.reuse_records++; | |
| writer->stats.frames_saved += depth; | |
| return 0; | |
| } | |
| static int | |
| write_stack_replace_suffix(BinaryWriter *writer, StackCacheEntry *bucket, | |
| Py_uhash_t hash, const uint32_t *stack, | |
| size_t depth, size_t shared) | |
| { | |
| SampleBlock *block = writer->block; | |
| ByteBuffer *frames = &block->columns[COLUMN_FRAME_IDS]; | |
| size_t new_frames = depth - shared; | |
| if (column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, | |
| STACK_REPLACE_SUFFIX) < 0 || | |
| column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, shared) < 0 || | |
| column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, | |
| new_frames) < 0) { | |
| return -1; | |
| } | |
| /* Append the new inner frames, outermost first */ | |
| size_t stack_offset = frames->size / sizeof(uint32_t); | |
| if (new_frames != 0) { | |
| if (buffer_reserve( | |
| frames, frames->size + new_frames * sizeof(uint32_t)) < 0) { | |
| return -1; | |
| } | |
| uint32_t *new_frame_ids = (uint32_t *)(frames->data + frames->size); | |
| for (size_t i = 0; i < new_frames; i++) { | |
| uint32_t frame = stack[new_frames - 1 - i]; | |
| new_frame_ids[i] = frame; | |
| if (frame > block->max_frame) { | |
| block->max_frame = frame; | |
| } | |
| } | |
| frames->size += new_frames * sizeof(uint32_t); | |
| block->size += new_frames * sizeof(uint32_t); | |
| } | |
| /* Complete stacks are cached so later samples can REUSE them */ | |
| if (shared == 0) { | |
| if (bucket != NULL) { | |
| stack_cache_insert(bucket, (StackCacheEntry){ | |
| hash, (uint32_t)stack_offset, (uint32_t)depth}); | |
| } | |
| writer->stats.full_records++; | |
| } | |
| else { | |
| writer->stats.suffix_records++; | |
| } | |
| writer->stats.total_frames_written += new_frames; | |
| writer->stats.frames_saved += shared; | |
| return 0; | |
| } | |
| static int | |
| write_stack(BinaryWriter *writer, BlockStream *stream, int new_stream, | |
| const uint32_t *stack, size_t depth) | |
| { | |
| SampleBlock *block = writer->block; | |
| size_t shared = common_outer_frames(stream, stack, depth); | |
| /* Same stack as this stream's previous sample: extend its REPEAT */ | |
| if (!new_stream && shared == depth && | |
| shared == stream->prev_stack_depth) { | |
| return write_stack_repeat(writer, stream, depth); | |
| } | |
| /* Enough new frames to be worth looking up an earlier complete stack */ | |
| Py_uhash_t hash = 0; | |
| StackCacheEntry *bucket = NULL; | |
| StackCacheEntry *cached = NULL; | |
| if (depth - shared >= STACK_REUSE_MIN_FRAMES) { | |
| hash = stack_hash(stack, depth); | |
| bucket = block->cache[hash % STACK_CACHE_BUCKET_COUNT]; | |
| cached = stack_cache_lookup(bucket, &block->columns[COLUMN_FRAME_IDS], | |
| hash, stack, depth); | |
| } | |
| int result; | |
| if (cached != NULL) { | |
| /* Stack already stored in this block: point to its frames */ | |
| result = write_stack_reuse(writer, cached, depth); | |
| } | |
| else { | |
| /* Keep the shared outer frames and write only the new inner ones */ | |
| result = write_stack_replace_suffix(writer, bucket, hash, stack, | |
| depth, shared); | |
| } | |
| if (result < 0) { | |
| return -1; | |
| } | |
| stream->current_repeat = UINT32_MAX; | |
| memcpy(stream->prev_stack, stack, depth * sizeof(uint32_t)); | |
| stream->prev_stack_depth = depth; | |
| return 0; | |
| } | |
| size_t | |
| binary_writer_buffered_size(BinaryWriter *writer) | |
| { | |
| SampleBlock *block = writer->block; | |
| if (block == NULL || block->sample_count == 0) { | |
| return 0; | |
| } | |
| size_t size = block->columns[COLUMN_STREAM_TABLE].size | |
| + block->columns[COLUMN_SAMPLE_STREAM_IDS].size | |
| + block->columns[COLUMN_TIMESTAMP_RESIDUALS].size | |
| + block->columns[COLUMN_STACK_INSTRUCTIONS].size; | |
| const Repeat *repeats = (const Repeat *)block->repeats.data; | |
| size_t repeat_count = block->repeats.size / sizeof(Repeat); | |
| uint8_t bytes[MAX_VARINT_SIZE]; | |
| for (size_t i = 0; i < repeat_count; i++) { | |
| size += encode_varint_u64(bytes, STACK_REPEAT) | |
| + encode_varint_u64(bytes, repeats[i].count); | |
| } | |
| uint8_t common_flags, bitmap_flags; | |
| block_status_flags(block, &common_flags, &bitmap_flags); | |
| size += (size_t)_Py_popcount32(bitmap_flags) | |
| * status_bitmap_size(block->sample_count); | |
| size += block->columns[COLUMN_FRAME_IDS].size / sizeof(uint32_t) | |
| * block_frame_width(block); | |
| return size; | |
| } | |
| static int | |
| block_add_stream(BinaryWriter *writer, ThreadEntry *entry) | |
| { | |
| SampleBlock *block = writer->block; | |
| if (GROW_ARRAY(block->streams, block->stream_count, | |
| block->stream_capacity, BlockStream) < 0) { | |
| return -1; | |
| } | |
| /* Stream table entry: thread_id (8 bytes) + interpreter_id (4 bytes) */ | |
| uint8_t stream_entry[STREAM_ENTRY_SIZE]; | |
| memcpy(stream_entry + STM_OFF_THREAD_ID, | |
| &entry->thread_id, sizeof(entry->thread_id)); | |
| memcpy(stream_entry + STM_OFF_INTERPRETER_ID, | |
| &entry->interpreter_id, sizeof(entry->interpreter_id)); | |
| if (column_write(block, COLUMN_STREAM_TABLE, | |
| stream_entry, sizeof(stream_entry)) < 0) { | |
| return -1; | |
| } | |
| BlockStream *stream = &block->streams[block->stream_count]; | |
| stream->thread_index = (size_t)(entry - writer->thread_entries); | |
| stream->prev_timestamp = block->base_timestamp - writer->sample_interval_us; | |
| stream->current_repeat = UINT32_MAX; | |
| stream->prev_stack_depth = 0; | |
| entry->block_stream_index = block->stream_count++; | |
| return 0; | |
| } | |
| static int | |
| block_add_sample(BinaryWriter *writer, ThreadEntry *entry, | |
| uint64_t timestamp, uint8_t status, | |
| const uint32_t *stack, size_t depth) | |
| { | |
| static_assert(BLOCK_FLUSH_SIZE + MAX_SAMPLE_SIZE + MAX_BITMAP_PADDING == | |
| MAX_BLOCK_SIZE, | |
| "block flush size needs headroom for one complete sample"); | |
| SampleBlock *block = writer->block; | |
| /* Start a new block if this sample might not fit */ | |
| if (block->size >= BLOCK_FLUSH_SIZE || | |
| (entry->block_stream_index == UINT32_MAX && | |
| block->stream_count == MAX_BLOCK_STREAMS)) { | |
| if (block_flush(writer) < 0) { | |
| return -1; | |
| } | |
| } | |
| if (block->sample_count == 0) { | |
| block->base_timestamp = timestamp; | |
| } | |
| int new_stream = entry->block_stream_index == UINT32_MAX; | |
| if (new_stream && block_add_stream(writer, entry) < 0) { | |
| return -1; | |
| } | |
| BlockStream *stream = &block->streams[entry->block_stream_index]; | |
| /* Store the difference from the expected timestamp */ | |
| uint64_t residual = | |
| timestamp - stream->prev_timestamp - writer->sample_interval_us; | |
| uint8_t residual_bytes[MAX_VARINT_SIZE]; | |
| size_t residual_size = | |
| encode_varint_i64(residual_bytes, (int64_t)residual); | |
| if (column_write_varint(block, COLUMN_SAMPLE_STREAM_IDS, | |
| entry->block_stream_index) < 0 || | |
| column_write(block, COLUMN_TIMESTAMP_RESIDUALS, | |
| residual_bytes, residual_size) < 0 || | |
| column_write(block, COLUMN_STATUS_BITMAPS, | |
| &status, sizeof(status)) < 0) { | |
| return -1; | |
| } | |
| if (write_stack(writer, stream, new_stream, stack, depth) < 0) { | |
| return -1; | |
| } | |
| stream->prev_timestamp = timestamp; | |
| block->sample_count++; | |
| writer->total_samples++; | |
| return 0; | |
| } | |
| static void | |
| block_destroy(SampleBlock *block) | |
| { | |
| if (block == NULL) { | |
| return; | |
| } | |
| for (unsigned i = 0; i < COLUMN_COUNT; i++) { | |
| buffer_clear(&block->columns[i]); | |
| } | |
| buffer_clear(&block->repeats); | |
| buffer_clear(&block->transformed); | |
| buffer_clear(&block->compressed); | |
| buffer_clear(&block->output); | |
| PyMem_Free(block->streams); | |
| PyMem_Free(block); | |
| } | |
| BinaryWriter * | |
| binary_writer_create(PyObject *path, uint64_t sample_interval_us, int compression_type, | |
| uint64_t start_time_us, int profiling_mode, | |
| int capture_features) | |
| { | |
| if (compression_type != COMPRESSION_NONE && | |
| compression_type != COMPRESSION_ZSTD) { | |
| PyErr_SetString(PyExc_ValueError, "invalid compression type"); | |
| return NULL; | |
| } | |
| BinaryWriter *writer = PyMem_Calloc(1, sizeof(BinaryWriter)); | |
| if (!writer) { | |
| PyErr_NoMemory(); | |
| return NULL; | |
| } | |
| writer->start_time_us = start_time_us; | |
| writer->sample_interval_us = sample_interval_us; | |
| writer->compression_type = compression_type; | |
| writer->profiling_mode = profiling_mode; | |
| writer->capture_features = capture_features; | |
| writer->block = PyMem_Calloc(1, sizeof(SampleBlock)); | |
| if (writer->block == NULL) { | |
| PyErr_NoMemory(); | |
| goto error; | |
| } | |
| writer->block->streams = PyMem_Malloc( | |
| INITIAL_BLOCK_STREAM_CAPACITY * sizeof(BlockStream)); | |
| if (writer->block->streams == NULL) { | |
| PyErr_NoMemory(); | |
| goto error; | |
| } | |
| writer->block->stream_capacity = INITIAL_BLOCK_STREAM_CAPACITY; | |
| writer->string_hash = _Py_hashtable_new_full( | |
| string_hash_func, | |
| string_compare_func, | |
| string_key_destroy, /* Key destroy: decref the Python string */ | |
| NULL, /* Value destroy: values are just indices, not pointers */ | |
| NULL /* Use default allocator */ | |
| ); | |
| if (!writer->string_hash) { | |
| PyErr_NoMemory(); | |
| goto error; | |
| } | |
| writer->strings = PyMem_Malloc(INITIAL_STRING_CAPACITY * sizeof(char *)); | |
| if (!writer->strings) { | |
| PyErr_NoMemory(); | |
| goto error; | |
| } | |
| writer->string_lengths = PyMem_Malloc(INITIAL_STRING_CAPACITY * sizeof(size_t)); | |
| if (!writer->string_lengths) { | |
| PyErr_NoMemory(); | |
| goto error; | |
| } | |
| writer->string_capacity = INITIAL_STRING_CAPACITY; | |
| writer->frame_hash = _Py_hashtable_new_full( | |
| frame_key_hash_func, | |
| frame_key_compare_func, | |
| frame_key_destroy, /* Key destroy: free the FrameKey */ | |
| NULL, /* Value destroy: values are just indices, not pointers */ | |
| NULL /* Use default allocator */ | |
| ); | |
| if (!writer->frame_hash) { | |
| PyErr_NoMemory(); | |
| goto error; | |
| } | |
| writer->frame_entries = PyMem_Malloc(INITIAL_FRAME_CAPACITY * sizeof(FrameEntry)); | |
| if (!writer->frame_entries) { | |
| PyErr_NoMemory(); | |
| goto error; | |
| } | |
| writer->frame_capacity = INITIAL_FRAME_CAPACITY; | |
| writer->thread_entries = PyMem_Malloc(INITIAL_THREAD_CAPACITY * sizeof(ThreadEntry)); | |
| if (!writer->thread_entries) { | |
| PyErr_NoMemory(); | |
| goto error; | |
| } | |
| writer->thread_capacity = INITIAL_THREAD_CAPACITY; | |
| if (compression_type == COMPRESSION_ZSTD) { | |
| if (writer_init_zstd(writer) < 0) { | |
| goto error; | |
| } | |
| } | |
| writer->fp = Py_fopen(path, "wb"); | |
| if (!writer->fp) { | |
| goto error; | |
| } | |
| /* Hint sequential write pattern to kernel for better I/O scheduling */ | |
| #if defined(__linux__) && defined(POSIX_FADV_SEQUENTIAL) | |
| { | |
| int fd = fileno(writer->fp); | |
| if (fd >= 0) { | |
| (void)posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); | |
| } | |
| } | |
| #endif | |
| uint8_t header[FILE_HEADER_PLACEHOLDER_SIZE] = {0}; | |
| if (fwrite_checked_allow_threads(header, FILE_HEADER_PLACEHOLDER_SIZE, writer->fp) < 0) { | |
| goto error; | |
| } | |
| return writer; | |
| error: | |
| binary_writer_destroy(writer); | |
| return NULL; | |
| } | |
| /* Build a frame stack from Python frame list by interning all strings and frames. | |
| * Returns 0 on success, -1 on error. */ | |
| static int | |
| build_frame_stack(BinaryWriter *writer, PyObject *frame_list, | |
| uint32_t *curr_stack, size_t *curr_depth) | |
| { | |
| Py_ssize_t stack_depth = PyList_Size(frame_list); | |
| *curr_depth = (stack_depth < MAX_STACK_DEPTH) ? stack_depth : MAX_STACK_DEPTH; | |
| for (Py_ssize_t k = 0; k < (Py_ssize_t)*curr_depth; k++) { | |
| /* Use unchecked accessors since we control the data structures */ | |
| PyObject *frame_info = PyList_GET_ITEM(frame_list, k); | |
| /* Get filename, location, funcname, opcode from FrameInfo using unchecked access */ | |
| PyObject *filename = PyStructSequence_GET_ITEM(frame_info, 0); | |
| PyObject *location = PyStructSequence_GET_ITEM(frame_info, 1); | |
| PyObject *funcname = PyStructSequence_GET_ITEM(frame_info, 2); | |
| PyObject *opcode_obj = PyStructSequence_GET_ITEM(frame_info, 3); | |
| /* Extract location fields (can be None for synthetic frames) */ | |
| int32_t lineno = LOCATION_NOT_AVAILABLE; | |
| int32_t end_lineno = LOCATION_NOT_AVAILABLE; | |
| int32_t column = LOCATION_NOT_AVAILABLE; | |
| int32_t end_column = LOCATION_NOT_AVAILABLE; | |
| if (location != Py_None) { | |
| /* LocationInfo is a struct sequence or tuple with: | |
| * (lineno, end_lineno, column, end_column) */ | |
| PyObject *lineno_obj = PyTuple_Check(location) ? | |
| PyTuple_GET_ITEM(location, 0) : | |
| PyStructSequence_GET_ITEM(location, 0); | |
| PyObject *end_lineno_obj = PyTuple_Check(location) ? | |
| PyTuple_GET_ITEM(location, 1) : | |
| PyStructSequence_GET_ITEM(location, 1); | |
| PyObject *column_obj = PyTuple_Check(location) ? | |
| PyTuple_GET_ITEM(location, 2) : | |
| PyStructSequence_GET_ITEM(location, 2); | |
| PyObject *end_column_obj = PyTuple_Check(location) ? | |
| PyTuple_GET_ITEM(location, 3) : | |
| PyStructSequence_GET_ITEM(location, 3); | |
| PYLONG_TO_INT32_OR_DEFAULT(lineno_obj, lineno, LOCATION_NOT_AVAILABLE); | |
| PYLONG_TO_INT32_OR_DEFAULT(end_lineno_obj, end_lineno, LOCATION_NOT_AVAILABLE); | |
| PYLONG_TO_INT32_OR_DEFAULT(column_obj, column, LOCATION_NOT_AVAILABLE); | |
| PYLONG_TO_INT32_OR_DEFAULT(end_column_obj, end_column, LOCATION_NOT_AVAILABLE); | |
| } | |
| /* Extract opcode (can be None) */ | |
| uint8_t opcode = OPCODE_NONE; | |
| if (opcode_obj != Py_None) { | |
| long opcode_long = PyLong_AsLong(opcode_obj); | |
| if (UNLIKELY(PyErr_Occurred() != NULL)) { | |
| PyErr_Clear(); | |
| opcode = OPCODE_NONE; | |
| } else if (opcode_long >= 0 && opcode_long <= MAX_REAL_OPCODE) { | |
| opcode = (uint8_t)opcode_long; | |
| } | |
| } | |
| /* Intern filename */ | |
| uint32_t filename_idx; | |
| if (writer_intern_string(writer, filename, &filename_idx) < 0) { | |
| return -1; | |
| } | |
| /* Intern funcname */ | |
| uint32_t funcname_idx; | |
| if (writer_intern_string(writer, funcname, &funcname_idx) < 0) { | |
| return -1; | |
| } | |
| /* Intern frame with full location info */ | |
| FrameEntry frame_entry = { | |
| .filename_idx = filename_idx, | |
| .funcname_idx = funcname_idx, | |
| .lineno = lineno, | |
| .end_lineno = end_lineno, | |
| .column = column, | |
| .end_column = end_column, | |
| .opcode = opcode | |
| }; | |
| uint32_t frame_idx; | |
| if (writer_intern_frame(writer, &frame_entry, &frame_idx) < 0) { | |
| return -1; | |
| } | |
| curr_stack[k] = frame_idx; | |
| } | |
| return 0; | |
| } | |
| /* Process a single thread's sample. | |
| * Returns 0 on success, -1 on error. */ | |
| static int | |
| process_thread_sample(BinaryWriter *writer, PyObject *thread_info, | |
| uint32_t interpreter_id, uint64_t timestamp_us) | |
| { | |
| if (writer->total_samples == UINT64_MAX) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| PyErr_SetString(PyExc_OverflowError, "too many samples for binary format"); | |
| return -1; | |
| } | |
| PyObject *thread_id_obj = PyStructSequence_GET_ITEM(thread_info, 0); | |
| PyObject *status_obj = PyStructSequence_GET_ITEM(thread_info, 1); | |
| PyObject *frame_list = PyStructSequence_GET_ITEM(thread_info, 2); | |
| uint64_t thread_id = PyLong_AsUnsignedLongLong(thread_id_obj); | |
| if (thread_id == (uint64_t)-1 && PyErr_Occurred()) { | |
| return -1; | |
| } | |
| long status_long = PyLong_AsLong(status_obj); | |
| if (status_long == -1 && PyErr_Occurred()) { | |
| return -1; | |
| } | |
| if (status_long < 0 || status_long > UINT8_MAX) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| PyErr_Format(PyExc_OverflowError, | |
| "thread status %ld exceeds the binary format range", | |
| status_long); | |
| return -1; | |
| } | |
| uint8_t status = (uint8_t)status_long; | |
| ThreadEntry *entry = writer_get_or_create_thread_entry( | |
| writer, thread_id, interpreter_id); | |
| if (!entry) { | |
| return -1; | |
| } | |
| /* Process frames and build current stack */ | |
| uint32_t curr_stack[MAX_STACK_DEPTH]; | |
| size_t curr_depth; | |
| if (build_frame_stack(writer, frame_list, curr_stack, &curr_depth) < 0) { | |
| return -1; | |
| } | |
| return block_add_sample( | |
| writer, entry, timestamp_us, status, curr_stack, curr_depth); | |
| } | |
| int | |
| binary_writer_write_sample(BinaryWriter *writer, PyObject *stack_frames, uint64_t timestamp_us) | |
| { | |
| if (!PyList_Check(stack_frames)) { | |
| PyErr_SetString(PyExc_TypeError, "stack_frames must be a list"); | |
| return -1; | |
| } | |
| Py_ssize_t num_interpreters = PyList_GET_SIZE(stack_frames); | |
| for (Py_ssize_t i = 0; i < num_interpreters; i++) { | |
| PyObject *interp_info = PyList_GET_ITEM(stack_frames, i); | |
| PyObject *interp_id_obj = PyStructSequence_GET_ITEM(interp_info, 0); | |
| PyObject *threads = PyStructSequence_GET_ITEM(interp_info, 1); | |
| unsigned long long interp_id_long = PyLong_AsUnsignedLongLong(interp_id_obj); | |
| if (interp_id_long == (unsigned long long)-1 && PyErr_Occurred()) { | |
| return -1; | |
| } | |
| /* Bounds check: interpreter_id is stored as uint32_t in binary format */ | |
| if (interp_id_long > UINT32_MAX) { | |
| writer->state = BINARY_WRITER_LIMIT_REACHED; | |
| PyErr_Format(PyExc_OverflowError, | |
| "interpreter_id %llu exceeds maximum value %u", | |
| interp_id_long, UINT32_MAX); | |
| return -1; | |
| } | |
| uint32_t interpreter_id = (uint32_t)interp_id_long; | |
| Py_ssize_t num_threads = PyList_GET_SIZE(threads); | |
| for (Py_ssize_t j = 0; j < num_threads; j++) { | |
| PyObject *thread_info = PyList_GET_ITEM(threads, j); | |
| if (process_thread_sample(writer, thread_info, interpreter_id, timestamp_us) < 0) { | |
| return -1; | |
| } | |
| } | |
| } | |
| return 0; | |
| } | |
| int | |
| binary_writer_finalize(BinaryWriter *writer) | |
| { | |
| if (block_flush(writer) < 0) { | |
| return -1; | |
| } | |
| /* Use 64-bit file position for >2GB files */ | |
| file_offset_t string_table_offset = FTELL64(writer->fp); | |
| if (string_table_offset < 0) { | |
| PyErr_SetFromErrno(PyExc_IOError); | |
| return -1; | |
| } | |
| /* Release GIL during potentially large writes */ | |
| for (size_t i = 0; i < writer->string_count; i++) { | |
| uint8_t len_buf[10]; | |
| size_t len_size = encode_varint_u32(len_buf, (uint32_t)writer->string_lengths[i]); | |
| if (fwrite_checked_allow_threads(len_buf, len_size, writer->fp) < 0 || | |
| fwrite_checked_allow_threads(writer->strings[i], writer->string_lengths[i], writer->fp) < 0) { | |
| return -1; | |
| } | |
| } | |
| file_offset_t frame_table_offset = FTELL64(writer->fp); | |
| if (frame_table_offset < 0) { | |
| PyErr_SetFromErrno(PyExc_IOError); | |
| return -1; | |
| } | |
| for (size_t i = 0; i < writer->frame_count; i++) { | |
| FrameEntry *entry = &writer->frame_entries[i]; | |
| uint8_t buf[64]; /* Increased buffer for additional fields */ | |
| size_t pos = encode_varint_u32(buf, entry->filename_idx); | |
| pos += encode_varint_u32(buf + pos, entry->funcname_idx); | |
| pos += encode_varint_i32(buf + pos, entry->lineno); | |
| /* Delta encode end_lineno: store (end_lineno - lineno) as zigzag. | |
| * When lineno is -1, store delta as 0 (result will be -1). */ | |
| int32_t end_lineno_delta = 0; | |
| if (entry->lineno != LOCATION_NOT_AVAILABLE && | |
| entry->end_lineno != LOCATION_NOT_AVAILABLE) { | |
| end_lineno_delta = entry->end_lineno - entry->lineno; | |
| } | |
| pos += encode_varint_i32(buf + pos, end_lineno_delta); | |
| pos += encode_varint_i32(buf + pos, entry->column); | |
| /* Delta encode end_column: store (end_column - column) as zigzag. | |
| * When column is -1, store delta as 0 (result will be -1). */ | |
| int32_t end_column_delta = 0; | |
| if (entry->column != LOCATION_NOT_AVAILABLE && | |
| entry->end_column != LOCATION_NOT_AVAILABLE) { | |
| end_column_delta = entry->end_column - entry->column; | |
| } | |
| pos += encode_varint_i32(buf + pos, end_column_delta); | |
| buf[pos++] = entry->opcode; | |
| if (fwrite_checked_allow_threads(buf, pos, writer->fp) < 0) { | |
| return -1; | |
| } | |
| } | |
| if (writer->has_profile_stats) { | |
| uint8_t profile_stats[PROFILE_STATS_SIZE] = {0}; | |
| uint32_t version = PROFILE_STATS_VERSION; | |
| uint32_t size = PROFILE_STATS_SIZE; | |
| memcpy(profile_stats + PST_OFF_DURATION, | |
| &writer->duration_sec, PST_SIZE_DURATION); | |
| memcpy(profile_stats + PST_OFF_SAMPLE_RATE, | |
| &writer->sample_rate, PST_SIZE_SAMPLE_RATE); | |
| memcpy(profile_stats + PST_OFF_ERROR_RATE, | |
| &writer->error_rate, PST_SIZE_ERROR_RATE); | |
| memcpy(profile_stats + PST_OFF_MISSED_SAMPLES, | |
| &writer->missed_samples, PST_SIZE_MISSED_SAMPLES); | |
| memcpy(profile_stats + PST_OFF_PRESENT, | |
| &writer->profile_stats_present, PST_SIZE_PRESENT); | |
| memcpy(profile_stats + PST_OFF_MAGIC, | |
| PROFILE_STATS_MAGIC, PROFILE_STATS_MAGIC_SIZE); | |
| memcpy(profile_stats + PST_OFF_VERSION, &version, PST_SIZE_VERSION); | |
| memcpy(profile_stats + PST_OFF_SIZE, &size, PST_SIZE_SIZE); | |
| if (fwrite_checked_allow_threads( | |
| profile_stats, PROFILE_STATS_SIZE, writer->fp) < 0) { | |
| return -1; | |
| } | |
| } | |
| /* Footer: string_count(4) + frame_count(4) + file_size(8) + checksum(16) */ | |
| file_offset_t footer_offset = FTELL64(writer->fp); | |
| if (footer_offset < 0) { | |
| PyErr_SetFromErrno(PyExc_IOError); | |
| return -1; | |
| } | |
| uint64_t file_size = (uint64_t)footer_offset + FILE_FOOTER_SIZE; | |
| uint8_t footer[FILE_FOOTER_SIZE] = {0}; | |
| /* Cast size_t to uint32_t before memcpy to ensure correct bytes are copied | |
| * on both little-endian and big-endian systems (size_t is 8 bytes on 64-bit) */ | |
| uint32_t string_count_u32 = (uint32_t)writer->string_count; | |
| uint32_t frame_count_u32 = (uint32_t)writer->frame_count; | |
| memcpy(footer + FTR_OFF_STRINGS, &string_count_u32, FTR_SIZE_STRINGS); | |
| memcpy(footer + FTR_OFF_FRAMES, &frame_count_u32, FTR_SIZE_FRAMES); | |
| memcpy(footer + FTR_OFF_FILE_SIZE, &file_size, FTR_SIZE_FILE_SIZE); | |
| /* checksum (FTR_OFF_CHECKSUM..FILE_FOOTER_SIZE-1): placeholder zeros */ | |
| if (fwrite_checked_allow_threads(footer, FILE_FOOTER_SIZE, writer->fp) < 0) { | |
| return -1; | |
| } | |
| if (FSEEK64(writer->fp, 0, SEEK_SET) < 0) { | |
| PyErr_SetFromErrno(PyExc_IOError); | |
| return -1; | |
| } | |
| /* Convert file offsets and counts to fixed-width types for portable header format. | |
| * This ensures correct behavior on both little-endian and big-endian systems. */ | |
| uint64_t string_table_offset_u64 = (uint64_t)string_table_offset; | |
| uint64_t frame_table_offset_u64 = (uint64_t)frame_table_offset; | |
| uint32_t thread_count_u32 = (uint32_t)writer->thread_count; | |
| uint32_t compression_type_u32 = (uint32_t)writer->compression_type; | |
| uint32_t profiling_config_u32 = (uint32_t)(writer->profiling_mode + 1); | |
| if (writer->capture_features >= 0) { | |
| profiling_config_u32 |= PROFILING_CONFIG_FEATURES_KNOWN; | |
| profiling_config_u32 |= (uint32_t)writer->capture_features | |
| << PROFILING_CONFIG_FEATURES_SHIFT; | |
| } | |
| uint8_t header[FILE_HEADER_SIZE] = {0}; | |
| uint32_t magic = BINARY_FORMAT_MAGIC; | |
| uint32_t version = BINARY_FORMAT_VERSION; | |
| memcpy(header + HDR_OFF_MAGIC, &magic, HDR_SIZE_MAGIC); | |
| memcpy(header + HDR_OFF_VERSION, &version, HDR_SIZE_VERSION); | |
| header[HDR_OFF_PY_MAJOR] = PY_MAJOR_VERSION; | |
| header[HDR_OFF_PY_MINOR] = PY_MINOR_VERSION; | |
| header[HDR_OFF_PY_MICRO] = PY_MICRO_VERSION; | |
| memcpy(header + HDR_OFF_START_TIME, &writer->start_time_us, HDR_SIZE_START_TIME); | |
| memcpy(header + HDR_OFF_INTERVAL, &writer->sample_interval_us, HDR_SIZE_INTERVAL); | |
| memcpy(header + HDR_OFF_SAMPLES, &writer->total_samples, HDR_SIZE_SAMPLES); | |
| memcpy(header + HDR_OFF_THREADS, &thread_count_u32, HDR_SIZE_THREADS); | |
| memcpy(header + HDR_OFF_STR_TABLE, &string_table_offset_u64, HDR_SIZE_STR_TABLE); | |
| memcpy(header + HDR_OFF_FRAME_TABLE, &frame_table_offset_u64, HDR_SIZE_FRAME_TABLE); | |
| memcpy(header + HDR_OFF_COMPRESSION, &compression_type_u32, HDR_SIZE_COMPRESSION); | |
| memcpy(header + HDR_OFF_CONFIG, &profiling_config_u32, HDR_SIZE_CONFIG); | |
| if (fwrite_checked_allow_threads(header, FILE_HEADER_SIZE, writer->fp) < 0) { | |
| return -1; | |
| } | |
| if (Py_fclose(writer->fp) != 0) { | |
| writer->fp = NULL; | |
| PyErr_SetFromErrno(PyExc_IOError); | |
| return -1; | |
| } | |
| writer->fp = NULL; | |
| return 0; | |
| } | |
| int | |
| binary_writer_set_stats(BinaryWriter *writer, double duration_sec, | |
| double sample_rate, double error_rate, | |
| double missed_samples, uint32_t present) | |
| { | |
| if (!isfinite(duration_sec) || duration_sec < 0.0) { | |
| PyErr_SetString(PyExc_ValueError, | |
| "duration must be a finite non-negative value"); | |
| return -1; | |
| } | |
| if (!isfinite(sample_rate) || sample_rate < 0.0) { | |
| PyErr_SetString(PyExc_ValueError, | |
| "sample rate must be a finite non-negative value"); | |
| return -1; | |
| } | |
| if ((present & PROFILE_STATS_ERROR_RATE) && | |
| (!isfinite(error_rate) || error_rate < 0.0)) { | |
| PyErr_SetString(PyExc_ValueError, | |
| "error rate must be a finite non-negative value"); | |
| return -1; | |
| } | |
| if ((present & PROFILE_STATS_MISSED) && !isfinite(missed_samples)) { | |
| PyErr_SetString(PyExc_ValueError, | |
| "missed samples must be a finite value"); | |
| return -1; | |
| } | |
| writer->duration_sec = duration_sec; | |
| writer->sample_rate = sample_rate; | |
| writer->error_rate = error_rate; | |
| writer->missed_samples = missed_samples; | |
| writer->profile_stats_present = present; | |
| writer->has_profile_stats = 1; | |
| return 0; | |
| } | |
| void | |
| binary_writer_destroy(BinaryWriter *writer) | |
| { | |
| if (!writer) { | |
| return; | |
| } | |
| if (writer->fp) { | |
| Py_fclose(writer->fp); | |
| } | |
| block_destroy(writer->block); | |
| #ifdef HAVE_ZSTD | |
| if (writer->zstd.cctx) { | |
| ZSTD_freeCCtx(writer->zstd.cctx); | |
| } | |
| #endif | |
| if (writer->string_hash) { | |
| _Py_hashtable_destroy(writer->string_hash); | |
| } | |
| if (writer->strings) { | |
| for (size_t i = 0; i < writer->string_count; i++) { | |
| PyMem_Free(writer->strings[i]); | |
| } | |
| PyMem_Free(writer->strings); | |
| } | |
| PyMem_Free(writer->string_lengths); | |
| if (writer->frame_hash) { | |
| _Py_hashtable_destroy(writer->frame_hash); | |
| } | |
| PyMem_Free(writer->frame_entries); | |
| PyMem_Free(writer->thread_entries); | |
| PyMem_Free(writer); | |
| } | |
| #undef PYLONG_TO_INT32_OR_DEFAULT |