diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index c982f76431c52d..9ab20be70614de 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1724,6 +1724,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dont_inherit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(duration_sec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); @@ -1737,6 +1738,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(endpos)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(entrypoint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(env)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(error_rate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); @@ -1921,6 +1923,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minutes)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(missed_samples)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); @@ -2053,6 +2056,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rounding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sample_interval_us)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sample_rate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(script)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index c430d0fb99bc41..51d9fbe89b3423 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -447,6 +447,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(dont_inherit) STRUCT_FOR_ID(dst) STRUCT_FOR_ID(dst_dir_fd) + STRUCT_FOR_ID(duration_sec) STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) @@ -460,6 +461,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(endpos) STRUCT_FOR_ID(entrypoint) STRUCT_FOR_ID(env) + STRUCT_FOR_ID(error_rate) STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) @@ -644,6 +646,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(milliseconds) STRUCT_FOR_ID(minute) STRUCT_FOR_ID(minutes) + STRUCT_FOR_ID(missed_samples) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) @@ -776,6 +779,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(rounding) STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sample_interval_us) + STRUCT_FOR_ID(sample_rate) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) STRUCT_FOR_ID(script) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 5a194478f56231..88ca09e6ba245f 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1722,6 +1722,7 @@ extern "C" { INIT_ID(dont_inherit), \ INIT_ID(dst), \ INIT_ID(dst_dir_fd), \ + INIT_ID(duration_sec), \ INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ @@ -1735,6 +1736,7 @@ extern "C" { INIT_ID(endpos), \ INIT_ID(entrypoint), \ INIT_ID(env), \ + INIT_ID(error_rate), \ INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ @@ -1919,6 +1921,7 @@ extern "C" { INIT_ID(milliseconds), \ INIT_ID(minute), \ INIT_ID(minutes), \ + INIT_ID(missed_samples), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ @@ -2051,6 +2054,7 @@ extern "C" { INIT_ID(rounding), \ INIT_ID(salt), \ INIT_ID(sample_interval_us), \ + INIT_ID(sample_rate), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ INIT_ID(script), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 25529c5ca713a9..3c4d7d664537a8 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1568,6 +1568,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(duration_sec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(eager_start); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1620,6 +1624,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(error_rate); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(errors); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2356,6 +2364,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(missed_samples); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mod); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2884,6 +2896,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(sample_rate); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sched_priority); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/InternalDocs/profiling_binary_format.md b/InternalDocs/profiling_binary_format.md index 7e4592a0d89705..51b1944aa5c8a0 100644 --- a/InternalDocs/profiling_binary_format.md +++ b/InternalDocs/profiling_binary_format.md @@ -34,7 +34,7 @@ by 10-50x compared to text formats while also enabling faster I/O. ## File Layout -The file consists of five sections: +The file consists of five required sections and one optional extension: ``` +------------------+ Offset 0 @@ -47,6 +47,8 @@ The file consists of five sections: | String Table | Variable size +------------------+ frame_table_offset | Frame Table | Variable size ++------------------+ file_size - 64 (when stats are present) +| Profile Stats | 32 bytes (optional) +------------------+ file_size - 32 | Footer | 32 bytes (fixed) +------------------+ file_size @@ -354,6 +356,38 @@ location. Zigzag encoding ensures these small negative values encode efficiently (−1 becomes 1, which is one byte) rather than requiring the maximum varint length. +## Profile Statistics + +New files can store measured duration, sampling rate, error rate, and missed +sample percentage in an optional 56-byte extension immediately before the +footer. Older readers ignore these +bytes after parsing the declared number of frame-table entries, and newer +readers treat a missing extension as unavailable statistics. + +``` + Offset Size Type Description ++--------+------+---------+----------------------------------------+ +| 0 | 8 | double | Measured duration (seconds) | +| 8 | 8 | double | Measured sample rate (samples/second) | +| 16 | 8 | double | Failed sample percentage | +| 24 | 8 | double | Missed sample percentage | +| 32 | 4 | uint32 | Optional field presence flags | +| 36 | 4 | uint32 | Reserved | +| 40 | 8 | bytes | Signature ("TACHSTAT") | +| 48 | 4 | uint32 | Extension version (1) | +| 52 | 4 | uint32 | Extension size (56) | ++--------+------+---------+----------------------------------------+ +``` + +Putting the signature, version, and size at the end lets readers discover +the extension from its fixed position relative to the footer while allowing +future versions to add fields before that trailer. Multi-byte values use the +same native byte order as the rest of the file and are byte-swapped by +cross-endian readers. + +Readers also accept the original 32-byte extension, which only contains the +duration and sampling rate. + ## Footer ``` @@ -448,8 +482,9 @@ compress less; higher levels (6+) compress more but slow down writing. Level 4. Flush remaining buffered data and finalize compression 5. Write the string table (length-prefixed strings in index order) 6. Write the frame table (varint-encoded entries in index order) -7. Write the footer with final counts -8. Seek to offset 0 and write the header with actual values +7. Write measured profile statistics, when available +8. Write the footer with final counts +9. Seek to offset 0 and write the header with actual values The writer maintains two dictionaries: one mapping strings to indices, one mapping (filename_idx, funcname_idx, lineno) tuples to frame indices. These @@ -461,12 +496,13 @@ enable O(1) lookup during interning. if the magic appears byte-swapped) 2. Validate version and read remaining header fields (byte-swapping if needed) 3. Seek to end − 32 and read the footer (byte-swapping counts if needed) -4. Allocate string array of `string_count` elements -5. Parse the string table, populating the array -6. Allocate frame array of `frame_count * 3` uint32 elements -7. Parse the frame table, populating the array -8. If compressed, decompress the sample data region -9. Iterate through samples, resolving indices to strings/frames +4. Read measured profile statistics when the optional extension is present +5. Allocate string array of `string_count` elements +6. Parse the string table, populating the array +7. Allocate the frame array +8. Parse the frame table, populating the array +9. If compressed, decompress the sample data region +10. Iterate through samples, resolving indices to strings/frames (byte-swapping thread_id and interpreter_id if needed) The reader builds lookup arrays rather than dictionaries since it only needs @@ -530,11 +566,10 @@ one write() call (or feeds through the compression stream). ## Future Considerations -The format reserves space for future extensions. The 12 reserved bytes in -the header could hold additional metadata. The 16-byte checksum field in -the footer is currently unused. The version field allows incompatible -changes with graceful rejection. New compression types could be added -(compression_type > 1). +The optional profile-statistics block provides an extensible metadata area. +The 16-byte checksum field in the footer is currently unused. The version +field allows incompatible changes with graceful rejection. New compression +types could be added (compression_type > 1). Any changes that alter the meaning of existing fields or the parsing logic should increment the version number to prevent older readers from diff --git a/Lib/profiling/sampling/binary_collector.py b/Lib/profiling/sampling/binary_collector.py index afbbc829269067..ec6020078985ca 100644 --- a/Lib/profiling/sampling/binary_collector.py +++ b/Lib/profiling/sampling/binary_collector.py @@ -87,6 +87,13 @@ def collect_failed_sample(self): """Record a failed sample attempt (no-op for binary format).""" pass + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, + error_rate=None, missed_samples=None, **kwargs): + """Persist measured statistics for later replay.""" + self._writer.set_stats( + duration_sec, sample_rate, error_rate, missed_samples + ) + def export(self, filename=None): """Finalize and close the binary file. diff --git a/Lib/profiling/sampling/binary_reader.py b/Lib/profiling/sampling/binary_reader.py index a29dad91ae339d..e611dfd45523f1 100644 --- a/Lib/profiling/sampling/binary_reader.py +++ b/Lib/profiling/sampling/binary_reader.py @@ -125,6 +125,8 @@ def convert_binary_to_format(input_file, output_file, output_format, # Replay samples through collector count = reader.replay_samples(collector, progress_callback) + if hasattr(collector, "set_replay_stats"): + collector.set_replay_stats(info) # Export to target format collector.export(output_file) diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index b47b166c6cd940..83af313e1efcbd 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -776,6 +776,8 @@ def progress_callback(current, total): ) count = reader.replay_samples(collector, progress_callback) + if hasattr(collector, "set_replay_stats"): + collector.set_replay_stats(info) print() if args.format == "pstats": diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index dd86d5d54a9aaa..8de460856666d7 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -103,7 +103,6 @@ def collect(self, stack_frames, timestamps_us=None): """Override to track thread status statistics before processing frames.""" # Weight is number of timestamps (samples with identical stack) weight = len(timestamps_us) if timestamps_us else 1 - # Increment sample count by weight self._sample_count += weight @@ -148,6 +147,21 @@ def set_stats(self, sample_interval_usec, duration_sec, sample_rate, "mode": mode } + def set_replay_stats(self, info): + """Restore measured statistics stored in a binary profile.""" + duration_sec = info.get("duration_sec") + sample_rate = info.get("sample_rate") + if duration_sec is None or sample_rate is None: + return + self.set_stats( + self.sample_interval_usec, + duration_sec, + sample_rate, + error_rate=info.get("error_rate"), + missed_samples=info.get("missed_samples"), + mode=self.stats.get("mode"), + ) + def export(self, filename): flamegraph_data = self._convert_to_flamegraph_format() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index e4963dca9c9663..253f419dae3c2e 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -1580,6 +1580,96 @@ def test_timestamp_preservation_with_rle(self): self.assertEqual(ts_collector.all_timestamps, expected_timestamps) +class TestBinaryReplayToFlamegraph(BinaryFormatTestBase): + def test_replay_includes_persisted_stats(self): + frames = [ + make_frame("hot.py", 99, "hot_func"), + make_frame("main.py", 1, "main"), + ] + samples = [ + [ + make_interpreter( + 0, + [make_thread(1, frames, THREAD_STATUS_HAS_GIL)], + ) + ] + for _ in range(5) + ] + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as file: + bin_path = file.name + self.temp_files.append(bin_path) + collector = BinaryCollector(bin_path, 2000, compression="none") + for sample in samples: + collector.collect(sample) + collector.set_stats( + 2000, 1.25, 4.0, error_rate=2.5, missed_samples=1.5 + ) + collector.export(None) + + with BinaryReader(bin_path) as reader: + info = reader.get_info() + self.assertEqual(info["duration_sec"], 1.25) + self.assertEqual(info["sample_rate"], 4.0) + self.assertEqual(info["error_rate"], 2.5) + self.assertEqual(info["missed_samples"], 1.5) + + with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as file: + html_path = file.name + self.temp_files.append(html_path) + + convert_binary_to_format(bin_path, html_path, "flamegraph") + + with open(html_path, encoding="utf-8") as file: + content = file.read() + self.assertIn('"duration_sec": 1.25', content) + self.assertIn('"sample_rate": 4.0', content) + self.assertIn('"error_rate": 2.5', content) + self.assertIn('"missed_samples": 1.5', content) + + def test_legacy_binary_has_no_measured_stats(self): + frame = make_frame("hot.py", 99, "hot_func") + bin_path = self.create_binary_file([ + [make_interpreter(0, [make_thread(1, [frame])])] + ]) + + with BinaryReader(bin_path) as reader: + info = reader.get_info() + + self.assertIsNone(info["duration_sec"]) + self.assertIsNone(info["sample_rate"]) + self.assertIsNone(info["error_rate"]) + self.assertIsNone(info["missed_samples"]) + + def test_original_stats_extension_remains_readable(self): + frame = make_frame("hot.py", 99, "hot_func") + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as file: + bin_path = file.name + self.temp_files.append(bin_path) + collector = BinaryCollector(bin_path, 2000, compression="none") + collector.collect([ + make_interpreter(0, [make_thread(1, [frame])]) + ]) + collector.set_stats(2000, 1.25, 4.0) + collector.export(None) + + with open(bin_path, "rb") as file: + data = file.read() + stats = data[-88:-32] + footer = bytearray(data[-32:]) + old_stats = stats[:16] + b"TACHSTAT" + struct.pack("=II", 1, 32) + old_data = bytearray(data[:-88] + old_stats + footer) + struct.pack_into("=Q", old_data, -24, len(old_data)) + with open(bin_path, "wb") as file: + file.write(old_data) + + with BinaryReader(bin_path) as reader: + info = reader.get_info() + self.assertEqual(info["duration_sec"], 1.25) + self.assertEqual(info["sample_rate"], 4.0) + self.assertIsNone(info["error_rate"]) + self.assertIsNone(info["missed_samples"]) + + class TestBinaryReplayToJsonl(BinaryFormatTestBase): """Tests for binary -> JSONL replay via convert_binary_to_format.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 13e112f78014b9..069a72eb1c88a6 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -1159,6 +1159,30 @@ def test_flamegraph_collector_stats_accumulation(self): collector.collect(stack_frames_gc) self.assertEqual(collector.samples_with_gc_frames, 2) + def test_flamegraph_collector_restores_replay_stats(self): + """Replay restores measured values from binary metadata.""" + collector = FlamegraphCollector(1000) + collector.set_replay_stats({ + "duration_sec": 1.25, + "sample_rate": 4.0, + "error_rate": 2.5, + "missed_samples": 1.5, + }) + + self.assertEqual(collector.stats["duration_sec"], 1.25) + self.assertEqual(collector.stats["sample_rate"], 4.0) + self.assertEqual(collector.stats["error_rate"], 2.5) + self.assertEqual(collector.stats["missed_samples"], 1.5) + + def test_flamegraph_collector_leaves_legacy_replay_stats_unavailable(self): + collector = FlamegraphCollector(1000) + original_stats = collector.stats.copy() + collector.set_replay_stats({ + "duration_sec": None, + "sample_rate": None, + }) + self.assertEqual(collector.stats, original_stats) + def test_flamegraph_collector_per_thread_stats(self): """Test per-thread statistics tracking in FlamegraphCollector.""" collector = FlamegraphCollector(sample_interval_usec=1000) diff --git a/Misc/NEWS.d/next/Library/2026-07-19-12-10-00.gh-issue-154060.cR6tYn.rst b/Misc/NEWS.d/next/Library/2026-07-19-12-10-00.gh-issue-154060.cR6tYn.rst new file mode 100644 index 00000000000000..a9df0e6ad39963 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-12-10-00.gh-issue-154060.cR6tYn.rst @@ -0,0 +1,2 @@ +Store measured duration and sampling rate in Tachyon binary profiles so they +remain available in replayed flamegraphs. diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index 8c34668ab15ae7..615e3626e1cbc6 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -91,6 +91,37 @@ static_assert(SAMPLE_HEADER_FIXED_SIZE == 13, static_assert(FILE_FOOTER_SIZE == 32, "FILE_FOOTER_SIZE must remain 32"); +/* Optional profiling statistics immediately precede the footer. The + * signature and size live at the end so readers can discover extensions + * without changing the fixed header or moving streamed sample data. */ +#define PROFILE_STATS_MAGIC "TACHSTAT" +#define PROFILE_STATS_MAGIC_SIZE 8 +#define PROFILE_STATS_VERSION 1 +#define PST_OFF_DURATION 0 +#define PST_SIZE_DURATION 8 +#define PST_OFF_SAMPLE_RATE (PST_OFF_DURATION + PST_SIZE_DURATION) +#define PST_SIZE_SAMPLE_RATE 8 +#define PST_OFF_ERROR_RATE (PST_OFF_SAMPLE_RATE + PST_SIZE_SAMPLE_RATE) +#define PST_SIZE_ERROR_RATE 8 +#define PST_OFF_MISSED_SAMPLES (PST_OFF_ERROR_RATE + PST_SIZE_ERROR_RATE) +#define PST_SIZE_MISSED_SAMPLES 8 +#define PST_OFF_PRESENT (PST_OFF_MISSED_SAMPLES + PST_SIZE_MISSED_SAMPLES) +#define PST_SIZE_PRESENT 4 +#define PST_OFF_RESERVED (PST_OFF_PRESENT + PST_SIZE_PRESENT) +#define PST_SIZE_RESERVED 4 +#define PST_OFF_MAGIC (PST_OFF_RESERVED + PST_SIZE_RESERVED) +#define PST_OFF_VERSION (PST_OFF_MAGIC + PROFILE_STATS_MAGIC_SIZE) +#define PST_SIZE_VERSION 4 +#define PST_OFF_SIZE (PST_OFF_VERSION + PST_SIZE_VERSION) +#define PST_SIZE_SIZE 4 +#define PROFILE_STATS_SIZE (PST_OFF_SIZE + PST_SIZE_SIZE) +#define PROFILE_STATS_V1_SIZE 32 +#define PROFILE_STATS_ERROR_RATE 0x01 +#define PROFILE_STATS_MISSED 0x02 + +static_assert(PROFILE_STATS_SIZE == 56, + "PROFILE_STATS_SIZE must remain 56"); + /* Minimum on-disk bytes of a string (1) and frame (7) table entry. */ #define MIN_STRING_ENTRY_SIZE 1 #define MIN_FRAME_ENTRY_SIZE 7 @@ -261,6 +292,12 @@ typedef struct { uint64_t start_time_us; uint64_t sample_interval_us; uint64_t total_samples; + double duration_sec; + double sample_rate; + double error_rate; + double missed_samples; + uint32_t profile_stats_present; + int has_profile_stats; /* String hash table: PyObject* -> uint32_t index */ _Py_hashtable_t *string_hash; @@ -328,6 +365,12 @@ typedef struct { uint64_t sample_interval_us; uint64_t sample_count; uint32_t thread_count; + double duration_sec; + double sample_rate; + double error_rate; + double missed_samples; + uint32_t profile_stats_present; + int has_profile_stats; uint64_t string_table_offset; uint64_t frame_table_offset; @@ -556,6 +599,16 @@ int binary_writer_write_sample( */ int binary_writer_finalize(BinaryWriter *writer); +/* Store measured statistics to write during finalization. */ +int binary_writer_set_stats( + BinaryWriter *writer, + double duration_sec, + double sample_rate, + double error_rate, + double missed_samples, + uint32_t present +); + /* * Destroy a binary writer and free all resources. * Safe to call even if writer is partially initialized. diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 07352f94cc805d..4e5ccaaabc7270 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -12,6 +12,7 @@ #include "binary_io.h" #include "_remote_debugging.h" #include "pycore_bitutils.h" /* _Py_bswap32, _Py_bswap64 for cross-endian reading */ +#include #include #ifdef HAVE_ZSTD @@ -125,6 +126,88 @@ reader_parse_footer(BinaryReader *reader, const uint8_t *data, size_t file_size) return 0; } +static inline int +reader_parse_profile_stats(BinaryReader *reader, const uint8_t *data, + size_t file_size) +{ + size_t minimum_size = FILE_FOOTER_SIZE + PROFILE_STATS_V1_SIZE; + if (file_size < minimum_size) { + return 0; + } + + const uint8_t *stats_end = data + file_size - FILE_FOOTER_SIZE; + const uint8_t *stats_tail = stats_end - PROFILE_STATS_MAGIC_SIZE - + PST_SIZE_VERSION - PST_SIZE_SIZE; + if (memcmp(stats_tail, + PROFILE_STATS_MAGIC, PROFILE_STATS_MAGIC_SIZE) != 0) { + return 0; + } + + uint32_t version, size; + memcpy(&version, stats_tail + PROFILE_STATS_MAGIC_SIZE, + PST_SIZE_VERSION); + memcpy(&size, stats_tail + PROFILE_STATS_MAGIC_SIZE + PST_SIZE_VERSION, + PST_SIZE_SIZE); + version = SWAP32_IF(reader->needs_swap, version); + size = SWAP32_IF(reader->needs_swap, size); + + if (size < PROFILE_STATS_V1_SIZE || + size > file_size - FILE_FOOTER_SIZE) { + PyErr_SetString(PyExc_ValueError, + "Invalid profiling statistics size"); + return -1; + } + if (version != PROFILE_STATS_VERSION) { + return 0; + } + + const uint8_t *trailer = stats_end - size; + uint64_t duration_bits, sample_rate_bits; + memcpy(&duration_bits, trailer + PST_OFF_DURATION, PST_SIZE_DURATION); + memcpy(&sample_rate_bits, trailer + PST_OFF_SAMPLE_RATE, + PST_SIZE_SAMPLE_RATE); + duration_bits = SWAP64_IF(reader->needs_swap, duration_bits); + sample_rate_bits = SWAP64_IF(reader->needs_swap, sample_rate_bits); + memcpy(&reader->duration_sec, &duration_bits, sizeof(duration_bits)); + memcpy(&reader->sample_rate, &sample_rate_bits, sizeof(sample_rate_bits)); + + if (!isfinite(reader->duration_sec) || reader->duration_sec < 0.0 || + !isfinite(reader->sample_rate) || reader->sample_rate < 0.0) { + PyErr_SetString(PyExc_ValueError, + "Invalid profiling statistics values"); + return -1; + } + reader->has_profile_stats = 1; + + if (size >= PROFILE_STATS_SIZE) { + uint64_t error_rate_bits, missed_samples_bits; + uint32_t present; + memcpy(&error_rate_bits, trailer + PST_OFF_ERROR_RATE, + PST_SIZE_ERROR_RATE); + memcpy(&missed_samples_bits, trailer + PST_OFF_MISSED_SAMPLES, + PST_SIZE_MISSED_SAMPLES); + memcpy(&present, trailer + PST_OFF_PRESENT, PST_SIZE_PRESENT); + error_rate_bits = SWAP64_IF(reader->needs_swap, error_rate_bits); + missed_samples_bits = SWAP64_IF(reader->needs_swap, + missed_samples_bits); + present = SWAP32_IF(reader->needs_swap, present); + memcpy(&reader->error_rate, &error_rate_bits, + sizeof(error_rate_bits)); + memcpy(&reader->missed_samples, &missed_samples_bits, + sizeof(missed_samples_bits)); + if (((present & PROFILE_STATS_ERROR_RATE) && + (!isfinite(reader->error_rate) || reader->error_rate < 0.0)) || + ((present & PROFILE_STATS_MISSED) && + !isfinite(reader->missed_samples))) { + PyErr_SetString(PyExc_ValueError, + "Invalid profiling statistics values"); + return -1; + } + reader->profile_stats_present = present; + } + return 0; +} + #ifdef HAVE_ZSTD /* Maximum decompression buffer size to prevent memory exhaustion (1GB) */ #define MAX_DECOMPRESS_SIZE (1ULL << 30) @@ -538,6 +621,9 @@ binary_reader_open(PyObject *path) if (reader_parse_footer(reader, data, file_size) < 0) { goto error; } + if (reader_parse_profile_stats(reader, data, file_size) < 0) { + goto error; + } /* Validate table offsets are within file bounds */ if (reader->string_table_offset > file_size) { @@ -1270,8 +1356,40 @@ binary_reader_get_info(BinaryReader *reader) if (py_version == NULL) { return NULL; } + PyObject *duration = reader->has_profile_stats + ? PyFloat_FromDouble(reader->duration_sec) : Py_NewRef(Py_None); + if (duration == NULL) { + Py_DECREF(py_version); + return NULL; + } + PyObject *sample_rate = reader->has_profile_stats + ? PyFloat_FromDouble(reader->sample_rate) : Py_NewRef(Py_None); + if (sample_rate == NULL) { + Py_DECREF(py_version); + Py_DECREF(duration); + return NULL; + } + PyObject *error_rate = + reader->profile_stats_present & PROFILE_STATS_ERROR_RATE + ? PyFloat_FromDouble(reader->error_rate) : Py_NewRef(Py_None); + if (error_rate == NULL) { + Py_DECREF(py_version); + Py_DECREF(duration); + Py_DECREF(sample_rate); + return NULL; + } + PyObject *missed_samples = + reader->profile_stats_present & PROFILE_STATS_MISSED + ? PyFloat_FromDouble(reader->missed_samples) : Py_NewRef(Py_None); + if (missed_samples == NULL) { + Py_DECREF(py_version); + Py_DECREF(duration); + Py_DECREF(sample_rate); + Py_DECREF(error_rate); + return NULL; + } return Py_BuildValue( - "{s:I, s:N, s:K, s:K, s:K, s:I, s:I, s:I, s:i}", + "{s:I, s:N, s:K, s:K, s:K, s:I, s:I, s:I, s:i, s:N, s:N, s:N, s:N}", "version", BINARY_FORMAT_VERSION, "python_version", py_version, "start_time_us", reader->start_time_us, @@ -1280,7 +1398,11 @@ binary_reader_get_info(BinaryReader *reader) "thread_count", reader->thread_count, "string_count", reader->strings_count, "frame_count", reader->frames_count, - "compression_type", reader->compression_type + "compression_type", reader->compression_type, + "duration_sec", duration, + "sample_rate", sample_rate, + "error_rate", error_rate, + "missed_samples", missed_samples ); } diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index e2ac40d96b39be..753d0b0cc966d0 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -12,6 +12,7 @@ #include "binary_io.h" #include "_remote_debugging.h" #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE +#include #include #ifdef HAVE_ZSTD @@ -1146,6 +1147,30 @@ binary_writer_finalize(BinaryWriter *writer) } } + if (writer->has_profile_stats) { + uint8_t profile_stats[PROFILE_STATS_SIZE] = {0}; + uint32_t version = PROFILE_STATS_VERSION; + uint32_t size = PROFILE_STATS_SIZE; + memcpy(profile_stats + PST_OFF_DURATION, + &writer->duration_sec, PST_SIZE_DURATION); + memcpy(profile_stats + PST_OFF_SAMPLE_RATE, + &writer->sample_rate, PST_SIZE_SAMPLE_RATE); + memcpy(profile_stats + PST_OFF_ERROR_RATE, + &writer->error_rate, PST_SIZE_ERROR_RATE); + memcpy(profile_stats + PST_OFF_MISSED_SAMPLES, + &writer->missed_samples, PST_SIZE_MISSED_SAMPLES); + memcpy(profile_stats + PST_OFF_PRESENT, + &writer->profile_stats_present, PST_SIZE_PRESENT); + memcpy(profile_stats + PST_OFF_MAGIC, + PROFILE_STATS_MAGIC, PROFILE_STATS_MAGIC_SIZE); + memcpy(profile_stats + PST_OFF_VERSION, &version, PST_SIZE_VERSION); + memcpy(profile_stats + PST_OFF_SIZE, &size, PST_SIZE_SIZE); + if (fwrite_checked_allow_threads( + profile_stats, PROFILE_STATS_SIZE, writer->fp) < 0) { + return -1; + } + } + /* Footer: string_count(4) + frame_count(4) + file_size(8) + checksum(16) */ file_offset_t footer_offset = FTELL64(writer->fp); if (footer_offset < 0) { @@ -1207,6 +1232,41 @@ binary_writer_finalize(BinaryWriter *writer) return 0; } +int +binary_writer_set_stats(BinaryWriter *writer, double duration_sec, + double sample_rate, double error_rate, + double missed_samples, uint32_t present) +{ + if (!isfinite(duration_sec) || duration_sec < 0.0) { + PyErr_SetString(PyExc_ValueError, + "duration must be a finite non-negative value"); + return -1; + } + if (!isfinite(sample_rate) || sample_rate < 0.0) { + PyErr_SetString(PyExc_ValueError, + "sample rate must be a finite non-negative value"); + return -1; + } + if ((present & PROFILE_STATS_ERROR_RATE) && + (!isfinite(error_rate) || error_rate < 0.0)) { + PyErr_SetString(PyExc_ValueError, + "error rate must be a finite non-negative value"); + return -1; + } + if ((present & PROFILE_STATS_MISSED) && !isfinite(missed_samples)) { + PyErr_SetString(PyExc_ValueError, + "missed samples must be a finite value"); + return -1; + } + writer->duration_sec = duration_sec; + writer->sample_rate = sample_rate; + writer->error_rate = error_rate; + writer->missed_samples = missed_samples; + writer->profile_stats_present = present; + writer->has_profile_stats = 1; + return 0; +} + void binary_writer_destroy(BinaryWriter *writer) { diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index ad66d947d19c52..efbf529922b33d 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -873,6 +873,103 @@ _remote_debugging_BinaryWriter_write_sample(PyObject *self, PyObject *const *arg return return_value; } +PyDoc_STRVAR(_remote_debugging_BinaryWriter_set_stats__doc__, +"set_stats($self, /, duration_sec, sample_rate, error_rate=None,\n" +" missed_samples=None)\n" +"--\n" +"\n" +"Store measured profile statistics in the binary file."); + +#define _REMOTE_DEBUGGING_BINARYWRITER_SET_STATS_METHODDEF \ + {"set_stats", _PyCFunction_CAST(_remote_debugging_BinaryWriter_set_stats), METH_FASTCALL|METH_KEYWORDS, _remote_debugging_BinaryWriter_set_stats__doc__}, + +static PyObject * +_remote_debugging_BinaryWriter_set_stats_impl(BinaryWriterObject *self, + double duration_sec, + double sample_rate, + PyObject *error_rate, + PyObject *missed_samples); + +static PyObject * +_remote_debugging_BinaryWriter_set_stats(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(duration_sec), &_Py_ID(sample_rate), &_Py_ID(error_rate), &_Py_ID(missed_samples), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"duration_sec", "sample_rate", "error_rate", "missed_samples", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_stats", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + double duration_sec; + double sample_rate; + PyObject *error_rate = Py_None; + PyObject *missed_samples = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + duration_sec = PyFloat_AS_DOUBLE(args[0]); + } + else + { + duration_sec = PyFloat_AsDouble(args[0]); + if (duration_sec == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + sample_rate = PyFloat_AS_DOUBLE(args[1]); + } + else + { + sample_rate = PyFloat_AsDouble(args[1]); + if (sample_rate == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + error_rate = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + missed_samples = args[3]; +skip_optional_pos: + return_value = _remote_debugging_BinaryWriter_set_stats_impl((BinaryWriterObject *)self, duration_sec, sample_rate, error_rate, missed_samples); + +exit: + return return_value; +} + PyDoc_STRVAR(_remote_debugging_BinaryWriter_finalize__doc__, "finalize($self, /)\n" "--\n" @@ -1588,4 +1685,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=22511c10d9224b28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5f20a08be0d0ed5a input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 4f365f303f14c7..708632bf603ddc 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -1781,6 +1781,53 @@ _remote_debugging_BinaryWriter_write_sample_impl(BinaryWriterObject *self, Py_RETURN_NONE; } +/*[clinic input] +_remote_debugging.BinaryWriter.set_stats + duration_sec: double + sample_rate: double + error_rate: object = None + missed_samples: object = None + +Store measured profile statistics in the binary file. +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_BinaryWriter_set_stats_impl(BinaryWriterObject *self, + double duration_sec, + double sample_rate, + PyObject *error_rate, + PyObject *missed_samples) +/*[clinic end generated code: output=28ab1bdd7c631a97 input=1646e7182f4c2259]*/ +{ + if (!self->writer) { + PyErr_SetString(PyExc_ValueError, "Writer is closed"); + return NULL; + } + uint32_t present = 0; + double error_rate_value = 0.0; + double missed_samples_value = 0.0; + if (error_rate != Py_None) { + error_rate_value = PyFloat_AsDouble(error_rate); + if (error_rate_value == -1.0 && PyErr_Occurred()) { + return NULL; + } + present |= PROFILE_STATS_ERROR_RATE; + } + if (missed_samples != Py_None) { + missed_samples_value = PyFloat_AsDouble(missed_samples); + if (missed_samples_value == -1.0 && PyErr_Occurred()) { + return NULL; + } + present |= PROFILE_STATS_MISSED; + } + if (binary_writer_set_stats(self->writer, duration_sec, sample_rate, + error_rate_value, missed_samples_value, + present) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + /* Finalize the writer, cache total_samples, and destroy it. * * The cache assignment must happen AFTER binary_writer_finalize(): finalize @@ -1928,6 +1975,7 @@ static PyGetSetDef BinaryWriter_getset[] = { static PyMethodDef BinaryWriter_methods[] = { _REMOTE_DEBUGGING_BINARYWRITER_WRITE_SAMPLE_METHODDEF + _REMOTE_DEBUGGING_BINARYWRITER_SET_STATS_METHODDEF _REMOTE_DEBUGGING_BINARYWRITER_FINALIZE_METHODDEF _REMOTE_DEBUGGING_BINARYWRITER_CLOSE_METHODDEF _REMOTE_DEBUGGING_BINARYWRITER___ENTER___METHODDEF