diff --git a/CLAUDE.md b/CLAUDE.md index 6b2370cd..7d6dff01 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1946,13 +1946,25 @@ time span `[t, t+1)`; the worker reports `sub_frame`, `sub_frames` and else the worker takes `totalFrames`; Soft Telecine's last cycle filled as `pipe_frames` does). `FrameMap::Decimate` declares radius `cycle + 1`. Which frame of the target's cycle goes depends on the picture: each frame - is tagged with its index, the script reads the tag where the target would - be, picks between the two outputs the walk computed (`after_drop`) and - prints `PREVIEW_DECIMATE:corr=N` for `PREVIEW_INFO`. Stabilize's pitch + is tagged with its index, and a `FrameEval` at the very end + (`_preview_decimate_pick`, `prop_src` the frame where the target would + be) picks between the two outputs the walk computed (`after_drop`) and + prints `PREVIEW_DECIMATE:corr=N` for `PREVIEW_INFO` when vspipe requests + the frame. **No template requests a frame while the script is + evaluated** (task #65): the first version read the tag with `get_frame()` + there, so the target's frame was ready the moment vspipe asked, and + **R78's vspipe counts a frame done before writing it**: its main thread + tore the script down under the frame callback (SIGSEGV, 2-3% of x86 IVTC + previews under load). R80 fixed vspipe (upstream 4ff3f8a0); until the + bundle has it, any preview can still hit it, only far more rarely. + `test_259` allows only the border colour's one-frame swatch. Stabilize's pitch lookup starts at `preview_source_offset` (`first + skip - pad`). `integration_ivtc_preview_test.dart` (heavy) checks previews against a - lossless encode by the film frame they hold - (docs/reports/issue-46-ivtc-preview). + lossless encode by the film frame they hold, and repeats the decimating + preview 36 times (docs/reports/issue-46-ivtc-preview, + docs/reports/task-65-ivtc-preview-teardown). An IVTC test needs a source + with pulldown: on progressive frames IVTC keeps each frame unchanged + (`_telecineFixture` in the parity test). - A source frame with no output of its own (down-conversion, decimation) shows the output just before it, as the preview always has. diff --git a/app/test/integration_high_bit_depth_filters_test.dart b/app/test/integration_high_bit_depth_filters_test.dart index 6084e504..0806c182 100644 --- a/app/test/integration_high_bit_depth_filters_test.dart +++ b/app/test/integration_high_bit_depth_filters_test.dart @@ -294,6 +294,39 @@ Future _shakenFixture(String pixFmt) async { return path; } +/// The master with 3:2 pulldown, at [pixFmt], for IVTC (task #65): its first +/// 20 frames as film, telecined to 25 slots, kept at the file's rate and +/// length (which `_job` declares). The master itself has no pulldown, so VFM +/// matches every frame to itself and VDecimate only drops whole frames: since +/// the preview runs VDecimate in the encode's phase (task #46), IVTC of frame +/// [_previewFrame] is that source frame unchanged (0.000 levels from the +/// passthrough). Here frame 12 is the cycle's mixed-field slot, which the +/// passthrough shows combed and IVTC field-matches (5.2 levels apart; 8 vs +/// 10-bit 0.19). Built from the master, so every depth holds one picture. +Future _telecineFixture(String pixFmt) async { + final path = '$_outDir/telecined_$pixFmt.mkv'; + if (File(path).existsSync()) return path; + final master = await _master(); + final result = await Process.run( + WorkerHarness.ffmpegPath, + [ + '-hide_banner', '-loglevel', 'error', '-y', + '-i', master, + '-vf', 'setfield=tff,trim=end_frame=20,' + 'telecine=first_field=t:pattern=23,' + 'setpts=N/(${_fps.toInt()}*TB),setfield=tff', + '-r', '${_fps.toInt()}', + '-c:v', 'ffv1', '-pix_fmt', pixFmt, + path, + ], + environment: WorkerHarness.ffmpegEnv, + ); + expect(result.exitCode, 0, + reason: 'telecined fixture generation failed for $pixFmt: ' + '${result.stderr}'); + return path; +} + /// The fixture [pass] runs on, at [pixFmt]. Future _fixtureFor(_Pass pass, String pixFmt) => (pass.fixture ?? _fixture)(pixFmt); @@ -495,8 +528,9 @@ class _Pass { /// master lacks (null: [_fixture]): [_shakenFixture] for a stabiliser, which /// has nothing to do on a picture with no global motion; [_scratchedFixture] /// for DeScratch (task #34); [_crawlFixture] for the dot-crawl and rainbow - /// filters (task #37). The passthrough reference is rendered from the same - /// fixture. + /// filters (task #37); [_telecineFixture] for IVTC, which keeps a frame + /// with no pulldown unchanged (task #65). The passthrough reference is + /// rendered from the same fixture. final Future Function(String pixFmt)? fixture; /// Whether the pass needs a working KNLMeansCL OpenCL device. Without one the @@ -617,6 +651,9 @@ List<_Pass> _passes() => [ ), // vivtc.VFM is 8-bit only and runs on a downconverted metrics clip, taking // its pixels from the full-depth clip via clip2 (issue: 10-bit ProRes IVTC). + // On the telecined fixture: the master has no pulldown, so IVTC kept its + // frame unchanged once the preview decimated in the encode's phase + // (task #46, #65), and passed only while the preview showed the wrong one. _Pass( 'deinterlace_ivtc', _only( @@ -627,6 +664,7 @@ List<_Pass> _passes() => [ ), ), tolerance: 3.0, + fixture: _telecineFixture, ), // DeScratch is 8-bit only: the pass analyses an 8-bit copy and applies @@ -798,7 +836,8 @@ List<_Pass> _passes() => [ // This fixture holds no fade, and on a frame that is not fading the fix // changes ~0.1 levels by design — so it cannot be required to change // the picture; the parity check is what this case is for (the fix's - // effect is integration_classical_filters_test's). + // effect is integration_classical_filters_test's). The telecined + // fixture, so the parity run goes through real field matching. _Pass( 'ivtc_fix_fades', _only( @@ -810,6 +849,7 @@ List<_Pass> _passes() => [ ), ), changesPicture: false, + fixture: _telecineFixture, ), // Deflicker measures frame brightness against a windowed average, so its diff --git a/app/test/integration_ivtc_preview_test.dart b/app/test/integration_ivtc_preview_test.dart index 880a6388..9387be8c 100644 --- a/app/test/integration_ivtc_preview_test.dart +++ b/app/test/integration_ivtc_preview_test.dart @@ -302,4 +302,47 @@ void main() { retime: true); }, timeout: const Timeout(Duration(minutes: 10))); }); + + // Task #65: in the 2026-11-16 nightly, decimating IVTC previews exited 1 + // on x86: vspipe R78 segfaulted in teardown, under its own frame callback, + // because the preview read VDecimate's drop with get_frame() during script + // evaluation and its one frame came back at once (2-3% of previews with + // four running under CPU load; 0 run one at a time). It is read in the + // graph now (test_259 keeps it there). This runs the preview again and + // again, three at a time, at every cycle phase: too few runs to catch a + // 2% fault reliably, so test_259 is the guard and this the smoke test. + test('IVTC preview exits cleanly, run after run (task #65)', () async { + final job = VideoJob( + id: const Uuid().v4(), + inputPath: _prores.path, + outputPath: '$_outDir/repeat.mkv', + processingPipeline: const ProcessingPipeline(deinterlace: _ivtc), + qtgmcParameters: _ivtc, + encodingSettings: const EncodingSettings( + codec: VideoCodec.ffv1, container: ContainerFormat.mkv, audioMode: AudioMode.none), + detectedFieldOrder: FieldOrder.topFieldFirst, + inputWidth: _prores.width, + inputHeight: _prores.height, + inputPixelFormat: _prores.pixFmt, + inputFrameRate: 30000 / 1001, + totalFrames: _prores.slots, + ).toJson(); + final failures = []; + final frames = [for (var i = 0; i < 36; i++) (i * 7) % _prores.slots]; + for (var i = 0; i < frames.length; i += 3) { + final runs = await Future.wait([ + for (final f in frames.skip(i).take(3)) + WorkerHarness.runPreview(job, frame: f, label: 'repeat_$f') + .then((pv) => (f, pv)), + ]); + for (final (f, pv) in runs) { + if (!pv.success) { + failures.add('frame $f: ${pv.error} | ${pv.errorTail}'); + } else if (!pv.logs.contains('decimation dropped a frame at or before the target: ')) { + failures.add('frame $f: no decimation report'); + } + } + } + expect(failures, isEmpty, reason: '${failures.length} of ${frames.length} previews'); + }, timeout: const Timeout(Duration(minutes: 10))); } diff --git a/app/test/preview_error_tail_test.dart b/app/test/preview_error_tail_test.dart new file mode 100644 index 00000000..20998c7a --- /dev/null +++ b/app/test/preview_error_tail_test.dart @@ -0,0 +1,41 @@ +// The heavy tests report a failed preview by PreviewResult.errorTail. It must +// keep the worker's own error line, which carries vspipe's exit status, even +// when more than four lines follow it (task #65: the nightly showed only +// plugin warnings and vspipe's frame count). +import 'package:flutter_test/flutter_test.dart'; + +import 'support/worker_harness.dart'; + +void main() { + const status = 'Error generating preview: vspipe exited with signal 11 ' + '(SIGSEGV) — a native crash; no Python error was printed'; + + test('keeps the worker error line when it has scrolled out of the tail', () { + final logs = [ + status, + 'Core freed but 3 filter instances still exist', + 'Plugin already loaded (com.vapoursynth.a)', + 'Plugin already loaded (com.vapoursynth.b)', + 'Plugin already loaded (com.vapoursynth.c)', + 'Output 1 frames in 0.00 seconds (1000.00 fps)', + ].join('\n'); + final tail = previewErrorTail(logs, 'preview exited with 1'); + expect(tail, startsWith(status)); + expect(tail, endsWith('Output 1 frames in 0.00 seconds (1000.00 fps)')); + expect(tail, isNot(contains('Core freed'))); + }); + + test('does not repeat a worker error line already in the tail', () { + final logs = ['warning', status, 'Traceback line'].join('\n'); + expect(previewErrorTail(logs, null), 'warning | $status | Traceback line'); + }); + + test('without the worker line, the last four lines', () { + final logs = List.generate(6, (i) => 'line $i').join('\n'); + expect(previewErrorTail(logs, null), 'line 2 | line 3 | line 4 | line 5'); + }); + + test('no logs: the error', () { + expect(previewErrorTail('\n \n', 'timed out'), 'timed out'); + }); +} diff --git a/app/test/support/worker_harness.dart b/app/test/support/worker_harness.dart index 4895657b..9541caae 100644 --- a/app/test/support/worker_harness.dart +++ b/app/test/support/worker_harness.dart @@ -1212,6 +1212,27 @@ class JobResult { : 'FAILED: $error (exit code: $exitCode)'; } +/// The last few lines of a failed preview's stderr (the VapourSynth error, +/// without the traceback noise above it), led by the worker's own +/// `Error generating preview:` line when that has scrolled out of them. That +/// line carries vspipe's exit status (task #56 F9b); in the 2026-11-16 +/// nightly the tail held only plugin warnings and vspipe's "Output 1 frames" +/// line, so the status that said "crashed after writing" was never shown +/// (task #65). +String previewErrorTail(String logs, String? error) { + final lines = logs + .split('\n') + .map((l) => l.trim()) + .where((l) => l.isNotEmpty) + .toList(); + // A preview killed at its timeout has no logs; say so rather than nothing. + if (lines.isEmpty) return error ?? ''; + final tail = lines.length <= 4 ? lines : lines.sublist(lines.length - 4); + final head = lines.indexWhere((l) => l.contains('Error generating preview:')); + if (head < 0 || head >= lines.length - tail.length) return tail.join(' | '); + return [lines[head], '…', ...tail].join(' | '); +} + /// Outcome of a `--preview` run: the PNG bytes, or an error plus the worker's /// stderr (which carries the VapourSynth traceback for a failed pass). class PreviewResult { @@ -1223,18 +1244,9 @@ class PreviewResult { bool get success => png != null && png!.isNotEmpty; - /// The last few stderr lines — the VapourSynth error, without the traceback - /// noise above it. - String get errorTail { - final lines = logs - .split('\n') - .map((l) => l.trim()) - .where((l) => l.isNotEmpty) - .toList(); - // A preview killed at its timeout has no logs; say so rather than nothing. - if (lines.isEmpty) return error ?? ''; - return lines.length <= 4 ? lines.join(' | ') : lines.sublist(lines.length - 4).join(' | '); - } + /// The worker's own error line and the last few stderr lines; see + /// [previewErrorTail]. + String get errorTail => previewErrorTail(logs, error); @override String toString() => success ? 'SUCCESS (${png!.length} bytes)' : 'FAILED: $error\n$errorTail'; diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index 4edee339..750f7ac1 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -5181,3 +5181,38 @@ Report: [docs/reports/task-60-bifrost-v4](reports/task-60-bifrost-v4/index.html) v3.0 (one run each). - **The earlier 2026-08-15 note "Bifrost is 8-bit only", and its include-farm note, describe v3.0.** + +### The IVTC preview crashed vspipe in teardown; IVTC parity needs pulldown (task #65, 2026-11-18) + +Report: [docs/reports/task-65-ivtc-preview-teardown](reports/task-65-ivtc-preview-teardown/index.html). +Both faults came in with task #46 (the preview runs VDecimate in the encode's phase). + +- **vspipe R78 counts a frame done before writing it.** `frameDoneCallback` + increments `completedFrames` without the mutex, then writes the frame, and + takes the lock only to notify. `outputNode`'s `condition.wait(lock, pred)` + checks the predicate as soon as it is called, so if the frame completes + before the main thread reaches `wait`, main sees "all done", calls + `freeScript` and the callback segfaults (`outputFrame`, + `frameDoneCallback`; main in Python's GC under `vpy4_freeScript`). Fixed + upstream in 4ff3f8a0, first released in R80 ("Counting at entry … let the + waiter return while the last frame was still being written"). +- **A one-frame preview hits it when its frame is ready at once.** #46's + preview read VDecimate's drop with `get_frame()` while the script was + evaluated, which computed the target's graph in advance. With four + previews at a time under CPU load, that template failed 23/1320 (Linux + x64), 21/1320 (x64-v2) and 4/1320 (macOS x64) IVTC previews, every Linux + failure a vspipe core in teardown. The drop is now read in the graph (a + `FrameEval` with `prop_src`): 0/2640 on Linux and 0/1320 on macOS. Run one + at a time, both templates were 0/2280; on Apple Silicon 0/600 (Rosetta). +- **It isn't gone.** Bwdif (no decimation) crashed the same way 2 times in + 1200 on macOS x64, one with each template. Until the bundle has R80+, any + preview can. +- **IVTC needs pulldown to test.** The high-bit-depth parity master is + progressive, so IVTC of frame 12 is frame 12 unchanged (0.000 levels from + the passthrough); it "changed the picture" only while the preview showed + the wrong frame. The case now runs on the master telecined 3:2 (5.2 levels; + 8 vs 10-bit 0.19). +- **Concurrent previews of one job id share a scratch directory** + (`vapourbox_preview_`). The app gives every preview a fresh id; the + first repro run gave every loop one id and measured the collision, not the + crash. diff --git a/docs/reports/README.md b/docs/reports/README.md index a7dc42ea..b2f27c7d 100644 --- a/docs/reports/README.md +++ b/docs/reports/README.md @@ -120,3 +120,4 @@ to it. | 2026-11-15 | [9-bit sources: Dehalo's YAHR failed](2026-11-9-bit-sources/index.html) | Built: 9-bit sources decode as 10-bit. CTMF has no 9-bit format, and havsfunc reaches it through `MinBlur(r>=2)` and directly: YAHR, FineDehalo contra, MCTemporalDenoise pfMode 2 and ContraSharpening r2 failed at every size, HQDeringmod and SMDegrain contrasharp at HD. The conversion is lossless (limited range an exact shift); the Noise Reduction CTMF block's own 9-bit guard is removed. `original` output of a 9-bit source is 10-bit (+1.2% FFV1). Windows/Linux by the nightly run only | | 2026-11-16 | [Task #49: one output capability table, and the output conformance matrix](task-49-output-conformance/index.html) | Built: `output_capabilities.json` read by worker and app; a 444-case generated heavy test. 425/446 pass; three deterministic faults (19 cases) and an intermittent `pipe_source` wrong-frame fault filed as task #58 | | 2026-11-17 | [Task #60: Bifrost v3.0 → v4.0](task-60-bifrost-v4/index.html) | Built: every platform takes Bifrost 4.0 from its PyPI wheel. v3.0's chroma past the last whole 4x4 block was uninitialised (2–3 results in 3 runs at width 98/718, 720x486); v4.0 gives one, identical to v3.0 inside whole blocks. Runs natively at 8–16 bits, luma bit-exact, 0.1% of chroma decisions differing from 8-bit at the threshold; the 8-bit copy is gone. 4:2:0 at 720x486 with `interlaced=True` is still refused (task #57) | +| 2026-11-18 | [Task #65: the IVTC preview's vspipe crash, and IVTC parity](task-65-ivtc-preview-teardown/index.html) | Fixed: vspipe R78 counts a frame done before writing it, so a one-frame preview whose frame is ready at once can be torn down under its callback (SIGSEGV; fixed upstream in R80). Task #46's evaluation-time `get_frame()` made the IVTC preview's frame ready at once: 2–3% of x86 IVTC previews under load crashed, 0/3960 with the read moved into the graph. Any preview can still hit it rarely (2/1200 Bwdif previews on macOS x64) until the bundle has R80+. The IVTC parity case needed a telecined fixture | diff --git a/docs/reports/task-65-ivtc-preview-teardown/data/backtrace-example.txt b/docs/reports/task-65-ivtc-preview-teardown/data/backtrace-example.txt new file mode 100644 index 00000000..f6d65fba --- /dev/null +++ b/docs/reports/task-65-ivtc-preview-teardown/data/backtrace-example.txt @@ -0,0 +1,137 @@ +== core /repro/cores/core.vspipe-bin.11891 (/deps/linux-x64/vapoursynth/vspipe-bin) after ivtc pass=2 frame=1 rc=0 +[New LWP 11951] +[New LWP 11949] +[New LWP 11950] +[New LWP 11948] +[New LWP 11891] +[New LWP 11951] +[New LWP 11949] +[New LWP 11950] +[New LWP 11948] +[New LWP 11891] +[Thread debugging using libthread_db enabled] +Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". +Core was generated by `/deps/linux-x64/vapoursynth/vspipe-bin -c y4'. +Program terminated with signal SIGSEGV, Segmentation fault. +#0 0x000055e589eec04b in outputFrame(VSFrame const*, VSPipeOutputData*) () +[Thread debugging using libthread_db enabled] +Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". +Core was generated by `/deps/linux-x64/vapoursynth/vspipe-bin -c y4'. +Program terminated with signal SIGSEGV, Segmentation fault. +#0 0x000055e589eec04b in outputFrame(VSFrame const*, VSPipeOutputData*) () +[Current thread is 1 (Thread 0x7f93f2ffe6c0 (LWP 11951))] +Signal Stop Print Pass to program Description +SIGSEGV Yes Yes Yes Segmentation fault + +Thread 5 (Thread 0x7f94048bc740 (LWP 11891)): +#0 0x00007f9402bb2a5b in type_is_gc () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#1 0x00007f9402c442f5 in visit_decref () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#2 0x00007f9402b9560e in dict_traverse () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#3 0x00007f9402c441f6 in deduce_unreachable () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#4 0x00007f9402c427f5 in gc_collect_main.llvm () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#5 0x00007f9402d74275 in gc_collect () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#6 0x00007f9402b98876 in cfunction_vectorcall_FASTCALL_KEYWORDS.llvm.16519472471476773533 () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#7 0x00007f9402b6b486 in PyObject_Vectorcall () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#8 0x00007f94022aced2 in vpy4_freeScript(VSScript*) () from /deps/linux-x64/vapoursynth/vapoursynth.abi3.so +#9 0x00007f94049e71ee in freeScript(VSScript*) () from /deps/linux-x64/vapoursynth/libvsscript.so +#10 0x000055e589ee3a10 in main () + +Thread 4 (Thread 0x7f93fade76c0 (LWP 11948)): +#0 0x00007f9404298e31 in __futex_abstimed_wait_common64 (private=32659, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:57 +#1 __futex_abstimed_wait_common (cancel=true, private=32659, abstime=0x0, clockid=0, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:87 +#2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0x55e5a205ec64, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0x0, private=private@entry=0) at ./nptl/futex-internal.c:139 +#3 0x00007f940429b8ad in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x55e5a205eb68, cond=0x55e5a205ec38) at ./nptl/pthread_cond_wait.c:503 +#4 ___pthread_cond_wait (cond=0x55e5a205ec38, mutex=0x55e5a205eb68) at ./nptl/pthread_cond_wait.c:627 +#5 0x00007f9402130975 in VSThreadPool::runTasks(bool&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#6 0x00007f94046ecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#7 0x00007f940429cb64 in start_thread (arg=) at ./nptl/pthread_create.c:447 +#8 0x00007f9404329e8c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 + +Thread 3 (Thread 0x7f93f3fff6c0 (LWP 11950)): +#0 0x00007f9404298e31 in __futex_abstimed_wait_common64 (private=32659, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:57 +#1 __futex_abstimed_wait_common (cancel=true, private=32659, abstime=0x0, clockid=0, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:87 +#2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0x55e5a205ec64, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0x0, private=private@entry=0) at ./nptl/futex-internal.c:139 +#3 0x00007f940429b8ad in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x55e5a205eb68, cond=0x55e5a205ec38) at ./nptl/pthread_cond_wait.c:503 +#4 ___pthread_cond_wait (cond=0x55e5a205ec38, mutex=0x55e5a205eb68) at ./nptl/pthread_cond_wait.c:627 +#5 0x00007f9402130975 in VSThreadPool::runTasks(bool&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#6 0x00007f94046ecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#7 0x00007f940429cb64 in start_thread (arg=) at ./nptl/pthread_create.c:447 +#8 0x00007f9404329e8c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 + +Thread 2 (Thread 0x7f93f9de66c0 (LWP 11949)): +#0 0x00007f9404298e31 in __futex_abstimed_wait_common64 (private=32659, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:57 +#1 __futex_abstimed_wait_common (cancel=true, private=32659, abstime=0x0, clockid=0, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:87 +#2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0x55e5a205ec64, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0x0, private=private@entry=0) at ./nptl/futex-internal.c:139 +#3 0x00007f940429b8ad in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x55e5a205eb68, cond=0x55e5a205ec38) at ./nptl/pthread_cond_wait.c:503 +#4 ___pthread_cond_wait (cond=0x55e5a205ec38, mutex=0x55e5a205eb68) at ./nptl/pthread_cond_wait.c:627 +#5 0x00007f9402130975 in VSThreadPool::runTasks(bool&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#6 0x00007f94046ecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#7 0x00007f940429cb64 in start_thread (arg=) at ./nptl/pthread_create.c:447 +#8 0x00007f9404329e8c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 + +Thread 1 (Thread 0x7f93f2ffe6c0 (LWP 11951)): +#0 0x000055e589eec04b in outputFrame(VSFrame const*, VSPipeOutputData*) () +#1 0x000055e589eecaad in frameDoneCallback(void*, VSFrame const*, int, VSNode*, char const*) [clone .lto_priv.0] () +#2 0x00007f940212ef8b in VSThreadPool::returnFrame(VSFrameContext*, vs_intrusive_ptr const&, std::unique_lock&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#3 0x00007f9402131b25 in VSThreadPool::runTasks(bool&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#4 0x00007f94046ecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#5 0x00007f940429cb64 in start_thread (arg=) at ./nptl/pthread_create.c:447 +#6 0x00007f9404329e8c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 +[Current thread is 1 (Thread 0x7f93f2ffe6c0 (LWP 11951))] +Signal Stop Print Pass to program Description +SIGSEGV Yes Yes Yes Segmentation fault + +Thread 5 (Thread 0x7f94048bc740 (LWP 11891)): +#0 0x00007f9402bb2a5b in type_is_gc () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#1 0x00007f9402c442f5 in visit_decref () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#2 0x00007f9402b9560e in dict_traverse () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#3 0x00007f9402c441f6 in deduce_unreachable () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#4 0x00007f9402c427f5 in gc_collect_main.llvm () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#5 0x00007f9402d74275 in gc_collect () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#6 0x00007f9402b98876 in cfunction_vectorcall_FASTCALL_KEYWORDS.llvm.16519472471476773533 () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#7 0x00007f9402b6b486 in PyObject_Vectorcall () from /deps/linux-x64/python/lib/libpython3.12.so.1.0 +#8 0x00007f94022aced2 in vpy4_freeScript(VSScript*) () from /deps/linux-x64/vapoursynth/vapoursynth.abi3.so +#9 0x00007f94049e71ee in freeScript(VSScript*) () from /deps/linux-x64/vapoursynth/libvsscript.so +#10 0x000055e589ee3a10 in main () + +Thread 4 (Thread 0x7f93fade76c0 (LWP 11948)): +#0 0x00007f9404298e31 in __futex_abstimed_wait_common64 (private=32659, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:57 +#1 __futex_abstimed_wait_common (cancel=true, private=32659, abstime=0x0, clockid=0, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:87 +#2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0x55e5a205ec64, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0x0, private=private@entry=0) at ./nptl/futex-internal.c:139 +#3 0x00007f940429b8ad in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x55e5a205eb68, cond=0x55e5a205ec38) at ./nptl/pthread_cond_wait.c:503 +#4 ___pthread_cond_wait (cond=0x55e5a205ec38, mutex=0x55e5a205eb68) at ./nptl/pthread_cond_wait.c:627 +#5 0x00007f9402130975 in VSThreadPool::runTasks(bool&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#6 0x00007f94046ecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#7 0x00007f940429cb64 in start_thread (arg=) at ./nptl/pthread_create.c:447 +#8 0x00007f9404329e8c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 + +Thread 3 (Thread 0x7f93f3fff6c0 (LWP 11950)): +#0 0x00007f9404298e31 in __futex_abstimed_wait_common64 (private=32659, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:57 +#1 __futex_abstimed_wait_common (cancel=true, private=32659, abstime=0x0, clockid=0, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:87 +#2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0x55e5a205ec64, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0x0, private=private@entry=0) at ./nptl/futex-internal.c:139 +#3 0x00007f940429b8ad in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x55e5a205eb68, cond=0x55e5a205ec38) at ./nptl/pthread_cond_wait.c:503 +#4 ___pthread_cond_wait (cond=0x55e5a205ec38, mutex=0x55e5a205eb68) at ./nptl/pthread_cond_wait.c:627 +#5 0x00007f9402130975 in VSThreadPool::runTasks(bool&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#6 0x00007f94046ecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#7 0x00007f940429cb64 in start_thread (arg=) at ./nptl/pthread_create.c:447 +#8 0x00007f9404329e8c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 + +Thread 2 (Thread 0x7f93f9de66c0 (LWP 11949)): +#0 0x00007f9404298e31 in __futex_abstimed_wait_common64 (private=32659, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:57 +#1 __futex_abstimed_wait_common (cancel=true, private=32659, abstime=0x0, clockid=0, expected=0, futex_word=0x55e5a205ec64) at ./nptl/futex-internal.c:87 +#2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0x55e5a205ec64, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0x0, private=private@entry=0) at ./nptl/futex-internal.c:139 +#3 0x00007f940429b8ad in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x55e5a205eb68, cond=0x55e5a205ec38) at ./nptl/pthread_cond_wait.c:503 +#4 ___pthread_cond_wait (cond=0x55e5a205ec38, mutex=0x55e5a205eb68) at ./nptl/pthread_cond_wait.c:627 +#5 0x00007f9402130975 in VSThreadPool::runTasks(bool&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#6 0x00007f94046ecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#7 0x00007f940429cb64 in start_thread (arg=) at ./nptl/pthread_create.c:447 +#8 0x00007f9404329e8c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 + +Thread 1 (Thread 0x7f93f2ffe6c0 (LWP 11951)): +#0 0x000055e589eec04b in outputFrame(VSFrame const*, VSPipeOutputData*) () +#1 0x000055e589eecaad in frameDoneCallback(void*, VSFrame const*, int, VSNode*, char const*) [clone .lto_priv.0] () +#2 0x00007f940212ef8b in VSThreadPool::returnFrame(VSFrameContext*, vs_intrusive_ptr const&, std::unique_lock&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#3 0x00007f9402131b25 in VSThreadPool::runTasks(bool&) () from /deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#4 0x00007f94046ecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#5 0x00007f940429cb64 in start_thread (arg=) at ./nptl/pthread_create.c:447 +#6 0x00007f9404329e8c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 diff --git a/docs/reports/task-65-ivtc-preview-teardown/data/summary-linux-x64-v2.txt b/docs/reports/task-65-ivtc-preview-teardown/data/summary-linux-x64-v2.txt new file mode 100644 index 00000000..6b28815c --- /dev/null +++ b/docs/reports/task-65-ivtc-preview-teardown/data/summary-linux-x64-v2.txt @@ -0,0 +1,6 @@ +main ivtc (4 loops, CPU load on): runs=600 failures=14 vspipe-cores=14 +main ivtc-avi (4 loops, CPU load on): runs=720 failures=7 vspipe-cores=7 +main bwdif (4 loops, CPU load on): runs=600 failures=0 vspipe-cores=0 +branch ivtc (4 loops, CPU load on): runs=600 failures=0 vspipe-cores=0 +branch ivtc-avi (4 loops, CPU load on): runs=720 failures=0 vspipe-cores=0 +branch bwdif (4 loops, CPU load on): runs=600 failures=0 vspipe-cores=0 diff --git a/docs/reports/task-65-ivtc-preview-teardown/data/summary-linux-x64.txt b/docs/reports/task-65-ivtc-preview-teardown/data/summary-linux-x64.txt new file mode 100644 index 00000000..81174d4c --- /dev/null +++ b/docs/reports/task-65-ivtc-preview-teardown/data/summary-linux-x64.txt @@ -0,0 +1,6 @@ +main ivtc (4 loops, CPU load on): runs=600 failures=15 vspipe-cores=15 +main ivtc-avi (4 loops, CPU load on): runs=720 failures=8 vspipe-cores=8 +main bwdif (4 loops, CPU load on): runs=600 failures=0 vspipe-cores=0 +branch ivtc (4 loops, CPU load on): runs=600 failures=0 vspipe-cores=0 +branch ivtc-avi (4 loops, CPU load on): runs=720 failures=0 vspipe-cores=0 +branch bwdif (4 loops, CPU load on): runs=600 failures=0 vspipe-cores=0 diff --git a/docs/reports/task-65-ivtc-preview-teardown/data/summary-macos-x64.txt b/docs/reports/task-65-ivtc-preview-teardown/data/summary-macos-x64.txt new file mode 100644 index 00000000..a526ca90 --- /dev/null +++ b/docs/reports/task-65-ivtc-preview-teardown/data/summary-macos-x64.txt @@ -0,0 +1,6 @@ +main ivtc (4 loops, CPU load on): runs=600 failures=3 vspipe-cores=0 +main ivtc-avi (4 loops, CPU load on): runs=720 failures=1 vspipe-cores=0 +main bwdif (4 loops, CPU load on): runs=600 failures=1 vspipe-cores=0 +branch ivtc (4 loops, CPU load on): runs=600 failures=0 vspipe-cores=0 +branch ivtc-avi (4 loops, CPU load on): runs=720 failures=0 vspipe-cores=0 +branch bwdif (4 loops, CPU load on): runs=600 failures=1 vspipe-cores=0 diff --git a/docs/reports/task-65-ivtc-preview-teardown/index.html b/docs/reports/task-65-ivtc-preview-teardown/index.html new file mode 100644 index 00000000..a055dac8 --- /dev/null +++ b/docs/reports/task-65-ivtc-preview-teardown/index.html @@ -0,0 +1,162 @@ + + + + + +IVTC preview crash + + + +
+

Task #65: the IVTC preview's vspipe crash, and IVTC parity

+

2026-11-18 · GitHub-hosted runners (ubuntu-24.04, macos-15-intel) and macOS arm64 (M1 Pro) · deps 1.17.0-rc3 · scripts in scripts/, raw counts and a backtrace in data/

+ +

Outcome

+
    +
  • vspipe R78 counts a frame done before it writes it. A one-frame run whose frame is ready almost +immediately can have its main thread tear the script down while the frame callback is still writing: SIGSEGV in +outputFrame / frameDoneCallback. Fixed upstream in 4ff3f8a0, first released in R80.
  • +
  • Task #46's IVTC preview made its frame ready at once: it read VDecimate's drop with +get_frame() while the script was evaluated, which computed the target's graph in advance. Under load, +2–3% of x86 IVTC previews crashed. The read is now made inside the graph (a FrameEval with +prop_src), giving 0 crashes in 3960 IVTC previews. It picks the same frame: 120 of 120 +previews were byte-identical to the old template's on macOS arm64.
  • +
  • Not gone for every preview: Bwdif, which doesn't decimate, crashed 2 times in 1200 runs on macOS x64, +once with each template. Only a VapourSynth upgrade to R80+ removes it (filed as a follow-up).
  • +
  • The 8 vs 10-bit IVTC parity case had no pulldown to remove. Its master is progressive, so IVTC +keeps frame 12 unchanged (0.000 levels from the passthrough). It passed only while the preview showed the wrong frame (the +bug #46 fixed). It now runs on the master telecined 3:2.
  • +
+ +

1. What the nightly showed (2026-11-16, main dc9e36c)

+ + + + + + +
Failurearm64 macx64 macx64-v2 macWin x64Win x64-v2Linux x64Linux x64-v2
parity deinterlace_ivtc: "same frame as a passthrough"✗✗✗✗✗✗✗
IVTC on 10-bit ProRes: preview exited with 1✓✓✗✓✓✗✗
IVTC at the minimum radius: preview exited with 1✓✓✓✓✓✓✗
parity ivtc_fix_fades: preview exited with 1✓✓✓✓✓✓✗
+

The error tail showed three "plugin already loaded" warnings (task #61) and vspipe's own Output 1 frames +line, without the exit status. Every failure path in vspipe that can follow that line prints Error: first, +and none had, so this pointed to a native crash after the frame. The test harness now always shows the worker's +Error generating preview: line, which carries the status (task #56 F9b).

+ +

2. Reproducing it

+

scripts/preview_loop.sh runs the worker's preview over every source frame of a job, keeps each failure's +stderr, and runs gdb over any core. scripts/compare_templates.sh runs it with main's template and with this +branch's (each a copy of the debug worker with its own templates/), four loops at a time, with stress-ng +holding the CPUs at 60%. scripts/repro-ivtc-preview.yml is the temporary workflow that ran both on CI. +The jobs:

+
    +
  • ivtc: prores422_10bit_telecine.mov (720x480 10-bit 4:2:2, 30 frames) through IVTC, the nightly's case;
  • +
  • ivtc-avi: hard_telecine_test.avi (8-bit 4:2:0, 90 frames) through IVTC;
  • +
  • bwdif: the ProRes clip through Bwdif, the control (no decimation, no evaluation-time read).
  • +
+ +

One loop at a time: nothing

+ + + + + + +
Bundle (old template)ivtcivtc-avibwdif
linux-x640 / 3000 / 2700 / 300
linux-x64-v20 / 3000 / 2700 / 300
macos-x640 / 3000 / 2700 / 300
macos-x64-v20 / 3000 / 2700 / 300
+

The same held locally, an x86_64 worker under Rosetta on Apple Silicon: 0 in 600, sequential and four at a +time. The nightly ran its previews while the rest of the suite ran.

+ +

Four loops at a time under CPU load

+ + + + + + + + +
BundleTemplateivtcivtc-avibwdifvspipe cores
linux-x64main15 / 6008 / 7200 / 60023
this branch0 / 6000 / 7200 / 6000
linux-x64-v2main14 / 6007 / 7200 / 60021
this branch0 / 6000 / 7200 / 6000
macos-x64main3 / 6001 / 7201 / 600(no core dumps)
this branch0 / 6000 / 7201 / 600
+

Every failure was vspipe exited with signal 11 (SIGSEGV). macos-x64-v2 had not been scheduled by the +time the others finished. Raw counts: data/summary-*.txt.

+

A first run gave all four loops one job id. The worker names its preview scratch directory after the +job id (vapourbox_preview_<id>), so the loops deleted each other's files, and the failures it +counted (about 25% of runs, Bwdif included) were that collision, not the crash. That run's numbers are not used.

+ +

3. Where it crashes

+

Of the 44 Linux cores, 40 show the crashing thread in outputFrame (23) or +frameDoneCallback (17) while the main thread is in freeScript → vpy4_freeScript → Python's +garbage collector. The remaining 4 are unsymbolised: the process was already unmapping its libraries. Example: +data/backtrace-example.txt.

+

vspipe R78's frameDoneCallback (abridged):

+
if (rnode == data->node) data->completedFrames++;   // no lock, first thing
+...
+outputFrame(frame, data);                             // writes the frame
+...
+if (data->totalFrames == data->completedFrames ...) {
+    std::lock_guard<std::mutex> lock(data->mutex);
+    data->condition.notify_one();
+}
+

and outputNode:

+
std::unique_lock<std::mutex> lock(data->mutex);
+for (n ...) getFrameAsync(n, node, frameDoneCallback, data);
+data->condition.wait(lock, [&]() { return data->totalFrames == data->completedFrames ...; });
+// ... then output closes and the script is freed
+

wait evaluates its predicate before it first sleeps. If the one frame completes between +getFrameAsync and that check, the predicate is already true while the callback is still inside +outputFrame, and main goes on to free what the callback is using. Upstream's +4ff3f8a0 ("Fix many rare issues found by AI", 2026-09-06, in R80) moved the increment to the end of the callback, +under the mutex, with the comment "Counting at entry, as this used to, let the waiter return while the last frame was +still being written."

+

The race needs the frame to finish within the main thread's few instructions between request and check, or +while it is descheduled. Task #46's evaluation-time get_frame() made that likely: the target's frames +were already in the cache, so the output frame came back at once. CPU load, which deschedules main, raised the rate +from 0 to 2–3%.

+ +

4. The change

+
    +
  • _preview_decimate_out keeps the frame where the target would be (c[_j], still tagged) +instead of reading it, and _preview_decimate_pick at the end of the script returns +FrameEval(c[keep], pick, prop_src=probe). The pick chooses c[keep] or +c[after_drop] from the tag and prints PREVIEW_DECIMATE:corr=N, which the worker's stderr +reader already collects. The constant cases (the cycle's last frame, past the clip's end) still decide when the script is evaluated, without a frame.
  • +
  • Same picks: on macOS arm64 the old and new templates gave byte-identical PNGs and the same drop report for all 30 +ProRes and 90 AVI frames.
  • +
  • test_259: no get_frame( in the template outside the border colour's one-frame BlankClip +swatch. A heavy smoke test repeats the ProRes preview 36 times, three at a time. At a 2% rate it would +miss the fault about half the time, so test_259 is the guard.
  • +
+ +

5. The parity case

+ + + + +
Fixture (frame 12, 352x288)IVTC 10-bit vs passthrough 10-bitIVTC 8-bit vs 10-bit
master (progressive, flagged TFF)0.0000.192
master's first 20 frames telecined 3:2 to 25 slots5.2320.194
+

Mean absolute difference in 10-bit RGB levels. Measured with an x86_64 worker under Rosetta (macos-x64 1.17.0-rc2). +The heavy test on macOS arm64 with 1.17.0-rc3 gave 0.191 for both IVTC cases (tolerance 3.0 and 2.0). +ivtc_fix_fades moved to the same fixture, so its parity goes through real field matching.

+ +

Limits

+
    +
  • The rates are for four previews at a time with CPUs held at 60% on hosted runners; they say the fault is real and +which template triggers it, not how often a user sees it.
  • +
  • No Windows run: its vspipe is the same R78 source, in the wheel.
  • +
  • Apple Silicon was not run under load natively. arm64 never failed in the nightly.
  • +
+
+ + diff --git a/docs/reports/task-65-ivtc-preview-teardown/scripts/compare_templates.sh b/docs/reports/task-65-ivtc-preview-teardown/scripts/compare_templates.sh new file mode 100755 index 00000000..180b0a5b --- /dev/null +++ b/docs/reports/task-65-ivtc-preview-teardown/scripts/compare_templates.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Task #65: the IVTC preview loop with main's template and with this +# branch's, side by side, each as 4 concurrent loops under CPU load (the +# nightly ran its previews beside the rest of the heavy suite). Run from the +# repository root after `cd worker && cargo build`: +# +# compare_templates.sh +# +# Each side is a copy of the debug worker with its own templates/ beside it, +# which the worker searches first. Writes /summary.txt. +set -u +REF="$1"; OUT="$2" +HERE="$(cd "$(dirname "$0")" && pwd)" +BIN=worker/target/debug +mkdir -p "$OUT" +for side in main branch; do + rm -rf "${BIN:?}/${side:?}"; mkdir -p "$BIN/$side" + cp "$BIN/vapourbox-worker" "$BIN/$side/" + cp -R worker/templates "$BIN/$side/templates" +done +git show "$REF:worker/templates/pipeline_template.vpy" > "$BIN/main/templates/pipeline_template.vpy" +if cmp -s "$BIN/main/templates/pipeline_template.vpy" "$BIN/branch/templates/pipeline_template.vpy"; then + echo "note: the branch's template is main's" | tee "$OUT/summary.txt" +fi + +CPUS=$(getconf _NPROCESSORS_ONLN) +LOAD="" +if command -v stress-ng >/dev/null; then + stress-ng --cpu "$CPUS" --cpu-load 60 --timeout 0 >/dev/null 2>&1 & + LOAD=$! +fi +PASSES="${PASSES:-5}" +for side in main branch; do + for mode in ivtc ivtc-avi bwdif; do + cores_before=$(ls "${CORE_DIR:-/nonexistent}" 2>/dev/null | grep -vc '\.bt$') + pids=() + passes=$([ "$mode" = ivtc-avi ] && echo $(( (PASSES + 2) / 3 )) || echo "$PASSES") + for k in 1 2 3 4; do + bash "$HERE/preview_loop.sh" "$BIN/$side/vapourbox-worker" "$mode" "$passes" \ + "$OUT/$side-$mode-$k" > "$OUT/$side-$mode-$k.log" 2>&1 & + pids+=($!) + done + wait "${pids[@]}" + runs=0; fails=0 + for k in 1 2 3 4; do + line=$(tail -n1 "$OUT/$side-$mode-$k.log") + r=$(echo "$line" | sed -n 's/.*runs=\([0-9]*\).*/\1/p'); f=$(echo "$line" | sed -n 's/.*failures=\([0-9]*\).*/\1/p') + runs=$((runs + ${r:-0})); fails=$((fails + ${f:-0})) + grep '^FAIL' "$OUT/$side-$mode-$k.log" | head -5 + done + cores=$(( $(ls "${CORE_DIR:-/nonexistent}" 2>/dev/null | grep -vc '\.bt$') - cores_before )) + echo "$side $mode (4 loops, CPU load ${LOAD:+on}): runs=$runs failures=$fails vspipe-cores=$cores" | tee -a "$OUT/summary.txt" + done +done +[ -n "$LOAD" ] && kill "$LOAD" +cat "$OUT/summary.txt" diff --git a/docs/reports/task-65-ivtc-preview-teardown/scripts/preview_loop.sh b/docs/reports/task-65-ivtc-preview-teardown/scripts/preview_loop.sh new file mode 100755 index 00000000..9cd1f638 --- /dev/null +++ b/docs/reports/task-65-ivtc-preview-teardown/scripts/preview_loop.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Run the worker's preview over and over and count the runs that fail +# (task #65). Run from the repository root: +# +# preview_loop.sh +# +# picks the job: +# ivtc the ProRes 10-bit telecine fixture through IVTC (decimating), +# the preview the 2026-11-16 nightly failed on +# ivtc-avi hard_telecine_test.avi through IVTC (decimating, 8-bit) +# bwdif the ProRes fixture through Bwdif (the control: no decimation) +# +# Each pass previews every source frame. A run that exits non-zero keeps +# its stderr in /fail---.err; if a core file +# appears (the caller sets ulimit -c and core_pattern), every thread's +# backtrace goes beside it (needs gdb). The last line printed is +# " runs=N failures=M". +set -u +WORKER="$1"; MODE="$2"; PASSES="$3"; OUT="$4" +ROOT="$(pwd)" +mkdir -p "$OUT" +JOB="$OUT/job-$MODE.json" +CORES="${CORE_DIR:-$OUT/cores}" + +case "$MODE" in + ivtc|bwdif) + SRC="$ROOT/Tests/TestResources/prores422_10bit_telecine.mov" + FMT=yuv422p10le; FRAMES=30 + METHOD=$([ "$MODE" = bwdif ] && echo bwdif || echo ivtc) ;; + ivtc-avi) + SRC="$ROOT/Tests/TestResources/hard_telecine_test.avi" + FMT=yuv420p; FRAMES=90; METHOD=ivtc ;; + *) echo "unknown mode $MODE" >&2; exit 2 ;; +esac + +# Each loop its own job id: the worker's preview scratch directory is named +# after it, so concurrent loops sharing one deleted each other's files. +ID=$(uuidgen 2>/dev/null || python3 -c 'import uuid; print(uuid.uuid4())') +cat > "$JOB" </dev/null | grep -vc '\.bt$') + "$WORKER" --config "$JOB" --preview --frame "$f" > "$OUT/last.png" 2> "$OUT/last.err" + rc=$? + if [ $rc -ne 0 ]; then + fails=$((fails + 1)) + cp "$OUT/last.err" "$OUT/fail-$MODE-$pass-$f.err" + echo "FAIL $MODE pass=$pass frame=$f rc=$rc: $(grep -m1 'Error generating preview' "$OUT/last.err")" + fi + after=$(ls "$CORES" 2>/dev/null | grep -vc '\.bt$') + if [ "$after" -gt "$before" ] && command -v gdb >/dev/null; then + for core in $(ls -t "$CORES"/core.* | grep -v '\.bt$' | head -n $((after - before))); do + [ -e "$core.bt" ] && continue + exe=$(file "$core" | sed -n "s/.*execfn: '\([^']*\)'.*/\1/p") + echo "== core $core ($exe) after $MODE pass=$pass frame=$f rc=$rc" > "$core.bt" + gdb -batch -nx -ex 'info signals SIGSEGV' -ex 'thread apply all bt 40' \ + "$exe" "$core" >> "$core.bt" 2>&1 + sed -n '1,200p' "$core.bt" + done + fi + done + echo "$MODE pass $pass: runs=$runs failures=$fails" +done +echo "$MODE runs=$runs failures=$fails" diff --git a/docs/reports/task-65-ivtc-preview-teardown/scripts/repro-ivtc-preview.yml b/docs/reports/task-65-ivtc-preview-teardown/scripts/repro-ivtc-preview.yml new file mode 100644 index 00000000..0955ab6f --- /dev/null +++ b/docs/reports/task-65-ivtc-preview-teardown/scripts/repro-ivtc-preview.yml @@ -0,0 +1,97 @@ +name: Repro IVTC preview exit (task 65, temporary) + +# TEMPORARY (task #65): loops the decimating IVTC preview on x86 with core +# dumps, to measure how often vspipe exits non-zero after writing its frame +# and where. Delete before the task's PR is merged. + +on: + push: + branches: [stuart/65-nightly-2026-11-16-main-dc9e36c-ivtc-hea] + workflow_dispatch: + +permissions: + contents: read + actions: read + +jobs: + linux: + strategy: + fail-fast: false + matrix: + asset: [linux-x64, linux-x64-v2] + name: linux (${{ matrix.asset }}) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v5 + with: {fetch-depth: 0} + - name: System packages + run: sudo apt-get update && sudo apt-get install -y gdb file stress-ng + - name: Deps + env: {GH_TOKEN: '${{ github.token }}'} + run: | + TAG=$(python3 -c "import json;print(json.load(open('app/assets/deps-version.json'))['releaseTag'])") + bash .github/scripts/install-deps-bundle.sh "${{ matrix.asset }}" "$TAG" "${TAG#deps-v}" deps/linux-x64 + mkdir -p "$HOME/.local/share/VapourBox" + ln -sfn "$GITHUB_WORKSPACE/deps" "$HOME/.local/share/VapourBox/deps" + - uses: dtolnay/rust-toolchain@stable + - name: Build worker + run: cd worker && cargo build + - name: Loops + run: | + mkdir -p "$GITHUB_WORKSPACE/repro/cores" + sudo sysctl -w kernel.core_pattern="$GITHUB_WORKSPACE/repro/cores/core.%e.%p" + ulimit -c unlimited + export CORE_DIR="$GITHUB_WORKSPACE/repro/cores" + bash docs/reports/task-65-ivtc-preview-teardown/scripts/compare_templates.sh \ + origin/main repro + echo "## ${{ matrix.asset }}" >> "$GITHUB_STEP_SUMMARY" + cat repro/summary.txt >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v5 + if: always() + with: + name: repro-${{ matrix.asset }} + path: | + repro/**/*.log + repro/**/*.err + repro/summary.txt + repro/cores/*.bt + if-no-files-found: ignore + + macos: + strategy: + fail-fast: false + matrix: + asset: [macos-x64, macos-x64-v2] + name: macos (${{ matrix.asset }}) + runs-on: macos-15-intel + env: + VAPOURBOX_DEPS_DIR: ${{ github.workspace }}/deps/macos-x64 + steps: + - uses: actions/checkout@v5 + with: {fetch-depth: 0} + - name: Deps + env: {GH_TOKEN: '${{ github.token }}'} + run: | + TAG=$(python3 -c "import json;print(json.load(open('app/assets/deps-version.json'))['releaseTag'])") + bash .github/scripts/install-deps-bundle.sh "${{ matrix.asset }}" "$TAG" "${TAG#deps-v}" deps/macos-x64 + xattr -cr deps/macos-x64 2>/dev/null || true + - uses: dtolnay/rust-toolchain@stable + - name: Build worker + run: cd worker && cargo build && brew install stress-ng + - name: Loops + run: | + bash docs/reports/task-65-ivtc-preview-teardown/scripts/compare_templates.sh origin/main repro + echo "## ${{ matrix.asset }}" >> "$GITHUB_STEP_SUMMARY" + cat repro/summary.txt >> "$GITHUB_STEP_SUMMARY" + - name: Crash reports + if: always() + run: python3 .github/scripts/print-crash-reports.py + - uses: actions/upload-artifact@v5 + if: always() + with: + name: repro-${{ matrix.asset }} + path: | + repro/**/*.log + repro/**/*.err + repro/summary.txt + if-no-files-found: ignore diff --git a/worker/templates/pipeline_template.vpy b/worker/templates/pipeline_template.vpy index 716cab82..9a46a264 100644 --- a/worker/templates/pipeline_template.vpy +++ b/worker/templates/pipeline_template.vpy @@ -64,6 +64,16 @@ clip = create_pipe_clip( # goes depends on the picture: each frame is tagged with its index, and the # tag of the frame where the target would be says whether one went at or # before it. The worker computed the output for both cases. +# +# That tag is read inside the graph (_preview_decimate_pick, at the end), when +# vspipe asks for the preview frame, never with get_frame() while the script +# is evaluated (task #65): that computed the target's graph in advance, so +# vspipe's one frame came back at once, and R78's vspipe can then tear the +# script down while its frame callback is still writing (it counts the frame +# done before writing it). It segfaulted in 2-3% of x86 IVTC previews under +# load; 0 in 2640 with the read in the graph. +_dec_probe = None +_dec_target = -1 _dec_corr = 0 @@ -89,15 +99,32 @@ def _preview_decimate_in(c): def _preview_decimate_out(c): - global _dec_corr + global _dec_probe, _dec_target, _dec_corr _t, _cycle = {{PREVIEW_DECIMATE_TARGET}}, {{PREVIEW_DECIMATE_CYCLE}} _j = _t // _cycle * (_cycle - 1) + _t % _cycle if _t % _cycle == _cycle - 1 or _j >= c.num_frames: + # The cycle's last frame, or past the clip's end: a frame went before + # it whatever the picture. _dec_corr = 1 + print(f'PREVIEW_DECIMATE:corr={_dec_corr}', file=sys.stderr) else: - _dec_corr = 0 if c.get_frame(_j).props['_PreviewSlot'] == _t else 1 - print(f'PREVIEW_DECIMATE:corr={_dec_corr}', file=sys.stderr) + _dec_probe, _dec_target = c[_j], _t return core.std.RemoveFrameProps(c, '_PreviewSlot') + + +def _preview_decimate_pick(c, keep, after_drop): + # The frame to show: c[keep] if the frame at the target's place is the + # target (nothing dropped at or before it), else c[after_drop]. Decided + # from _dec_probe's tag when the frame is requested. + if _dec_probe is None: + return c[after_drop if _dec_corr else keep] + _shown = (c[keep], c[after_drop]) + + def _pick(n, f): + corr = 0 if f.props['_PreviewSlot'] == _dec_target else 1 + print(f'PREVIEW_DECIMATE:corr={corr}', file=sys.stderr, flush=True) + return _shown[corr] + return core.std.FrameEval(_shown[0], _pick, prop_src=_dec_probe) {{/PREVIEW_ONLY}} # 4:1:1 (NTSC DV) becomes 4:2:2 before anything else, losslessly: horizontal @@ -3155,8 +3182,10 @@ clip = _pad_to(clip, _canvas_w if _canvas_w > 0 else clip.width, _canvas_h if _c # source frames around the requested frame and passes the deterministic output # index it maps to (after any frame-rate change such as QTGMC double-rate). # Clamp defensively in case the decode came up short. -target_frame = max(0, min({{PREVIEW_OUTPUT_INDEX_AFTER_DROP}} if _dec_corr else {{PREVIEW_OUTPUT_INDEX}}, - clip.num_frames - 1)) -clip = clip[target_frame] +# A decimation's two candidates, without and with a frame dropped at or +# before the target, are picked between in the graph (_preview_decimate_pick). +_keep, _after_drop = [max(0, min(_i, clip.num_frames - 1)) + for _i in ({{PREVIEW_OUTPUT_INDEX}}, {{PREVIEW_OUTPUT_INDEX_AFTER_DROP}})] +clip = _preview_decimate_pick(clip, _keep, _after_drop) {{/PREVIEW_ONLY}} clip.set_output() diff --git a/worker/tests/filter_integration_test.rs b/worker/tests/filter_integration_test.rs index ce88fa9f..9f585dcf 100644 --- a/worker/tests/filter_integration_test.rs +++ b/worker/tests/filter_integration_test.rs @@ -1979,7 +1979,7 @@ fn test_55_preview_selects_exact_frame() { let script = std::fs::read_to_string(&script_path).expect("read preview script"); assert!( - script.contains("min(7 if _dec_corr else 7,"), + script.contains("for _i in (7, 7)]"), "preview must select the exact computed output index, script was:\n{}", script ); @@ -9308,7 +9308,7 @@ fn test_251_ivtc_and_soft_telecine_preview_decimates_in_the_encodes_phase() { "_skip = 0\n", "_pad, _real, _len = 1, 11, ", &format!("_t, _cycle = 7, 5\n"), - &format!("min({} if _dec_corr else {},", d.after_drop.0, out.local_index), + &format!("for _i in ({}, {})]", out.local_index, d.after_drop.0), ] { assert!(preview.contains(value), "{name}: preview lacks {value:?}"); } @@ -9331,7 +9331,7 @@ fn test_251_ivtc_and_soft_telecine_preview_decimates_in_the_encodes_phase() { unblend_origin: 0, decimate: None::, source_frame_offset: 0, }; let preview = std::fs::read_to_string(generator.generate_preview(&plain, ¶ms).unwrap()).unwrap(); - assert!(preview.contains("min(10 if _dec_corr else 10,")); + assert!(preview.contains("for _i in (10, 10)]")); assert!(!preview.contains("core.vivtc.VDecimate(")); } @@ -9876,3 +9876,56 @@ fn test_258_field_splits_are_padded() { assert!(problems.is_empty(), "{}", problems.join("\n")); assert!(sites >= 3, "the scan found only {sites} field splits"); } + +#[test] +fn test_259_the_template_requests_no_frame_while_it_is_evaluated() { + // Task #65. The IVTC / Soft Telecine preview read VDecimate's drop with + // `get_frame()` while the script was evaluated. That computed the + // target's graph in advance, so vspipe's one frame came back at once, + // and R78's vspipe counts a frame done before writing it: its main + // thread tore the script down under the callback (SIGSEGV in + // frameDoneCallback / outputFrame, 2-3% of x86 IVTC previews under + // load). The drop is read in the graph (`_preview_decimate_pick`'s + // FrameEval). The one frame the template requests itself is the border + // colour's swatch, a one-frame BlankClip that reaches no source. + let template = std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("templates/pipeline_template.vpy"), + ) + .unwrap() + .replace("\r\n", "\n"); + let calls: Vec<&str> = template + .lines() + .filter(|l| !l.trim_start().starts_with('#') && l.contains("get_frame(")) + .collect(); + assert_eq!(calls, vec![" frame = swatch.get_frame(0)"], + "a frame requested while the script is evaluated: {calls:?}"); + + // The preview of a decimating job picks its frame in the graph. + let mut job = create_base_job("test_259_decimate_pick"); + job.qtgmc_parameters.enabled = true; + job.qtgmc_parameters.method = DeinterlaceMethod::Ivtc; + job.qtgmc_parameters.tff = Some(true); + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: job.qtgmc_parameters.clone(), + ..ProcessingPipeline::default() + }); + let pipeline = job.effective_pipeline(); + let out = pipeline.preview_output_in(0, 34, 40, 0, Some((13, 44))); + let params = PreviewParams { + width: 720, height: 480, pix_fmt: "yuv420p".into(), num_frames: 13, + fps_num: 30000, fps_den: 1001, + output_index: out.local_index as i32, + rate_pad: out.rate_pad, unblend_pad: 0, unblend_origin: 0, + decimate: out.decimate, source_frame_offset: 34, + }; + let generator = ScriptGenerator::new().unwrap(); + let preview = std::fs::read_to_string(generator.generate_preview(&job, ¶ms).unwrap()).unwrap(); + for needle in [ + "_dec_probe, _dec_target = c[_j], _t", + "core.std.FrameEval(_shown[0], _pick, prop_src=_dec_probe)", + "clip = _preview_decimate_pick(clip, _keep, _after_drop)", + ] { + assert!(preview.contains(needle), "preview lacks {needle:?}"); + } + assert!(!preview.contains("get_frame(_j)")); +}