From 08b3ac67d23a5ce848a26664f7d4f0da45b3cf7a Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:36:11 +0000 Subject: [PATCH 1/9] perf(camera): scaled IDCT for instant MJPEG bubble Decode Linux MJPEG at the largest FFmpeg lowres that still covers the instant overlay. Studio holds full resolution, and the first captured frame stays native so VideoInfo does not shrink. Blur coverage stays at least 640x360. --- crates/camera-ffmpeg/src/lib.rs | 11 + crates/camera-ffmpeg/src/linux.rs | 275 ++++++++++++++++-- crates/camera-ffmpeg/src/macos.rs | 8 + crates/camera-ffmpeg/src/mjpeg_idct.rs | 70 +++++ crates/camera-ffmpeg/src/windows.rs | 8 + crates/recording/src/feeds/camera.rs | 145 ++++++++- .../src/instant_recording/linux_camera.rs | 43 +++ crates/recording/src/sources/camera.rs | 3 + 8 files changed, 536 insertions(+), 27 deletions(-) create mode 100644 crates/camera-ffmpeg/src/mjpeg_idct.rs diff --git a/crates/camera-ffmpeg/src/lib.rs b/crates/camera-ffmpeg/src/lib.rs index 3ec20823a6b..cf0bafa056b 100644 --- a/crates/camera-ffmpeg/src/lib.rs +++ b/crates/camera-ffmpeg/src/lib.rs @@ -1,3 +1,5 @@ +mod mjpeg_idct; + #[cfg(target_os = "macos")] mod macos; #[cfg(target_os = "macos")] @@ -17,4 +19,13 @@ pub trait CapturedFrameExt { /// Creates an ffmpeg video frame from the native frame. /// Only size, format, and data are set. fn as_ffmpeg(&self) -> Result; + + /// Like [`as_ffmpeg`](Self::as_ffmpeg), but MJPEG may decode with a scaled + /// IDCT when `max_width`×`max_height` is smaller than the frame. The + /// result still covers that rectangle. Other pixel formats are unchanged. + fn as_ffmpeg_covering( + &self, + max_width: u32, + max_height: u32, + ) -> Result; } diff --git a/crates/camera-ffmpeg/src/linux.rs b/crates/camera-ffmpeg/src/linux.rs index 89e42e2a85b..293cd42922a 100644 --- a/crates/camera-ffmpeg/src/linux.rs +++ b/crates/camera-ffmpeg/src/linux.rs @@ -3,6 +3,7 @@ use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo}; use std::cell::RefCell; use crate::CapturedFrameExt; +use crate::mjpeg_idct::idct_lowres; #[derive(thiserror::Error, Debug)] pub enum AsFFmpegError { @@ -24,19 +25,27 @@ pub enum AsFFmpegError { struct MjpegDecoder { decoder: ffmpeg::codec::decoder::Video, + lowres: i32, } impl MjpegDecoder { - fn new() -> Result { + fn new(lowres: i32) -> Result { let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG) .ok_or_else(|| AsFFmpegError::MjpegDecodeError("MJPEG codec not found".to_string()))?; + let max_lowres = codec.max_lowres().clamp(0, 3); + let lowres = lowres.clamp(0, max_lowres); - let decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec); + let mut decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec); + if lowres > 0 { + unsafe { + (*decoder_context.as_mut_ptr()).lowres = lowres; + } + } let decoder = decoder_context.decoder().video().map_err(|e| { AsFFmpegError::MjpegDecodeError(format!("Failed to create decoder: {e}")) })?; - Ok(Self { decoder }) + Ok(Self { decoder, lowres }) } fn decode(&mut self, bytes: &[u8]) -> Result { @@ -60,35 +69,73 @@ thread_local! { static MJPEG_DECODER: RefCell> = const { RefCell::new(None) }; } -fn decode_mjpeg(bytes: &[u8]) -> Result { +fn decode_mjpeg(bytes: &[u8], lowres: i32) -> Result { MJPEG_DECODER.with(|decoder_cell| { - let mut decoder_opt = decoder_cell.borrow_mut(); - - if decoder_opt.is_none() { - *decoder_opt = Some(MjpegDecoder::new()?); + let mut slot = decoder_cell.borrow_mut(); + if slot.as_ref().is_none_or(|decoder| decoder.lowres != lowres) { + tracing::debug!(lowres, "MJPEG IDCT scale"); + *slot = Some(MjpegDecoder::new(lowres)?); } - - decoder_opt.as_mut().unwrap().decode(bytes) + slot.as_mut() + .expect("MJPEG decoder is inserted before use") + .decode(bytes) }) } +fn mjpeg_frame_covering( + bytes: &[u8], + source_width: u32, + source_height: u32, + max_width: u32, + max_height: u32, +) -> Result { + let lowres = idct_lowres(source_width, source_height, max_width, max_height); + if lowres == 0 { + return decode_mjpeg(bytes, 0); + } + match decode_mjpeg(bytes, lowres) { + Ok(frame) if frame.width() >= max_width && frame.height() >= max_height => Ok(frame), + _ => decode_mjpeg(bytes, 0), + } +} + impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { - let native = self.native(); - let format = native.format; - let bytes = native.bytes.as_slice(); - - match fourcc_str(format.fourcc).as_str() { - "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2), - "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2), - "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3), - "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3), - "NV12" => copy_nv12(bytes, format), - "YU12" => copy_yuv420(bytes, format, false), - "YV12" => copy_yuv420(bytes, format, true), - "MJPG" | "JPEG" => decode_mjpeg(bytes), - other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())), - } + frame_from_native(self, None) + } + + fn as_ffmpeg_covering( + &self, + max_width: u32, + max_height: u32, + ) -> Result { + frame_from_native(self, Some((max_width, max_height))) + } +} + +fn frame_from_native( + frame: &CapturedFrame, + cover: Option<(u32, u32)>, +) -> Result { + let native = frame.native(); + let format = native.format; + let bytes = native.bytes.as_slice(); + + match fourcc_str(format.fourcc).as_str() { + "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2), + "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2), + "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3), + "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3), + "NV12" => copy_nv12(bytes, format), + "YU12" => copy_yuv420(bytes, format, false), + "YV12" => copy_yuv420(bytes, format, true), + "MJPG" | "JPEG" => match cover { + Some((max_width, max_height)) => { + mjpeg_frame_covering(bytes, format.width, format.height, max_width, max_height) + } + None => decode_mjpeg(bytes, 0), + }, + other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())), } } @@ -247,3 +294,181 @@ fn require_len( fn fourcc_str(fourcc: [u8; 4]) -> String { String::from_utf8_lossy(&fourcc).to_string() } + +#[cfg(test)] +mod tests { + use super::{MjpegDecoder, decode_mjpeg, mjpeg_frame_covering}; + use crate::mjpeg_idct::idct_lowres; + use ffmpeg::Packet; + use ffmpeg::format::Pixel; + use ffmpeg::frame::Video as FFVideo; + + fn encode_mjpeg(width: u32, height: u32) -> Vec { + let codec = ffmpeg::codec::encoder::find(ffmpeg::codec::Id::MJPEG).unwrap(); + let mut encoder = ffmpeg::codec::context::Context::new_with_codec(codec) + .encoder() + .video() + .unwrap(); + encoder.set_width(width); + encoder.set_height(height); + encoder.set_format(Pixel::YUVJ420P); + encoder.set_time_base((1, 30)); + encoder.set_qmin(4); + encoder.set_qmax(4); + let mut encoder = encoder.open().unwrap(); + + let mut frame = FFVideo::new(Pixel::YUVJ420P, width, height); + let luma_stride = frame.stride(0); + let luma = frame.data_mut(0); + for y in 0..height as usize { + for x in 0..width as usize { + let nx = x as f32 / width as f32; + let ny = y as f32 / height as f32; + let value = 70.0 + + 40.0 * (nx * std::f32::consts::TAU * 2.0).sin() + + 25.0 * (ny * std::f32::consts::TAU * 3.0).sin() + + 18.0 * ((nx + ny) * std::f32::consts::TAU * 5.0).sin() + + ((x * 13 + y * 7) & 7) as f32 + - 3.0; + luma[y * luma_stride + x] = value.clamp(0.0, 255.0) as u8; + } + } + frame.data_mut(1).fill(110); + frame.data_mut(2).fill(150); + frame.set_pts(Some(0)); + encoder.send_frame(&frame).unwrap(); + encoder.send_eof().unwrap(); + let mut packet = Packet::empty(); + encoder.receive_packet(&mut packet).unwrap(); + packet.data().unwrap().to_vec() + } + + fn scale_bgra(frame: &FFVideo, width: u32, height: u32) -> FFVideo { + let mut scaler = ffmpeg::software::scaling::Context::get( + frame.format(), + frame.width(), + frame.height(), + Pixel::BGRA, + width, + height, + ffmpeg::software::scaling::Flags::BILINEAR, + ) + .unwrap(); + let mut output = FFVideo::empty(); + scaler.run(frame, &mut output).unwrap(); + output + } + + fn bgra_delta(full: &FFVideo, scaled: &FFVideo) -> (f64, u8) { + let width = full.width().min(scaled.width()) as usize; + let height = full.height().min(scaled.height()) as usize; + let mut sum = 0u64; + let mut max = 0u8; + let mut count = 0u64; + for y in 0..height { + let full_row = &full.data(0)[y * full.stride(0)..]; + let scaled_row = &scaled.data(0)[y * scaled.stride(0)..]; + for x in 0..width { + for channel in 0..3 { + let delta = full_row[x * 4 + channel].abs_diff(scaled_row[x * 4 + channel]); + sum += u64::from(delta); + max = max.max(delta); + count += 1; + } + } + } + ((sum as f64) / count.max(1) as f64, max) + } + + #[test] + fn scaled_idct_covers_the_bubble_and_stays_close_to_full_then_scale() { + ffmpeg::init().unwrap(); + let bytes = encode_mjpeg(640, 360); + let scaled = mjpeg_frame_covering(&bytes, 640, 360, 200, 120).unwrap(); + assert_eq!((scaled.width(), scaled.height()), (320, 180)); + let full = decode_mjpeg(&bytes, 0).unwrap(); + assert_eq!((full.width(), full.height()), (640, 360)); + let (mean, max) = bgra_delta(&scale_bgra(&full, 200, 120), &scale_bgra(&scaled, 200, 120)); + assert!( + max <= 6 && mean <= 1.5, + "scaled IDCT vs full decode + scale: mean {mean:.3} max {max}" + ); + } + + #[test] + fn full_resolution_request_restores_the_decoder_after_a_scaled_frame() { + ffmpeg::init().unwrap(); + let bytes = encode_mjpeg(640, 360); + let scaled = mjpeg_frame_covering(&bytes, 640, 360, 200, 120).unwrap(); + assert_eq!((scaled.width(), scaled.height()), (320, 180)); + let full = mjpeg_frame_covering(&bytes, 640, 360, 640, 360).unwrap(); + assert_eq!((full.width(), full.height()), (640, 360)); + let untouched = decode_mjpeg(&bytes, 0).unwrap(); + assert_eq!((untouched.width(), untouched.height()), (640, 360)); + } + + #[test] + #[ignore = "release wall-time measurement for the instant bubble"] + fn bubble_scaled_idct_is_faster_than_full_decode_and_scale() { + ffmpeg::init().unwrap(); + let bytes = encode_mjpeg(1920, 1080); + let bubble = 324u32; + let lowres = idct_lowres(1920, 1080, bubble, bubble); + assert_eq!(lowres, 1); + let mut full_decoder = None; + let mut scaled_decoder = None; + let warmup = |lowres: i32, slot: &mut Option| { + for _ in 0..4 { + if slot.as_ref().is_none_or(|decoder| decoder.lowres != lowres) { + *slot = Some(MjpegDecoder::new(lowres).unwrap()); + } + slot.as_mut().unwrap().decode(&bytes).unwrap(); + } + }; + warmup(0, &mut full_decoder); + warmup(lowres, &mut scaled_decoder); + let full_frame = full_decoder.as_mut().unwrap().decode(&bytes).unwrap(); + let scaled_frame = scaled_decoder.as_mut().unwrap().decode(&bytes).unwrap(); + assert_eq!((scaled_frame.width(), scaled_frame.height()), (960, 540)); + let (mean, max) = bgra_delta( + &scale_bgra(&full_frame, bubble, bubble), + &scale_bgra(&scaled_frame, bubble, bubble), + ); + + let iterations = 40; + let time = |slot: &mut Option, source: &FFVideo| { + let mut scaler = ffmpeg::software::scaling::Context::get( + source.format(), + source.width(), + source.height(), + Pixel::BGRA, + bubble, + bubble, + ffmpeg::software::scaling::Flags::BILINEAR, + ) + .unwrap(); + let mut output = FFVideo::empty(); + let start = std::time::Instant::now(); + for _ in 0..iterations { + let frame = slot.as_mut().unwrap().decode(&bytes).unwrap(); + scaler.run(&frame, &mut output).unwrap(); + } + start.elapsed().as_secs_f64() * 1e3 / iterations as f64 + }; + let full_ms = time(&mut full_decoder, &full_frame); + let scaled_ms = time(&mut scaled_decoder, &scaled_frame); + let speedup = full_ms / scaled_ms; + println!( + "mjpeg-idct bubble 1920x1080 -> {bubble}: full+scale {full_ms:.3}ms scaled+scale {scaled_ms:.3}ms speedup {speedup:.2}x bgra_mean {mean:.3} max {max} jpeg_bytes {}", + bytes.len() + ); + assert!( + speedup >= 1.20, + "expected a bubble-size win, got {speedup:.2}x ({full_ms:.3}ms -> {scaled_ms:.3}ms)" + ); + assert!( + max <= 8, + "bubble pixel delta too large: mean {mean:.3} max {max}" + ); + } +} diff --git a/crates/camera-ffmpeg/src/macos.rs b/crates/camera-ffmpeg/src/macos.rs index a6c11b602bd..0857655cd66 100644 --- a/crates/camera-ffmpeg/src/macos.rs +++ b/crates/camera-ffmpeg/src/macos.rs @@ -62,6 +62,14 @@ impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { sample_buf_as_ffmpeg(self.native().sample_buf()) } + + fn as_ffmpeg_covering( + &self, + _max_width: u32, + _max_height: u32, + ) -> Result { + self.as_ffmpeg() + } } // Standalone so consumers holding a bare retained CMSampleBuffer (e.g. the diff --git a/crates/camera-ffmpeg/src/mjpeg_idct.rs b/crates/camera-ffmpeg/src/mjpeg_idct.rs new file mode 100644 index 00000000000..caefe378e04 --- /dev/null +++ b/crates/camera-ffmpeg/src/mjpeg_idct.rs @@ -0,0 +1,70 @@ +/// Largest FFmpeg MJPEG `lowres` whose decoded frame still covers +/// `target_width`×`target_height`. +/// +/// `0` is full resolution, `1` is 1/2, `2` is 1/4, `3` is 1/8. Dimensions use +/// the decoder's ceiling division, so the choice cannot undershoot the frame +/// FFmpeg actually returns. +pub(crate) fn idct_lowres( + source_width: u32, + source_height: u32, + target_width: u32, + target_height: u32, +) -> i32 { + if source_width == 0 || source_height == 0 || target_width == 0 || target_height == 0 { + return 0; + } + let mut lowres = 0; + for candidate in 1..=3 { + let (width, height) = scaled_dimensions(source_width, source_height, candidate); + if width < target_width || height < target_height || width == 0 || height == 0 { + break; + } + lowres = candidate; + } + lowres +} + +pub(crate) fn scaled_dimensions(source_width: u32, source_height: u32, lowres: i32) -> (u32, u32) { + if lowres <= 0 { + return (source_width, source_height); + } + let shift = u32::try_from(lowres).unwrap_or(3).min(3); + let divisor = 1u32 << shift; + ( + source_width.saturating_add(divisor - 1) / divisor, + source_height.saturating_add(divisor - 1) / divisor, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bubble_on_1080p_uses_half_idct() { + assert_eq!(idct_lowres(1920, 1080, 324, 324), 1); + assert_eq!(scaled_dimensions(1920, 1080, 1), (960, 540)); + } + + #[test] + fn smaller_bubble_uses_quarter_idct() { + assert_eq!(idct_lowres(1920, 1080, 216, 216), 2); + assert_eq!(scaled_dimensions(1920, 1080, 2), (480, 270)); + assert_eq!(scaled_dimensions(1920, 1080, 3), (240, 135)); + } + + #[test] + fn studio_and_large_overlay_stay_full_resolution() { + assert_eq!(idct_lowres(1920, 1080, 1920, 1080), 0); + assert_eq!(idct_lowres(1920, 1080, 648, 648), 0); + assert_eq!(idct_lowres(1280, 720, 432, 432), 0); + assert_eq!(idct_lowres(1920, 1080, 0, 324), 0); + } + + #[test] + fn ceiling_division_matches_odd_mjpeg_lowres() { + assert_eq!(scaled_dimensions(1919, 1079, 1), (960, 540)); + assert_eq!(scaled_dimensions(641, 481, 1), (321, 241)); + assert_eq!(idct_lowres(1919, 1079, 960, 540), 1); + } +} diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 3b1ee47581c..8f2a217a1e4 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -230,6 +230,14 @@ fn src_y(y: usize, height: usize, is_bottom_up: bool) -> usize { } impl CapturedFrameExt for CapturedFrame { + fn as_ffmpeg_covering( + &self, + _max_width: u32, + _max_height: u32, + ) -> Result { + self.as_ffmpeg() + } + fn as_ffmpeg(&self) -> Result { let native = self.native(); let width = native.width; diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index e70151f06b8..011f3206588 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -69,6 +69,7 @@ pub struct CameraFeed { pub struct NativeCameraIngress { senders: std::sync::atomic::AtomicUsize, mailbox_full: std::sync::atomic::AtomicU64, + mjpeg_budget: Arc, } impl NativeCameraIngress { @@ -347,6 +348,97 @@ impl CameraFeedLock { pub fn video_info(&self) -> &VideoInfo { &self.video_info } + + /// Pins MJPEG decode at full resolution for the guard's lifetime. + /// Studio camera tracks hold this so an instant overlay budget cannot shrink them. + pub fn hold_full_resolution(&self) -> MjpegFullResHold { + MjpegOverlayBudget::hold_full(&self.native_ingress.mjpeg_budget) + } + + /// Allows a scaled MJPEG IDCT that still covers `width`×`height`. + /// A studio full-resolution hold overrides this. + pub fn hold_instant_overlay(&self, width: u32, height: u32) -> MjpegOverlayHold { + MjpegOverlayBudget::hold_overlay(&self.native_ingress.mjpeg_budget, width, height) + } +} + +/// Shared with the Linux capture thread. Studio increments `full_res_holds`; +/// instant mode publishes the overlay rectangle. Full resolution wins. +/// The mutex keeps an older hold from publishing over, or clearing, a newer one. +#[derive(Debug, Default)] +struct MjpegOverlayBudget { + state: std::sync::Mutex, +} + +#[derive(Debug, Default)] +struct MjpegOverlayState { + full_res_holds: usize, + overlay: u64, + overlay_generation: u64, +} + +impl MjpegOverlayBudget { + fn lock(&self) -> std::sync::MutexGuard<'_, MjpegOverlayState> { + self.state.lock().unwrap_or_else(|error| error.into_inner()) + } + + fn overlay_target(&self) -> Option<(u32, u32)> { + let state = self.lock(); + if state.full_res_holds > 0 { + return None; + } + let width = (state.overlay >> 32) as u32; + let height = state.overlay as u32; + (width > 0 && height > 0).then_some((width, height)) + } + + fn hold_full(budget: &Arc) -> MjpegFullResHold { + budget.lock().full_res_holds += 1; + MjpegFullResHold { + budget: Arc::clone(budget), + } + } + + fn hold_overlay(budget: &Arc, width: u32, height: u32) -> MjpegOverlayHold { + let mut state = budget.lock(); + state.overlay_generation = state.overlay_generation.wrapping_add(1); + let generation = state.overlay_generation; + state.overlay = if width == 0 || height == 0 { + 0 + } else { + (u64::from(width) << 32) | u64::from(height) + }; + drop(state); + MjpegOverlayHold { + budget: Arc::clone(budget), + generation, + } + } +} + +pub struct MjpegFullResHold { + budget: Arc, +} + +impl Drop for MjpegFullResHold { + fn drop(&mut self) { + let holds = &mut self.budget.lock().full_res_holds; + *holds = holds.saturating_sub(1); + } +} + +pub struct MjpegOverlayHold { + budget: Arc, + generation: u64, +} + +impl Drop for MjpegOverlayHold { + fn drop(&mut self) { + let mut state = self.budget.lock(); + if state.overlay_generation == self.generation { + state.overlay = 0; + } + } } impl Deref for CameraFeedLock { @@ -1270,7 +1362,7 @@ async fn setup_camera( recipient: Recipient, _native_recipient: Recipient, _ffmpeg_sender_count: Arc, - _native_ingress: Arc, + native_ingress: Arc, _cancel_rx: &mpsc::Receiver<()>, ) -> Result { let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?; @@ -1279,6 +1371,7 @@ async fn setup_camera( let (ready_tx, ready_rx) = oneshot::channel(); let mut ready_signal = Some(ready_tx); + let mjpeg_budget = Arc::clone(&native_ingress.mjpeg_budget); let capture_handle = camera .start_capturing(format.clone(), move |frame| { @@ -1287,7 +1380,19 @@ async fn setup_camera( let timestamp = Timestamp::Instant(std::time::Instant::now()); - let Ok(mut ff_frame) = frame.as_ffmpeg() else { + // The first frame stays full resolution: VideoInfo is taken from it, + // and studio records that size. Scaled IDCT starts on later frames, + // and only while an instant overlay budget is set with no studio hold. + let overlay = if ready_signal.is_some() { + None + } else { + mjpeg_budget.overlay_target() + }; + let decoded = match overlay { + Some((width, height)) => frame.as_ffmpeg_covering(width, height), + None => frame.as_ffmpeg(), + }; + let Ok(mut ff_frame) = decoded else { return; }; @@ -3117,3 +3222,39 @@ mod ingress_tests { } } } + +#[cfg(test)] +mod mjpeg_budget_tests { + use std::sync::Arc; + + use super::*; + + #[test] + fn studio_hold_overrides_instant_overlay_until_both_are_released() { + let budget = Arc::new(MjpegOverlayBudget::default()); + assert!(budget.overlay_target().is_none()); + + let overlay = MjpegOverlayBudget::hold_overlay(&budget, 324, 324); + assert_eq!(budget.overlay_target(), Some((324, 324))); + + let studio = MjpegOverlayBudget::hold_full(&budget); + assert!(budget.overlay_target().is_none()); + drop(studio); + assert_eq!(budget.overlay_target(), Some((324, 324))); + + let replacement = MjpegOverlayBudget::hold_overlay(&budget, 200, 180); + drop(overlay); + assert_eq!(budget.overlay_target(), Some((200, 180))); + drop(replacement); + assert!(budget.overlay_target().is_none()); + } + + #[test] + fn empty_overlay_does_not_enable_scaled_decode() { + let budget = Arc::new(MjpegOverlayBudget::default()); + let hold = MjpegOverlayBudget::hold_overlay(&budget, 0, 324); + assert!(budget.overlay_target().is_none()); + drop(hold); + assert!(budget.overlay_target().is_none()); + } +} diff --git a/crates/recording/src/instant_recording/linux_camera.rs b/crates/recording/src/instant_recording/linux_camera.rs index 84607cb807b..f91f79a3916 100644 --- a/crates/recording/src/instant_recording/linux_camera.rs +++ b/crates/recording/src/instant_recording/linux_camera.rs @@ -476,6 +476,24 @@ pub(super) struct Config { pub(super) struct PreparedCamera { camera_feed: Arc, frames: PreparedCameraFrames, + mjpeg_overlay: camera::MjpegOverlayHold, +} + +/// Portable blur downscales the camera to this before the matte. Scaled IDCT +/// has to cover it, or an instant recording with blur upscales a bubble-sized +/// decode into the segmenter. +const INSTANT_BLUR_COVERAGE: (u32, u32) = (640, 360); + +fn instant_mjpeg_coverage(presentation: &LinuxCameraPresentation, blur: bool) -> (u32, u32) { + let rect = presentation.rect; + if blur { + ( + rect.width.max(INSTANT_BLUR_COVERAGE.0), + rect.height.max(INSTANT_BLUR_COVERAGE.1), + ) + } else { + (rect.width, rect.height) + } } struct PreparedCameraFrames { @@ -497,6 +515,9 @@ impl PreparedCamera { if let Some(reference) = reference_size { presentation = presentation.resolve(reference, (info.width, info.height))?; } + let (coverage_width, coverage_height) = + instant_mjpeg_coverage(&presentation, processed.is_some()); + let mjpeg_overlay = camera_feed.hold_instant_overlay(coverage_width, coverage_height); let receiver = if let Some(processed) = processed { anyhow::ensure!( Arc::ptr_eq(&processed.camera_feed, &camera_feed), @@ -530,6 +551,7 @@ impl PreparedCamera { Self { camera_feed, frames, + mjpeg_overlay, }, timestamps, )) @@ -561,6 +583,7 @@ pub(super) struct CameraCompositeSource { info: VideoInfo, _camera_feed: Arc, _compositor_stop: tokio_util::sync::DropGuard, + _mjpeg_overlay: camera::MjpegOverlayHold, } impl VideoSource for CameraCompositeSource { @@ -578,6 +601,7 @@ impl VideoSource for CameraCompositeSource { let PreparedCamera { camera_feed, frames, + mjpeg_overlay, } = config.camera; let PreparedCameraFrames { receiver: camera_rx, @@ -609,6 +633,7 @@ impl VideoSource for CameraCompositeSource { info, _camera_feed: camera_feed, _compositor_stop: compositor_stop, + _mjpeg_overlay: mjpeg_overlay, }) } @@ -2914,6 +2939,24 @@ mod tests { assert_eq!(camera_freshness_anchor(&frame, now), captured_at); } + #[test] + fn instant_mjpeg_coverage_is_the_overlay_unless_blur_needs_more() { + let presentation = LinuxCameraPresentation::default_for(1920, 1080); + assert_eq!(presentation.rect.width, 324); + assert_eq!(instant_mjpeg_coverage(&presentation, false), (324, 324)); + assert_eq!(instant_mjpeg_coverage(&presentation, true), (640, 360)); + let large = LinuxCameraPresentation { + rect: LinuxCameraRect { + x: 0, + y: 0, + width: 800, + height: 800, + }, + ..LinuxCameraPresentation::default_for(1920, 1080) + }; + assert_eq!(instant_mjpeg_coverage(&large, true), (800, 800)); + } + #[test] fn overlay_rect_is_bottom_right_and_safe_for_tiny_frames() { assert_eq!(overlay_rect(100, 50), (84, 34, 15)); diff --git a/crates/recording/src/sources/camera.rs b/crates/recording/src/sources/camera.rs index 5a6ee33e4f0..d1c16f06529 100644 --- a/crates/recording/src/sources/camera.rs +++ b/crates/recording/src/sources/camera.rs @@ -67,6 +67,7 @@ pub struct Camera { stop_tx: Option>, stopped: Arc, original_video_info: VideoInfo, + _full_res: camera::MjpegFullResHold, } impl VideoSource for Camera { @@ -83,6 +84,7 @@ impl VideoSource for Camera { { let (tx, rx) = flume::bounded(256); let stall_health_tx = ctx.health_tx().clone(); + let full_res = feed_lock.hold_full_resolution(); let original_video_info = *feed_lock.video_info(); let original_width = original_video_info.width; @@ -272,6 +274,7 @@ impl VideoSource for Camera { stop_tx: Some(stop_tx), stopped, original_video_info, + _full_res: full_res, }) } From b508392cf230397b83fc96b980fa323d1329cabf Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:22:13 +0000 Subject: [PATCH 2/9] fix(camera): gate mjpeg_idct module to Linux Clippy -D warnings fails on macOS and Windows because idct_lowres and scaled_dimensions are only used by the Linux capture path. --- crates/camera-ffmpeg/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/camera-ffmpeg/src/lib.rs b/crates/camera-ffmpeg/src/lib.rs index cf0bafa056b..2808e48b615 100644 --- a/crates/camera-ffmpeg/src/lib.rs +++ b/crates/camera-ffmpeg/src/lib.rs @@ -1,3 +1,4 @@ +#[cfg(target_os = "linux")] mod mjpeg_idct; #[cfg(target_os = "macos")] From 7be02649b583096530550ea79effaa8eeeaa208c Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:23:48 +0000 Subject: [PATCH 3/9] fix(camera): gate overlay_target to Linux Clippy -D warnings fails on macOS and Windows because overlay_target is only called from the Linux capture path. Keep the method for tests so the budget checks still compile on every target. --- crates/recording/src/feeds/camera.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 011f3206588..c5f65067c4e 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -382,6 +382,7 @@ impl MjpegOverlayBudget { self.state.lock().unwrap_or_else(|error| error.into_inner()) } + #[cfg(any(target_os = "linux", test))] fn overlay_target(&self) -> Option<(u32, u32)> { let state = self.lock(); if state.full_res_holds > 0 { From a1cf2207da106acb616751fae2ce3f5da2422c91 Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:06:24 +0800 Subject: [PATCH 4/9] chore: rerun interrupted CI checks From 4993f3c1aab41d22482f732417d73379025c1a8c Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:17:38 +0800 Subject: [PATCH 5/9] test: run camera frame conversion tests in Linux CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b500ac245b1..38ebae31129 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -306,7 +306,7 @@ jobs: cargo test --locked -p cap-recording --lib -- --test-threads=1 cargo test --locked -p cap-recording --test x11_shm_capture -- --test-threads=1 cargo test --locked -p scap-targets --lib --test linux_targets - cargo test --locked -p cap-enc-ffmpeg + cargo test --locked -p cap-enc-ffmpeg -p cap-camera-ffmpeg build-gpui: name: Build GPUI Desktop From 78145f51a7307bf999d826e016a1ec0943187790 Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:44:51 +0800 Subject: [PATCH 6/9] fix: repair Linux camera and screenshot CI setup --- .github/workflows/instant-reliability.yml | 7 +++++-- scripts/linux-desktop-harness/scenario-screenshot.sh | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/instant-reliability.yml b/.github/workflows/instant-reliability.yml index af99f5174a4..6bf6b4de713 100644 --- a/.github/workflows/instant-reliability.yml +++ b/.github/workflows/instant-reliability.yml @@ -316,8 +316,11 @@ jobs: run: | set -euo pipefail sudo apt-get update -qq - # v4l2loopback needs videodev, which is in the extra kernel modules. - sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \ + if ! sudo modprobe videodev 2>/dev/null; then + sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" + sudo modprobe videodev + fi + sudo apt-get install -y -qq "linux-headers-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \ dbus ffmpeg pulseaudio pulseaudio-utils libasound2-plugins >/dev/null chmod +x "$RUNNER_TEMP/build/cap-gpui" "$RUNNER_TEMP/build/real-app-tests" sudo modprobe v4l2loopback video_nr=42 card_label="Cap Test Camera" exclusive_caps=1 diff --git a/scripts/linux-desktop-harness/scenario-screenshot.sh b/scripts/linux-desktop-harness/scenario-screenshot.sh index 512d63c9367..2ef6f75cf75 100755 --- a/scripts/linux-desktop-harness/scenario-screenshot.sh +++ b/scripts/linux-desktop-harness/scenario-screenshot.sh @@ -5,6 +5,12 @@ export XDG_RUNTIME_DIR=/run/user/$(id -u) export DBUS_SESSION_BUS_ADDRESS=unix:path=$XDG_RUNTIME_DIR/bus out=${HARNESS_OUT:-/out} eval "$(systemctl --user show-environment | grep -E "^(WAYLAND_DISPLAY|XDG_SESSION_TYPE|XDG_CURRENT_DESKTOP|XDG_SESSION_DESKTOP|LIBGL_ALWAYS_SOFTWARE|HYPRLAND_INSTANCE_SIGNATURE|NIRI_SOCKET)=" | sed 's/^/export /')" +expected_size=${SCREEN:-1920x1080} +if [[ -n ${HYPRLAND_INSTANCE_SIGNATURE:-} ]]; then + expected_size=$(hyprctl -j monitors | python3 -c 'import json, sys +monitor, = json.load(sys.stdin) +print("{}x{}".format(monitor["width"], monitor["height"]))') || exit 1 +fi : >"$out/result.txt" result() { printf '%s=%s\n' "$1" "$2" >>"$out/result.txt"; } @@ -81,7 +87,7 @@ if data[:8] == b"\x89PNG\r\n\x1a\n": print(*struct.unpack(">II", data[16:24]))' "$shot") echo "screenshot: ${shot:-none} ${size:-}" >>"$out/actions.log" result screenshot_taken "$([[ -n $size ]] && echo yes || echo no)" -result screenshot_size "$([[ ${size// /x} == "${SCREEN:-1920x1080}" ]] && echo yes || echo "${size// /x}")" +result screenshot_size "$([[ ${size// /x} == "$expected_size" ]] && echo yes || echo "${size// /x}")" result capture_failed "$(grep -ac 'screenshot capture failed' "$out/cap.log")" if [[ -z $shot ]]; then find "$HOME" /tmp -newer "$out/profile/store" -iname "Screenshot*.png" -printf "portal file: %s bytes %p\n" 2>/dev/null >>"$out/actions.log" From 5fd46c3317781af58bbcdbcd3e8479ae8ad74a8b Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:11:16 +0800 Subject: [PATCH 7/9] fix: configure current Hyprland in the Linux harness --- scripts/linux-desktop-harness/arch-up.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/linux-desktop-harness/arch-up.sh b/scripts/linux-desktop-harness/arch-up.sh index 6a1a8028862..043d83d2e23 100755 --- a/scripts/linux-desktop-harness/arch-up.sh +++ b/scripts/linux-desktop-harness/arch-up.sh @@ -30,6 +30,17 @@ debug { disable_logs = false } exec-once = sh -c '$export_session' exec-once = waybar CONF + cat >"$HOME/.config/hypr/hyprland.lua" <"$HOME/.config/hypr/xdph.conf" < Date: Sun, 1 Nov 2026 02:21:10 +0800 Subject: [PATCH 8/9] fix: retry transient native dependency download failures --- scripts/setup.js | 32 ++++++++++++++++++++++++++++---- scripts/setup.test.mjs | 20 +++++++++++++++++++- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/scripts/setup.js b/scripts/setup.js index 10eb59b69ed..de6b12d54c1 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -698,10 +698,34 @@ async function signMacOSDylib(filePath) { } async function downloadBytes(url) { - const response = await fetch(url); - if (!response.ok) - throw new Error(`Failed to download ${url}: HTTP ${response.status}`); - return response.arrayBuffer(); + for (let attempt = 0; ; attempt++) { + try { + const response = await fetch(url); + if (!response.ok) + throw new Error(`Failed to download ${url}: HTTP ${response.status}`); + return await response.arrayBuffer(); + } catch (error) { + const cause = error instanceof Error ? error.cause : undefined; + const retryable = + cause instanceof Error && + "code" in cause && + [ + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_SOCKET", + "ECONNRESET", + "ECONNREFUSED", + "ETIMEDOUT", + "EAI_AGAIN", + "ENOTFOUND", + ].includes(String(cause.code)); + if (attempt === 2 || !retryable) throw error; + console.warn( + `Download connection failed; retrying (${attempt + 2}/3): ${url}`, + ); + await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1))); + } + } } async function fileExists(path) { diff --git a/scripts/setup.test.mjs b/scripts/setup.test.mjs index 75060c4a2fb..5c21ad9c131 100644 --- a/scripts/setup.test.mjs +++ b/scripts/setup.test.mjs @@ -64,6 +64,7 @@ test( } let failNative = true; + let networkFailures = 0; const requests = []; const server = createServer((request, response) => { requests.push(request.url); @@ -71,6 +72,11 @@ test( response.writeHead(503).end("Service Unavailable\n"); return; } + if (request.url === nativeUrl && networkFailures > 0) { + networkFailures--; + request.socket.destroy(); + return; + } const archive = archives.get(request.url); response.writeHead(archive ? 200 : 404); response.end(archive ?? "Unexpected setup download"); @@ -126,7 +132,19 @@ globalThis.fetch = (input, options) => { assert.match(failed.stderr, /HTTP 503/); failNative = false; - for (const expectedRequests of [[nativeUrl, onnxUrl], []]) { + networkFailures = 3; + requests.length = 0; + const disconnected = await runSetup(); + assert.equal(disconnected.code, 1, disconnected.stderr); + assert.deepEqual(requests, [nativeUrl, nativeUrl, nativeUrl]); + assert.equal(existsSync(cache), false); + assert.match(disconnected.stderr, /UND_ERR_SOCKET/); + + networkFailures = 2; + for (const expectedRequests of [ + [nativeUrl, nativeUrl, nativeUrl, onnxUrl], + [], + ]) { requests.length = 0; const result = await runSetup(); assert.equal(result.code, 0, result.stderr); From 3d524e955be1a0c208f0ea45fd7cfa4fcfa2fa24 Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:40:44 +0800 Subject: [PATCH 9/9] fix: allow busy Wayland compositors to enumerate displays --- .../src/platform/linux/wayland.rs | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/crates/scap-targets/src/platform/linux/wayland.rs b/crates/scap-targets/src/platform/linux/wayland.rs index c4b8bcdbf6a..8c0118f92bb 100644 --- a/crates/scap-targets/src/platform/linux/wayland.rs +++ b/crates/scap-targets/src/platform/linux/wayland.rs @@ -15,6 +15,8 @@ use wayland_protocols::xdg::xdg_output::zv1::client::{zxdg_output_manager_v1, zx use super::{DisplayImpl, LogicalBounds, LogicalPosition, LogicalSize}; +const QUERY_TIMEOUT: Duration = Duration::from_secs(1); + #[derive(Clone, Copy, Default)] struct OutputGeometry { position: (i32, i32), @@ -108,7 +110,7 @@ pub(super) fn displays() -> Option> { runtime.join(display) }; let connection = Connection::from_socket(UnixStream::connect(socket).ok()?).ok()?; - query(&connection, Duration::from_millis(250)) + query(&connection, QUERY_TIMEOUT) } fn output_uuid(name: &str) -> Option { @@ -453,6 +455,45 @@ mod tests { assert!(started.elapsed() < Duration::from_secs(1)); } + #[test] + fn busy_compositor_can_finish_display_enumeration_after_a_quarter_second() { + use std::io::{Read, Write}; + + let (client, mut server) = UnixStream::pair().unwrap(); + let (finished_tx, finished_rx) = std::sync::mpsc::channel(); + let responder = std::thread::spawn(move || -> std::io::Result<()> { + for reply in 0..2 { + loop { + let mut header = [0u8; 8]; + server.read_exact(&mut header)?; + let object = u32::from_ne_bytes(header[..4].try_into().unwrap()); + let message = u32::from_ne_bytes(header[4..].try_into().unwrap()); + let mut payload = vec![0; (message >> 16) as usize - header.len()]; + server.read_exact(&mut payload)?; + if object != 1 || message & 0xffff != 0 { + continue; + } + if reply == 0 { + std::thread::sleep(Duration::from_millis(350)); + } + let callback = u32::from_ne_bytes(payload[..4].try_into().unwrap()); + server.write_all(&callback.to_ne_bytes())?; + server.write_all(&(12u32 << 16).to_ne_bytes())?; + server.write_all(&0u32.to_ne_bytes())?; + break; + } + } + let _ = finished_rx.recv(); + Ok(()) + }); + let connection = Connection::from_socket(client).unwrap(); + let result = query(&connection, QUERY_TIMEOUT); + drop(connection); + let _ = finished_tx.send(()); + responder.join().unwrap().unwrap(); + assert!(result.is_some()); + } + #[test] fn wayland_session_does_not_use_xwayland_dimensions() { assert!(super::super::prefers_wayland_environment(