diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b500ac245b1..38ebae31129 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -306,7 +306,7 @@ jobs: cargo test --locked -p cap-recording --lib -- --test-threads=1 cargo test --locked -p cap-recording --test x11_shm_capture -- --test-threads=1 cargo test --locked -p scap-targets --lib --test linux_targets - cargo test --locked -p cap-enc-ffmpeg + cargo test --locked -p cap-enc-ffmpeg -p cap-camera-ffmpeg build-gpui: name: Build GPUI Desktop diff --git a/.github/workflows/instant-reliability.yml b/.github/workflows/instant-reliability.yml index af99f5174a4..6bf6b4de713 100644 --- a/.github/workflows/instant-reliability.yml +++ b/.github/workflows/instant-reliability.yml @@ -316,8 +316,11 @@ jobs: run: | set -euo pipefail sudo apt-get update -qq - # v4l2loopback needs videodev, which is in the extra kernel modules. - sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \ + if ! sudo modprobe videodev 2>/dev/null; then + sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" + sudo modprobe videodev + fi + sudo apt-get install -y -qq "linux-headers-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \ dbus ffmpeg pulseaudio pulseaudio-utils libasound2-plugins >/dev/null chmod +x "$RUNNER_TEMP/build/cap-gpui" "$RUNNER_TEMP/build/real-app-tests" sudo modprobe v4l2loopback video_nr=42 card_label="Cap Test Camera" exclusive_caps=1 diff --git a/crates/camera-ffmpeg/src/lib.rs b/crates/camera-ffmpeg/src/lib.rs index 3ec20823a6b..2808e48b615 100644 --- a/crates/camera-ffmpeg/src/lib.rs +++ b/crates/camera-ffmpeg/src/lib.rs @@ -1,3 +1,6 @@ +#[cfg(target_os = "linux")] +mod mjpeg_idct; + #[cfg(target_os = "macos")] mod macos; #[cfg(target_os = "macos")] @@ -17,4 +20,13 @@ pub trait CapturedFrameExt { /// Creates an ffmpeg video frame from the native frame. /// Only size, format, and data are set. fn as_ffmpeg(&self) -> Result; + + /// Like [`as_ffmpeg`](Self::as_ffmpeg), but MJPEG may decode with a scaled + /// IDCT when `max_width`×`max_height` is smaller than the frame. The + /// result still covers that rectangle. Other pixel formats are unchanged. + fn as_ffmpeg_covering( + &self, + max_width: u32, + max_height: u32, + ) -> Result; } diff --git a/crates/camera-ffmpeg/src/linux.rs b/crates/camera-ffmpeg/src/linux.rs index ad28a4be34e..03bbf2bdbeb 100644 --- a/crates/camera-ffmpeg/src/linux.rs +++ b/crates/camera-ffmpeg/src/linux.rs @@ -3,6 +3,7 @@ use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo}; use std::cell::RefCell; use crate::CapturedFrameExt; +use crate::mjpeg_idct::idct_lowres; #[derive(thiserror::Error, Debug)] pub enum AsFFmpegError { @@ -24,19 +25,27 @@ pub enum AsFFmpegError { struct MjpegDecoder { decoder: ffmpeg::codec::decoder::Video, + lowres: i32, } impl MjpegDecoder { - fn new() -> Result { + fn new(lowres: i32) -> Result { let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG) .ok_or_else(|| AsFFmpegError::MjpegDecodeError("MJPEG codec not found".to_string()))?; - - let decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec); + let max_lowres = codec.max_lowres().clamp(0, 3); + let lowres = lowres.clamp(0, max_lowres); + + let mut decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec); + if lowres > 0 { + unsafe { + (*decoder_context.as_mut_ptr()).lowres = lowres; + } + } let decoder = decoder_context.decoder().video().map_err(|e| { AsFFmpegError::MjpegDecodeError(format!("Failed to create decoder: {e}")) })?; - Ok(Self { decoder }) + Ok(Self { decoder, lowres }) } fn decode(&mut self, bytes: &[u8]) -> Result { @@ -60,26 +69,61 @@ thread_local! { static MJPEG_DECODER: RefCell> = const { RefCell::new(None) }; } -fn decode_mjpeg(bytes: &[u8]) -> Result { +fn decode_mjpeg(bytes: &[u8], lowres: i32) -> Result { MJPEG_DECODER.with(|decoder_cell| { - let mut decoder_opt = decoder_cell.borrow_mut(); - - if decoder_opt.is_none() { - *decoder_opt = Some(MjpegDecoder::new()?); + let mut slot = decoder_cell.borrow_mut(); + if slot.as_ref().is_none_or(|decoder| decoder.lowres != lowres) { + tracing::debug!(lowres, "MJPEG IDCT scale"); + *slot = Some(MjpegDecoder::new(lowres)?); } - - decoder_opt.as_mut().unwrap().decode(bytes) + slot.as_mut() + .expect("MJPEG decoder is inserted before use") + .decode(bytes) }) } +fn mjpeg_frame_covering( + bytes: &[u8], + source_width: u32, + source_height: u32, + max_width: u32, + max_height: u32, +) -> Result { + let lowres = idct_lowres(source_width, source_height, max_width, max_height); + if lowres == 0 { + return decode_mjpeg(bytes, 0); + } + match decode_mjpeg(bytes, lowres) { + Ok(frame) if frame.width() >= max_width && frame.height() >= max_height => Ok(frame), + _ => decode_mjpeg(bytes, 0), + } +} + impl CapturedFrameExt for CapturedFrame<'_> { fn as_ffmpeg(&self) -> Result { let native = self.native(); frame_from_bytes(native.bytes, native.format) } + + fn as_ffmpeg_covering( + &self, + max_width: u32, + max_height: u32, + ) -> Result { + let native = self.native(); + frame_from_bytes_covering(native.bytes, native.format, Some((max_width, max_height))) + } } fn frame_from_bytes(bytes: &[u8], format: NativeFrameFormat) -> Result { + frame_from_bytes_covering(bytes, format, None) +} + +fn frame_from_bytes_covering( + bytes: &[u8], + format: NativeFrameFormat, + cover: Option<(u32, u32)>, +) -> Result { match fourcc_str(format.fourcc).as_str() { "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2), "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2), @@ -88,7 +132,12 @@ fn frame_from_bytes(bytes: &[u8], format: NativeFrameFormat) -> Result copy_nv12(bytes, format), "YU12" => copy_yuv420(bytes, format, false), "YV12" => copy_yuv420(bytes, format, true), - "MJPG" | "JPEG" => decode_mjpeg(bytes), + "MJPG" | "JPEG" => match cover { + Some((max_width, max_height)) => { + mjpeg_frame_covering(bytes, format.width, format.height, max_width, max_height) + } + None => decode_mjpeg(bytes, 0), + }, other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())), } } @@ -310,9 +359,184 @@ fn fourcc_str(fourcc: [u8; 4]) -> String { #[cfg(test)] mod tests { - use super::{AsFFmpegError, copy_nv12, copy_packed, frame_from_bytes}; + use super::{ + AsFFmpegError, MjpegDecoder, copy_nv12, copy_packed, decode_mjpeg, frame_from_bytes, + frame_from_bytes_covering, mjpeg_frame_covering, + }; + use crate::mjpeg_idct::idct_lowres; use cap_camera::NativeFrameFormat; + use ffmpeg::Packet; use ffmpeg::format::Pixel; + use ffmpeg::frame::Video as FFVideo; + + fn encode_mjpeg(width: u32, height: u32) -> Vec { + let codec = ffmpeg::codec::encoder::find(ffmpeg::codec::Id::MJPEG).unwrap(); + let mut encoder = ffmpeg::codec::context::Context::new_with_codec(codec) + .encoder() + .video() + .unwrap(); + encoder.set_width(width); + encoder.set_height(height); + encoder.set_format(Pixel::YUVJ420P); + encoder.set_time_base((1, 30)); + encoder.set_qmin(4); + encoder.set_qmax(4); + let mut encoder = encoder.open().unwrap(); + + let mut frame = FFVideo::new(Pixel::YUVJ420P, width, height); + let luma_stride = frame.stride(0); + let luma = frame.data_mut(0); + for y in 0..height as usize { + for x in 0..width as usize { + let nx = x as f32 / width as f32; + let ny = y as f32 / height as f32; + let value = 70.0 + + 40.0 * (nx * std::f32::consts::TAU * 2.0).sin() + + 25.0 * (ny * std::f32::consts::TAU * 3.0).sin() + + 18.0 * ((nx + ny) * std::f32::consts::TAU * 5.0).sin() + + ((x * 13 + y * 7) & 7) as f32 + - 3.0; + luma[y * luma_stride + x] = value.clamp(0.0, 255.0) as u8; + } + } + frame.data_mut(1).fill(110); + frame.data_mut(2).fill(150); + frame.set_pts(Some(0)); + encoder.send_frame(&frame).unwrap(); + encoder.send_eof().unwrap(); + let mut packet = Packet::empty(); + encoder.receive_packet(&mut packet).unwrap(); + packet.data().unwrap().to_vec() + } + + fn scale_bgra(frame: &FFVideo, width: u32, height: u32) -> FFVideo { + let mut scaler = ffmpeg::software::scaling::Context::get( + frame.format(), + frame.width(), + frame.height(), + Pixel::BGRA, + width, + height, + ffmpeg::software::scaling::Flags::BILINEAR, + ) + .unwrap(); + let mut output = FFVideo::empty(); + scaler.run(frame, &mut output).unwrap(); + output + } + + fn bgra_delta(full: &FFVideo, scaled: &FFVideo) -> (f64, u8) { + let width = full.width().min(scaled.width()) as usize; + let height = full.height().min(scaled.height()) as usize; + let mut sum = 0u64; + let mut max = 0u8; + let mut count = 0u64; + for y in 0..height { + let full_row = &full.data(0)[y * full.stride(0)..]; + let scaled_row = &scaled.data(0)[y * scaled.stride(0)..]; + for x in 0..width { + for channel in 0..3 { + let delta = full_row[x * 4 + channel].abs_diff(scaled_row[x * 4 + channel]); + sum += u64::from(delta); + max = max.max(delta); + count += 1; + } + } + } + ((sum as f64) / count.max(1) as f64, max) + } + + #[test] + fn scaled_idct_covers_the_bubble_and_stays_close_to_full_then_scale() { + ffmpeg::init().unwrap(); + let bytes = encode_mjpeg(640, 360); + let scaled = mjpeg_frame_covering(&bytes, 640, 360, 200, 120).unwrap(); + assert_eq!((scaled.width(), scaled.height()), (320, 180)); + let full = decode_mjpeg(&bytes, 0).unwrap(); + assert_eq!((full.width(), full.height()), (640, 360)); + let (mean, max) = bgra_delta(&scale_bgra(&full, 200, 120), &scale_bgra(&scaled, 200, 120)); + assert!( + max <= 6 && mean <= 1.5, + "scaled IDCT vs full decode + scale: mean {mean:.3} max {max}" + ); + } + + #[test] + fn full_resolution_request_restores_the_decoder_after_a_scaled_frame() { + ffmpeg::init().unwrap(); + let bytes = encode_mjpeg(640, 360); + let scaled = mjpeg_frame_covering(&bytes, 640, 360, 200, 120).unwrap(); + assert_eq!((scaled.width(), scaled.height()), (320, 180)); + let full = mjpeg_frame_covering(&bytes, 640, 360, 640, 360).unwrap(); + assert_eq!((full.width(), full.height()), (640, 360)); + let untouched = decode_mjpeg(&bytes, 0).unwrap(); + assert_eq!((untouched.width(), untouched.height()), (640, 360)); + } + + #[test] + #[ignore = "release wall-time measurement for the instant bubble"] + fn bubble_scaled_idct_is_faster_than_full_decode_and_scale() { + ffmpeg::init().unwrap(); + let bytes = encode_mjpeg(1920, 1080); + let bubble = 324u32; + let lowres = idct_lowres(1920, 1080, bubble, bubble); + assert_eq!(lowres, 1); + let mut full_decoder = None; + let mut scaled_decoder = None; + let warmup = |lowres: i32, slot: &mut Option| { + for _ in 0..4 { + if slot.as_ref().is_none_or(|decoder| decoder.lowres != lowres) { + *slot = Some(MjpegDecoder::new(lowres).unwrap()); + } + slot.as_mut().unwrap().decode(&bytes).unwrap(); + } + }; + warmup(0, &mut full_decoder); + warmup(lowres, &mut scaled_decoder); + let full_frame = full_decoder.as_mut().unwrap().decode(&bytes).unwrap(); + let scaled_frame = scaled_decoder.as_mut().unwrap().decode(&bytes).unwrap(); + assert_eq!((scaled_frame.width(), scaled_frame.height()), (960, 540)); + let (mean, max) = bgra_delta( + &scale_bgra(&full_frame, bubble, bubble), + &scale_bgra(&scaled_frame, bubble, bubble), + ); + + let iterations = 40; + let time = |slot: &mut Option, source: &FFVideo| { + let mut scaler = ffmpeg::software::scaling::Context::get( + source.format(), + source.width(), + source.height(), + Pixel::BGRA, + bubble, + bubble, + ffmpeg::software::scaling::Flags::BILINEAR, + ) + .unwrap(); + let mut output = FFVideo::empty(); + let start = std::time::Instant::now(); + for _ in 0..iterations { + let frame = slot.as_mut().unwrap().decode(&bytes).unwrap(); + scaler.run(&frame, &mut output).unwrap(); + } + start.elapsed().as_secs_f64() * 1e3 / iterations as f64 + }; + let full_ms = time(&mut full_decoder, &full_frame); + let scaled_ms = time(&mut scaled_decoder, &scaled_frame); + let speedup = full_ms / scaled_ms; + println!( + "mjpeg-idct bubble 1920x1080 -> {bubble}: full+scale {full_ms:.3}ms scaled+scale {scaled_ms:.3}ms speedup {speedup:.2}x bgra_mean {mean:.3} max {max} jpeg_bytes {}", + bytes.len() + ); + assert!( + speedup >= 1.20, + "expected a bubble-size win, got {speedup:.2}x ({full_ms:.3}ms -> {scaled_ms:.3}ms)" + ); + assert!( + max <= 8, + "bubble pixel delta too large: mean {mean:.3} max {max}" + ); + } fn format(fourcc: &[u8; 4], width: u32, height: u32, stride: usize) -> NativeFrameFormat { NativeFrameFormat { @@ -323,6 +547,26 @@ mod tests { } } + #[test] + fn frame_conversion_scales_only_mjpeg_with_an_overlay_budget() { + ffmpeg::init().unwrap(); + let bytes = encode_mjpeg(640, 360); + let native_format = format(b"MJPG", 640, 360, 0); + let scaled = frame_from_bytes_covering(&bytes, native_format, Some((200, 120))).unwrap(); + assert_eq!((scaled.width(), scaled.height()), (320, 180)); + let full = frame_from_bytes(&bytes, native_format).unwrap(); + assert_eq!((full.width(), full.height()), (640, 360)); + + let packed = frame_from_bytes_covering( + &[1, 2, 3, 4, 5, 6, 7, 8], + format(b"YUYV", 4, 1, 8), + Some((2, 1)), + ) + .unwrap(); + assert_eq!((packed.width(), packed.height()), (4, 1)); + assert_eq!(active_rows(&packed, 0, 8), vec![1, 2, 3, 4, 5, 6, 7, 8]); + } + fn active_rows(frame: &ffmpeg::frame::Video, plane: usize, row_width: usize) -> Vec { let stride = frame.stride(plane); let height = frame.plane_height(plane) as usize; diff --git a/crates/camera-ffmpeg/src/macos.rs b/crates/camera-ffmpeg/src/macos.rs index a6c11b602bd..0857655cd66 100644 --- a/crates/camera-ffmpeg/src/macos.rs +++ b/crates/camera-ffmpeg/src/macos.rs @@ -62,6 +62,14 @@ impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { sample_buf_as_ffmpeg(self.native().sample_buf()) } + + fn as_ffmpeg_covering( + &self, + _max_width: u32, + _max_height: u32, + ) -> Result { + self.as_ffmpeg() + } } // Standalone so consumers holding a bare retained CMSampleBuffer (e.g. the diff --git a/crates/camera-ffmpeg/src/mjpeg_idct.rs b/crates/camera-ffmpeg/src/mjpeg_idct.rs new file mode 100644 index 00000000000..caefe378e04 --- /dev/null +++ b/crates/camera-ffmpeg/src/mjpeg_idct.rs @@ -0,0 +1,70 @@ +/// Largest FFmpeg MJPEG `lowres` whose decoded frame still covers +/// `target_width`×`target_height`. +/// +/// `0` is full resolution, `1` is 1/2, `2` is 1/4, `3` is 1/8. Dimensions use +/// the decoder's ceiling division, so the choice cannot undershoot the frame +/// FFmpeg actually returns. +pub(crate) fn idct_lowres( + source_width: u32, + source_height: u32, + target_width: u32, + target_height: u32, +) -> i32 { + if source_width == 0 || source_height == 0 || target_width == 0 || target_height == 0 { + return 0; + } + let mut lowres = 0; + for candidate in 1..=3 { + let (width, height) = scaled_dimensions(source_width, source_height, candidate); + if width < target_width || height < target_height || width == 0 || height == 0 { + break; + } + lowres = candidate; + } + lowres +} + +pub(crate) fn scaled_dimensions(source_width: u32, source_height: u32, lowres: i32) -> (u32, u32) { + if lowres <= 0 { + return (source_width, source_height); + } + let shift = u32::try_from(lowres).unwrap_or(3).min(3); + let divisor = 1u32 << shift; + ( + source_width.saturating_add(divisor - 1) / divisor, + source_height.saturating_add(divisor - 1) / divisor, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bubble_on_1080p_uses_half_idct() { + assert_eq!(idct_lowres(1920, 1080, 324, 324), 1); + assert_eq!(scaled_dimensions(1920, 1080, 1), (960, 540)); + } + + #[test] + fn smaller_bubble_uses_quarter_idct() { + assert_eq!(idct_lowres(1920, 1080, 216, 216), 2); + assert_eq!(scaled_dimensions(1920, 1080, 2), (480, 270)); + assert_eq!(scaled_dimensions(1920, 1080, 3), (240, 135)); + } + + #[test] + fn studio_and_large_overlay_stay_full_resolution() { + assert_eq!(idct_lowres(1920, 1080, 1920, 1080), 0); + assert_eq!(idct_lowres(1920, 1080, 648, 648), 0); + assert_eq!(idct_lowres(1280, 720, 432, 432), 0); + assert_eq!(idct_lowres(1920, 1080, 0, 324), 0); + } + + #[test] + fn ceiling_division_matches_odd_mjpeg_lowres() { + assert_eq!(scaled_dimensions(1919, 1079, 1), (960, 540)); + assert_eq!(scaled_dimensions(641, 481, 1), (321, 241)); + assert_eq!(idct_lowres(1919, 1079, 960, 540), 1); + } +} diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 3b1ee47581c..8f2a217a1e4 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -230,6 +230,14 @@ fn src_y(y: usize, height: usize, is_bottom_up: bool) -> usize { } impl CapturedFrameExt for CapturedFrame { + fn as_ffmpeg_covering( + &self, + _max_width: u32, + _max_height: u32, + ) -> Result { + self.as_ffmpeg() + } + fn as_ffmpeg(&self) -> Result { let native = self.native(); let width = native.width; diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index e70151f06b8..c5f65067c4e 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -69,6 +69,7 @@ pub struct CameraFeed { pub struct NativeCameraIngress { senders: std::sync::atomic::AtomicUsize, mailbox_full: std::sync::atomic::AtomicU64, + mjpeg_budget: Arc, } impl NativeCameraIngress { @@ -347,6 +348,98 @@ impl CameraFeedLock { pub fn video_info(&self) -> &VideoInfo { &self.video_info } + + /// Pins MJPEG decode at full resolution for the guard's lifetime. + /// Studio camera tracks hold this so an instant overlay budget cannot shrink them. + pub fn hold_full_resolution(&self) -> MjpegFullResHold { + MjpegOverlayBudget::hold_full(&self.native_ingress.mjpeg_budget) + } + + /// Allows a scaled MJPEG IDCT that still covers `width`×`height`. + /// A studio full-resolution hold overrides this. + pub fn hold_instant_overlay(&self, width: u32, height: u32) -> MjpegOverlayHold { + MjpegOverlayBudget::hold_overlay(&self.native_ingress.mjpeg_budget, width, height) + } +} + +/// Shared with the Linux capture thread. Studio increments `full_res_holds`; +/// instant mode publishes the overlay rectangle. Full resolution wins. +/// The mutex keeps an older hold from publishing over, or clearing, a newer one. +#[derive(Debug, Default)] +struct MjpegOverlayBudget { + state: std::sync::Mutex, +} + +#[derive(Debug, Default)] +struct MjpegOverlayState { + full_res_holds: usize, + overlay: u64, + overlay_generation: u64, +} + +impl MjpegOverlayBudget { + fn lock(&self) -> std::sync::MutexGuard<'_, MjpegOverlayState> { + self.state.lock().unwrap_or_else(|error| error.into_inner()) + } + + #[cfg(any(target_os = "linux", test))] + fn overlay_target(&self) -> Option<(u32, u32)> { + let state = self.lock(); + if state.full_res_holds > 0 { + return None; + } + let width = (state.overlay >> 32) as u32; + let height = state.overlay as u32; + (width > 0 && height > 0).then_some((width, height)) + } + + fn hold_full(budget: &Arc) -> MjpegFullResHold { + budget.lock().full_res_holds += 1; + MjpegFullResHold { + budget: Arc::clone(budget), + } + } + + fn hold_overlay(budget: &Arc, width: u32, height: u32) -> MjpegOverlayHold { + let mut state = budget.lock(); + state.overlay_generation = state.overlay_generation.wrapping_add(1); + let generation = state.overlay_generation; + state.overlay = if width == 0 || height == 0 { + 0 + } else { + (u64::from(width) << 32) | u64::from(height) + }; + drop(state); + MjpegOverlayHold { + budget: Arc::clone(budget), + generation, + } + } +} + +pub struct MjpegFullResHold { + budget: Arc, +} + +impl Drop for MjpegFullResHold { + fn drop(&mut self) { + let holds = &mut self.budget.lock().full_res_holds; + *holds = holds.saturating_sub(1); + } +} + +pub struct MjpegOverlayHold { + budget: Arc, + generation: u64, +} + +impl Drop for MjpegOverlayHold { + fn drop(&mut self) { + let mut state = self.budget.lock(); + if state.overlay_generation == self.generation { + state.overlay = 0; + } + } } impl Deref for CameraFeedLock { @@ -1270,7 +1363,7 @@ async fn setup_camera( recipient: Recipient, _native_recipient: Recipient, _ffmpeg_sender_count: Arc, - _native_ingress: Arc, + native_ingress: Arc, _cancel_rx: &mpsc::Receiver<()>, ) -> Result { let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?; @@ -1279,6 +1372,7 @@ async fn setup_camera( let (ready_tx, ready_rx) = oneshot::channel(); let mut ready_signal = Some(ready_tx); + let mjpeg_budget = Arc::clone(&native_ingress.mjpeg_budget); let capture_handle = camera .start_capturing(format.clone(), move |frame| { @@ -1287,7 +1381,19 @@ async fn setup_camera( let timestamp = Timestamp::Instant(std::time::Instant::now()); - let Ok(mut ff_frame) = frame.as_ffmpeg() else { + // The first frame stays full resolution: VideoInfo is taken from it, + // and studio records that size. Scaled IDCT starts on later frames, + // and only while an instant overlay budget is set with no studio hold. + let overlay = if ready_signal.is_some() { + None + } else { + mjpeg_budget.overlay_target() + }; + let decoded = match overlay { + Some((width, height)) => frame.as_ffmpeg_covering(width, height), + None => frame.as_ffmpeg(), + }; + let Ok(mut ff_frame) = decoded else { return; }; @@ -3117,3 +3223,39 @@ mod ingress_tests { } } } + +#[cfg(test)] +mod mjpeg_budget_tests { + use std::sync::Arc; + + use super::*; + + #[test] + fn studio_hold_overrides_instant_overlay_until_both_are_released() { + let budget = Arc::new(MjpegOverlayBudget::default()); + assert!(budget.overlay_target().is_none()); + + let overlay = MjpegOverlayBudget::hold_overlay(&budget, 324, 324); + assert_eq!(budget.overlay_target(), Some((324, 324))); + + let studio = MjpegOverlayBudget::hold_full(&budget); + assert!(budget.overlay_target().is_none()); + drop(studio); + assert_eq!(budget.overlay_target(), Some((324, 324))); + + let replacement = MjpegOverlayBudget::hold_overlay(&budget, 200, 180); + drop(overlay); + assert_eq!(budget.overlay_target(), Some((200, 180))); + drop(replacement); + assert!(budget.overlay_target().is_none()); + } + + #[test] + fn empty_overlay_does_not_enable_scaled_decode() { + let budget = Arc::new(MjpegOverlayBudget::default()); + let hold = MjpegOverlayBudget::hold_overlay(&budget, 0, 324); + assert!(budget.overlay_target().is_none()); + drop(hold); + assert!(budget.overlay_target().is_none()); + } +} diff --git a/crates/recording/src/instant_recording/linux_camera.rs b/crates/recording/src/instant_recording/linux_camera.rs index 84607cb807b..f91f79a3916 100644 --- a/crates/recording/src/instant_recording/linux_camera.rs +++ b/crates/recording/src/instant_recording/linux_camera.rs @@ -476,6 +476,24 @@ pub(super) struct Config { pub(super) struct PreparedCamera { camera_feed: Arc, frames: PreparedCameraFrames, + mjpeg_overlay: camera::MjpegOverlayHold, +} + +/// Portable blur downscales the camera to this before the matte. Scaled IDCT +/// has to cover it, or an instant recording with blur upscales a bubble-sized +/// decode into the segmenter. +const INSTANT_BLUR_COVERAGE: (u32, u32) = (640, 360); + +fn instant_mjpeg_coverage(presentation: &LinuxCameraPresentation, blur: bool) -> (u32, u32) { + let rect = presentation.rect; + if blur { + ( + rect.width.max(INSTANT_BLUR_COVERAGE.0), + rect.height.max(INSTANT_BLUR_COVERAGE.1), + ) + } else { + (rect.width, rect.height) + } } struct PreparedCameraFrames { @@ -497,6 +515,9 @@ impl PreparedCamera { if let Some(reference) = reference_size { presentation = presentation.resolve(reference, (info.width, info.height))?; } + let (coverage_width, coverage_height) = + instant_mjpeg_coverage(&presentation, processed.is_some()); + let mjpeg_overlay = camera_feed.hold_instant_overlay(coverage_width, coverage_height); let receiver = if let Some(processed) = processed { anyhow::ensure!( Arc::ptr_eq(&processed.camera_feed, &camera_feed), @@ -530,6 +551,7 @@ impl PreparedCamera { Self { camera_feed, frames, + mjpeg_overlay, }, timestamps, )) @@ -561,6 +583,7 @@ pub(super) struct CameraCompositeSource { info: VideoInfo, _camera_feed: Arc, _compositor_stop: tokio_util::sync::DropGuard, + _mjpeg_overlay: camera::MjpegOverlayHold, } impl VideoSource for CameraCompositeSource { @@ -578,6 +601,7 @@ impl VideoSource for CameraCompositeSource { let PreparedCamera { camera_feed, frames, + mjpeg_overlay, } = config.camera; let PreparedCameraFrames { receiver: camera_rx, @@ -609,6 +633,7 @@ impl VideoSource for CameraCompositeSource { info, _camera_feed: camera_feed, _compositor_stop: compositor_stop, + _mjpeg_overlay: mjpeg_overlay, }) } @@ -2914,6 +2939,24 @@ mod tests { assert_eq!(camera_freshness_anchor(&frame, now), captured_at); } + #[test] + fn instant_mjpeg_coverage_is_the_overlay_unless_blur_needs_more() { + let presentation = LinuxCameraPresentation::default_for(1920, 1080); + assert_eq!(presentation.rect.width, 324); + assert_eq!(instant_mjpeg_coverage(&presentation, false), (324, 324)); + assert_eq!(instant_mjpeg_coverage(&presentation, true), (640, 360)); + let large = LinuxCameraPresentation { + rect: LinuxCameraRect { + x: 0, + y: 0, + width: 800, + height: 800, + }, + ..LinuxCameraPresentation::default_for(1920, 1080) + }; + assert_eq!(instant_mjpeg_coverage(&large, true), (800, 800)); + } + #[test] fn overlay_rect_is_bottom_right_and_safe_for_tiny_frames() { assert_eq!(overlay_rect(100, 50), (84, 34, 15)); diff --git a/crates/recording/src/sources/camera.rs b/crates/recording/src/sources/camera.rs index 5a6ee33e4f0..d1c16f06529 100644 --- a/crates/recording/src/sources/camera.rs +++ b/crates/recording/src/sources/camera.rs @@ -67,6 +67,7 @@ pub struct Camera { stop_tx: Option>, stopped: Arc, original_video_info: VideoInfo, + _full_res: camera::MjpegFullResHold, } impl VideoSource for Camera { @@ -83,6 +84,7 @@ impl VideoSource for Camera { { let (tx, rx) = flume::bounded(256); let stall_health_tx = ctx.health_tx().clone(); + let full_res = feed_lock.hold_full_resolution(); let original_video_info = *feed_lock.video_info(); let original_width = original_video_info.width; @@ -272,6 +274,7 @@ impl VideoSource for Camera { stop_tx: Some(stop_tx), stopped, original_video_info, + _full_res: full_res, }) } diff --git a/crates/scap-targets/src/platform/linux/wayland.rs b/crates/scap-targets/src/platform/linux/wayland.rs index c4b8bcdbf6a..8c0118f92bb 100644 --- a/crates/scap-targets/src/platform/linux/wayland.rs +++ b/crates/scap-targets/src/platform/linux/wayland.rs @@ -15,6 +15,8 @@ use wayland_protocols::xdg::xdg_output::zv1::client::{zxdg_output_manager_v1, zx use super::{DisplayImpl, LogicalBounds, LogicalPosition, LogicalSize}; +const QUERY_TIMEOUT: Duration = Duration::from_secs(1); + #[derive(Clone, Copy, Default)] struct OutputGeometry { position: (i32, i32), @@ -108,7 +110,7 @@ pub(super) fn displays() -> Option> { runtime.join(display) }; let connection = Connection::from_socket(UnixStream::connect(socket).ok()?).ok()?; - query(&connection, Duration::from_millis(250)) + query(&connection, QUERY_TIMEOUT) } fn output_uuid(name: &str) -> Option { @@ -453,6 +455,45 @@ mod tests { assert!(started.elapsed() < Duration::from_secs(1)); } + #[test] + fn busy_compositor_can_finish_display_enumeration_after_a_quarter_second() { + use std::io::{Read, Write}; + + let (client, mut server) = UnixStream::pair().unwrap(); + let (finished_tx, finished_rx) = std::sync::mpsc::channel(); + let responder = std::thread::spawn(move || -> std::io::Result<()> { + for reply in 0..2 { + loop { + let mut header = [0u8; 8]; + server.read_exact(&mut header)?; + let object = u32::from_ne_bytes(header[..4].try_into().unwrap()); + let message = u32::from_ne_bytes(header[4..].try_into().unwrap()); + let mut payload = vec![0; (message >> 16) as usize - header.len()]; + server.read_exact(&mut payload)?; + if object != 1 || message & 0xffff != 0 { + continue; + } + if reply == 0 { + std::thread::sleep(Duration::from_millis(350)); + } + let callback = u32::from_ne_bytes(payload[..4].try_into().unwrap()); + server.write_all(&callback.to_ne_bytes())?; + server.write_all(&(12u32 << 16).to_ne_bytes())?; + server.write_all(&0u32.to_ne_bytes())?; + break; + } + } + let _ = finished_rx.recv(); + Ok(()) + }); + let connection = Connection::from_socket(client).unwrap(); + let result = query(&connection, QUERY_TIMEOUT); + drop(connection); + let _ = finished_tx.send(()); + responder.join().unwrap().unwrap(); + assert!(result.is_some()); + } + #[test] fn wayland_session_does_not_use_xwayland_dimensions() { assert!(super::super::prefers_wayland_environment( diff --git a/scripts/linux-desktop-harness/arch-up.sh b/scripts/linux-desktop-harness/arch-up.sh index 6a1a8028862..043d83d2e23 100755 --- a/scripts/linux-desktop-harness/arch-up.sh +++ b/scripts/linux-desktop-harness/arch-up.sh @@ -30,6 +30,17 @@ debug { disable_logs = false } exec-once = sh -c '$export_session' exec-once = waybar CONF + cat >"$HOME/.config/hypr/hyprland.lua" <"$HOME/.config/hypr/xdph.conf" <"$out/result.txt" result() { printf '%s=%s\n' "$1" "$2" >>"$out/result.txt"; } @@ -81,7 +87,7 @@ if data[:8] == b"\x89PNG\r\n\x1a\n": print(*struct.unpack(">II", data[16:24]))' "$shot") echo "screenshot: ${shot:-none} ${size:-}" >>"$out/actions.log" result screenshot_taken "$([[ -n $size ]] && echo yes || echo no)" -result screenshot_size "$([[ ${size// /x} == "${SCREEN:-1920x1080}" ]] && echo yes || echo "${size// /x}")" +result screenshot_size "$([[ ${size// /x} == "$expected_size" ]] && echo yes || echo "${size// /x}")" result capture_failed "$(grep -ac 'screenshot capture failed' "$out/cap.log")" if [[ -z $shot ]]; then find "$HOME" /tmp -newer "$out/profile/store" -iname "Screenshot*.png" -printf "portal file: %s bytes %p\n" 2>/dev/null >>"$out/actions.log" diff --git a/scripts/setup.js b/scripts/setup.js index 10eb59b69ed..de6b12d54c1 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -698,10 +698,34 @@ async function signMacOSDylib(filePath) { } async function downloadBytes(url) { - const response = await fetch(url); - if (!response.ok) - throw new Error(`Failed to download ${url}: HTTP ${response.status}`); - return response.arrayBuffer(); + for (let attempt = 0; ; attempt++) { + try { + const response = await fetch(url); + if (!response.ok) + throw new Error(`Failed to download ${url}: HTTP ${response.status}`); + return await response.arrayBuffer(); + } catch (error) { + const cause = error instanceof Error ? error.cause : undefined; + const retryable = + cause instanceof Error && + "code" in cause && + [ + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_SOCKET", + "ECONNRESET", + "ECONNREFUSED", + "ETIMEDOUT", + "EAI_AGAIN", + "ENOTFOUND", + ].includes(String(cause.code)); + if (attempt === 2 || !retryable) throw error; + console.warn( + `Download connection failed; retrying (${attempt + 2}/3): ${url}`, + ); + await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1))); + } + } } async function fileExists(path) { diff --git a/scripts/setup.test.mjs b/scripts/setup.test.mjs index 75060c4a2fb..5c21ad9c131 100644 --- a/scripts/setup.test.mjs +++ b/scripts/setup.test.mjs @@ -64,6 +64,7 @@ test( } let failNative = true; + let networkFailures = 0; const requests = []; const server = createServer((request, response) => { requests.push(request.url); @@ -71,6 +72,11 @@ test( response.writeHead(503).end("Service Unavailable\n"); return; } + if (request.url === nativeUrl && networkFailures > 0) { + networkFailures--; + request.socket.destroy(); + return; + } const archive = archives.get(request.url); response.writeHead(archive ? 200 : 404); response.end(archive ?? "Unexpected setup download"); @@ -126,7 +132,19 @@ globalThis.fetch = (input, options) => { assert.match(failed.stderr, /HTTP 503/); failNative = false; - for (const expectedRequests of [[nativeUrl, onnxUrl], []]) { + networkFailures = 3; + requests.length = 0; + const disconnected = await runSetup(); + assert.equal(disconnected.code, 1, disconnected.stderr); + assert.deepEqual(requests, [nativeUrl, nativeUrl, nativeUrl]); + assert.equal(existsSync(cache), false); + assert.match(disconnected.stderr, /UND_ERR_SOCKET/); + + networkFailures = 2; + for (const expectedRequests of [ + [nativeUrl, nativeUrl, nativeUrl, onnxUrl], + [], + ]) { requests.length = 0; const result = await runSetup(); assert.equal(result.code, 0, result.stderr);