diff --git a/.github/workflows/instant-reliability.yml b/.github/workflows/instant-reliability.yml index af99f5174a4..6bf6b4de713 100644 --- a/.github/workflows/instant-reliability.yml +++ b/.github/workflows/instant-reliability.yml @@ -316,8 +316,11 @@ jobs: run: | set -euo pipefail sudo apt-get update -qq - # v4l2loopback needs videodev, which is in the extra kernel modules. - sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \ + if ! sudo modprobe videodev 2>/dev/null; then + sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" + sudo modprobe videodev + fi + sudo apt-get install -y -qq "linux-headers-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \ dbus ffmpeg pulseaudio pulseaudio-utils libasound2-plugins >/dev/null chmod +x "$RUNNER_TEMP/build/cap-gpui" "$RUNNER_TEMP/build/real-app-tests" sudo modprobe v4l2loopback video_nr=42 card_label="Cap Test Camera" exclusive_caps=1 diff --git a/apps/desktop-gpui/src/feeds.rs b/apps/desktop-gpui/src/feeds.rs index 14ce0bd60b0..c8333393972 100644 --- a/apps/desktop-gpui/src/feeds.rs +++ b/apps/desktop-gpui/src/feeds.rs @@ -311,15 +311,9 @@ impl CameraProcessingFactory { #[cfg(target_os = "linux")] impl RecordingCameraWorker { - fn publish( - &self, - image: &Arc, - dims: (usize, usize), - timestamp: cap_timestamp::Timestamp, - mask: Option, - ) { + fn accept(&self, timestamp: cap_timestamp::Timestamp) -> bool { if self.publisher.is_cancelled() { - return; + return false; } match validate_recording_camera_frame( timestamp, @@ -327,13 +321,31 @@ impl RecordingCameraWorker { self.epoch, self.current_epoch.load(Ordering::Acquire), ) { - Ok(true) => {} - Ok(false) => return, + Ok(true) => true, + Ok(false) => false, Err(error) => { self.publisher.fail(error); - return; + false } } + } + + fn publish_sample(&self, frame: cap_recording::instant_recording::LinuxProcessedCameraFrame) { + if self.accept(frame.timestamp) { + self.publisher.publish(frame); + } + } + + fn publish( + &self, + image: &Arc, + dims: (usize, usize), + timestamp: cap_timestamp::Timestamp, + mask: Option, + ) { + if !self.accept(timestamp) { + return; + } if image.as_bytes(0).is_none() { self.publisher .fail("Processed camera pixels unavailable".to_string()); @@ -341,9 +353,10 @@ impl RecordingCameraWorker { } self.publisher.publish( cap_recording::instant_recording::LinuxProcessedCameraFrame { - bgra: cap_recording::instant_recording::LinuxCameraPixels::new(RenderImagePixels( - image.clone(), - )), + pixels: cap_recording::instant_recording::LinuxCameraPixels::new( + RenderImagePixels(image.clone()), + ), + format: cap_recording::instant_recording::LinuxCameraSampleFormat::Bgra, dimensions: (dims.0 as u32, dims.1 as u32), stride: dims.0 * 4, timestamp, @@ -355,6 +368,24 @@ impl RecordingCameraWorker { } } +#[cfg(target_os = "linux")] +fn yuyv_overlay_frame( + frame: &cap_recording::FFmpegVideoFrame, + blur: u8, + generation: u64, + processing: cap_recording::instant_recording::LinuxCameraProcessing, +) -> Option { + if blur != 0 { + return None; + } + cap_recording::instant_recording::LinuxProcessedCameraFrame::from_yuyv422( + &frame.inner, + frame.timestamp, + generation, + processing, + ) +} + #[cfg(target_os = "linux")] struct RenderImagePixels(Arc); @@ -562,12 +593,29 @@ fn run_camera_preview_worker(config: CameraPreviewWorkerConfig) { .publisher .fail("Requested camera blur is unavailable".to_string()); } + #[cfg(target_os = "linux")] + let published_yuyv = recording.as_ref().is_some_and(|recording| { + let Some(sample) = yuyv_overlay_frame( + &frame, + requested_blur, + recording.generation, + recording.processing, + ) else { + return false; + }; + recording.publish_sample(sample); + true + }); + #[cfg(target_os = "linux")] + if published_yuyv && !active.load(Ordering::Acquire) { + continue; + } let max_dims = blur_mode.map(|_| crate::camera_blur_portable::MAX_DIMS); let Some((mut image, dims)) = camera_preview_image(&frame.inner, &mut scaler, requested_mirror, max_dims) else { #[cfg(target_os = "linux")] - if let Some(recording) = &recording { + if !published_yuyv && let Some(recording) = &recording { recording .publisher .fail("Camera frame conversion failed".to_string()); @@ -657,7 +705,10 @@ fn run_camera_preview_worker(config: CameraPreviewWorkerConfig) { } } #[cfg(target_os = "linux")] - if effects_ready && let Some(recording) = &recording { + if !published_yuyv + && effects_ready + && let Some(recording) = &recording + { recording.publish(&image, dims, frame.timestamp, applied_mask); } if active.load(Ordering::Acquire) { @@ -2206,6 +2257,62 @@ mod tests { assert!(scaler.is_some()); } + #[cfg(target_os = "linux")] + #[test] + fn blur_off_yuyv_recording_keeps_the_packed_plane() { + use cap_recording::instant_recording::{ + LinuxCameraBlur, LinuxCameraProcessing, LinuxCameraSampleFormat, + }; + use cap_timestamp::Timestamp; + + let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::YUYV422, 4, 2); + let stride = frame.stride(0); + let row = 8; + frame.data_mut(0).fill(0xAB); + for y in 0..2 { + frame.data_mut(0)[y * stride..y * stride + row] + .copy_from_slice(&[10, 20, 30, 40, 50, 60, 70, 80]); + } + let timestamp = Timestamp::Instant(std::time::Instant::now()); + let processing = LinuxCameraProcessing { + mirrored: true, + blur: LinuxCameraBlur::Off, + }; + let packed = cap_recording::FFmpegVideoFrame { + inner: frame, + timestamp, + }; + let sample = yuyv_overlay_frame(&packed, 0, 4, processing).expect("yuyv sample"); + assert_eq!(sample.format, LinuxCameraSampleFormat::Yuyv422); + assert_eq!(sample.dimensions, (4, 2)); + assert_eq!(sample.stride, row); + assert_eq!(sample.generation, 4); + assert_eq!(sample.processing, processing); + assert!(sample.mask.is_none()); + assert_eq!( + &*sample.pixels, + &[ + 10, 20, 30, 40, 50, 60, 70, 80, 10, 20, 30, 40, 50, 60, 70, 80 + ] + ); + + let mut blurred = packed; + assert!(yuyv_overlay_frame(&blurred, 1, 4, processing).is_none()); + blurred.inner = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, 4, 2); + assert!(yuyv_overlay_frame(&blurred, 0, 4, processing).is_none()); + let heavy = LinuxCameraProcessing { + blur: LinuxCameraBlur::Heavy, + ..processing + }; + let mut yuyv = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::YUYV422, 4, 2); + yuyv.data_mut(0).fill(1); + let source = cap_recording::FFmpegVideoFrame { + inner: yuyv, + timestamp, + }; + assert!(yuyv_overlay_frame(&source, 0, 4, heavy).is_none()); + } + #[cfg(target_os = "linux")] #[test] fn recording_camera_pixels_share_the_preview_image() { diff --git a/crates/recording/src/instant_recording.rs b/crates/recording/src/instant_recording.rs index 6c0102a11fc..7b48083b99e 100644 --- a/crates/recording/src/instant_recording.rs +++ b/crates/recording/src/instant_recording.rs @@ -7,8 +7,8 @@ mod linux_camera; pub use linux_camera::{ LINUX_CAMERA_MAX_MASK_AGE, LinuxCameraBlur, LinuxCameraEffect, LinuxCameraMaskReceipt, LinuxCameraPixels, LinuxCameraPresentation, LinuxCameraPresentationError, - LinuxCameraProcessing, LinuxCameraPublisher, LinuxCameraRect, LinuxCameraShape, - LinuxProcessedCameraFrame, LinuxProcessedCameraSource, + LinuxCameraProcessing, LinuxCameraPublisher, LinuxCameraRect, LinuxCameraSampleFormat, + LinuxCameraShape, LinuxProcessedCameraFrame, LinuxProcessedCameraSource, }; use crate::{ diff --git a/crates/recording/src/instant_recording/linux_camera.rs b/crates/recording/src/instant_recording/linux_camera.rs index 84607cb807b..271164284ed 100644 --- a/crates/recording/src/instant_recording/linux_camera.rs +++ b/crates/recording/src/instant_recording/linux_camera.rs @@ -227,9 +227,16 @@ impl std::fmt::Debug for LinuxCameraPixels { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LinuxCameraSampleFormat { + Bgra, + Yuyv422, +} + #[derive(Debug, Clone)] pub struct LinuxProcessedCameraFrame { - pub bgra: LinuxCameraPixels, + pub pixels: LinuxCameraPixels, + pub format: LinuxCameraSampleFormat, pub dimensions: (u32, u32), pub stride: usize, pub timestamp: Timestamp, @@ -238,6 +245,49 @@ pub struct LinuxProcessedCameraFrame { pub mask: Option, } +impl LinuxProcessedCameraFrame { + /// Tight copy of an even-width `YUYV422` camera plane. Blur stays on the + /// BGRA path: a blurred sample is already filtered, so it cannot be a raw + /// YUYV plane. + pub fn from_yuyv422( + frame: &ffmpeg::frame::Video, + timestamp: Timestamp, + generation: u64, + processing: LinuxCameraProcessing, + ) -> Option { + if processing.blur != LinuxCameraBlur::Off || frame.format() != Pixel::YUYV422 { + return None; + } + let width = frame.width(); + let height = frame.height(); + if width == 0 || height == 0 || !width.is_multiple_of(2) { + return None; + } + let row = (width as usize).checked_mul(2)?; + let source_stride = frame.stride(0); + if source_stride < row { + return None; + } + let source = frame.data(0); + let mut packed = vec![0u8; row.checked_mul(height as usize)?]; + for y in 0..height as usize { + let start = y.checked_mul(source_stride)?; + let src = source.get(start..start + row)?; + packed[y * row..y * row + row].copy_from_slice(src); + } + Some(Self { + pixels: LinuxCameraPixels::new(packed), + format: LinuxCameraSampleFormat::Yuyv422, + dimensions: (width, height), + stride: row, + timestamp, + generation, + processing, + mask: None, + }) + } +} + pub const LINUX_CAMERA_MAX_MASK_AGE: Duration = Duration::from_millis(750); #[derive(Clone)] @@ -438,29 +488,52 @@ fn validate_processed_frame( _ => anyhow::bail!("Processed camera blur was not applied as requested"), } let (width, height) = frame.dimensions; + let bytes_per_pixel = match frame.format { + LinuxCameraSampleFormat::Bgra => 4usize, + LinuxCameraSampleFormat::Yuyv422 => { + anyhow::ensure!( + expected.blur == LinuxCameraBlur::Off, + "YUYV camera samples cannot carry background blur" + ); + anyhow::ensure!(width.is_multiple_of(2), "YUYV camera width must be even"); + 2 + } + }; let row = usize::try_from(width) .ok() - .and_then(|width| width.checked_mul(4)) + .and_then(|width| width.checked_mul(bytes_per_pixel)) .context("Processed camera row overflow")?; let size = frame .stride .checked_mul(height as usize) .context("Processed camera size overflow")?; anyhow::ensure!( - width > 0 && height > 0 && frame.stride >= row && frame.bgra.len() == size, - "Processed camera has invalid BGRA dimensions or stride" + width > 0 && height > 0 && frame.stride >= row && frame.pixels.len() == size, + "Processed camera has invalid dimensions or stride" ); Ok(()) } fn processed_to_ffmpeg(frame: &LinuxProcessedCameraFrame) -> anyhow::Result { let (width, height) = frame.dimensions; - let mut inner = ffmpeg::frame::Video::new(Pixel::BGRA, width, height); + // Blur-off YUYV is mirrored in the packed plane, then scaled once to the + // overlay. That is not the previous two-stage bilinear (full BGRA, then + // BGRA to RGBA): chroma and edge pixels can differ. + let (pixel, bytes_per_pixel, mirror) = match frame.format { + LinuxCameraSampleFormat::Bgra => (Pixel::BGRA, 4usize, false), + LinuxCameraSampleFormat::Yuyv422 => (Pixel::YUYV422, 2usize, frame.processing.mirrored), + }; + let mut inner = ffmpeg::frame::Video::new(pixel, width, height); let stride = inner.stride(0); - let row = width as usize * 4; + let row = width as usize * bytes_per_pixel; for y in 0..height as usize { - inner.data_mut(0)[y * stride..y * stride + row] - .copy_from_slice(&frame.bgra[y * frame.stride..y * frame.stride + row]); + let src = &frame.pixels[y * frame.stride..y * frame.stride + row]; + let dst = &mut inner.data_mut(0)[y * stride..y * stride + row]; + if mirror { + mirror_yuyv_row(src, dst); + } else { + dst.copy_from_slice(src); + } } Ok(FFmpegVideoFrame { inner, @@ -468,6 +541,17 @@ fn processed_to_ffmpeg(frame: &LinuxProcessedCameraFrame) -> anyhow::Result LinuxProcessedCameraFrame { LinuxProcessedCameraFrame { - bgra: LinuxCameraPixels::new([ + pixels: LinuxCameraPixels::new([ 10, 20, 30, 255, 40, 50, 60, 255, 99, 99, 99, 99, 70, 80, 90, 255, 100, 110, 120, 255, 99, 99, 99, 99u8, ]), + format: LinuxCameraSampleFormat::Bgra, dimensions: (2, 2), stride: 12, timestamp: Timestamp::Instant(now), @@ -2256,10 +2341,10 @@ mod tests { validate_processed_frame(&frame, frame.processing, 7, now, now).unwrap(); let converted = processed_to_ffmpeg(&frame).unwrap(); assert_eq!(converted.inner.format(), Pixel::BGRA); - assert_eq!(&converted.inner.data(0)[..8], &frame.bgra[..8]); + assert_eq!(&converted.inner.data(0)[..8], &frame.pixels[..8]); assert_eq!( &converted.inner.data(0)[converted.inner.stride(0)..][..8], - &frame.bgra[12..20] + &frame.pixels[12..20] ); assert!(matches!(converted.timestamp, Timestamp::Instant(timestamp) if timestamp == now)); let mut compositor = Compositor::new( @@ -2317,7 +2402,7 @@ mod tests { invalid.stride = 7; assert!(validate_processed_frame(&invalid, frame.processing, 7, now, now).is_err()); invalid = frame.clone(); - invalid.bgra = LinuxCameraPixels::new([0u8; 23]); + invalid.pixels = LinuxCameraPixels::new([0u8; 23]); assert!(validate_processed_frame(&invalid, frame.processing, 7, now, now).is_err()); invalid = frame.clone(); invalid.timestamp = Timestamp::SystemTime(std::time::SystemTime::now()); @@ -3007,11 +3092,12 @@ mod tests { for pixel in bytes.chunks_exact_mut(4) { pixel.copy_from_slice(&[11, 22, 33, 255]); } - let bgra = LinuxCameraPixels::new(bytes); - let shared_pixels = bgra.clone(); + let pixels = LinuxCameraPixels::new(bytes); + let shared_pixels = pixels.clone(); let source = shared_pixels.as_ptr(); let processed = LinuxProcessedCameraFrame { - bgra, + pixels, + format: LinuxCameraSampleFormat::Bgra, dimensions: (size, size), stride: row, timestamp: Timestamp::Instant(now), @@ -3023,7 +3109,7 @@ mod tests { mask: None, }; let converted = processed_to_ffmpeg(&processed).expect("processed frame"); - assert_eq!(processed.bgra.as_ptr(), source); + assert_eq!(processed.pixels.as_ptr(), source); assert_eq!(shared_pixels.as_ptr(), source); assert_ne!(converted.inner.data(0).as_ptr(), source); let camera_pixels = converted.inner.data(0).as_ptr(); @@ -3117,7 +3203,7 @@ mod tests { assert_eq!(camera_frame.inner.data(0).as_ptr(), camera_pixels); assert_eq!(shared_frame.data(0).as_ptr(), camera_pixels); assert_eq!(plane_buffer_refcount(&camera_frame.inner), 2); - assert_eq!(processed.bgra.as_ptr(), source); + assert_eq!(processed.pixels.as_ptr(), source); assert_eq!(shared_pixels.as_ptr(), source); assert_eq!(&shared_pixels[..4], &[11, 22, 33, 255]); } @@ -4687,4 +4773,455 @@ mod tests { ); } } + + fn solid_yuyv(width: u32, height: u32, y: u8, u: u8, v: u8) -> ffmpeg::frame::Video { + let mut frame = ffmpeg::frame::Video::new(Pixel::YUYV422, width, height); + for pixel in frame.data_mut(0).chunks_exact_mut(4) { + pixel.copy_from_slice(&[y, u, y, v]); + } + frame + } + + fn split_yuyv(width: u32, height: u32) -> ffmpeg::frame::Video { + let mut frame = ffmpeg::frame::Video::new(Pixel::YUYV422, width, height); + let stride = frame.stride(0); + let row = width as usize * 2; + let half_pairs = width as usize / 4; + let data = frame.data_mut(0); + for y in 0..height as usize { + for (pair, pixel) in data[y * stride..y * stride + row] + .chunks_exact_mut(4) + .enumerate() + { + let luma = if pair < half_pairs { 16 } else { 235 }; + pixel.copy_from_slice(&[luma, 128, luma, 128]); + } + } + frame + } + + fn overlay_at(width: u32, height: u32, shape: LinuxCameraShape) -> Compositor { + Compositor::new( + VideoInfo::from_raw_ffmpeg(Pixel::RGBA, width.max(16), height.max(16), 30), + LinuxCameraPresentation { + rect: LinuxCameraRect { + x: 0, + y: 0, + width, + height, + }, + shape, + mirrored: false, + effect: LinuxCameraEffect::None, + }, + ) + } + + fn rgb_sum(frame: &ffmpeg::frame::Video, x: usize, y: usize) -> u16 { + let offset = y * frame.stride(0) + x * 4; + let pixel = &frame.data(0)[offset..offset + 3]; + u16::from(pixel[0]) + u16::from(pixel[1]) + u16::from(pixel[2]) + } + + fn max_channel_delta(left: &ffmpeg::frame::Video, right: &ffmpeg::frame::Video) -> u8 { + assert_eq!( + (left.width(), left.height()), + (right.width(), right.height()) + ); + let row = left.width() as usize * 4; + let mut max = 0u8; + for y in 0..left.height() as usize { + let left_row = &left.data(0)[y * left.stride(0)..][..row]; + let right_row = &right.data(0)[y * right.stride(0)..][..row]; + for (left_byte, right_byte) in left_row.iter().zip(right_row) { + max = max.max(left_byte.abs_diff(*right_byte)); + } + } + max + } + + fn yuyv_sample(frame: &ffmpeg::frame::Video, mirrored: bool) -> LinuxProcessedCameraFrame { + LinuxProcessedCameraFrame::from_yuyv422( + frame, + Timestamp::Instant(Instant::now()), + 1, + LinuxCameraProcessing { + mirrored, + blur: LinuxCameraBlur::Off, + }, + ) + .expect("yuyv sample") + } + + #[test] + fn mirrored_yuyv_row_keeps_distinct_luma_and_chroma() { + ffmpeg::init().expect("FFmpeg initializes"); + let mut source = ffmpeg::frame::Video::new(Pixel::YUYV422, 4, 1); + // Hand-written mirror of distinct Y0/U/Y1/V macropixels. Deriving the + // expected bytes from `mirror_yuyv_row`, or repeating luma and chroma + // inside each pair, lets a missed luma swap or a U/V exchange pass. + let row = [0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80]; + source.data_mut(0)[..row.len()].copy_from_slice(&row); + let frame = processed_to_ffmpeg(&yuyv_sample(&source, true)).expect("mirrored yuyv"); + assert!(frame.inner.stride(0) >= row.len()); + assert_eq!( + &frame.inner.data(0)[..row.len()], + &[0x70, 0x60, 0x50, 0x80, 0x30, 0x20, 0x10, 0x40] + ); + } + + #[test] + fn yuyv_overlay_scales_without_expanding_to_bgra_and_mirrors_once() { + ffmpeg::init().expect("FFmpeg initializes"); + let source = split_yuyv(8, 2); + let plain = yuyv_sample(&source, false); + assert_eq!(plain.format, LinuxCameraSampleFormat::Yuyv422); + assert_eq!(plain.pixels.len(), 8 * 2 * 2); + let stored = plain.pixels.to_vec(); + let converted = processed_to_ffmpeg(&plain).expect("yuyv frame"); + assert_eq!(converted.inner.format(), Pixel::YUYV422); + assert_eq!(&*plain.pixels, stored.as_slice()); + assert_eq!(&converted.inner.data(0)[..16], &stored[..16]); + + let mirrored = yuyv_sample(&source, true); + let mirrored_frame = processed_to_ffmpeg(&mirrored).expect("mirrored yuyv"); + let mut expected = vec![0u8; 16]; + mirror_yuyv_row(&stored[..16], &mut expected); + assert_eq!(&mirrored_frame.inner.data(0)[..16], expected.as_slice()); + assert_ne!(&mirrored_frame.inner.data(0)[..16], &stored[..16]); + + let mut plain_compositor = overlay_at( + 8, + 2, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + let plain_overlay = plain_compositor + .camera_overlay(&converted.inner, 1) + .expect("plain overlay") + .clone(); + assert_eq!( + plain_compositor + .camera_to_rgba + .as_ref() + .expect("scaler") + .source_format, + Pixel::YUYV422 + ); + assert_eq!( + plain_compositor + .camera_to_rgba + .as_ref() + .expect("scaler") + .destination_format, + Pixel::RGBA + ); + assert_eq!( + ( + plain_compositor + .camera_to_rgba + .as_ref() + .expect("scaler") + .destination_width, + plain_compositor + .camera_to_rgba + .as_ref() + .expect("scaler") + .destination_height + ), + (8, 2) + ); + let mut mirrored_compositor = overlay_at( + 8, + 2, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + let mirrored_overlay = mirrored_compositor + .camera_overlay(&mirrored_frame.inner, 1) + .expect("mirrored overlay") + .clone(); + assert!(rgb_sum(&plain_overlay, 0, 0) < rgb_sum(&plain_overlay, 7, 0)); + assert!(rgb_sum(&mirrored_overlay, 0, 0) > rgb_sum(&mirrored_overlay, 7, 0)); + + let colored = solid_yuyv(8, 2, 128, 200, 100); + let colored_frame = processed_to_ffmpeg(&yuyv_sample(&colored, false)).expect("color"); + let mut color_compositor = overlay_at( + 8, + 2, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + let color = color_compositor + .camera_overlay(&colored_frame.inner, 1) + .expect("color overlay"); + let pixel = &color.data(0)[..4]; + assert!(pixel[2] > pixel[0] && pixel[2] > pixel[1], "{pixel:?}"); + assert_eq!(pixel[3], 255); + + let mut round = overlay_at(8, 8, LinuxCameraShape::Round); + let round_source = + processed_to_ffmpeg(&yuyv_sample(&solid_yuyv(8, 8, 140, 128, 128), false)) + .expect("round source"); + let round_overlay = round + .camera_overlay(&round_source.inner, 1) + .expect("round overlay"); + assert_eq!(round_overlay.data(0)[3], 0); + let center = 4 * round_overlay.stride(0) + 4 * 4; + assert_eq!(round_overlay.data(0)[center + 3], 255); + + let wide = solid_yuyv(32, 18, 160, 128, 128); + let mut direct = overlay_at( + 16, + 9, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + let direct_frame = processed_to_ffmpeg(&yuyv_sample(&wide, false)).expect("direct"); + let direct_scaled = direct + .convert_camera(&direct_frame.inner, 16, 9) + .expect("direct scale"); + assert_eq!((direct_scaled.width(), direct_scaled.height()), (16, 9)); + let mut expand = + RgbaConverter::new(Pixel::YUYV422, 32, 18, Pixel::BGRA, 32, 18).expect("expand"); + let expanded = expand.convert(&wide).expect("full bgra"); + assert_eq!(expanded.format(), Pixel::BGRA); + let mut two_stage = overlay_at( + 16, + 9, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + let staged = two_stage + .convert_camera(&expanded, 16, 9) + .expect("staged scale"); + assert_eq!( + two_stage + .camera_to_rgba + .as_ref() + .expect("bgra scaler") + .source_format, + Pixel::BGRA + ); + let delta = max_channel_delta(&direct_scaled, &staged); + assert!( + delta <= 8, + "solid downscale drifted from the two-stage bilinear path by {delta}" + ); + + let now = Instant::now(); + let mut blurred = yuyv_sample(&solid_yuyv(4, 2, 16, 128, 128), false); + blurred.timestamp = Timestamp::Instant(now); + blurred.processing.blur = LinuxCameraBlur::Light; + blurred.mask = Some(LinuxCameraMaskReceipt { + generation: 1, + submitted_at: now, + completed_at: now, + }); + blurred.generation = 1; + let error = validate_processed_frame(&blurred, blurred.processing, 1, now, now) + .expect_err("blur cannot ride on yuyv"); + assert!(error.to_string().contains("YUYV"), "{error}"); + blurred.processing.blur = LinuxCameraBlur::Off; + blurred.mask = None; + blurred.dimensions.0 = 3; + let error = validate_processed_frame(&blurred, blurred.processing, 1, now, now) + .expect_err("odd yuyv"); + assert!(error.to_string().contains("even"), "{error}"); + assert!( + LinuxProcessedCameraFrame::from_yuyv422( + &ffmpeg::frame::Video::new(Pixel::BGRA, 4, 2), + Timestamp::Instant(now), + 1, + LinuxCameraProcessing { + mirrored: false, + blur: LinuxCameraBlur::Off, + }, + ) + .is_none() + ); + } + + #[test] + #[ignore = "release timing benchmark; run with --release --ignored --nocapture"] + fn yuyv_overlay_scale_benchmark() { + ffmpeg::init().expect("FFmpeg initializes"); + const WARMUP: usize = 8; + const RUNS: usize = 31; + let width = 1920u32; + let height = 1080u32; + let dst_w = 576u32; + let dst_h = 324u32; + let cpu = std::fs::read_to_string("/proc/cpuinfo") + .ok() + .and_then(|text| { + text.lines() + .find(|line| line.starts_with("model name")) + .and_then(|line| line.split(':').nth(1)) + .map(|value| value.trim().to_string()) + }) + .unwrap_or_else(|| "unknown".to_string()); + println!("cpu: {cpu}"); + println!( + "rustc 1.88.0; warmup {WARMUP} excluded; median is sorted index {} of {RUNS}", + RUNS / 2 + ); + println!("path: Compositor::convert_camera swscale Flags::BILINEAR"); + println!( + "before is full-frame YUYV->BGRA bilinear, a tight BGRA copy, then BGRA->RGBA overlay" + ); + println!("after is a tight YUYV copy plus one YUYV->RGBA overlay scale"); + println!("single-scale pixels are not bit-identical to that two-stage bilinear"); + + let mut yuyv = solid_yuyv(width, height, 140, 160, 90); + let row = width as usize * 2; + let stride = yuyv.stride(0); + for y in 0..height as usize { + for (x, pixel) in yuyv.data_mut(0)[y * stride..y * stride + row] + .chunks_exact_mut(4) + .enumerate() + { + let luma = 16u8.wrapping_add((x as u8).wrapping_mul(3).wrapping_add(y as u8)); + pixel[0] = luma; + pixel[2] = luma.wrapping_add(8); + } + } + let timestamp = Timestamp::Instant(Instant::now()); + let unmirrored = LinuxCameraProcessing { + mirrored: false, + blur: LinuxCameraBlur::Off, + }; + let mirrored = LinuxCameraProcessing { + mirrored: true, + blur: LinuxCameraBlur::Off, + }; + + let mut expand = + RgbaConverter::new(Pixel::YUYV422, width, height, Pixel::BGRA, width, height) + .expect("expand"); + let expanded = expand.convert(&yuyv).expect("warmup expand"); + let mut bgra_scaler = overlay_at( + dst_w, + dst_h, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + let mut yuyv_scaler = overlay_at( + dst_w, + dst_h, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + let bgra_scaled = bgra_scaler + .convert_camera(&expanded, dst_w, dst_h) + .expect("bgra scale"); + let yuyv_scaled = yuyv_scaler + .convert_camera(&yuyv, dst_w, dst_h) + .expect("yuyv scale"); + assert_eq!((bgra_scaled.width(), bgra_scaled.height()), (dst_w, dst_h)); + assert_eq!((yuyv_scaled.width(), yuyv_scaled.height()), (dst_w, dst_h)); + let delta = max_channel_delta(&bgra_scaled, &yuyv_scaled); + println!("max_abs_channel_delta scaler_only 1920x1080->576x324 = {delta}"); + + let millis = |sample: (u128, u128, u128)| { + ( + sample.0 as f64 / 1_000_000.0, + sample.1 as f64 / 1_000_000.0, + sample.2 as f64 / 1_000_000.0, + ) + }; + let print_ms = |label: &str, sample: (u128, u128, u128)| { + let (min, median, max) = millis(sample); + println!("{label} median {median:.3} ms min {min:.3} max {max:.3} n={RUNS}"); + }; + + print_ms( + "scaler BGRA 1920x1080 -> RGBA 576x324", + time_crop(WARMUP, RUNS, || { + let scaled = bgra_scaler + .convert_camera(std::hint::black_box(&expanded), dst_w, dst_h) + .expect("timed bgra scale"); + std::hint::black_box(scaled.data(0)[0]); + }), + ); + print_ms( + "scaler YUYV 1920x1080 -> RGBA 576x324", + time_crop(WARMUP, RUNS, || { + let scaled = yuyv_scaler + .convert_camera(std::hint::black_box(&yuyv), dst_w, dst_h) + .expect("timed yuyv scale"); + std::hint::black_box(scaled.data(0)[0]); + }), + ); + + let mut bgra_e2e = overlay_at( + dst_w, + dst_h, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + let mut yuyv_e2e = overlay_at( + dst_w, + dst_h, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + let mut yuyv_mirror_e2e = overlay_at( + dst_w, + dst_h, + LinuxCameraShape::RoundedRectangle { radius_pixels: 0 }, + ); + print_ms( + "e2e two-stage YUYV->full BGRA + copy + BGRA->overlay", + time_crop(WARMUP, RUNS, || { + let expanded = expand.convert(std::hint::black_box(&yuyv)).expect("expand"); + let row = width as usize * 4; + let mut owned = vec![0u8; row * height as usize]; + for y in 0..height as usize { + owned[y * row..y * row + row] + .copy_from_slice(&expanded.data(0)[y * expanded.stride(0)..][..row]); + } + let processed = LinuxProcessedCameraFrame { + pixels: LinuxCameraPixels::new(owned), + format: LinuxCameraSampleFormat::Bgra, + dimensions: (width, height), + stride: row, + timestamp, + generation: 1, + processing: unmirrored, + mask: None, + }; + let frame = processed_to_ffmpeg(&processed).expect("bgra ffmpeg"); + let scaled = bgra_e2e + .convert_camera(&frame.inner, dst_w, dst_h) + .expect("e2e bgra"); + std::hint::black_box(scaled.data(0)[0]); + }), + ); + print_ms( + "e2e direct YUYV copy + YUYV->overlay", + time_crop(WARMUP, RUNS, || { + let sample = LinuxProcessedCameraFrame::from_yuyv422( + std::hint::black_box(&yuyv), + timestamp, + 1, + unmirrored, + ) + .expect("sample"); + let frame = processed_to_ffmpeg(&sample).expect("yuyv ffmpeg"); + let scaled = yuyv_e2e + .convert_camera(&frame.inner, dst_w, dst_h) + .expect("e2e yuyv"); + std::hint::black_box(scaled.data(0)[0]); + }), + ); + print_ms( + "e2e direct mirrored YUYV copy + YUYV->overlay", + time_crop(WARMUP, RUNS, || { + let sample = LinuxProcessedCameraFrame::from_yuyv422( + std::hint::black_box(&yuyv), + timestamp, + 1, + mirrored, + ) + .expect("mirrored sample"); + let frame = processed_to_ffmpeg(&sample).expect("mirrored ffmpeg"); + let scaled = yuyv_mirror_e2e + .convert_camera(&frame.inner, dst_w, dst_h) + .expect("e2e mirror"); + std::hint::black_box(scaled.data(0)[0]); + }), + ); + } } diff --git a/crates/scap-targets/src/platform/linux/wayland.rs b/crates/scap-targets/src/platform/linux/wayland.rs index c4b8bcdbf6a..8c0118f92bb 100644 --- a/crates/scap-targets/src/platform/linux/wayland.rs +++ b/crates/scap-targets/src/platform/linux/wayland.rs @@ -15,6 +15,8 @@ use wayland_protocols::xdg::xdg_output::zv1::client::{zxdg_output_manager_v1, zx use super::{DisplayImpl, LogicalBounds, LogicalPosition, LogicalSize}; +const QUERY_TIMEOUT: Duration = Duration::from_secs(1); + #[derive(Clone, Copy, Default)] struct OutputGeometry { position: (i32, i32), @@ -108,7 +110,7 @@ pub(super) fn displays() -> Option> { runtime.join(display) }; let connection = Connection::from_socket(UnixStream::connect(socket).ok()?).ok()?; - query(&connection, Duration::from_millis(250)) + query(&connection, QUERY_TIMEOUT) } fn output_uuid(name: &str) -> Option { @@ -453,6 +455,45 @@ mod tests { assert!(started.elapsed() < Duration::from_secs(1)); } + #[test] + fn busy_compositor_can_finish_display_enumeration_after_a_quarter_second() { + use std::io::{Read, Write}; + + let (client, mut server) = UnixStream::pair().unwrap(); + let (finished_tx, finished_rx) = std::sync::mpsc::channel(); + let responder = std::thread::spawn(move || -> std::io::Result<()> { + for reply in 0..2 { + loop { + let mut header = [0u8; 8]; + server.read_exact(&mut header)?; + let object = u32::from_ne_bytes(header[..4].try_into().unwrap()); + let message = u32::from_ne_bytes(header[4..].try_into().unwrap()); + let mut payload = vec![0; (message >> 16) as usize - header.len()]; + server.read_exact(&mut payload)?; + if object != 1 || message & 0xffff != 0 { + continue; + } + if reply == 0 { + std::thread::sleep(Duration::from_millis(350)); + } + let callback = u32::from_ne_bytes(payload[..4].try_into().unwrap()); + server.write_all(&callback.to_ne_bytes())?; + server.write_all(&(12u32 << 16).to_ne_bytes())?; + server.write_all(&0u32.to_ne_bytes())?; + break; + } + } + let _ = finished_rx.recv(); + Ok(()) + }); + let connection = Connection::from_socket(client).unwrap(); + let result = query(&connection, QUERY_TIMEOUT); + drop(connection); + let _ = finished_tx.send(()); + responder.join().unwrap().unwrap(); + assert!(result.is_some()); + } + #[test] fn wayland_session_does_not_use_xwayland_dimensions() { assert!(super::super::prefers_wayland_environment( diff --git a/scripts/linux-desktop-harness/arch-up.sh b/scripts/linux-desktop-harness/arch-up.sh index 6a1a8028862..043d83d2e23 100755 --- a/scripts/linux-desktop-harness/arch-up.sh +++ b/scripts/linux-desktop-harness/arch-up.sh @@ -30,6 +30,17 @@ debug { disable_logs = false } exec-once = sh -c '$export_session' exec-once = waybar CONF + cat >"$HOME/.config/hypr/hyprland.lua" <"$HOME/.config/hypr/xdph.conf" <"$out/result.txt" result() { printf '%s=%s\n' "$1" "$2" >>"$out/result.txt"; } @@ -81,7 +87,7 @@ if data[:8] == b"\x89PNG\r\n\x1a\n": print(*struct.unpack(">II", data[16:24]))' "$shot") echo "screenshot: ${shot:-none} ${size:-}" >>"$out/actions.log" result screenshot_taken "$([[ -n $size ]] && echo yes || echo no)" -result screenshot_size "$([[ ${size// /x} == "${SCREEN:-1920x1080}" ]] && echo yes || echo "${size// /x}")" +result screenshot_size "$([[ ${size// /x} == "$expected_size" ]] && echo yes || echo "${size// /x}")" result capture_failed "$(grep -ac 'screenshot capture failed' "$out/cap.log")" if [[ -z $shot ]]; then find "$HOME" /tmp -newer "$out/profile/store" -iname "Screenshot*.png" -printf "portal file: %s bytes %p\n" 2>/dev/null >>"$out/actions.log"