diff --git a/.github/workflows/glass-probe.yml b/.github/workflows/glass-probe.yml index 57f960e09b4..b3bb695d920 100644 --- a/.github/workflows/glass-probe.yml +++ b/.github/workflows/glass-probe.yml @@ -143,7 +143,7 @@ jobs: db="/Library/Application Support/com.apple.TCC/TCC.db" now=$(date +%s) for client in "$app" /bin/bash /Users/runner/bin/Runner.Worker /Users/runner/bin/Runner.Listener; do - sudo sqlite3 "$db" "INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('kTCCServiceScreenCapture','$client',1,2,4,1,'UNUSED',0,$now);" + sudo sqlite3 -cmd ".timeout 10000" "$db" "INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('kTCCServiceScreenCapture','$client',1,2,4,1,'UNUSED',0,$now);" done sudo killall tccd || true sleep 2 diff --git a/.github/workflows/instant-reliability.yml b/.github/workflows/instant-reliability.yml index af99f5174a4..17afabc4c60 100644 --- a/.github/workflows/instant-reliability.yml +++ b/.github/workflows/instant-reliability.yml @@ -168,8 +168,8 @@ jobs: sort -u "$RUNNER_TEMP/capture-binaries" | grep '^/' | while read -r client; do for service in kTCCServiceScreenCapture kTCCServiceMicrophone; do row="INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('$service','$client',1,2,4,1,'UNUSED',0,$now);" - sudo sqlite3 "$db" "$row" - sqlite3 "$user_db" "$row" + sudo sqlite3 -cmd ".timeout 10000" "$db" "$row" + sqlite3 -cmd ".timeout 10000" "$user_db" "$row" done /usr/libexec/PlistBuddy -c "Add :$client date $until" "$approvals" >/dev/null 2>&1 || true echo "granted $client" @@ -316,8 +316,11 @@ jobs: run: | set -euo pipefail sudo apt-get update -qq - # v4l2loopback needs videodev, which is in the extra kernel modules. - sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \ + if ! sudo modprobe videodev 2>/dev/null; then + sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" + sudo modprobe videodev + fi + sudo apt-get install -y -qq "linux-headers-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \ dbus ffmpeg pulseaudio pulseaudio-utils libasound2-plugins >/dev/null chmod +x "$RUNNER_TEMP/build/cap-gpui" "$RUNNER_TEMP/build/real-app-tests" sudo modprobe v4l2loopback video_nr=42 card_label="Cap Test Camera" exclusive_caps=1 diff --git a/.github/workflows/picker-smoke.yml b/.github/workflows/picker-smoke.yml index 58bdd3eba8d..42c79de2fea 100644 --- a/.github/workflows/picker-smoke.yml +++ b/.github/workflows/picker-smoke.yml @@ -141,7 +141,7 @@ jobs: now=$(date +%s) for client in "$app" /bin/bash /Users/runner/bin/Runner.Worker /Users/runner/bin/Runner.Listener; do for service in kTCCServiceScreenCapture kTCCServiceAccessibility; do - sudo sqlite3 "$db" "INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('$service','$client',1,2,4,1,'UNUSED',0,$now);" + sudo sqlite3 -cmd ".timeout 10000" "$db" "INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('$service','$client',1,2,4,1,'UNUSED',0,$now);" done done sudo killall tccd || true diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.yml index 3420e677f8a..b50975dae0e 100644 --- a/.github/workflows/verify-recording.yml +++ b/.github/workflows/verify-recording.yml @@ -287,8 +287,8 @@ jobs: sort -u "$RUNNER_TEMP/capture-binaries" | grep '^/' | while read -r client; do for service in kTCCServiceScreenCapture kTCCServiceMicrophone; do row="INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('$service','$client',1,2,4,1,'UNUSED',0,$now);" - sudo sqlite3 "$db" "$row" - sqlite3 "$user_db" "$row" + sudo sqlite3 -cmd ".timeout 10000" "$db" "$row" + sqlite3 -cmd ".timeout 10000" "$user_db" "$row" done /usr/libexec/PlistBuddy -c "Add :$client date $until" "$approvals" >/dev/null 2>&1 || true echo "granted $client" diff --git a/crates/recording/src/sources/screen_capture/linux.rs b/crates/recording/src/sources/screen_capture/linux.rs index 7ca0ef3bed0..9b9b626496c 100644 --- a/crates/recording/src/sources/screen_capture/linux.rs +++ b/crates/recording/src/sources/screen_capture/linux.rs @@ -310,6 +310,7 @@ struct PipewireCaptureState { rate_limited: Arc, capture_clock: Instant, cadence_gate: FrameCadenceGate, + frame_pool: CaptureFramePool, /// Reads frames with a GPU-tiled modifier; `None` without a usable EGL. importer: Option, imported: Vec, @@ -565,6 +566,7 @@ fn capture_pipewire_node( rate_limited: rate_limited.clone(), capture_clock: started, cadence_gate: FrameCadenceGate::new(1_000_000_000 / i64::from(input.fps.max(1))), + frame_pool: CaptureFramePool::new(), importer, imported: Vec::new(), }; @@ -893,7 +895,12 @@ fn process_pipewire_frame( &mut state.imported, )? } else { - frame_from_pipewire_data(&mut datas[0], state.format, state.crop_bounds)? + frame_from_pipewire_data( + &mut datas[0], + state.format, + state.crop_bounds, + &mut state.frame_pool, + )? }; let Some(raw_frame) = raw_frame else { return Ok(Some(StallSendOutcome::StalledAndDropped { waited_ms: 0 })); @@ -939,10 +946,70 @@ fn prepare_pipewire_frame( .scale(&frame, output) } +const CAPTURE_FRAME_POOL_CAPACITY: usize = 4; + +struct CaptureFramePool { + key: Option<(ffmpeg::format::Pixel, u32, u32)>, + frames: Vec, +} + +impl CaptureFramePool { + fn new() -> Self { + Self { + key: None, + frames: Vec::new(), + } + } + + fn acquire( + &mut self, + format: ffmpeg::format::Pixel, + width: u32, + height: u32, + ) -> ffmpeg::frame::Video { + if self.key != Some((format, width, height)) { + self.key = Some((format, width, height)); + self.frames = (0..CAPTURE_FRAME_POOL_CAPACITY) + .map(|_| ffmpeg::frame::Video::new(format, width, height)) + .collect(); + } + + if let Some(index) = self.frames.iter_mut().position(capture_frame_writable) { + self.frames.swap_remove(index) + } else { + ffmpeg::frame::Video::new(format, width, height) + } + } + + fn share(&mut self, frame: ffmpeg::frame::Video) -> ffmpeg::frame::Video { + let mut shared = ffmpeg::frame::Video::empty(); + let status = unsafe { ffmpeg::ffi::av_frame_ref(shared.as_mut_ptr(), frame.as_ptr()) }; + if status >= 0 { + self.release(frame); + shared + } else { + frame + } + } + + fn release(&mut self, frame: ffmpeg::frame::Video) { + if self.key == Some((frame.format(), frame.width(), frame.height())) + && self.frames.len() < CAPTURE_FRAME_POOL_CAPACITY + { + self.frames.push(frame); + } + } +} + +fn capture_frame_writable(frame: &mut ffmpeg::frame::Video) -> bool { + unsafe { ffmpeg::ffi::av_frame_is_writable(frame.as_mut_ptr()) != 0 } +} + fn frame_from_pipewire_data( data: &mut spa::buffer::Data, format: spa::param::video::VideoInfoRaw, crop_bounds: Option, + pool: &mut CaptureFramePool, ) -> anyhow::Result> { let (pixel_format, bytes_per_pixel) = pipewire_pixel_format(format.format()).ok_or_else(|| { @@ -1002,35 +1069,41 @@ fn frame_from_pipewire_data( }; let row_bytes = crop_width * bytes_per_pixel; - let mut frame = ffmpeg::frame::Video::new(pixel_format, crop_width as u32, crop_height as u32); + let mut frame = pool.acquire(pixel_format, crop_width as u32, crop_height as u32); let target_stride = frame.stride(0); + let mut copy_error = None; if target_stride < row_bytes { - bail!( + copy_error = Some(anyhow!( "PipeWire target frame stride was too small: {} for {}x{}", target_stride, crop_width, crop_height - ); - } + )); + } else { + for y in 0..crop_height { + let source_start = offset + (crop_y + y) * source_stride + crop_x * bytes_per_pixel; + let source_end = source_start + row_bytes; + if source_end > source_limit { + copy_error = Some(anyhow!( + "PipeWire screen capture frame was too small: {} bytes for {}x{}", + source.len(), + source_width, + source_height + )); + break; + } - for y in 0..crop_height { - let source_start = offset + (crop_y + y) * source_stride + crop_x * bytes_per_pixel; - let source_end = source_start + row_bytes; - if source_end > source_limit { - bail!( - "PipeWire screen capture frame was too small: {} bytes for {}x{}", - source.len(), - source_width, - source_height - ); + let target_start = y * target_stride; + frame.data_mut(0)[target_start..target_start + row_bytes] + .copy_from_slice(&source[source_start..source_end]); } - - let target_start = y * target_stride; - frame.data_mut(0)[target_start..target_start + row_bytes] - .copy_from_slice(&source[source_start..source_end]); + } + if let Some(error) = copy_error { + pool.release(frame); + return Err(error); } - Ok(Some(frame)) + Ok(Some(pool.share(frame))) } /// A frame with a GPU-tiled modifier, read back through EGL as RGBA. @@ -2498,6 +2571,7 @@ pub struct X11Grabber { output: VideoInfo, scaler: Option, show_cursor: bool, + frame_pool: CaptureFramePool, } enum GrabBackend { @@ -2640,6 +2714,7 @@ impl X11Grabber { output, scaler: None, show_cursor, + frame_pool: CaptureFramePool::new(), }; grabber.refresh_window_pixmap()?; grabber.backend = grabber.initial_backend(allow_shm); @@ -2784,26 +2859,38 @@ impl X11Grabber { } None => (self.root, self.x, self.y), }; - let mut source = ffmpeg::frame::Video::new( + let mut source = self.frame_pool.acquire( self.source_pixel, u32::from(self.width), u32::from(self.height), ); - if self.uses_shm() { + let read = if self.uses_shm() { if let Err(shm_error) = self.read_shm_image(drawable, x, y, &mut source) { // Only abandon MIT-SHM once GetImage succeeds on the same frame; // if both fail the cause is the drawable/region, not SHM. - self.read_get_image(drawable, x, y, &mut source)?; - tracing::warn!( - error = %format!("{shm_error:#}"), - "X11 MIT-SHM capture failed, switching to GetImage" - ); - self.backend = GrabBackend::GetImage; + match self.read_get_image(drawable, x, y, &mut source) { + Ok(()) => { + tracing::warn!( + error = %format!("{shm_error:#}"), + "X11 MIT-SHM capture failed, switching to GetImage" + ); + self.backend = GrabBackend::GetImage; + Ok(()) + } + Err(error) => Err(error), + } + } else { + Ok(()) } } else { - self.read_get_image(drawable, x, y, &mut source)?; + self.read_get_image(drawable, x, y, &mut source) + }; + if let Err(error) = read { + self.frame_pool.release(source); + return Err(error); } + let source = self.frame_pool.share(source); let mut frame = prepare_pipewire_frame(source, &mut self.scaler, self.output)?; if self.show_cursor @@ -3421,10 +3508,10 @@ mod system_audio_tests { #[cfg(test)] mod pipewire_frame_tests { use super::{ - FrameScaler, LinuxCaptureSource, PipewireNodeInput, VideoInfo, capture_pipewire_node, - frame_from_pipewire_data, pipewire_dmabuf_format_param, pipewire_format_param, - prefers_wayland_environment, prepare_pipewire_frame, spa, wayland_area_matches_display, - wayland_video_info, + CAPTURE_FRAME_POOL_CAPACITY, CaptureFramePool, FrameScaler, LinuxCaptureSource, + PipewireNodeInput, VideoInfo, capture_pipewire_node, frame_from_pipewire_data, + pipewire_dmabuf_format_param, pipewire_format_param, prefers_wayland_environment, + prepare_pipewire_frame, spa, wayland_area_matches_display, wayland_video_info, }; use std::os::fd::AsRawFd; @@ -3567,7 +3654,8 @@ mod pipewire_frame_tests { height: height as u32, }); format.set_modifier(0); - let frame = frame_from_pipewire_data(data, format, None) + let mut pool = CaptureFramePool::new(); + let frame = frame_from_pipewire_data(data, format, None, &mut pool) .unwrap() .expect("a frame"); assert_eq!((frame.width(), frame.height()), (4, 3)); @@ -3681,6 +3769,90 @@ mod pipewire_frame_tests { assert!(!prefers_wayland_environment(false, true, Some("wayland"))); } + #[test] + fn capture_frame_pool_reuses_free_frames_and_resets_on_shape_change() { + let format = ffmpeg::format::Pixel::BGRZ; + let mut pool = CaptureFramePool::new(); + let mut first = pool.acquire(format, 4, 2); + let first_ptr = first.data(0).as_ptr(); + first.data_mut(0).fill(1); + let first = pool.share(first); + assert_eq!(first.data(0).as_ptr(), first_ptr); + assert!(first.data(0).iter().all(|byte| *byte == 1)); + + let mut second = pool.acquire(format, 4, 2); + assert_ne!(second.data(0).as_ptr(), first_ptr); + second.data_mut(0).fill(2); + assert!(first.data(0).iter().all(|byte| *byte == 1)); + let second = pool.share(second); + + drop(first); + let reused = pool.acquire(format, 4, 2); + assert_eq!(reused.data(0).as_ptr(), first_ptr); + assert!(second.data(0).iter().all(|byte| *byte == 2)); + let _reused = pool.share(reused); + + let resized = pool.acquire(format, 8, 2); + assert_eq!((resized.width(), resized.height()), (8, 2)); + assert_ne!(resized.data(0).as_ptr(), first_ptr); + let resized_ptr = resized.data(0).as_ptr(); + let _resized = pool.share(resized); + + let reformatted = pool.acquire(ffmpeg::format::Pixel::RGBZ, 8, 2); + assert_eq!(reformatted.format(), ffmpeg::format::Pixel::RGBZ); + assert_ne!(reformatted.data(0).as_ptr(), resized_ptr); + } + + #[test] + fn capture_frame_pool_keeps_held_frames_when_capacity_is_busy() { + let format = ffmpeg::format::Pixel::BGRZ; + let mut pool = CaptureFramePool::new(); + let mut held = Vec::new(); + let mut pointers = Vec::new(); + + for index in 0..CAPTURE_FRAME_POOL_CAPACITY { + let marker = (index + 1) as u8; + let mut frame = pool.acquire(format, 4, 2); + let pointer = frame.data(0).as_ptr(); + frame.data_mut(0).fill(marker); + let frame = pool.share(frame); + assert_eq!(frame.data(0).as_ptr(), pointer); + assert!(frame.data(0).iter().all(|byte| *byte == marker)); + pointers.push(pointer); + held.push(frame); + } + + let mut overflow = pool.acquire(format, 4, 2); + let overflow_ptr = overflow.data(0).as_ptr(); + assert!(!pointers.contains(&overflow_ptr)); + overflow.data_mut(0).fill(9); + let overflow = pool.share(overflow); + assert_eq!(overflow.data(0).as_ptr(), overflow_ptr); + assert!(overflow.data(0).iter().all(|byte| *byte == 9)); + assert_eq!(pool.frames.len(), CAPTURE_FRAME_POOL_CAPACITY); + assert!( + pool.frames + .iter() + .all(|frame| pointers.contains(&frame.data(0).as_ptr())) + ); + + for (index, frame) in held.iter().enumerate() { + let marker = (index + 1) as u8; + assert_eq!(frame.data(0).as_ptr(), pointers[index]); + assert!(frame.data(0).iter().all(|byte| *byte == marker)); + } + + let dropped_ptr = pointers.remove(0); + drop(held.remove(0)); + let reused = pool.acquire(format, 4, 2); + assert_eq!(reused.data(0).as_ptr(), dropped_ptr); + assert_eq!(pool.frames.len(), CAPTURE_FRAME_POOL_CAPACITY - 1); + for (frame, pointer) in held.iter().zip(&pointers) { + assert_eq!(frame.data(0).as_ptr(), *pointer); + } + let _overflow = overflow; + } + #[test] fn matching_pipewire_frames_reuse_owned_pixel_storage_without_a_scaler() { let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRZ, 16, 12); diff --git a/crates/scap-targets/src/platform/linux/wayland.rs b/crates/scap-targets/src/platform/linux/wayland.rs index c4b8bcdbf6a..8c0118f92bb 100644 --- a/crates/scap-targets/src/platform/linux/wayland.rs +++ b/crates/scap-targets/src/platform/linux/wayland.rs @@ -15,6 +15,8 @@ use wayland_protocols::xdg::xdg_output::zv1::client::{zxdg_output_manager_v1, zx use super::{DisplayImpl, LogicalBounds, LogicalPosition, LogicalSize}; +const QUERY_TIMEOUT: Duration = Duration::from_secs(1); + #[derive(Clone, Copy, Default)] struct OutputGeometry { position: (i32, i32), @@ -108,7 +110,7 @@ pub(super) fn displays() -> Option> { runtime.join(display) }; let connection = Connection::from_socket(UnixStream::connect(socket).ok()?).ok()?; - query(&connection, Duration::from_millis(250)) + query(&connection, QUERY_TIMEOUT) } fn output_uuid(name: &str) -> Option { @@ -453,6 +455,45 @@ mod tests { assert!(started.elapsed() < Duration::from_secs(1)); } + #[test] + fn busy_compositor_can_finish_display_enumeration_after_a_quarter_second() { + use std::io::{Read, Write}; + + let (client, mut server) = UnixStream::pair().unwrap(); + let (finished_tx, finished_rx) = std::sync::mpsc::channel(); + let responder = std::thread::spawn(move || -> std::io::Result<()> { + for reply in 0..2 { + loop { + let mut header = [0u8; 8]; + server.read_exact(&mut header)?; + let object = u32::from_ne_bytes(header[..4].try_into().unwrap()); + let message = u32::from_ne_bytes(header[4..].try_into().unwrap()); + let mut payload = vec![0; (message >> 16) as usize - header.len()]; + server.read_exact(&mut payload)?; + if object != 1 || message & 0xffff != 0 { + continue; + } + if reply == 0 { + std::thread::sleep(Duration::from_millis(350)); + } + let callback = u32::from_ne_bytes(payload[..4].try_into().unwrap()); + server.write_all(&callback.to_ne_bytes())?; + server.write_all(&(12u32 << 16).to_ne_bytes())?; + server.write_all(&0u32.to_ne_bytes())?; + break; + } + } + let _ = finished_rx.recv(); + Ok(()) + }); + let connection = Connection::from_socket(client).unwrap(); + let result = query(&connection, QUERY_TIMEOUT); + drop(connection); + let _ = finished_tx.send(()); + responder.join().unwrap().unwrap(); + assert!(result.is_some()); + } + #[test] fn wayland_session_does_not_use_xwayland_dimensions() { assert!(super::super::prefers_wayland_environment( diff --git a/scripts/linux-desktop-harness/arch-up.sh b/scripts/linux-desktop-harness/arch-up.sh index 6a1a8028862..043d83d2e23 100755 --- a/scripts/linux-desktop-harness/arch-up.sh +++ b/scripts/linux-desktop-harness/arch-up.sh @@ -30,6 +30,17 @@ debug { disable_logs = false } exec-once = sh -c '$export_session' exec-once = waybar CONF + cat >"$HOME/.config/hypr/hyprland.lua" <"$HOME/.config/hypr/xdph.conf" <"$out/result.txt" result() { printf '%s=%s\n' "$1" "$2" >>"$out/result.txt"; } @@ -81,7 +87,7 @@ if data[:8] == b"\x89PNG\r\n\x1a\n": print(*struct.unpack(">II", data[16:24]))' "$shot") echo "screenshot: ${shot:-none} ${size:-}" >>"$out/actions.log" result screenshot_taken "$([[ -n $size ]] && echo yes || echo no)" -result screenshot_size "$([[ ${size// /x} == "${SCREEN:-1920x1080}" ]] && echo yes || echo "${size// /x}")" +result screenshot_size "$([[ ${size// /x} == "$expected_size" ]] && echo yes || echo "${size// /x}")" result capture_failed "$(grep -ac 'screenshot capture failed' "$out/cap.log")" if [[ -z $shot ]]; then find "$HOME" /tmp -newer "$out/profile/store" -iname "Screenshot*.png" -printf "portal file: %s bytes %p\n" 2>/dev/null >>"$out/actions.log"