perf(camera): scaled IDCT for instant MJPEG bubble

.github/workflows/ci.yml

@@ -306,7 +306,7 @@ jobs:
           cargo test --locked -p cap-recording --lib -- --test-threads=1
           cargo test --locked -p cap-recording --test x11_shm_capture -- --test-threads=1
           cargo test --locked -p scap-targets --lib --test linux_targets
-          cargo test --locked -p cap-enc-ffmpeg
+          cargo test --locked -p cap-enc-ffmpeg -p cap-camera-ffmpeg
 
   build-gpui:
     name: Build GPUI Desktop

.github/workflows/instant-reliability.yml

@@ -316,8 +316,11 @@ jobs:
         run: |
           set -euo pipefail
           sudo apt-get update -qq
-          # v4l2loopback needs videodev, which is in the extra kernel modules.
-          sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \
+          if ! sudo modprobe videodev 2>/dev/null; then
+            sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)"
+            sudo modprobe videodev
+          fi
+          sudo apt-get install -y -qq "linux-headers-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \
             dbus ffmpeg pulseaudio pulseaudio-utils libasound2-plugins >/dev/null
           chmod +x "$RUNNER_TEMP/build/cap-gpui" "$RUNNER_TEMP/build/real-app-tests"
           sudo modprobe v4l2loopback video_nr=42 card_label="Cap Test Camera" exclusive_caps=1

crates/camera-ffmpeg/src/lib.rs

@@ -1,3 +1,6 @@
+#[cfg(target_os = "linux")]
+mod mjpeg_idct;
+
 #[cfg(target_os = "macos")]
 mod macos;
 #[cfg(target_os = "macos")]
@@ -17,4 +20,13 @@ pub trait CapturedFrameExt {
     /// Creates an ffmpeg video frame from the native frame.
     /// Only size, format, and data are set.
     fn as_ffmpeg(&self) -> Result<ffmpeg::frame::Video, AsFFmpegError>;
+
+    /// Like [`as_ffmpeg`](Self::as_ffmpeg), but MJPEG may decode with a scaled
+    /// IDCT when `max_width`×`max_height` is smaller than the frame. The
+    /// result still covers that rectangle. Other pixel formats are unchanged.
+    fn as_ffmpeg_covering(
+        &self,
+        max_width: u32,
+        max_height: u32,
+    ) -> Result<ffmpeg::frame::Video, AsFFmpegError>;
 }

crates/camera-ffmpeg/src/linux.rs

@@ -3,6 +3,7 @@ use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo};
 use std::cell::RefCell;
 
 use crate::CapturedFrameExt;
+use crate::mjpeg_idct::idct_lowres;
 
 #[derive(thiserror::Error, Debug)]
 pub enum AsFFmpegError {
@@ -24,19 +25,27 @@ pub enum AsFFmpegError {
 
 struct MjpegDecoder {
     decoder: ffmpeg::codec::decoder::Video,
+    lowres: i32,
 }
 
 impl MjpegDecoder {
-    fn new() -> Result<Self, AsFFmpegError> {
+    fn new(lowres: i32) -> Result<Self, AsFFmpegError> {
         let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG)
             .ok_or_else(|| AsFFmpegError::MjpegDecodeError("MJPEG codec not found".to_string()))?;
-
-        let decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec);
+        let max_lowres = codec.max_lowres().clamp(0, 3);
+        let lowres = lowres.clamp(0, max_lowres);
+
+        let mut decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec);
+        if lowres > 0 {
+            unsafe {
+                (*decoder_context.as_mut_ptr()).lowres = lowres;
+            }
+        }
         let decoder = decoder_context.decoder().video().map_err(|e| {
             AsFFmpegError::MjpegDecodeError(format!("Failed to create decoder: {e}"))
         })?;
 
-        Ok(Self { decoder })
+        Ok(Self { decoder, lowres })
     }
 
     fn decode(&mut self, bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
@@ -60,26 +69,61 @@ thread_local! {
     static MJPEG_DECODER: RefCell<Option<MjpegDecoder>> = const { RefCell::new(None) };
 }
 
-fn decode_mjpeg(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
+fn decode_mjpeg(bytes: &[u8], lowres: i32) -> Result<FFVideo, AsFFmpegError> {
     MJPEG_DECODER.with(|decoder_cell| {
-        let mut decoder_opt = decoder_cell.borrow_mut();
-
-        if decoder_opt.is_none() {
-            *decoder_opt = Some(MjpegDecoder::new()?);
+        let mut slot = decoder_cell.borrow_mut();
+        if slot.as_ref().is_none_or(|decoder| decoder.lowres != lowres) {
+            tracing::debug!(lowres, "MJPEG IDCT scale");
+            *slot = Some(MjpegDecoder::new(lowres)?);
         }
-
-        decoder_opt.as_mut().unwrap().decode(bytes)
+        slot.as_mut()
+            .expect("MJPEG decoder is inserted before use")
+            .decode(bytes)
     })
 }
 
+fn mjpeg_frame_covering(
+    bytes: &[u8],
+    source_width: u32,
+    source_height: u32,
+    max_width: u32,
+    max_height: u32,
+) -> Result<FFVideo, AsFFmpegError> {
+    let lowres = idct_lowres(source_width, source_height, max_width, max_height);
+    if lowres == 0 {
+        return decode_mjpeg(bytes, 0);
+    }
+    match decode_mjpeg(bytes, lowres) {
+        Ok(frame) if frame.width() >= max_width && frame.height() >= max_height => Ok(frame),
+        _ => decode_mjpeg(bytes, 0),
+    }
+}
+
 impl CapturedFrameExt for CapturedFrame<'_> {
     fn as_ffmpeg(&self) -> Result<FFVideo, AsFFmpegError> {
         let native = self.native();
         frame_from_bytes(native.bytes, native.format)
     }
+
+    fn as_ffmpeg_covering(
+        &self,
+        max_width: u32,
+        max_height: u32,
+    ) -> Result<FFVideo, AsFFmpegError> {
+        let native = self.native();
+        frame_from_bytes_covering(native.bytes, native.format, Some((max_width, max_height)))
+    }
 }
 
 fn frame_from_bytes(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpegError> {
+    frame_from_bytes_covering(bytes, format, None)
+}
+
+fn frame_from_bytes_covering(
+    bytes: &[u8],
+    format: NativeFrameFormat,
+    cover: Option<(u32, u32)>,
+) -> Result<FFVideo, AsFFmpegError> {
     match fourcc_str(format.fourcc).as_str() {
         "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2),
         "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2),
@@ -88,7 +132,12 @@ fn frame_from_bytes(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo,
         "NV12" => copy_nv12(bytes, format),
         "YU12" => copy_yuv420(bytes, format, false),
         "YV12" => copy_yuv420(bytes, format, true),
-        "MJPG" | "JPEG" => decode_mjpeg(bytes),
+        "MJPG" | "JPEG" => match cover {
+            Some((max_width, max_height)) => {
+                mjpeg_frame_covering(bytes, format.width, format.height, max_width, max_height)
+            }
+            None => decode_mjpeg(bytes, 0),
+        },
         other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())),
     }
 }
@@ -310,9 +359,184 @@ fn fourcc_str(fourcc: [u8; 4]) -> String {
 
 #[cfg(test)]
 mod tests {
-    use super::{AsFFmpegError, copy_nv12, copy_packed, frame_from_bytes};
+    use super::{
+        AsFFmpegError, MjpegDecoder, copy_nv12, copy_packed, decode_mjpeg, frame_from_bytes,
+        frame_from_bytes_covering, mjpeg_frame_covering,
+    };
+    use crate::mjpeg_idct::idct_lowres;
     use cap_camera::NativeFrameFormat;
+    use ffmpeg::Packet;
     use ffmpeg::format::Pixel;
+    use ffmpeg::frame::Video as FFVideo;
+
+    fn encode_mjpeg(width: u32, height: u32) -> Vec<u8> {
+        let codec = ffmpeg::codec::encoder::find(ffmpeg::codec::Id::MJPEG).unwrap();
+        let mut encoder = ffmpeg::codec::context::Context::new_with_codec(codec)
+            .encoder()
+            .video()
+            .unwrap();
+        encoder.set_width(width);
+        encoder.set_height(height);
+        encoder.set_format(Pixel::YUVJ420P);
+        encoder.set_time_base((1, 30));
+        encoder.set_qmin(4);
+        encoder.set_qmax(4);
+        let mut encoder = encoder.open().unwrap();
+
+        let mut frame = FFVideo::new(Pixel::YUVJ420P, width, height);
+        let luma_stride = frame.stride(0);
+        let luma = frame.data_mut(0);
+        for y in 0..height as usize {
+            for x in 0..width as usize {
+                let nx = x as f32 / width as f32;
+                let ny = y as f32 / height as f32;
+                let value = 70.0
+                    + 40.0 * (nx * std::f32::consts::TAU * 2.0).sin()
+                    + 25.0 * (ny * std::f32::consts::TAU * 3.0).sin()
+                    + 18.0 * ((nx + ny) * std::f32::consts::TAU * 5.0).sin()
+                    + ((x * 13 + y * 7) & 7) as f32
+                    - 3.0;
+                luma[y * luma_stride + x] = value.clamp(0.0, 255.0) as u8;
+            }
+        }
+        frame.data_mut(1).fill(110);
+        frame.data_mut(2).fill(150);
+        frame.set_pts(Some(0));
+        encoder.send_frame(&frame).unwrap();
+        encoder.send_eof().unwrap();
+        let mut packet = Packet::empty();
+        encoder.receive_packet(&mut packet).unwrap();
+        packet.data().unwrap().to_vec()
+    }
+
+    fn scale_bgra(frame: &FFVideo, width: u32, height: u32) -> FFVideo {
+        let mut scaler = ffmpeg::software::scaling::Context::get(
+            frame.format(),
+            frame.width(),
+            frame.height(),
+            Pixel::BGRA,
+            width,
+            height,
+            ffmpeg::software::scaling::Flags::BILINEAR,
+        )
+        .unwrap();
+        let mut output = FFVideo::empty();
+        scaler.run(frame, &mut output).unwrap();
+        output
+    }
+
+    fn bgra_delta(full: &FFVideo, scaled: &FFVideo) -> (f64, u8) {
+        let width = full.width().min(scaled.width()) as usize;
+        let height = full.height().min(scaled.height()) as usize;
+        let mut sum = 0u64;
+        let mut max = 0u8;
+        let mut count = 0u64;
+        for y in 0..height {
+            let full_row = &full.data(0)[y * full.stride(0)..];
+            let scaled_row = &scaled.data(0)[y * scaled.stride(0)..];
+            for x in 0..width {
+                for channel in 0..3 {
+                    let delta = full_row[x * 4 + channel].abs_diff(scaled_row[x * 4 + channel]);
+                    sum += u64::from(delta);
+                    max = max.max(delta);
+                    count += 1;
+                }
+            }
+        }
+        ((sum as f64) / count.max(1) as f64, max)
+    }
+
+    #[test]
+    fn scaled_idct_covers_the_bubble_and_stays_close_to_full_then_scale() {
+        ffmpeg::init().unwrap();
+        let bytes = encode_mjpeg(640, 360);
+        let scaled = mjpeg_frame_covering(&bytes, 640, 360, 200, 120).unwrap();
+        assert_eq!((scaled.width(), scaled.height()), (320, 180));
+        let full = decode_mjpeg(&bytes, 0).unwrap();
+        assert_eq!((full.width(), full.height()), (640, 360));
+        let (mean, max) = bgra_delta(&scale_bgra(&full, 200, 120), &scale_bgra(&scaled, 200, 120));
+        assert!(
+            max <= 6 && mean <= 1.5,
+            "scaled IDCT vs full decode + scale: mean {mean:.3} max {max}"
+        );
+    }
+
+    #[test]
+    fn full_resolution_request_restores_the_decoder_after_a_scaled_frame() {
+        ffmpeg::init().unwrap();
+        let bytes = encode_mjpeg(640, 360);
+        let scaled = mjpeg_frame_covering(&bytes, 640, 360, 200, 120).unwrap();
+        assert_eq!((scaled.width(), scaled.height()), (320, 180));
+        let full = mjpeg_frame_covering(&bytes, 640, 360, 640, 360).unwrap();
+        assert_eq!((full.width(), full.height()), (640, 360));
+        let untouched = decode_mjpeg(&bytes, 0).unwrap();
+        assert_eq!((untouched.width(), untouched.height()), (640, 360));
+    }
+
+    #[test]
+    #[ignore = "release wall-time measurement for the instant bubble"]
+    fn bubble_scaled_idct_is_faster_than_full_decode_and_scale() {
+        ffmpeg::init().unwrap();
+        let bytes = encode_mjpeg(1920, 1080);
+        let bubble = 324u32;
+        let lowres = idct_lowres(1920, 1080, bubble, bubble);
+        assert_eq!(lowres, 1);
+        let mut full_decoder = None;
+        let mut scaled_decoder = None;
+        let warmup = |lowres: i32, slot: &mut Option<MjpegDecoder>| {
+            for _ in 0..4 {
+                if slot.as_ref().is_none_or(|decoder| decoder.lowres != lowres) {
+                    *slot = Some(MjpegDecoder::new(lowres).unwrap());
+                }
+                slot.as_mut().unwrap().decode(&bytes).unwrap();
+            }
+        };
+        warmup(0, &mut full_decoder);
+        warmup(lowres, &mut scaled_decoder);
+        let full_frame = full_decoder.as_mut().unwrap().decode(&bytes).unwrap();
+        let scaled_frame = scaled_decoder.as_mut().unwrap().decode(&bytes).unwrap();
+        assert_eq!((scaled_frame.width(), scaled_frame.height()), (960, 540));
+        let (mean, max) = bgra_delta(
+            &scale_bgra(&full_frame, bubble, bubble),
+            &scale_bgra(&scaled_frame, bubble, bubble),
+        );
+
+        let iterations = 40;
+        let time = |slot: &mut Option<MjpegDecoder>, source: &FFVideo| {
+            let mut scaler = ffmpeg::software::scaling::Context::get(
+                source.format(),
+                source.width(),
+                source.height(),
+                Pixel::BGRA,
+                bubble,
+                bubble,
+                ffmpeg::software::scaling::Flags::BILINEAR,
+            )
+            .unwrap();
+            let mut output = FFVideo::empty();
+            let start = std::time::Instant::now();
+            for _ in 0..iterations {
+                let frame = slot.as_mut().unwrap().decode(&bytes).unwrap();
+                scaler.run(&frame, &mut output).unwrap();
+            }
+            start.elapsed().as_secs_f64() * 1e3 / iterations as f64
+        };
+        let full_ms = time(&mut full_decoder, &full_frame);
+        let scaled_ms = time(&mut scaled_decoder, &scaled_frame);
+        let speedup = full_ms / scaled_ms;
+        println!(
+            "mjpeg-idct bubble 1920x1080 -> {bubble}: full+scale {full_ms:.3}ms scaled+scale {scaled_ms:.3}ms speedup {speedup:.2}x bgra_mean {mean:.3} max {max} jpeg_bytes {}",
+            bytes.len()
+        );
+        assert!(
+            speedup >= 1.20,
+            "expected a bubble-size win, got {speedup:.2}x ({full_ms:.3}ms -> {scaled_ms:.3}ms)"
+        );
+        assert!(
+            max <= 8,
+            "bubble pixel delta too large: mean {mean:.3} max {max}"
+        );
+    }
 
     fn format(fourcc: &[u8; 4], width: u32, height: u32, stride: usize) -> NativeFrameFormat {
         NativeFrameFormat {
@@ -323,6 +547,26 @@ mod tests {
         }
     }
 
+    #[test]
+    fn frame_conversion_scales_only_mjpeg_with_an_overlay_budget() {
+        ffmpeg::init().unwrap();
+        let bytes = encode_mjpeg(640, 360);
+        let native_format = format(b"MJPG", 640, 360, 0);
+        let scaled = frame_from_bytes_covering(&bytes, native_format, Some((200, 120))).unwrap();
+        assert_eq!((scaled.width(), scaled.height()), (320, 180));
+        let full = frame_from_bytes(&bytes, native_format).unwrap();
+        assert_eq!((full.width(), full.height()), (640, 360));
+
+        let packed = frame_from_bytes_covering(
+            &[1, 2, 3, 4, 5, 6, 7, 8],
+            format(b"YUYV", 4, 1, 8),
+            Some((2, 1)),
+        )
+        .unwrap();
+        assert_eq!((packed.width(), packed.height()), (4, 1));
+        assert_eq!(active_rows(&packed, 0, 8), vec![1, 2, 3, 4, 5, 6, 7, 8]);
+    }
+
     fn active_rows(frame: &ffmpeg::frame::Video, plane: usize, row_width: usize) -> Vec<u8> {
         let stride = frame.stride(plane);
         let height = frame.plane_height(plane) as usize;

crates/camera-ffmpeg/src/macos.rs

@@ -62,6 +62,14 @@ impl CapturedFrameExt for CapturedFrame {
     fn as_ffmpeg(&self) -> Result<ffmpeg::frame::Video, AsFFmpegError> {
         sample_buf_as_ffmpeg(self.native().sample_buf())
     }
+
+    fn as_ffmpeg_covering(
+        &self,
+        _max_width: u32,
+        _max_height: u32,
+    ) -> Result<ffmpeg::frame::Video, AsFFmpegError> {
+        self.as_ffmpeg()
+    }
 }
 
 // Standalone so consumers holding a bare retained CMSampleBuffer (e.g. the