perf(recording): drain macOS encoder autorelease pools and recycle video frames

crates/camera/src/macos.rs

@@ -24,38 +24,40 @@ pub(super) fn list_cameras_impl() -> impl Iterator<Item = CameraInfo> {
 
 impl CameraInfo {
     pub(super) fn formats_impl(&self) -> Option<Vec<Format>> {
-        let device = find_device(self)?;
-
-        let mut ret = vec![];
-
-        for format in device.formats().iter() {
-            let desc = format.format_desc();
-            let width = desc.dims().width as u32;
-            let height = desc.dims().height as u32;
-            let pixel_format = fourcc_display(desc.media_sub_type());
-
-            for fr_range in format.video_supported_frame_rate_ranges().iter() {
-                // SAFETY: trust me bro it crashes on intel mac otherwise
-                let fr_range = unsafe {
-                    &*(fr_range as *const av::capture::device::FrameRateRange)
-                        .cast::<AVFrameRateRange>()
-                };
-
-                let min = unsafe { fr_range.minFrameDuration() };
-
-                ret.push(Format {
-                    native: format.retained(),
-                    info: FormatInfo {
-                        width,
-                        height,
-                        frame_rate: min.timescale as f32 / min.value as f32,
-                    },
-                    pixel_format: Some(pixel_format.clone()),
-                })
+        objc::ar_pool(|| {
+            let device = find_device(self)?;
+
+            let mut ret = vec![];
+
+            for format in device.formats().iter() {
+                let desc = format.format_desc();
+                let width = desc.dims().width as u32;
+                let height = desc.dims().height as u32;
+                let pixel_format = fourcc_display(desc.media_sub_type());
+
+                for fr_range in format.video_supported_frame_rate_ranges().iter() {
+                    // SAFETY: trust me bro it crashes on intel mac otherwise
+                    let fr_range = unsafe {
+                        &*(fr_range as *const av::capture::device::FrameRateRange)
+                            .cast::<AVFrameRateRange>()
+                    };
+
+                    let min = unsafe { fr_range.minFrameDuration() };
+
+                    ret.push(Format {
+                        native: format.retained(),
+                        info: FormatInfo {
+                            width,
+                            height,
+                            frame_rate: min.timescale as f32 / min.value as f32,
+                        },
+                        pixel_format: Some(pixel_format.clone()),
+                    })
+                }
             }
-        }
 
-        Some(ret)
+            Some(ret)
+        })
     }
 }
 
@@ -156,7 +158,7 @@ pub(super) fn start_capturing_impl(
     let input =
         av::capture::DeviceInput::with_device(&device).map_err(AVFoundationError::Static)?;
 
-    let queue = dispatch::Queue::new();
+    let queue = dispatch::Queue::serial_with_ar_pool();
     let mut missing_image_bufs: u64 = 0;
     let delegate =
         CallbackOutputDelegate::with(CallbackOutputDelegateInner::new(Box::new(move |data| {

crates/enc-ffmpeg/src/mux/segmented_stream.rs

@@ -465,15 +465,23 @@ impl SegmentedVideoEncoder {
 
     pub fn queue_frame(
         &mut self,
-        frame: frame::Video,
+        mut frame: frame::Video,
+        timestamp: Duration,
+    ) -> Result<(), QueueFrameError> {
+        self.queue_frame_reusable(&mut frame, timestamp)
+    }
+
+    pub fn queue_frame_reusable(
+        &mut self,
+        frame: &mut frame::Video,
         timestamp: Duration,
     ) -> Result<(), QueueFrameError> {
         // Encode with the frame's real capture-derived timestamp. The encoder
         // anchors pts at the first frame, so capture gaps (static content,
         // stream restarts, dropped frames) stay in the timeline instead of
         // compressing it and drifting video ahead of audio.
         self.encoder
-            .queue_frame_to(frame, timestamp, &mut self.mux)?;
+            .queue_frame_to_reusable(frame, timestamp, &mut self.mux)?;
 
         self.mux.frame_queued();
 

crates/enc-ffmpeg/src/video/h264.rs

@@ -952,6 +952,10 @@ impl H264Encoder {
             let converted = converted_frame.get_or_insert_with(|| {
                 frame::Video::new(self.output_format, self.output_width, self.output_height)
             });
+            if unsafe { ffmpeg::ffi::av_frame_make_writable(converted.as_mut_ptr()) } < 0 {
+                *converted =
+                    frame::Video::new(self.output_format, self.output_width, self.output_height);
+            }
             super::packed_yuv::convert_or_scale(frame, converted, converter)
                 .map_err(QueueFrameError::Converter)?;
             converted.set_pts(pts);
@@ -972,6 +976,15 @@ impl H264Encoder {
         mut frame: frame::Video,
         timestamp: Duration,
         output: &mut format::context::Output,
+    ) -> Result<(), QueueFrameError> {
+        self.queue_preconverted_frame_reusable(&mut frame, timestamp, output)
+    }
+
+    pub fn queue_preconverted_frame_reusable(
+        &mut self,
+        frame: &mut frame::Video,
+        timestamp: Duration,
+        output: &mut format::context::Output,
     ) -> Result<(), QueueFrameError> {
         trace!(
             "Encoding pre-converted frame: format={:?}, size={}x{}, expected={:?} {}x{}",
@@ -983,11 +996,40 @@ impl H264Encoder {
             self.output_height
         );
 
+        self.base.update_pts(frame, timestamp, &mut self.encoder);
+
         self.base
-            .update_pts(&mut frame, timestamp, &mut self.encoder);
+            .send_frame(frame, output, &mut self.encoder)
+            .map_err(QueueFrameError::Encode)?;
+
+        Ok(())
+    }
+
+    pub fn queue_frame_to_reusable<S: PacketSink + ?Sized>(
+        &mut self,
+        frame: &mut frame::Video,
+        timestamp: Duration,
+        output: &mut S,
+    ) -> Result<(), QueueFrameError> {
+        self.base.update_pts(frame, timestamp, &mut self.encoder);
+
+        let frame_to_send = if let Some(converter) = &mut self.converter {
+            let pts = frame.pts();
+            let converted = self.converted_frame_pool.as_mut().unwrap();
+            if unsafe { ffmpeg::ffi::av_frame_make_writable(converted.as_mut_ptr()) } < 0 {
+                *converted =
+                    frame::Video::new(self.output_format, self.output_width, self.output_height);
+            }
+            super::packed_yuv::convert_or_scale(frame, converted, converter)
+                .map_err(QueueFrameError::Converter)?;
+            converted.set_pts(pts);
+            converted as &frame::Video
+        } else {
+            frame as &frame::Video
+        };
 
         self.base
-            .send_frame(&frame, output, &mut self.encoder)
+            .send_frame(frame_to_send, output, &mut self.encoder)
             .map_err(QueueFrameError::Encode)?;
 
         Ok(())

crates/enc-ffmpeg/src/video/h264_packet.rs

@@ -172,6 +172,10 @@ impl H264PacketEncoder {
             let converted = converted_frame.get_or_insert_with(|| {
                 frame::Video::new(self.output_format, self.output_width, self.output_height)
             });
+            if unsafe { ffmpeg::ffi::av_frame_make_writable(converted.as_mut_ptr()) } < 0 {
+                *converted =
+                    frame::Video::new(self.output_format, self.output_width, self.output_height);
+            }
             super::packed_yuv::convert_or_scale(frame, converted, converter)
                 .map_err(EncodePacketError::Converter)?;
             converted.set_pts(pts);

crates/enc-ffmpeg/src/video/packed_yuv.rs

@@ -837,6 +837,93 @@ mod tests {
         assert!(!packets.is_empty());
     }
 
+    #[test]
+    fn reusable_queue_and_encode_preserve_plane_backing_and_detach_when_retained() {
+        ffmpeg::init().unwrap();
+        if ffmpeg::codec::encoder::find_by_name("libx264").is_none() {
+            return;
+        }
+
+        let directory = tempfile::tempdir().unwrap();
+        let path = directory.path().join("reusable.mp4");
+        let mut output = ffmpeg::format::output_as(&path, "mp4").unwrap();
+        let info = cap_media_info::VideoInfo::from_raw_ffmpeg(Pixel::BGRA, 64, 48, 30);
+        let mut queue_enc = super::super::h264::H264Encoder::builder(info)
+            .with_preset(super::super::h264::H264Preset::Ultrafast)
+            .with_crf(18)
+            .with_encoder_priority_override(&["libx264"])
+            .build(&mut output)
+            .unwrap();
+
+        let mut frame = frame::Video::new(Pixel::BGRA, 64, 48);
+        frame.data_mut(0).fill(0x24);
+        let src_ptr = frame.data(0).as_ptr();
+        let mut sink = crate::PacketQueue::new(ffmpeg::Rational(1, 90_000));
+        for idx in 0..4u64 {
+            queue_enc
+                .queue_frame_to_reusable(
+                    &mut frame,
+                    std::time::Duration::from_millis(idx * 33),
+                    &mut sink,
+                )
+                .unwrap();
+            assert_eq!(frame.data(0).as_ptr(), src_ptr);
+        }
+        queue_enc.flush_to(&mut sink).unwrap();
+        assert!(!sink.take().is_empty());
+
+        let mut pkt_enc = super::super::h264::H264Encoder::builder(info)
+            .with_preset(super::super::h264::H264Preset::Ultrafast)
+            .with_crf(18)
+            .with_encoder_priority_override(&["libx264"])
+            .build_standalone()
+            .unwrap();
+        let mut converted: Option<frame::Video> = None;
+        let mut packets = Vec::new();
+        pkt_enc
+            .encode_frame_reusable(
+                &mut frame,
+                &mut converted,
+                std::time::Duration::ZERO,
+                |pkt| {
+                    packets.push(pkt.data);
+                    Ok(())
+                },
+            )
+            .unwrap();
+
+        let mut retained = frame::Video::empty();
+        let old_y_byte = {
+            let conv = converted.as_ref().unwrap();
+            assert_eq!(
+                unsafe { ffmpeg::ffi::av_frame_ref(retained.as_mut_ptr(), conv.as_ptr()) },
+                0
+            );
+            retained.data(0)[0]
+        };
+
+        frame.data_mut(0).fill(0xee);
+        pkt_enc
+            .encode_frame_reusable(
+                &mut frame,
+                &mut converted,
+                std::time::Duration::from_millis(33),
+                |pkt| {
+                    packets.push(pkt.data);
+                    Ok(())
+                },
+            )
+            .unwrap();
+        assert_eq!(retained.data(0)[0], old_y_byte);
+        pkt_enc
+            .flush(|pkt| {
+                packets.push(pkt.data);
+                Ok(())
+            })
+            .unwrap();
+        assert!(!packets.is_empty());
+    }
+
     #[test]
     #[ignore = "release timing bench, run with --ignored --nocapture"]
     fn bench_packed_yuv_conversion() {