From fbbe4166047e2c9f9067341b57c3c8a01771c16a Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:25:44 +0000 Subject: [PATCH 1/3] fix(desktop-gpui): transcribe only retained source audio --- apps/desktop-gpui/src/editor_tabs.rs | 10 +- apps/desktop-gpui/src/transcription.rs | 439 ++++++------------ .../src/transcription/verify_tests.rs | 238 ++++++++-- crates/project/src/caption_timing.rs | 76 +++ crates/project/src/lib.rs | 5 +- 5 files changed, 441 insertions(+), 327 deletions(-) diff --git a/apps/desktop-gpui/src/editor_tabs.rs b/apps/desktop-gpui/src/editor_tabs.rs index a793647488b..3a808180d86 100644 --- a/apps/desktop-gpui/src/editor_tabs.rs +++ b/apps/desktop-gpui/src/editor_tabs.rs @@ -3176,6 +3176,7 @@ impl EditorWindow { }; let recording_durations = summary.clip_display_durations.clone(); let recording_duration = summary.recording_duration; + let timeline = self.project.timeline.clone(); let project_path = self.project_path.clone(); let model = self.selected_caption_model().name; let language = self.sidebar.caption_language; @@ -3187,9 +3188,16 @@ impl EditorWindow { cx.spawn_in(window, async move |this, cx| { let task_path = project_path.clone(); + let task_durations = recording_durations.clone(); let task = gpui_tokio::Tokio::spawn(cx, async move { tokio::task::spawn_blocking(move || { - transcription::transcribe_blocking(&task_path, model, language) + transcription::transcribe_blocking( + &task_path, + model, + language, + timeline.as_ref(), + &task_durations, + ) }) .await .map_err(|error| format!("Transcription task panicked: {error}"))? diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs index bc7ad223b00..d6ba04c8470 100644 --- a/apps/desktop-gpui/src/transcription.rs +++ b/apps/desktop-gpui/src/transcription.rs @@ -26,6 +26,7 @@ use std::{ collections::{HashMap, HashSet}, fs::File, io::{Read, Write}, + ops::Range, path::{Path, PathBuf}, sync::{ Arc, LazyLock, Mutex, PoisonError, @@ -40,7 +41,6 @@ use cap_project::{ ProjectConfiguration, RecordingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment, }; -use cap_rendering::Video; use ffmpeg::{ ChannelLayout, codec as avcodec, format::{self as avformat}, @@ -923,162 +923,23 @@ const DECODE_SAMPLE_RATE: u32 = 48_000; /// `WHISPER_SAMPLE_RATE` (`captions.rs:66`). const WHISPER_SAMPLE_RATE: u32 = 16_000; -/// `AudioData::from_file` (`crates/audio/src/audio_data.rs:22-95`) transcribed -/// against the same ffmpeg fork: any input decoded to packed f32 at 48kHz, -/// mono stays mono, everything else downmixes to stereo. -fn decode_audio_file(path: &Path) -> Result<(Vec, usize), String> { - let mut input_ctx = avformat::input(&path).map_err(|e| format!("Input Open / {e}"))?; - let input_stream = input_ctx - .streams() - .best(ffmpeg::media::Type::Audio) - .ok_or_else(|| "No Stream".to_string())?; - - let decoder_ctx = avcodec::Context::from_parameters(input_stream.parameters()) - .map_err(|e| format!("Decoder Parameters / {e}"))?; - let mut decoder = decoder_ctx - .decoder() - .audio() - .map_err(|e| format!("Set Parameters / {e}"))?; - - let source_channels = decoder.channels().max(1); - if decoder.channel_layout().is_empty() { - decoder.set_channel_layout(ChannelLayout::default(i32::from(source_channels))); - } - decoder.set_packet_time_base(input_stream.time_base()); - - let target_channels: u16 = if source_channels <= 1 { 1 } else { 2 }; - let target_channel_layout = ChannelLayout::default(i32::from(target_channels)); - let mut options = ffmpeg::Dictionary::new(); - options.set("filter_size", "128"); - options.set("cutoff", "0.97"); - - let mut resampler = resampling::Context::get_with( - decoder.format(), - decoder.channel_layout(), - decoder.rate(), - avformat::Sample::F32(avformat::sample::Type::Packed), - target_channel_layout, - DECODE_SAMPLE_RATE, - options, - ) - .map_err(|e| format!("Resampler / {e}"))?; - - let index = input_stream.index(); - - let mut decoded_frame = ffmpeg::frame::Audio::empty(); - let mut samples: Vec = Vec::new(); - - for (stream, packet) in input_ctx.packets() { - if stream.index() != index { - continue; - } - - decoder - .send_packet(&packet) - .map_err(|e| format!("Send Packet / {e}"))?; - - while decoder.receive_frame(&mut decoded_frame).is_ok() { - run_decode_resampler(&mut resampler, &decoded_frame, &mut samples)?; - } - } - - decoder.send_eof().map_err(|e| format!("Send EOF / {e}"))?; - - while decoder.receive_frame(&mut decoded_frame).is_ok() { - run_decode_resampler(&mut resampler, &decoded_frame, &mut samples)?; - } - - flush_decode_resampler(&mut resampler, &mut samples)?; - - Ok((samples, usize::from(target_channels))) -} - -fn run_decode_resampler( - resampler: &mut resampling::Context, - decoded_frame: &ffmpeg::frame::Audio, - samples: &mut Vec, -) -> Result<(), String> { - let target = *resampler.output(); - let capacity = decode_resample_capacity(resampler, decoded_frame.samples()); - let mut resampled_frame = - ffmpeg::frame::Audio::new(target.format, capacity, target.channel_layout); - - resampler - .run(decoded_frame, &mut resampled_frame) - .map_err(|e| format!("Run Resampler / {e}"))?; - - append_resampled_frame(samples, &resampled_frame) -} - -fn flush_decode_resampler( - resampler: &mut resampling::Context, - samples: &mut Vec, -) -> Result<(), String> { - for _ in 0..64 { - let Some(delay) = resampler.delay() else { - break; - }; - let target = *resampler.output(); - let capacity = delay - .output - .max(1) - .saturating_add(16) - .min(i64::from(i32::MAX)) as usize; - let mut resampled_frame = - ffmpeg::frame::Audio::new(target.format, capacity, target.channel_layout); - let remaining = resampler - .flush(&mut resampled_frame) - .map_err(|e| format!("Flush Resampler / {e}"))?; - - let output_samples = resampled_frame.samples(); - append_resampled_frame(samples, &resampled_frame)?; - - if remaining.is_none() || output_samples == 0 { - break; - } - } - Ok(()) -} - -fn decode_resample_capacity(resampler: &resampling::Context, input_samples: usize) -> usize { - let src_rate = u64::from(resampler.input().rate.max(1)); - let dst_rate = u64::from(resampler.output().rate.max(1)); - let pending_output_samples = resampler - .delay() - .map(|d| d.output.max(0) as u64) - .unwrap_or(0); - let resampled_from_input = (input_samples as u64) - .saturating_mul(dst_rate) - .div_ceil(src_rate); - - pending_output_samples - .saturating_add(resampled_from_input) - .saturating_add(16) - .min(i32::MAX as u64) as usize -} - -fn append_resampled_frame( - samples: &mut Vec, - frame: &ffmpeg::frame::Audio, -) -> Result<(), String> { - if frame.samples() == 0 { - return Ok(()); - } - - let byte_len = frame - .samples() - .saturating_mul(frame.channels() as usize) - .saturating_mul(std::mem::size_of::()); - let data = frame - .data(0) - .get(..byte_len) - .ok_or_else(|| "Resampled frame data shorter than expected".to_string())?; - - // SAFETY: ffmpeg's frame buffers are allocated with av_malloc alignment, - // and the slice length is a whole number of f32s by construction. - samples - .extend(unsafe { std::slice::from_raw_parts(data.as_ptr() as *const f32, byte_len / 4) }); - Ok(()) +fn decode_transcription_source( + path: &Path, + offset_secs: f64, + range: Range, +) -> Result { + let rate = f64::from(DECODE_SAMPLE_RATE); + let start = (range.start * rate).round() as i64; + let end = (range.end * rate).round() as i64; + let offset = (offset_secs * rate).round() as i64; + let source_start = (start + offset).max(0) as usize; + let source_end = (end + offset).max(0) as usize; + let decoded = cap_audio::AudioData::from_file_range(path, source_start, source_end)?; + Ok(TranscriptionAudioSource { + samples: decoded.samples().to_vec(), + channels: usize::from(decoded.channels()), + offset_secs: (start + offset - source_start as i64) as f64 / rate, + }) } /// `normalize_audio_for_transcription` (`captions.rs:885-915`). @@ -1114,112 +975,74 @@ fn normalize_audio_for_transcription(samples: &mut [f32]) -> f32 { gain } -/// The recording-directory arm of `extract_audio_from_video` -/// (`captions.rs:250-517`): per segment, decode system audio then mic -/// (`captions.rs:281-283`), downmix each to mono, average them together, -/// concatenate the segments, normalize, and write a 16kHz mono s16le WAV. -fn extract_project_audio(project_path: &Path, output_path: &Path) -> Result<(), String> { - ffmpeg::init().map_err(|e| format!("Failed to initialise ffmpeg: {e}"))?; - - let meta = RecordingMeta::load_for_project(project_path) - .map_err(|e| format!("Failed to read recording metadata: {e}"))?; - let Some(studio) = meta.studio_meta() else { - return Err("Only studio recordings can be transcribed".to_string()); +fn extract_project_audio( + meta: &RecordingMeta, + studio: &StudioRecordingMeta, + recording_durations: &[f64], + range: Range, + output_path: &Path, +) -> Result { + let tracks: Vec> = match studio { + StudioRecordingMeta::SingleSegment { segment } => { + vec![segment.audio.iter().map(|audio| (audio, 0.0)).collect()] + } + StudioRecordingMeta::MultipleSegments { inner } => inner + .segments + .iter() + .map(|segment| { + let offsets = segment.calculate_audio_offsets(); + [ + ( + segment.system_audio.as_ref(), + f64::from(offsets.system_audio), + ), + (segment.mic.as_ref(), f64::from(offsets.mic)), + ] + .into_iter() + .filter_map(|(audio, offset)| Some((audio?, offset))) + .collect() + }) + .collect(), }; - - studio.ensure_ordinary_media_access(&meta.project_path)?; + if tracks.len() != recording_durations.len() { + return Err("Recording durations do not match the recording metadata".to_string()); + } let mut final_samples = Vec::new(); let mut any_audio = false; - match studio { - StudioRecordingMeta::SingleSegment { segment } => { - let display_duration = Video::new( - segment.display.path.to_path(&meta.project_path), - segment.display.start_time.unwrap_or_default(), - ) - .map_err(|error| format!("Failed to read display video: {error}"))? - .duration; - let mut sources = Vec::new(); - if let Some(audio) = &segment.audio { - let path = meta.path(&audio.path); - if path.exists() { - match decode_audio_file(&path) { - Ok((samples, channels)) => { - any_audio = true; - sources.push(TranscriptionAudioSource { - samples, - channels, - offset_secs: 0.0, - }); - } - Err(error) => tracing::warn!( - path = %path.display(), - "Failed to process audio source: {error}" - ), - } - } - } - let take = TranscriptionAudioTake { - display_duration_secs: display_duration, - sources, - }; - append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE) - .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?; + let mut take_start = 0.0; + for (tracks, duration) in tracks.into_iter().zip(recording_durations) { + let start = (range.start - take_start).max(0.0); + let end = (range.end - take_start).min(*duration); + take_start += duration; + if start >= end { + continue; } - StudioRecordingMeta::MultipleSegments { inner } => { - for (segment_idx, segment) in inner.segments.iter().enumerate() { - let display_duration = Video::new( - segment.display.path.to_path(&meta.project_path), - segment.display.start_time.unwrap_or_default(), - ) - .map_err(|error| { - format!("Failed to read display video for segment {segment_idx}: {error}") - })? - .duration; - let offsets = segment.calculate_audio_offsets(); - let mut sources = Vec::new(); - let mut push_source = |audio: &cap_project::AudioMeta, offset_secs: f64| { - let path = meta.path(&audio.path); - if !path.exists() { - return; - } - match decode_audio_file(&path) { - Ok((samples, channels)) => { - any_audio = true; - sources.push(TranscriptionAudioSource { - samples, - channels, - offset_secs, - }); - } - Err(error) => tracing::warn!( - path = %path.display(), - "Failed to process audio source: {error}" - ), - } - }; - - if let Some(system_audio) = &segment.system_audio { - push_source(system_audio, f64::from(offsets.system_audio)); - } - if let Some(mic) = &segment.mic { - push_source(mic, f64::from(offsets.mic)); + let mut sources = Vec::new(); + for (audio, offset) in tracks { + let path = meta.path(&audio.path); + if !path.exists() { + continue; + } + match decode_transcription_source(&path, offset, start..end) { + Ok(source) => { + any_audio = true; + sources.push(source); } - let take = TranscriptionAudioTake { - display_duration_secs: display_duration, - sources, - }; - append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE) - .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?; + Err(error) => tracing::warn!( + path = %path.display(), + "Failed to process audio source: {error}" + ), } } + let take = TranscriptionAudioTake { + display_duration_secs: end - start, + sources, + }; + append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE) + .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?; } - if !any_audio { - return Err("No audio sources found in the recording metadata".to_string()); - } - - if final_samples.is_empty() { - return Err("Failed to process any audio sources".to_string()); + return Ok(false); } let gain = normalize_audio_for_transcription(&mut final_samples); @@ -1227,7 +1050,8 @@ fn extract_project_audio(project_path: &Path, output_path: &Path) -> Result<(), tracing::info!("Applied transcription audio gain: {gain:.2}x"); } - write_wav_16k_mono(&final_samples, output_path) + write_wav_16k_mono(&final_samples, output_path)?; + Ok(true) } /// The WAV encode half of the recording-directory arm @@ -1650,7 +1474,7 @@ fn process_with_parakeet( .collect::, _>>() .map_err(|e| format!("Failed to read audio file: {e}"))?; if samples.is_empty() { - return Err("No speech detected in the audio".to_string()); + return Ok(Vec::new()); } let mut tokens = Vec::new(); @@ -1678,7 +1502,7 @@ fn process_with_parakeet( tracing::info!("Parakeet produced {} timed tokens", tokens.len()); - caption_segments_from_tokens(tokens) + Ok(caption_segments_from_tokens(tokens)) } /// Parakeet's timed word tokens -> caption segments: drop empty tokens, @@ -1686,7 +1510,7 @@ fn process_with_parakeet( #[cfg(any(test, not(all(target_os = "macos", target_arch = "x86_64"))))] fn caption_segments_from_tokens( tokens: impl IntoIterator, -) -> Result, String> { +) -> Vec { let words = normalize_caption_words( tokens .into_iter() @@ -1699,10 +1523,6 @@ fn caption_segments_from_tokens( .collect(), ); - if words.is_empty() { - return Err("No speech detected in the audio".to_string()); - } - let mut segments = Vec::new(); for (chunk_idx, chunk) in caption_word_chunks(&words).into_iter().enumerate() { segments.push(CaptionSegment { @@ -1714,7 +1534,7 @@ fn caption_segments_from_tokens( }); } - Ok(segments) + segments } #[cfg(all(target_os = "macos", target_arch = "x86_64"))] @@ -1737,6 +1557,8 @@ pub fn transcribe_blocking( project_path: &Path, model: &str, language: &str, + timeline: Option<&TimelineConfiguration>, + recording_durations: &[f64], ) -> Result, String> { let model_file = model_path(model); if !project_path.exists() { @@ -1752,30 +1574,42 @@ pub fn transcribe_blocking( )); } - transcribe_project_with(project_path, |audio_path| { - let _guard = lock_transcription_worker_slot(); - if is_parakeet_model(model) { - tracing::info!("Using Parakeet TDT engine"); - let result = process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref()); - schedule_parakeet_release(); - result - } else { - tracing::info!("Using Whisper engine"); - let hints = transcription_hints(); - let context = get_whisper_context(model_file.to_string_lossy().as_ref())?; - let result = process_with_whisper(audio_path, context, language, &hints); - release_whisper_context_after_transcription(); - result - } - }) + let _guard = lock_transcription_worker_slot(); + let result = + transcribe_project_with(project_path, timeline, recording_durations, |audio_path| { + if is_parakeet_model(model) { + tracing::info!("Using Parakeet TDT engine"); + process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref()) + } else { + tracing::info!("Using Whisper engine"); + let hints = transcription_hints(); + let context = get_whisper_context(model_file.to_string_lossy().as_ref())?; + process_with_whisper(audio_path, context, language, &hints) + } + }); + if is_parakeet_model(model) { + schedule_parakeet_release(); + } else { + release_whisper_context_after_transcription(); + } + result } /// Extracts the project's transcription audio to a temporary 16 kHz mono WAV /// and runs `engine` on it. fn transcribe_project_with( project_path: &Path, - engine: impl FnOnce(&Path) -> Result, String>, + timeline: Option<&TimelineConfiguration>, + recording_durations: &[f64], + mut engine: impl FnMut(&Path) -> Result, String>, ) -> Result, String> { + ffmpeg::init().map_err(|e| format!("Failed to initialise ffmpeg: {e}"))?; + let meta = RecordingMeta::load_for_project(project_path) + .map_err(|e| format!("Failed to read recording metadata: {e}"))?; + let Some(studio) = meta.studio_meta() else { + return Err("Only studio recordings can be transcribed".to_string()); + }; + studio.ensure_ordinary_media_access(&meta.project_path)?; let temp_dir = std::env::temp_dir().join(format!( "cap-gpui-transcription-{}-{}", std::process::id(), @@ -1786,8 +1620,34 @@ fn transcribe_project_with( let audio_path = temp_dir.join("audio.wav"); let result = (|| { - extract_audio_from_video(project_path, &audio_path)?; - engine(&audio_path) + let mut segments = Vec::new(); + let mut any_audio = false; + for range in cap_project::transcription_source_ranges(timeline, recording_durations) { + if !extract_project_audio( + &meta, + studio, + recording_durations, + range.clone(), + &audio_path, + )? { + continue; + } + any_audio = true; + for mut segment in engine(&audio_path)? { + segment.id = format!("segment-{}", segments.len()); + segment.start += range.start as f32; + segment.end += range.start as f32; + for word in &mut segment.words { + word.start += range.start as f32; + word.end += range.start as f32; + } + segments.push(segment); + } + } + if !any_audio { + return Err("No audio sources found in the selected timeline".to_string()); + } + Ok(segments) })(); let _ = std::fs::remove_dir_all(&temp_dir); @@ -1799,21 +1659,6 @@ fn transcribe_project_with( Ok(segments) } -/// `resolve_audio_extraction_source` (`captions.rs:221-243`), directory arm -/// only -- the gpui editor always opens `.cap` bundles, so a plain media file -/// is refused rather than decoded. -fn extract_audio_from_video(video_path: &Path, output_path: &Path) -> Result<(), String> { - let metadata = std::fs::metadata(video_path) - .map_err(|e| format!("Failed to read video path metadata: {e}"))?; - if !metadata.is_dir() { - return Err("Video path is not a recording directory".to_string()); - } - if !video_path.join("recording-meta.json").is_file() { - return Err("Recording directory is missing recording-meta.json".to_string()); - } - extract_project_audio(video_path, output_path) -} - // --------------------------------------------------------------------------- // Caption words & chunking // --------------------------------------------------------------------------- diff --git a/apps/desktop-gpui/src/transcription/verify_tests.rs b/apps/desktop-gpui/src/transcription/verify_tests.rs index 39a2af3bec2..e00f2dbf842 100644 --- a/apps/desktop-gpui/src/transcription/verify_tests.rs +++ b/apps/desktop-gpui/src/transcription/verify_tests.rs @@ -36,6 +36,8 @@ use cap_verify::{ report::{self, Case, Tier}, }; +use cap_rendering::Video; + use super::*; const MAX_WORD_SECS: f32 = 2.5; @@ -335,10 +337,15 @@ fn takes_fixture() -> FixtureSpec { /// segments. fn check_project(case: &mut Case, project: &Path, takes: &Takes) { let mut tokens = Vec::new(); - let result = transcribe_project_with(project, |audio_path| { - tokens = fake_tokens(audio_path)?; - caption_segments_from_tokens(tokens.clone()) - }); + let result = transcribe_project_with( + project, + None, + &takes.iter().map(|take| take.0).collect::>(), + |audio_path| { + tokens = fake_tokens(audio_path)?; + Ok(caption_segments_from_tokens(tokens.clone())) + }, + ); let segments = match result { Ok(segments) => segments, Err(e) => { @@ -526,7 +533,7 @@ fn instant_project_is_refused() { ), } let mut called = false; - let result = transcribe_project_with(&project, |_| { + let result = transcribe_project_with(&project, None, &[], |_| { called = true; Ok(Vec::new()) }); @@ -606,33 +613,25 @@ fn segmentation_case(id: &str, unicode: bool) { unicode, ); let reference = reference_segments(&tokens); - match caption_segments_from_tokens(tokens.clone()) { - Err(e) if reference.is_empty() && e.contains("No speech") => empty += 1, - Err(e) => { - failed += 1; + let product = caption_segments_from_tokens(tokens.clone()); + if product.is_empty() { + empty += 1; + } + segments += product.len(); + let problems = compare_segments(&product, &reference, 0.0); + if !problems.is_empty() { + failed += 1; + if failed <= 3 { case.problem( "caption-segments", - format!("seed {seed}: unexpected error {e}"), + format!( + "seed {seed}: {} problems (first: {}); tokens {:?}", + problems.len(), + problems[0], + tokens.iter().map(|t| t.text.as_str()).collect::>() + ), ); } - Ok(product) => { - segments += product.len(); - let problems = compare_segments(&product, &reference, 0.0); - if !problems.is_empty() { - failed += 1; - if failed <= 3 { - case.problem( - "caption-segments", - format!( - "seed {seed}: {} problems (first: {}); tokens {:?}", - problems.len(), - problems[0], - tokens.iter().map(|t| t.text.as_str()).collect::>() - ), - ); - } - } - } } } case.note(format!( @@ -650,3 +649,186 @@ fn segmentation_matches_reference() { fn segmentation_matches_reference_unicode() { segmentation_case("captions/app/segmentation/unicode", true); } + +#[test] +fn trimmed_transcription_extracts_only_used_audio_and_restores_source_times() { + let spec = takes_fixture(); + let project = fixture::ensure_fixture(&spec, fixture::write_fixture).expect("fixture"); + let takes = multi_segment_takes(&project, &spec); + let durations: Vec = takes.iter().map(|take| take.0).collect(); + for (clips, expected_ranges, expected_starts) in [ + ( + serde_json::json!([ + {"recordingSegment": 0, "start": 1.0, "end": 3.0, "timescale": 2.0} + ]), + vec![1.0..3.0], + vec![0.125], + ), + ( + serde_json::json!([ + {"recordingSegment": 1, "start": 1.0, "end": 3.0, "timescale": 0.5}, + {"recordingSegment": 0, "start": 3.0, "end": 4.0, "timescale": 2.0}, + {"recordingSegment": 0, "start": 0.5, "end": 1.5, "timescale": 1.0}, + {"recordingSegment": 0, "start": 1.0, "end": 2.0, "timescale": 1.0}, + {"recordingSegment": 0, "start": 3.0, "end": 4.0, "timescale": 1.0} + ]), + vec![0.5..2.0, 3.0..4.0, durations[0] + 1.0..durations[0] + 3.0], + vec![0.5, 4.125, 4.75, 6.75], + ), + ( + serde_json::json!([ + {"recordingSegment":0,"start":4.0,"end":durations[0],"timescale":1.0}, + {"recordingSegment":1,"start":0.0,"end":1.0,"timescale":1.0} + ]), + vec![4.0..durations[0] + 1.0], + vec![0.25], + ), + ] { + let timeline: TimelineConfiguration = serde_json::from_value(serde_json::json!({ + "segments": clips, "zoomSegments": [] + })) + .unwrap(); + let mut called = 0; + let segments = transcribe_project_with(&project, Some(&timeline), &durations, |path| { + let range = &expected_ranges[called]; + let audio = media::decode_audio(path, Some(MARKER_RATE)).unwrap(); + let duration = audio.mono().len() as f64 / f64::from(MARKER_RATE); + assert!( + (duration - (range.end - range.start)).abs() < 0.002, + "extracted {duration}s, expected {}s", + range.end - range.start + ); + called += 1; + Ok(vec![CaptionSegment { + id: "segment-0".into(), + start: 0.25, + end: 0.5, + text: "word".into(), + words: vec![CaptionWord { + text: "word".into(), + start: 0.25, + end: 0.5, + }], + }]) + }) + .unwrap(); + assert_eq!(called, expected_ranges.len()); + for (segment, range) in segments.iter().zip(expected_ranges) { + assert!((f64::from(segment.start) - range.start - 0.25).abs() < 1e-5); + assert!((f64::from(segment.end) - range.start - 0.5).abs() < 1e-5); + assert_eq!(segment.words[0].start, segment.start); + assert_eq!(segment.words[0].end, segment.end); + } + let ids: std::collections::HashSet<_> = + segments.iter().map(|segment| &segment.id).collect(); + assert_eq!(ids.len(), segments.len()); + let tracks = derive_caption_track_segments(&segments, &timeline, &durations); + assert_eq!(tracks.len(), expected_starts.len()); + for (track, expected) in tracks.iter().zip(expected_starts) { + assert!((track.start - expected).abs() < 1e-5); + } + } +} + +#[test] +fn trimmed_sources_match_full_audio_with_positive_and_negative_offsets() { + let spec = takes_fixture(); + let project = fixture::ensure_fixture(&spec, fixture::write_fixture).unwrap(); + let meta = RecordingMeta::load_for_project(&project).unwrap(); + let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else { + panic!("studio") + }; + let segment = &inner.segments[0]; + for audio in [ + segment.mic.as_ref().unwrap(), + segment.system_audio.as_ref().unwrap(), + ] { + let path = meta.path(&audio.path); + let decoded = cap_audio::AudioData::from_file(&path).unwrap(); + for offset_secs in [-0.4, 0.3] { + let full = cap_editor::assemble_transcription_audio( + &[TranscriptionAudioTake { + display_duration_secs: 5.0, + sources: vec![TranscriptionAudioSource { + samples: decoded.samples().to_vec(), + channels: usize::from(decoded.channels()), + offset_secs, + }], + }], + DECODE_SAMPLE_RATE, + ) + .unwrap(); + for range in [0.0..1.0, 1.25..2.5, 3.0..4.0] { + let source = + decode_transcription_source(&path, offset_secs, range.clone()).unwrap(); + let trimmed = cap_editor::assemble_transcription_audio( + &[TranscriptionAudioTake { + display_duration_secs: range.end - range.start, + sources: vec![source], + }], + DECODE_SAMPLE_RATE, + ) + .unwrap(); + let start = (range.start * f64::from(DECODE_SAMPLE_RATE)).round() as usize; + let reference = &full[start..start + trimmed.len()]; + let worst = trimmed + .iter() + .zip(reference) + .fold(0.0_f32, |worst, (a, b)| worst.max((a - b).abs())); + assert!( + worst < 0.001, + "offset {offset_secs}, range {range:?}, worst sample error {worst}" + ); + } + } + } +} + +#[test] +fn transcription_keeps_speech_after_empty_ranges_and_cleans_up_on_failure() { + let spec = takes_fixture(); + let project = fixture::ensure_fixture(&spec, fixture::write_fixture).unwrap(); + let takes = multi_segment_takes(&project, &spec); + let durations: Vec<_> = takes.iter().map(|take| take.0).collect(); + let timeline = serde_json::from_value(serde_json::json!({"segments": [ + {"start":0.0,"end":1.0,"timescale":1.0}, + {"start":3.0,"end":4.0,"timescale":1.0} + ], "zoomSegments": []})) + .unwrap(); + let mut calls = 0; + let mut temporary = PathBuf::new(); + let result = transcribe_project_with(&project, Some(&timeline), &durations, |path| { + temporary = path.to_path_buf(); + calls += 1; + if calls == 1 { + return Ok(Vec::new()); + } + Ok(caption_segments_from_tokens([CaptionWord { + text: "speech".into(), + start: 0.1, + end: 0.2, + }])) + }) + .unwrap(); + assert_eq!(calls, 2); + assert!((result[0].words[0].start - 3.1).abs() < 1e-5); + assert!(!temporary.parent().unwrap().exists()); + let error = transcribe_project_with(&project, Some(&timeline), &durations, |path| { + temporary = path.to_path_buf(); + Err("test engine failure".into()) + }) + .unwrap_err(); + assert_eq!(error, "test engine failure"); + assert!(!temporary.parent().unwrap().exists()); + let empty = transcribe_project_with(&project, Some(&timeline), &durations, |_| Ok(Vec::new())) + .unwrap_err(); + assert_eq!(empty, "No speech detected in the audio"); + assert!(caption_segments_from_tokens(Vec::new()).is_empty()); + let empty_timeline = + serde_json::from_value(serde_json::json!({"segments":[],"zoomSegments":[]})).unwrap(); + let error = transcribe_project_with(&project, Some(&empty_timeline), &durations, |_| { + panic!("an empty timeline must not run the model") + }) + .unwrap_err(); + assert!(error.contains("No audio sources")); +} diff --git a/crates/project/src/caption_timing.rs b/crates/project/src/caption_timing.rs index 63700fc2abd..6b6b5e62227 100644 --- a/crates/project/src/caption_timing.rs +++ b/crates/project/src/caption_timing.rs @@ -152,6 +152,40 @@ fn build_source_to_edited_mappings( .collect() } +pub fn transcription_source_ranges( + timeline: Option<&TimelineConfiguration>, + recording_durations: &[f64], +) -> Vec> { + let Some(timeline) = timeline else { + let duration = recording_durations.iter().sum(); + return if duration > 0.0 { + vec![0.0..duration] + } else { + Vec::new() + }; + }; + let mut ranges: Vec<_> = build_source_to_edited_mappings(timeline, recording_durations) + .into_iter() + .zip(&timeline.segments) + .filter_map(|(mapping, segment)| { + let duration = *recording_durations.get(segment.recording_clip as usize)?; + let offset = mapping.source_start - segment.start; + let start = segment.start.max(0.0); + let end = segment.end.min(duration); + (start < end).then_some(offset + start..offset + end) + }) + .collect(); + ranges.sort_by(|a, b| a.start.total_cmp(&b.start)); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start <= last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + fn map_time_range_within_mapping( start: f64, end: f64, @@ -429,6 +463,48 @@ mod tests { use super::*; use crate::CaptionsData; + #[test] + fn transcription_ranges_follow_source_cuts_independent_of_speed_and_clip_order() { + let timeline = serde_json::from_value(serde_json::json!({ + "segments": [ + {"recordingSegment":1,"start":1.0,"end":3.0,"timescale":0.5}, + {"recordingSegment":0,"start":5.0,"end":6.0,"timescale":2.0}, + {"recordingSegment":0,"start":1.0,"end":3.0,"timescale":1.0}, + {"recordingSegment":0,"start":2.0,"end":4.0,"timescale":1.0}, + {"recordingSegment":0,"start":4.0,"end":4.5,"timescale":1.0}, + {"recordingSegment":0,"start":1.0,"end":3.0,"timescale":2.0} + ], "zoomSegments": [] + })) + .unwrap(); + assert_eq!( + transcription_source_ranges(Some(&timeline), &[10.0, 4.0]), + vec![1.0..4.5, 5.0..6.0, 11.0..13.0] + ); + } + + #[test] + fn transcription_ranges_preserve_empty_timelines_and_clip_to_sources() { + let mut timeline: TimelineConfiguration = + serde_json::from_value(serde_json::json!({"segments": [], "zoomSegments": []})) + .unwrap(); + assert!(transcription_source_ranges(Some(&timeline), &[30.0]).is_empty()); + assert_eq!( + transcription_source_ranges(None, &[30.0, 10.0]), + vec![0.0..40.0] + ); + timeline.segments = serde_json::from_value(serde_json::json!([ + {"recordingSegment":0,"start":-1.0,"end":4.0,"timescale":1.0}, + {"recordingSegment":0,"start":8.0,"end":40.0,"timescale":1.0}, + {"recordingSegment":2,"start":0.0,"end":3.0,"timescale":1.0}, + {"recordingSegment":0,"start":5.0,"end":5.0,"timescale":1.0} + ])) + .unwrap(); + assert_eq!( + transcription_source_ranges(Some(&timeline), &[10.0]), + vec![0.0..4.0, 8.0..10.0] + ); + } + fn project(source_timed: bool, start: f32, end: f32) -> ProjectConfiguration { ProjectConfiguration { timeline: Some(serde_json::from_value(serde_json::json!({ diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index dea389ee13b..6e82e759a3f 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -6,7 +6,10 @@ pub mod keyboard; mod meta; pub use animated_gradient::*; -pub use caption_timing::{derive_caption_track_segments, source_caption_id, synchronize_captions}; +pub use caption_timing::{ + derive_caption_track_segments, source_caption_id, synchronize_captions, + transcription_source_ranges, +}; pub use configuration::*; pub use cursor::*; pub use keyboard::*; From 8d32ae6c501e04970b91da7c1d7ced7ec8e96666 Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:59:36 +0000 Subject: [PATCH 2/3] fix(desktop-gpui): don't apply captions from stale trim ranges --- apps/desktop-gpui/src/editor_tabs.rs | 31 ++++++++++++--- crates/project/src/caption_timing.rs | 57 ++++++++++++++++++++++++++++ crates/project/src/lib.rs | 2 +- 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/apps/desktop-gpui/src/editor_tabs.rs b/apps/desktop-gpui/src/editor_tabs.rs index 3a808180d86..9a4841a5332 100644 --- a/apps/desktop-gpui/src/editor_tabs.rs +++ b/apps/desktop-gpui/src/editor_tabs.rs @@ -25,7 +25,7 @@ use cap_project::{ BackgroundBlurConfig, BackgroundBlurMode, CameraShape, CameraXPosition, CameraYPosition, CaptionSegment, CaptionSettings, CornerStyle, CursorAnimationStyle, CursorRippleConfig, KeyboardData, KeyboardSettings, ProjectConfiguration, RecordingMeta, ShadowConfiguration, - StereoMode, + StereoMode, transcription_ranges_cover, transcription_source_ranges, }; use gpui::{ AnyElement, Bounds, Context, EntityId, FontWeight, Hsla, InteractiveElement, IntoElement, @@ -3177,6 +3177,8 @@ impl EditorWindow { let recording_durations = summary.clip_display_durations.clone(); let recording_duration = summary.recording_duration; let timeline = self.project.timeline.clone(); + let transcribed_ranges = + transcription_source_ranges(timeline.as_ref(), &recording_durations); let project_path = self.project_path.clone(); let model = self.selected_caption_model().name; let language = self.sidebar.caption_language; @@ -3207,10 +3209,19 @@ impl EditorWindow { Err(error) => Err(format!("Transcription task failed: {error}")), }; - match result { - Ok(segments) => { - transcription::finish_generation(&project_path, None); - let _ = this.update_in(cx, |this, window, cx| { + let result = match result { + Ok(segments) => this + .update_in(cx, |this, window, cx| { + let current_ranges = transcription_source_ranges( + this.project.timeline.as_ref(), + &recording_durations, + ); + if !transcription_ranges_cover(&transcribed_ranges, ¤t_ranges) { + return Err( + "The timeline changed during generation. Generate captions again." + .to_string(), + ); + } this.apply_generated_captions( segments, &recording_durations, @@ -3218,7 +3229,15 @@ impl EditorWindow { window, cx, ); - }); + Ok(()) + }) + .unwrap_or(Ok(())), + Err(error) => Err(error), + }; + + match result { + Ok(()) => { + transcription::finish_generation(&project_path, None); } Err(error) => { tracing::error!("caption generation failed: {error}"); diff --git a/crates/project/src/caption_timing.rs b/crates/project/src/caption_timing.rs index 6b6b5e62227..5c32d1a66b5 100644 --- a/crates/project/src/caption_timing.rs +++ b/crates/project/src/caption_timing.rs @@ -186,6 +186,17 @@ pub fn transcription_source_ranges( merged } +pub fn transcription_ranges_cover( + transcribed: &[std::ops::Range], + required: &[std::ops::Range], +) -> bool { + required.iter().all(|range| { + transcribed + .iter() + .any(|source| source.start <= range.start && source.end >= range.end) + }) +} + fn map_time_range_within_mapping( start: f64, end: f64, @@ -505,6 +516,52 @@ mod tests { ); } + #[test] + fn transcription_coverage_accepts_identical_ranges() { + let ranges = [1.0..4.0, 6.0..10.0]; + assert!(transcription_ranges_cover(&ranges, &ranges)); + } + + #[test] + fn transcription_coverage_accepts_further_trims_and_reordered_clips() { + let timeline = serde_json::from_value(serde_json::json!({ + "segments": [ + {"start":7.0,"end":9.0,"timescale":2.0}, + {"start":2.0,"end":3.0,"timescale":0.5} + ], "zoomSegments": [] + })) + .unwrap(); + let current = transcription_source_ranges(Some(&timeline), &[10.0]); + assert!(transcription_ranges_cover(&[1.0..4.0, 6.0..10.0], ¤t)); + } + + #[test] + fn transcription_coverage_rejects_audio_restored_by_undo() { + let timeline = serde_json::from_value(serde_json::json!({ + "segments": [{"start":0.0,"end":10.0,"timescale":1.0}], + "zoomSegments": [] + })) + .unwrap(); + let current = transcription_source_ranges(Some(&timeline), &[10.0]); + assert!(!transcription_ranges_cover( + &[0.0..4.0, 6.0..10.0], + ¤t + )); + assert!(!transcription_ranges_cover(&[1.0..10.0], ¤t)); + assert!(!transcription_ranges_cover(&[0.0..9.0], ¤t)); + assert!(!transcription_ranges_cover(&[], ¤t)); + assert!(!transcription_ranges_cover( + &[0.0..4.0], + &[0.0..4.0, 6.0..10.0] + )); + } + + #[test] + fn transcription_coverage_accepts_an_empty_timeline() { + assert!(transcription_ranges_cover(&[0.0..10.0], &[])); + assert!(transcription_ranges_cover(&[], &[])); + } + fn project(source_timed: bool, start: f32, end: f32) -> ProjectConfiguration { ProjectConfiguration { timeline: Some(serde_json::from_value(serde_json::json!({ diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 6e82e759a3f..e4cead314a4 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -8,7 +8,7 @@ mod meta; pub use animated_gradient::*; pub use caption_timing::{ derive_caption_track_segments, source_caption_id, synchronize_captions, - transcription_source_ranges, + transcription_ranges_cover, transcription_source_ranges, }; pub use configuration::*; pub use cursor::*; From 4ce9922ca0b21fbfd6b0d63e4860717a39bea8f8 Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:47:39 +0000 Subject: [PATCH 3/3] fix(desktop-gpui): transcribe kept ranges in one model run --- apps/desktop-gpui/src/transcription.rs | 109 ++++++++--- .../src/transcription/verify_tests.rs | 183 +++++++++++++++--- 2 files changed, 241 insertions(+), 51 deletions(-) diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs index d6ba04c8470..4fe8db8570f 100644 --- a/apps/desktop-gpui/src/transcription.rs +++ b/apps/desktop-gpui/src/transcription.rs @@ -980,7 +980,7 @@ fn extract_project_audio( studio: &StudioRecordingMeta, recording_durations: &[f64], range: Range, - output_path: &Path, + final_samples: &mut Vec, ) -> Result { let tracks: Vec> = match studio { StudioRecordingMeta::SingleSegment { segment } => { @@ -1007,7 +1007,7 @@ fn extract_project_audio( if tracks.len() != recording_durations.len() { return Err("Recording durations do not match the recording metadata".to_string()); } - let mut final_samples = Vec::new(); + let initial_len = final_samples.len(); let mut any_audio = false; let mut take_start = 0.0; for (tracks, duration) in tracks.into_iter().zip(recording_durations) { @@ -1038,20 +1038,13 @@ fn extract_project_audio( display_duration_secs: end - start, sources, }; - append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE) + append_transcription_audio(final_samples, &take, DECODE_SAMPLE_RATE) .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?; } if !any_audio { - return Ok(false); + final_samples.truncate(initial_len); } - - let gain = normalize_audio_for_transcription(&mut final_samples); - if (gain - 1.0).abs() > 0.01 { - tracing::info!("Applied transcription audio gain: {gain:.2}x"); - } - - write_wav_16k_mono(&final_samples, output_path)?; - Ok(true) + Ok(any_audio) } /// The WAV encode half of the recording-directory arm @@ -1551,6 +1544,63 @@ fn process_with_parakeet( static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); +struct TranscriptionAudioOffset { + concatenated_start: f64, + source_start: f64, + length: f64, +} + +fn restore_source_timestamps( + segments: Vec, + offsets: &[TranscriptionAudioOffset], +) -> Vec { + let map_times = |start: f32, end: f32| { + let start = f64::from(start); + let index = offsets + .partition_point(|offset| offset.concatenated_start <= start) + .checked_sub(1)?; + let offset = &offsets[index]; + let local_start = start - offset.concatenated_start; + if local_start >= offset.length { + return None; + } + let local_end = + (f64::from(end) - offset.concatenated_start).clamp(local_start, offset.length); + Some(( + (offset.source_start + local_start) as f32, + (offset.source_start + local_end) as f32, + )) + }; + segments + .into_iter() + .filter_map(|mut segment| { + let word_count = segment.words.len(); + segment.words = segment + .words + .into_iter() + .filter_map(|mut word| { + (word.start, word.end) = map_times(word.start, word.end)?; + Some(word) + }) + .collect(); + if word_count != segment.words.len() { + segment.text = caption_text_from_words(&segment.words); + } + (segment.start, segment.end) = match (segment.words.first(), segment.words.last()) { + (Some(first), Some(last)) => (first.start, last.end), + _ if word_count == 0 => map_times(segment.start, segment.end)?, + _ => return None, + }; + Some(segment) + }) + .enumerate() + .map(|(index, mut segment)| { + segment.id = format!("segment-{index}"); + segment + }) + .collect() +} + /// `transcribe_audio` (`captions.rs:1315-1439`), minus the Tauri command /// plumbing. Blocking -- run it on the tokio blocking pool. pub fn transcribe_blocking( @@ -1601,7 +1651,7 @@ fn transcribe_project_with( project_path: &Path, timeline: Option<&TimelineConfiguration>, recording_durations: &[f64], - mut engine: impl FnMut(&Path) -> Result, String>, + engine: impl FnOnce(&Path) -> Result, String>, ) -> Result, String> { ffmpeg::init().map_err(|e| format!("Failed to initialise ffmpeg: {e}"))?; let meta = RecordingMeta::load_for_project(project_path) @@ -1620,34 +1670,35 @@ fn transcribe_project_with( let audio_path = temp_dir.join("audio.wav"); let result = (|| { - let mut segments = Vec::new(); - let mut any_audio = false; + let mut samples = Vec::new(); + let mut offsets = Vec::new(); for range in cap_project::transcription_source_ranges(timeline, recording_durations) { + let start = samples.len(); if !extract_project_audio( &meta, studio, recording_durations, range.clone(), - &audio_path, + &mut samples, )? { continue; } - any_audio = true; - for mut segment in engine(&audio_path)? { - segment.id = format!("segment-{}", segments.len()); - segment.start += range.start as f32; - segment.end += range.start as f32; - for word in &mut segment.words { - word.start += range.start as f32; - word.end += range.start as f32; - } - segments.push(segment); - } + offsets.push(TranscriptionAudioOffset { + concatenated_start: start as f64 / f64::from(DECODE_SAMPLE_RATE), + source_start: range.start, + length: (samples.len() - start) as f64 / f64::from(DECODE_SAMPLE_RATE), + }); } - if !any_audio { + if offsets.is_empty() { return Err("No audio sources found in the selected timeline".to_string()); } - Ok(segments) + let gain = normalize_audio_for_transcription(&mut samples); + if (gain - 1.0).abs() > 0.01 { + tracing::info!("Applied transcription audio gain: {gain:.2}x"); + } + write_wav_16k_mono(&samples, &audio_path)?; + drop(samples); + Ok(restore_source_timestamps(engine(&audio_path)?, &offsets)) })(); let _ = std::fs::remove_dir_all(&temp_dir); diff --git a/apps/desktop-gpui/src/transcription/verify_tests.rs b/apps/desktop-gpui/src/transcription/verify_tests.rs index e00f2dbf842..f247012ba6a 100644 --- a/apps/desktop-gpui/src/transcription/verify_tests.rs +++ b/apps/desktop-gpui/src/transcription/verify_tests.rs @@ -690,29 +690,41 @@ fn trimmed_transcription_extracts_only_used_audio_and_restores_source_times() { .unwrap(); let mut called = 0; let segments = transcribe_project_with(&project, Some(&timeline), &durations, |path| { - let range = &expected_ranges[called]; + let expected_duration: f64 = expected_ranges + .iter() + .map(|range| range.end - range.start) + .sum(); let audio = media::decode_audio(path, Some(MARKER_RATE)).unwrap(); let duration = audio.mono().len() as f64 / f64::from(MARKER_RATE); assert!( - (duration - (range.end - range.start)).abs() < 0.002, - "extracted {duration}s, expected {}s", - range.end - range.start + (duration - expected_duration).abs() < 0.002, + "extracted {duration}s, expected {expected_duration}s" ); called += 1; - Ok(vec![CaptionSegment { - id: "segment-0".into(), - start: 0.25, - end: 0.5, - text: "word".into(), - words: vec![CaptionWord { - text: "word".into(), - start: 0.25, - end: 0.5, - }], - }]) + let mut offset = 0.0; + Ok(expected_ranges + .iter() + .map(|range| { + let start = offset + 0.25; + let end = offset + 0.5; + offset += (range.end - range.start) as f32; + CaptionSegment { + id: "segment-0".into(), + start, + end, + text: "word".into(), + words: vec![CaptionWord { + text: "word".into(), + start, + end, + }], + } + }) + .collect()) }) .unwrap(); - assert_eq!(called, expected_ranges.len()); + assert_eq!(called, 1); + assert_eq!(segments.len(), expected_ranges.len()); for (segment, range) in segments.iter().zip(expected_ranges) { assert!((f64::from(segment.start) - range.start - 0.25).abs() < 1e-5); assert!((f64::from(segment.end) - range.start - 0.5).abs() < 1e-5); @@ -800,17 +812,14 @@ fn transcription_keeps_speech_after_empty_ranges_and_cleans_up_on_failure() { let result = transcribe_project_with(&project, Some(&timeline), &durations, |path| { temporary = path.to_path_buf(); calls += 1; - if calls == 1 { - return Ok(Vec::new()); - } Ok(caption_segments_from_tokens([CaptionWord { text: "speech".into(), - start: 0.1, - end: 0.2, + start: 1.1, + end: 1.2, }])) }) .unwrap(); - assert_eq!(calls, 2); + assert_eq!(calls, 1); assert!((result[0].words[0].start - 3.1).abs() < 1e-5); assert!(!temporary.parent().unwrap().exists()); let error = transcribe_project_with(&project, Some(&timeline), &durations, |path| { @@ -832,3 +841,133 @@ fn transcription_keeps_speech_after_empty_ranges_and_cleans_up_on_failure() { .unwrap_err(); assert!(error.contains("No audio sources")); } + +#[test] +fn source_timestamps_restore_words_in_two_ranges() { + let offsets = [ + TranscriptionAudioOffset { + concatenated_start: 0.0, + source_start: 10.0, + length: 2.0, + }, + TranscriptionAudioOffset { + concatenated_start: 2.0, + source_start: 20.0, + length: 3.0, + }, + ]; + let segments = restore_source_timestamps( + caption_segments_from_tokens([ + CaptionWord { + text: "first".into(), + start: 0.25, + end: 0.5, + }, + CaptionWord { + text: "second".into(), + start: 2.25, + end: 2.5, + }, + ]), + &offsets, + ); + assert_eq!(segments.len(), 1); + let segment = &segments[0]; + assert_eq!((segment.start, segment.end), (10.25, 20.5)); + assert_eq!( + (segment.words[0].start, segment.words[0].end), + (10.25, 10.5) + ); + assert_eq!( + (segment.words[1].start, segment.words[1].end), + (20.25, 20.5) + ); +} + +#[test] +fn source_timestamps_clamp_words_and_wordless_segments_at_junctions() { + let offsets = [ + TranscriptionAudioOffset { + concatenated_start: 0.0, + source_start: 10.0, + length: 2.0, + }, + TranscriptionAudioOffset { + concatenated_start: 2.0, + source_start: 20.0, + length: 3.0, + }, + ]; + let mut segments = caption_segments_from_tokens([ + CaptionWord { + text: "across".into(), + start: 1.75, + end: 2.25, + }, + CaptionWord { + text: "junction".into(), + start: 2.0, + end: 2.25, + }, + ]); + segments.push(CaptionSegment { + start: 1.75, + end: 2.25, + text: "no words".into(), + ..Default::default() + }); + let segments = restore_source_timestamps(segments, &offsets); + assert_eq!(segments.len(), 2); + assert_eq!( + (segments[0].words[0].start, segments[0].words[0].end), + (11.75, 12.0) + ); + assert_eq!( + (segments[0].words[1].start, segments[0].words[1].end), + (20.0, 20.25) + ); + assert_eq!((segments[0].start, segments[0].end), (11.75, 20.25)); + assert_eq!((segments[1].start, segments[1].end), (11.75, 12.0)); +} + +#[test] +fn source_timestamps_for_one_range_match_a_source_offset() { + let offsets = [TranscriptionAudioOffset { + concatenated_start: 0.0, + source_start: 37.0, + length: 4.0, + }]; + let segments = caption_segments_from_tokens([CaptionWord { + text: "word".into(), + start: 0.25, + end: 0.5, + }]); + let mut reference = segments.clone(); + for segment in &mut reference { + segment.start += 37.0; + segment.end += 37.0; + for word in &mut segment.words { + word.start += 37.0; + word.end += 37.0; + } + } + let segments = restore_source_timestamps(segments, &offsets); + assert!(compare_segments(&segments, &reference, 0.0).is_empty()); +} + +#[test] +fn source_timestamps_with_empty_ranges_produce_no_captions() { + let mut segments = caption_segments_from_tokens([CaptionWord { + text: "word".into(), + start: 0.25, + end: 0.5, + }]); + segments.push(CaptionSegment { + start: 0.25, + end: 0.5, + text: "no words".into(), + ..Default::default() + }); + assert!(restore_source_timestamps(segments, &[]).is_empty()); + assert!(restore_source_timestamps(Vec::new(), &[]).is_empty()); +}