From ddee1bcf87454c620ca84e67b4406c5f418b97c0 Mon Sep 17 00:00:00 2001 From: Hongyuan Du Date: Sat, 24 Oct 2026 21:45:47 +0800 Subject: [PATCH 1/2] fix(asr): bound EOS word times and report decoded progress Bound runner result copies before postprocessing without changing tail decoding. Preserve backfill provenance through ITN and speaker handling, retain written order for equal timestamps, and cap exported intervals to original-rate input samples. Report real decoded progress and cover EOF, EOU delivery, fractional durations, ITN remapping and result consumers with regression tests. Signed-off-by: Hongyuan Du --- app/live_transcript.cpp | 2 + app/transcribe.cpp | 8 +- docs/asr/configuration.md | 31 ++ server/http/http_server.cpp | 5 +- src/asr/decoders/decoder.h | 3 + src/asr/postproc/itn_align.cpp | 60 ++-- src/asr/recognizer.cpp | 63 +++- src/asr/recognizer.h | 4 + src/asr/runner.cpp | 30 +- src/asr/runner.h | 5 +- src/asr/types.h | 11 +- src/asr/word_time.h | 66 ++++ src/common/subtitles.cpp | 9 +- src/common/subtitles.h | 2 + tests/c/asr_c_smoke.c | 27 +- tests/cpp/asr/CMakeLists.txt | 10 + tests/cpp/asr/test_cache_stream_geometry.cpp | 13 +- tests/cpp/asr/test_live_transcript.cpp | 10 + tests/cpp/asr/test_result_timing.cpp | 316 +++++++++++++++++++ tests/cpp/common/test_subtitles.cpp | 31 ++ 20 files changed, 657 insertions(+), 49 deletions(-) create mode 100644 src/asr/word_time.h create mode 100644 tests/cpp/asr/test_result_timing.cpp diff --git a/app/live_transcript.cpp b/app/live_transcript.cpp index 160be857..e9cd55a2 100644 --- a/app/live_transcript.cpp +++ b/app/live_transcript.cpp @@ -12,6 +12,8 @@ namespace { double speaker_ready_time(const asr::Word& word) { + if (!word.has_audio_position) + return 0.0; // No speaker window can make a padding-only position valid. return std::max(word.end_time / 1000.0, word.start_time / 1000.0 + 0.160); } diff --git a/app/transcribe.cpp b/app/transcribe.cpp index 8ba4f1f8..4fb0ab0a 100644 --- a/app/transcribe.cpp +++ b/app/transcribe.cpp @@ -48,6 +48,7 @@ struct Transcript { std::string target_language; float confidence = 0.0f; float audio_seconds = 0.0f; + int32_t audio_extent_ms = -1; std::vector languages; std::vector words; }; @@ -285,6 +286,7 @@ append_result(Transcript& transcript, const asr::Result& result) { } transcript.confidence = alternative.confidence; transcript.audio_seconds = std::max(transcript.audio_seconds, result.audio_processed); + transcript.audio_extent_ms = std::max(transcript.audio_extent_ms, result.audio_extent_ms); for (const auto& language : alternative.language_codes) if (std::find(transcript.languages.begin(), transcript.languages.end(), language) == transcript.languages.end()) @@ -304,6 +306,7 @@ transcribe_one(asr::Recognizer& recognizer, const Options& options, const fs::pa const asr::AsrRequestOptions request = make_request_options(options); Transcript transcript; + transcript.audio_extent_ms = asr::real_audio_extent_ms(audio.samples.size(), audio.sample_rate); if (!options.stream) { append_result( transcript, recognizer.recognize( @@ -506,6 +509,8 @@ transcribe_live(asr::Recognizer& recognizer, const Options& options) { emit_turns(presenter.finish(), final_audio_processed); transcript.audio_seconds = std::max( transcript.audio_seconds, captured_samples / static_cast(microphone.sample_rate())); + transcript.audio_extent_ms = + asr::real_audio_extent_ms(captured_samples, microphone.sample_rate()); terminal.stopped(transcript.audio_seconds); return transcript; } @@ -563,7 +568,8 @@ render(const Transcript& t, OutputFormat format, const fs::path& source) { if (vtt) output << "WEBVTT\n\n"; const auto cues = nemo_speech::subtitle::make_cues( - t.words, t.text, static_cast(t.audio_seconds * 1000)); + t.words, t.text, + t.audio_extent_ms >= 0 ? t.audio_extent_ms : static_cast(t.audio_seconds * 1000)); for (size_t i = 0; i < cues.size(); ++i) { if (!vtt) output << i + 1 << '\n'; diff --git a/docs/asr/configuration.md b/docs/asr/configuration.md index fb9878f7..f1385c57 100644 --- a/docs/asr/configuration.md +++ b/docs/asr/configuration.md @@ -222,6 +222,37 @@ together. For Riva-compatible gRPC clients, a `runtime_config["force_eou"] = "true"` message finalizes the current utterance once the audio received so far is decoded. +## Time coordinates and word timestamps + +`audio_processed` is the real decoded progress in seconds, not the amount of +audio already queued by the caller. Lookahead and synthetic EOS padding do not +advance it beyond real decoded audio. At end-of-stream it reports the complete +input duration, including when the input was resampled. Publishing an utterance +does not restart the absolute stream clock. Input cannot be appended after +`finish()`. + +Word timestamps are approximate decoder-frame positions, not forced alignment. +They keep the model's nominal frame grid; the causal first-chunk correction for +the decoded frontier must not be subtracted from every word. The cache-aware +runner bounds result copies to the real encoder-frame range before ITN or +diarization; it does not change decoding state or discard tail tokens. The +shared result builder additionally bounds milliseconds to original-rate input +samples, since the last encoder interval can straddle EOF. Subtitle exporters use +the same sample-derived integer-millisecond extent instead of independently +rounding the floating-point progress field. + +EOS padding may be needed to emit a real trailing word. If its nominal position +is entirely outside the real encoder range, its text is retained on the last +real encoder interval. Such backfilled words remain internally marked through +ITN, receive no speaker attribution, and do not wait for future speaker evidence. +This is an approximate display interval, **not** recovered phonetic alignment. +Partly overlapping spans retain their original onset. ITN can split an integer +frame interval into zero-width spans; these remain valid rather than being +artificially extended. Integer-millisecond rounding never extends past the input. + +Subtitle reading-time extensions also remain within the recording. A +zero-duration recording produces no positive-duration subtitle cue. + ## Postprocessing: profanity, ITN, PnC Postprocessing runs on the final transcript in this order: **profanity → ITN → diff --git a/server/http/http_server.cpp b/server/http/http_server.cpp index c151d3b7..ce0060f4 100644 --- a/server/http/http_server.cpp +++ b/server/http/http_server.cpp @@ -280,7 +280,10 @@ transcript_response( return alternative.transcript + "\n"; if (format == "srt" || format == "vtt") { const bool vtt = format == "vtt"; - int audio_ms = std::max(1, static_cast(std::round(result.audio_processed * 1000))); + const int audio_ms = + result.audio_extent_ms >= 0 + ? result.audio_extent_ms + : std::max(0, static_cast(std::round(result.audio_processed * 1000))); std::vector words; words.reserve(alternative.words.size()); for (const auto& word : alternative.words) diff --git a/src/asr/decoders/decoder.h b/src/asr/decoders/decoder.h index f48a239c..3369959c 100644 --- a/src/asr/decoders/decoder.h +++ b/src/asr/decoders/decoder.h @@ -103,6 +103,9 @@ struct WordTiming { int64_t start_frame = 0; int64_t end_frame = 0; float confidence = 1.0f; + // Internal provenance: false when an EOS-only emission was moved onto + // real audio. Its bounded display interval must not imply speaker evidence. + bool has_audio_position = true; }; // One N-best hypothesis. Times use the same encoder-frame clock as WordTiming. diff --git a/src/asr/postproc/itn_align.cpp b/src/asr/postproc/itn_align.cpp index a6e9d598..bb75398b 100644 --- a/src/asr/postproc/itn_align.cpp +++ b/src/asr/postproc/itn_align.cpp @@ -67,6 +67,12 @@ average_confidence(const std::vector& timings, const std::vector(indices.size()); } +bool +have_audio_positions(const std::vector& timings, const std::vector& indices) { + return std::all_of( + indices.begin(), indices.end(), [&](int i) { return timings[i].has_audio_position; }); +} + std::vector split_ws(const std::string& s) { std::vector out; @@ -141,7 +147,8 @@ update_word_timings(std::vector& timings, const std::string& alignme if (tokens.empty()) return; - std::vector result; + std::vector> result; + size_t output_order = 0; int cursor = 0; const int n = static_cast(timings.size()); @@ -164,6 +171,8 @@ update_word_timings(std::vector& timings, const std::string& alignme continue; // no written words, or no spoken span matched (skip, like riva) const auto& outs = it->second; + const size_t output_base = output_order; + output_order += outs.size(); std::vector spoken_used(spoken.size(), false); std::vector output_used(outs.size(), false); @@ -173,7 +182,7 @@ update_word_timings(std::vector& timings, const std::string& alignme continue; WordTiming exact = timings[spoken[si]]; exact.word = outs[oi]; - result.push_back(std::move(exact)); + result.emplace_back(output_base + oi, std::move(exact)); spoken_used[si] = true; output_used[oi] = true; break; @@ -192,10 +201,13 @@ update_word_timings(std::vector& timings, const std::string& alignme continue; if (remaining_output.size() == 1 && starts_with_digit(outs[remaining_output.front()])) { - result.push_back( - {outs[remaining_output.front()], timings[remaining_spoken.front()].start_frame, - timings[remaining_spoken.back()].end_frame, - average_confidence(timings, remaining_spoken)}); + result.emplace_back( + output_base + remaining_output.front(), + WordTiming{ + outs[remaining_output.front()], timings[remaining_spoken.front()].start_frame, + timings[remaining_spoken.back()].end_frame, + average_confidence(timings, remaining_spoken), + have_audio_positions(timings, remaining_spoken)}); continue; } @@ -207,13 +219,15 @@ update_word_timings(std::vector& timings, const std::string& alignme const int64_t span = end - start; const float confidence = average_confidence(timings, remaining_spoken); for (size_t i = 0; i < remaining_output.size(); ++i) { - result.push_back( - {outs[remaining_output[i]], - start + span * static_cast(i) / - static_cast(remaining_output.size()), - start + span * static_cast(i + 1) / - static_cast(remaining_output.size()), - confidence}); + result.emplace_back( + output_base + remaining_output[i], + WordTiming{ + outs[remaining_output[i]], + start + span * static_cast(i) / + static_cast(remaining_output.size()), + start + span * static_cast(i + 1) / + static_cast(remaining_output.size()), + confidence, have_audio_positions(timings, remaining_spoken)}); } continue; } @@ -225,18 +239,26 @@ update_word_timings(std::vector& timings, const std::string& alignme const size_t take = base + (i < extra ? 1 : 0); std::vector slice( remaining_spoken.begin() + offset, remaining_spoken.begin() + offset + take); - result.push_back( - {outs[remaining_output[i]], timings[slice.front()].start_frame, - timings[slice.back()].end_frame, average_confidence(timings, slice)}); + result.emplace_back( + output_base + remaining_output[i], + WordTiming{ + outs[remaining_output[i]], timings[slice.front()].start_frame, + timings[slice.back()].end_frame, average_confidence(timings, slice), + have_audio_positions(timings, slice)}); offset += take; } } if (!result.empty()) { - std::sort(result.begin(), result.end(), [](const WordTiming& a, const WordTiming& b) { - return a.start_frame < b.start_frame; + // EOS backfill can give distinct words identical frame positions. + // Keep the written order for ties, not the order in which matches were found. + std::sort(result.begin(), result.end(), [](const auto& a, const auto& b) { + if (a.second.start_frame != b.second.start_frame) + return a.second.start_frame < b.second.start_frame; + return a.first < b.first; }); - timings = std::move(result); + timings.clear(); + for (auto& entry : result) timings.push_back(std::move(entry.second)); } } diff --git a/src/asr/recognizer.cpp b/src/asr/recognizer.cpp index ce9bdb28..0cfb7d8c 100644 --- a/src/asr/recognizer.cpp +++ b/src/asr/recognizer.cpp @@ -355,6 +355,8 @@ RecognitionStream::~RecognitionStream() { void RecognitionStream::push(const float* samples, size_t n, int sample_rate) { + if (input_finished_) + throw std::invalid_argument("cannot push audio after finish"); if (n == 0) return; if (samples == nullptr) @@ -371,6 +373,7 @@ RecognitionStream::push(const float* samples, size_t n, int sample_rate) { } else if (rate != input_sample_rate_) { throw std::invalid_argument("input sample rate cannot change within a stream"); } + input_samples_received_ += n; if (coordinate_ingress_) pending_cohort_target_ = recognizer_->arrive_streaming_ingress(); // The diarizer consumes the same model-rate audio the runner does, so it @@ -401,7 +404,12 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const r.is_final = is_final; r.stability = is_final ? 1.0f : 0.0f; r.channel_tag = 1; - r.audio_processed = u.audio_processed_sec; + r.audio_extent_ms = real_audio_extent_ms(input_samples_received_, input_sample_rate_); + const double received_sec = + input_sample_rate_ > 0 ? static_cast(input_samples_received_) / input_sample_rate_ + : 0.0; + r.audio_processed = static_cast( + input_finished_ ? received_sec : std::min(u.audio_processed_sec, received_sec)); if (!is_final && !interim_words_) { // Interim results contain only the raw top hypothesis. @@ -415,7 +423,6 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const // Detected languages are per-stream (shared by every alternative). const auto langs = runner_->detected_languages(); const bool want_off = opts_.needs_word_timings(); - const double ms = recognizer_->ms_per_enc_frame(); const std::string lang0 = langs.empty() ? std::string() : langs.front(); std::string postproc_language = opts_.language_code; std::string normalized_requested = postproc_language; @@ -447,8 +454,10 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const continue; Word ww; ww.word = w.word; - ww.start_time = static_cast(w.start_frame * ms + 0.5); - ww.end_time = static_cast(w.end_frame * ms + 0.5); + const auto time = word_time_(w); + ww.start_time = time.start_ms; + ww.end_time = time.end_ms; + ww.has_audio_position = time.has_audio_position; ww.confidence = w.confidence; ww.language_code = lang0; alt.words.push_back(std::move(ww)); @@ -479,11 +488,13 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const for (const auto& w : words) { Word ww; ww.word = w.word; - ww.start_time = static_cast(w.start_frame * ms + 0.5); - ww.end_time = static_cast(w.end_frame * ms + 0.5); + const auto time = word_time_(w); + ww.start_time = time.start_ms; + ww.end_time = time.end_ms; + ww.has_audio_position = time.has_audio_position; ww.confidence = w.confidence; ww.language_code = lang0; - if (tag_speakers && diar_) { + if (tag_speakers && diar_ && ww.has_audio_position) { // Transducer punctuation can extend a word timestamp into // the next turn. Anchor attribution to the word onset and // average two diar frames to reject single-frame noise. @@ -509,11 +520,25 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const return r; } +BoundedWordTime +RecognitionStream::word_time_(const WordTiming& word) const { + auto time = bound_word_time( + word.start_frame, word.end_frame, recognizer_->ms_per_enc_frame(), input_samples_received_, + input_sample_rate_); + time.has_audio_position = time.has_audio_position && word.has_audio_position; + return time; +} + void RecognitionStream::flush_diar_deficit_(const StreamingUpdate& u) { if (!diar_ || u.words.empty()) return; - const double end_sec = u.words.back().end_frame * recognizer_->ms_per_enc_frame() / 1000.0; + double end_sec = 0.0; + for (const auto& word : u.words) { + const auto time = word_time_(word); + if (time.has_audio_position) + end_sec = std::max(end_sec, time.end_ms / 1000.0); + } const auto target = static_cast(std::ceil(end_sec / diar_->seconds_per_frame())); if (target > diar_->n_frames()) { const int64_t before = diar_->n_frames(); @@ -538,13 +563,16 @@ RecognitionStream::next() { last_processed_sec_ = std::max(last_processed_sec_, u.audio_processed_sec); auto result = build_result_(u, /*is_final=*/true); if (diar_ && !u.words.empty()) { - // This final's words are tagged and later words start at or after - // its last one. Not audio_processed_sec: the cache runner sets it - // to all pushed audio, which can hold utterances not decoded yet. - const double last_start_sec = - u.words.back().start_frame * recognizer_->ms_per_enc_frame() / 1000.0; - diar_->hold_frames_from( - static_cast(last_start_sec / diar_->seconds_per_frame())); + // Retain the last observed word's attribution window; a nominal + // padding-only position must not evict real speech history. + for (auto it = u.words.rbegin(); it != u.words.rend(); ++it) { + const auto time = word_time_(*it); + if (time.has_audio_position) { + diar_->hold_frames_from(static_cast( + (time.start_ms / 1000.0) / diar_->seconds_per_frame())); + break; + } + } } result.late_punctuation = std::exchange(late_punctuation_, {}); return result; @@ -568,6 +596,7 @@ RecognitionStream::next() { Result RecognitionStream::finish() { + input_finished_ = true; const ScopedBatchCohort cohort_scope( pending_cohort_target_ > 0 ? pending_cohort_target_ : current_batch_cohort_target()); pending_cohort_target_ = 0; @@ -599,6 +628,10 @@ RecognitionStream::refresh_speaker_tags(Result& result) const { if (!diar_ || result.alternatives.empty()) return; for (auto& word : result.alternatives.front().words) { + if (!word.has_audio_position) { + word.speaker_tag = 0; + continue; + } const int speaker = diar_->speaker_for_word_time(word.start_time / 1000.0, word.end_time / 1000.0); word.speaker_tag = speaker >= 0 ? speaker + 1 : 0; diff --git a/src/asr/recognizer.h b/src/asr/recognizer.h index 8b7add09..360dc00b 100644 --- a/src/asr/recognizer.h +++ b/src/asr/recognizer.h @@ -19,6 +19,7 @@ #include "runner.h" #include "runtime.h" #include "types.h" +#include "word_time.h" namespace nemo_speech::asr { @@ -97,6 +98,7 @@ class RecognitionStream { private: Result build_result_(const StreamingUpdate& u, bool is_final) const; + BoundedWordTime word_time_(const WordTiming& word) const; // Extends diarization to cover all words in a final result. void flush_diar_deficit_(const StreamingUpdate& u); @@ -106,9 +108,11 @@ class RecognitionStream { std::unique_ptr diar_; AsrRequestOptions opts_; int input_sample_rate_ = 0; + uint64_t input_samples_received_ = 0; // Original rate; never counts padding. std::unique_ptr resampler_; std::vector resampled_audio_; bool resampler_flushed_ = false; + bool input_finished_ = false; bool coordinate_ingress_ = true; int pending_cohort_target_ = 0; // High-water mark of audio_processed_sec returned by next(). A step that diff --git a/src/asr/runner.cpp b/src/asr/runner.cpp index 74c63ca6..7faf3430 100644 --- a/src/asr/runner.cpp +++ b/src/asr/runner.cpp @@ -16,6 +16,7 @@ #include "greedy_ctc_decoder.h" #include "recognizer.h" +#include "word_time.h" #ifdef NEMO_SPEECH_WITH_FLASHLIGHT #include "flashlight_decoder.h" #endif @@ -972,6 +973,22 @@ CacheStreamRunner::process_one_chunk() { } } +int64_t +CacheStreamRunner::decoded_mel_frames() const { + return std::min( + total_mel_frames_produced_, enc_cfg_.cache_decoded_mel_frames(total_frames_emitted_)); +} + +void +CacheStreamRunner::bound_word_timings(StreamingUpdate& update) const { + const int64_t frames = total_mel_frames_produced_ > 0 + ? (total_mel_frames_produced_ - 1) / enc_cfg_.subsampling_factor + 1 + : 0; + for (auto& word : update.words) bound_word_frames(word, frames); + for (auto& alternative : update.extra_alternatives) + for (auto& word : alternative.words) bound_word_frames(word, frames); +} + StreamingUpdate CacheStreamRunner::step() { StreamingUpdate update; @@ -1031,8 +1048,12 @@ CacheStreamRunner::step() { // per-utterance buffers are already cleared. if (!update.is_final) update.transcript_so_far = transcript_; + // Received audio can include lookahead or whole later utterances. Report + // only real decoded coverage, consistently with BufferedStreamRunner. + const size_t decoded_samples = + std::min(audio_end, static_cast(decoded_mel_frames()) * model_->fe().hop_length()); update.audio_processed_sec = - static_cast(audio_end) / static_cast(model_->fe_config().sample_rate); + static_cast(decoded_samples) / static_cast(model_->fe_config().sample_rate); // A pending force_eou() must fire even on a chunk-less step. if (!update.is_final) poll_endpoint(update, /*after_chunk=*/false); @@ -1040,6 +1061,7 @@ CacheStreamRunner::step() { update.words = head_->word_timings(); if (head_) update.late_punctuation = head_->take_late_punctuation(); + bound_word_timings(update); trim_buffers(); return update; } @@ -1065,7 +1087,7 @@ CacheStreamRunner::poll_endpoint(StreamingUpdate& update, bool after_chunk) { // frames. Causal encoder frame f ends at mel f*sub (inclusive), so // the shorter first chunk shifts the exclusive frontier by sub-1. const double mel_ms = 1000.0 * hop / sample_rate; - const int64_t decoded_mel = enc_cfg_.cache_decoded_mel_frames(total_frames_emitted_); + const int64_t decoded_mel = decoded_mel_frames(); scan_vad_speech(*vad_, decoded_mel, vad_scan_frame_, vad_speech_seen_frame_); now_ms = static_cast(decoded_mel) * mel_ms; last_speech_ms = (vad_speech_seen_frame_ < 0) @@ -1117,7 +1139,7 @@ CacheStreamRunner::trim_buffers() { // masker's neighbourhood scans never cross the trimmed base. const int64_t margin = std::max(256, vad_masker_ ? 2 * vad_masker_->max_scan() : 0); - const int64_t decoded_mel = enc_cfg_.cache_decoded_mel_frames(total_frames_emitted_); + const int64_t decoded_mel = decoded_mel_frames(); const int64_t g = decoded_mel - margin; if (g > 0) vad_->discard_timeline_before(g); @@ -1133,6 +1155,7 @@ CacheStreamRunner::finalize() { static_cast(audio_base_ + audio_buf_.size()) / static_cast(model_->fe_config().sample_rate), opts_.max_alternatives); + bound_word_timings(update); return update; } finalizing_ = true; @@ -1195,6 +1218,7 @@ CacheStreamRunner::finalize() { opts_.max_alternatives); if (head_) update.late_punctuation += head_->take_late_punctuation(/*end_of_stream=*/true); + bound_word_timings(update); return update; } diff --git a/src/asr/runner.h b/src/asr/runner.h index dc33e224..d7df14a3 100644 --- a/src/asr/runner.h +++ b/src/asr/runner.h @@ -61,7 +61,8 @@ struct StreamingUpdate { bool is_final = false; // Mean token posterior for greedy CTC; 1.0 when the decoder has no posterior. float confidence = 1.0f; - // Seconds of audio processed. + // Real decoded progress, excluding lookahead and synthetic EOS padding. + // finalize() reports the complete real input duration. float audio_processed_sec = 0.0f; // Punctuation for the previous final (Decoder::take_late_punctuation); // not part of this update's text. @@ -280,6 +281,8 @@ class CacheStreamRunner final : public AsrRunner { private: int next_chunk_mel_frames() const; + int64_t decoded_mel_frames() const; + void bound_word_timings(StreamingUpdate& update) const; void process_one_chunk(); void upload_attn_mask(); void zero_caches(); diff --git a/src/asr/types.h b/src/asr/types.h index d5597f29..da594f5d 100644 --- a/src/asr/types.h +++ b/src/asr/types.h @@ -50,6 +50,9 @@ struct Word { // proto: WordInfo // proto WordInfo.speaker_tag: 1-based speaker id, set only when // diarization was requested (0 = untagged). int32_t speaker_tag = 0; + // Internal attribution guard. EOS-only emissions can have a bounded + // display interval without evidence for assigning a speaker at that time. + bool has_audio_position = true; }; struct Alternative { // proto: SpeechRecognitionAlternative @@ -64,8 +67,12 @@ struct Result { // proto: Streaming/SpeechRecognitionResult std::vector alternatives; bool is_final = true; float stability = 0.0f; - int32_t channel_tag = 1; // mono channel - float audio_processed = 0.0f; // seconds (proto audio_processed) + int32_t channel_tag = 1; // mono channel + // Decoded progress in seconds; the complete input duration at EOF. + float audio_processed = 0.0f; // proto audio_processed + // Internal, sample-derived input extent for display bounds. Unlike decoded + // progress it can include queued audio. -1 means unavailable to a producer. + int32_t audio_extent_ms = -1; // Streaming only: sentence punctuation emitted after the previous final's // endpoint. Append it to that final; it is not part of this result's text. std::string late_punctuation; diff --git a/src/asr/word_time.h b/src/asr/word_time.h new file mode 100644 index 00000000..0e608643 --- /dev/null +++ b/src/asr/word_time.h @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include +#include +#include + +#include "decoder.h" + +namespace nemo_speech::asr { + +struct BoundedWordTime { + int32_t start_ms = 0; + int32_t end_ms = 0; + bool has_audio_position = false; +}; + +// Project a result copy, never the decoder's frame clock or recurrent state. +// The exclusive frame limit counts causal outputs covering real Mel frames. +inline void +bound_word_frames(WordTiming& word, int64_t frame_limit) { + word.has_audio_position = word.has_audio_position && frame_limit > 0 && + word.start_frame < frame_limit && word.end_frame > 0; + if (frame_limit <= 0) { + word.start_frame = word.end_frame = 0; + return; + } + word.start_frame = std::clamp(word.start_frame, 0, frame_limit - 1); + word.end_frame = std::clamp(word.end_frame, word.start_frame + 1, frame_limit); +} + +inline int32_t +real_audio_extent_ms(uint64_t samples, int rate) { + if (rate <= 0) + return 0; + const long double ms = static_cast(samples) * 1000.0L / rate; + return static_cast( + std::min(ms, static_cast(std::numeric_limits::max()))); +} + +// The final encoder interval can straddle EOF even after frame bounding. +// Bound milliseconds to original-rate input samples too. Preserve legitimate +// zero-width ITN spans; this projection does not invent phonetic alignment. +inline BoundedWordTime +bound_word_time( + int64_t start_frame, int64_t end_frame, double frame_ms, uint64_t input_samples, + int input_rate) { + if (input_rate <= 0 || input_samples == 0 || frame_ms <= 0.0 || !std::isfinite(frame_ms)) + return {}; + const double audio_end = + static_cast(static_cast(input_samples) * 1000.0L / input_rate); + const double limit = real_audio_extent_ms(input_samples, input_rate); + const double start = start_frame * frame_ms; + const double end = end_frame * frame_ms; + const auto project = [limit](double t) { + return static_cast(std::clamp(std::floor(t + 0.5), 0.0, limit)); + }; + const int32_t bounded_start = project(start); + return { + bounded_start, std::max(bounded_start, project(end)), + end_frame >= start_frame && start < audio_end && (end > 0.0 || start >= 0.0)}; +} + +} // namespace nemo_speech::asr diff --git a/src/common/subtitles.cpp b/src/common/subtitles.cpp index 893f9af8..ff4617f4 100644 --- a/src/common/subtitles.cpp +++ b/src/common/subtitles.cpp @@ -168,10 +168,12 @@ make_speaker_turns(const std::vector& words) { std::vector make_cues(const std::vector& words, const std::string& fallback_text, int audio_ms) { + if (audio_ms <= 0) + return {}; if (words.empty()) { if (fallback_text.empty()) return {}; - return {{0, std::max(1, audio_ms), fallback_text}}; + return {{0, audio_ms, fallback_text}}; } std::vector groups; @@ -217,8 +219,9 @@ make_cues(const std::vector& words, const std::string& fallback_text, int cues.reserve(groups.size()); for (size_t i = 0; i < groups.size(); ++i) { auto cue = groups[i].cue; + cue.start_ms = std::clamp(cue.start_ms, 0, audio_ms - 1); const int available_end = - i + 1 < groups.size() ? groups[i + 1].cue.start_ms : std::max(cue.end_ms, audio_ms); + i + 1 < groups.size() ? std::min(audio_ms, groups[i + 1].cue.start_ms) : audio_ms; const int words_in_cue = static_cast(groups[i].end - groups[i].begin); const int readable_ms = std::max( kMinimumCueDurationMs, (words_in_cue * 60000 + kWordsPerMinute - 1) / kWordsPerMinute); @@ -227,7 +230,7 @@ make_cues(const std::vector& words, const std::string& fallback_text, int std::max(cue.end_ms, desired_end), std::min(available_end, cue.start_ms + kMaximumCueDurationMs)); if (cue.end_ms <= cue.start_ms) - cue.end_ms = cue.start_ms + 1; + cue.end_ms = std::min(audio_ms, cue.start_ms + 1); cues.push_back(std::move(cue)); } return cues; diff --git a/src/common/subtitles.h b/src/common/subtitles.h index 31c92daa..b22e89d9 100644 --- a/src/common/subtitles.h +++ b/src/common/subtitles.h @@ -34,6 +34,8 @@ bool attaches_to_previous(const std::string& text); std::vector make_speaker_turns(const std::vector& words); +// Input word intervals are already bounded by the recognizer. Reading-time +// extensions must stay inside audio_ms too; zero-duration audio yields no cues. std::vector make_cues( const std::vector& words, const std::string& fallback_text, int audio_ms); diff --git a/tests/c/asr_c_smoke.c b/tests/c/asr_c_smoke.c index 965d217a..5f66182a 100644 --- a/tests/c/asr_c_smoke.c +++ b/tests/c/asr_c_smoke.c @@ -45,6 +45,26 @@ read_wav_f32(const char* path, size_t* n_out) { return out; } +static int +check_times(const nemo_speech_asr_result* result, size_t real_samples) { + const double duration_ms = real_samples / 16.0; + const float progress = nemo_speech_asr_result_audio_processed(result); + if (progress < 0 || progress > real_samples / 16000.0 + 1e-5) { + fprintf(stderr, "C API progress exceeds the real input\n"); + return 0; + } + const size_t count = nemo_speech_asr_result_word_count(result, 0); + for (size_t i = 0; i < count; ++i) { + const int32_t start = nemo_speech_asr_result_word_start_time(result, 0, i); + const int32_t end = nemo_speech_asr_result_word_end_time(result, 0, i); + if (start < 0 || end < start || end > duration_ms) { + fprintf(stderr, "C API word timestamp exceeds the real input\n"); + return 0; + } + } + return 1; +} + int main(int argc, char** argv) { printf("nemo_speech_asr_version: %s\n", nemo_speech_asr_version()); @@ -107,6 +127,7 @@ main(int argc, char** argv) { } printf("offline: %s\n", nemo_speech_asr_result_transcript(r, 0)); printf("offline words: %zu\n", nemo_speech_asr_result_word_count(r, 0)); + int timing_ok = check_times(r, n); nemo_speech_asr_result_destroy(r); /* ---- streaming (160 ms chunks) ---- */ @@ -140,6 +161,7 @@ main(int argc, char** argv) { nemo_speech_asr_stream_push_f32(st, audio + pos, cn, 16000); nemo_speech_asr_result* sr = NULL; while (nemo_speech_asr_stream_next(st, &sr) == NEMO_SPEECH_ASR_OK && sr) { + timing_ok &= check_times(sr, pos + cn); if (nemo_speech_asr_result_is_final(sr)) { strncat( streamed, nemo_speech_asr_result_transcript(sr, 0), @@ -152,6 +174,7 @@ main(int argc, char** argv) { nemo_speech_asr_stream_finish(st); nemo_speech_asr_result* sr = NULL; while (nemo_speech_asr_stream_next(st, &sr) == NEMO_SPEECH_ASR_OK && sr) { + timing_ok &= check_times(sr, n); if (nemo_speech_asr_result_is_final(sr)) { strncat( streamed, nemo_speech_asr_result_transcript(sr, 0), @@ -165,6 +188,6 @@ main(int argc, char** argv) { free(audio); nemo_speech_asr_destroy(rec); - printf("[C ABI smoke OK]\n"); - return 0; + printf(timing_ok ? "[C ABI smoke OK]\n" : "[C ABI timing FAILED]\n"); + return timing_ok ? 0 : 1; } diff --git a/tests/cpp/asr/CMakeLists.txt b/tests/cpp/asr/CMakeLists.txt index 4f072841..6dffddd1 100644 --- a/tests/cpp/asr/CMakeLists.txt +++ b/tests/cpp/asr/CMakeLists.txt @@ -40,6 +40,16 @@ add_executable(test_postproc test_postproc.cpp) target_link_libraries(test_postproc PRIVATE nemo_speech_asr Threads::Threads) add_test(NAME asr_postprocessing COMMAND test_postproc) +add_executable(test_result_timing test_result_timing.cpp) +target_link_libraries(test_result_timing PRIVATE nemo_speech_asr) +set(result_timing_args) +if(DEFINED ENV{NEMO_SPEECH_TEST_CACHE_AWARE_RNNT_MODEL} AND + NOT "$ENV{NEMO_SPEECH_TEST_CACHE_AWARE_RNNT_MODEL}" STREQUAL "") + list(APPEND result_timing_args "$ENV{NEMO_SPEECH_TEST_CACHE_AWARE_RNNT_MODEL}" + "${CMAKE_SOURCE_DIR}/test_files/asr/wav/test/jfk.wav") +endif() +add_test(NAME result_timing COMMAND test_result_timing ${result_timing_args}) + add_executable(test_audio_resampler test_audio_resampler.cpp) target_link_libraries(test_audio_resampler PRIVATE nemo_speech_common) diff --git a/tests/cpp/asr/test_cache_stream_geometry.cpp b/tests/cpp/asr/test_cache_stream_geometry.cpp index db967d26..1e451bd6 100644 --- a/tests/cpp/asr/test_cache_stream_geometry.cpp +++ b/tests/cpp/asr/test_cache_stream_geometry.cpp @@ -94,10 +94,16 @@ check_model(const std::string& path, int gpu, int right) { CacheStreamRunner runner(model, cfg); runner.set_prompt_index(prompt); runner.feed_audio(audio.data(), first_samples - 1); - runner.step(); + require( + runner.step().audio_processed_sec == 0.0f, + "received lookahead must not count as decoded progress"); require(runner.chunks_processed() == 0, "first chunk ran before its last stable mel frame"); runner.feed_audio(audio.data() + first_samples - 1, 1); - runner.step(); + const auto first_update = runner.step(); + require( + std::fabs(first_update.audio_processed_sec - first_mel * hop / float(fe.sample_rate)) < + 1e-5f, + "first decoded progress must use the causal mel frontier"); require( runner.chunks_processed() == 1, "first chunk incorrectly waits for steady-state length"); @@ -187,6 +193,9 @@ check_model(const std::string& path, int gpu, int right) { } } const auto final = runner.finalize(); + require( + std::fabs(final.audio_processed_sec - size / float(fe.sample_rate)) < 1e-5f, + "EOS padding must not inflate final audio duration"); ids.insert(ids.end(), final.new_token_ids.begin(), final.new_token_ids.end()); require(final.is_final, "EOF not final"); require(runner.finalize().new_token_ids.empty(), "EOF duplicated tokens"); diff --git a/tests/cpp/asr/test_live_transcript.cpp b/tests/cpp/asr/test_live_transcript.cpp index bc20eb61..2ae25142 100644 --- a/tests/cpp/asr/test_live_transcript.cpp +++ b/tests/cpp/asr/test_live_transcript.cpp @@ -516,6 +516,16 @@ test_delayed_punctuation_in_revision_buffer() { int main() { + { + nemo_speech::cli::LiveDiarizationBuffer buffer; + auto late = word("Late", 560, 640, 0); + late.has_audio_position = false; + buffer.push(final_result(alternative("Late", {late}), 0.64f)); + const auto ready = buffer.update(0.01, [](Result&) {}); + check( + ready.size() == 1 && ready[0].alternatives[0].words[0].speaker_tag == 0, + "padding-only text does not wait for a nonexistent speaker window"); + } test_canonical_single_speaker_text(); test_silence_final_settles_without_more_speech(); test_punctuation_stays_with_previous_speaker(); diff --git a/tests/cpp/asr/test_result_timing.cpp b/tests/cpp/asr/test_result_timing.cpp new file mode 100644 index 00000000..e3819429 --- /dev/null +++ b/tests/cpp/asr/test_result_timing.cpp @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// No arguments: projection policy. Optional MODEL.gguf AUDIO.wav [GPU] [DIAR.gguf] [VAD.gguf] +// exercises actual SDK results, short EOF, input packets, resampling and EOU drain. +#include +#include +#include +#include +#include +#include +#include + +#include "itn_align.h" +#include "recognizer.h" +#include "word_time.h" + +using namespace nemo_speech::asr; + +static void +require(bool ok, const char* message) { + if (!ok) + throw std::runtime_error(message); +} + +static void +check_projection() { + WordTiming late_word{"late", 14, 15, 1.0f}; + bound_word_frames(late_word, 8); + require( + late_word.start_frame == 7 && late_word.end_frame == 8 && !late_word.has_audio_position, + "EOS text gets the last real interval, not speaker evidence"); + bound_word_frames(late_word, 8); + require(!late_word.has_audio_position, "projection must preserve provenance when repeated"); + WordTiming partial_word{"partial", 6, 15, 1.0f}; + bound_word_frames(partial_word, 8); + require( + partial_word.start_frame == 6 && partial_word.end_frame == 8 && + partial_word.has_audio_position, + "partly real word keeps its original onset"); + WordTiming empty_word{"empty", 0, 1, 1.0f}; + bound_word_frames(empty_word, 0); + require( + empty_word.start_frame == 0 && empty_word.end_frame == 0 && !empty_word.has_audio_position, + "empty input has no real frame to borrow"); + for (uint64_t samples = 256; samples <= 32000; samples += 17) { + const int64_t mels = (samples - 256) / 160 + 1; + const int64_t frames = (mels - 1) / 8 + 1; + WordTiming word{"tail", frames + 5, frames + 6, 1.0f}; + bound_word_frames(word, frames); + const auto time = bound_word_time(word.start_frame, word.end_frame, 80, samples, 16000); + require( + time.start_ms < time.end_ms && time.end_ms <= samples * 1000.0 / 16000, + "frame bounding plus sample bounding handles fractional tail frames"); + } + const auto empty = bound_word_time(0, 1, 80.0, 0, 16000); + require( + real_audio_extent_ms(28224, 44100) == 640 && real_audio_extent_ms(1001, 44100) == 22, + "all exporters must use the same sample-derived millisecond bound"); + require(empty.start_ms == 0 && empty.end_ms == 0 && !empty.has_audio_position, "empty bounds"); + const auto normal = bound_word_time(2, 4, 80.0, 16000, 16000); + const auto long_audio = bound_word_time(89999, 90000, 80.0, 7200ULL * 44100, 44100); + require( + long_audio.start_ms == 7199920 && long_audio.end_ms == 7200000, + "long recordings retain absolute time without per-chunk rounding drift"); + require( + normal.start_ms == 160 && normal.end_ms == 320 && normal.has_audio_position, + "valid nominal word times must not receive the decode-frontier offset"); + const auto partial = bound_word_time(7, 15, 80.0, 10240, 16000); + require( + partial.start_ms == 560 && partial.end_ms == 640 && partial.has_audio_position, + "partly observed word keeps its real interval"); + const auto late = bound_word_time(14, 15, 80.0, 10240, 16000); + require( + late.start_ms == 640 && late.end_ms == 640 && !late.has_audio_position, + "millisecond safeguard rejects unbounded coordinates from other producers"); + const auto point = bound_word_time(2, 2, 80.0, 16000, 16000); + require(point.has_audio_position, "valid zero-width ITN spans still have an audio position"); + const auto fraction = bound_word_time(0, 2, 80.0, 1001, 44100); + require(fraction.end_ms == 22, "millisecond rounding must not exceed actual source duration"); + const auto large = bound_word_time( + std::numeric_limits::max() - 1, std::numeric_limits::max(), 80.0, 16000, + 16000); + require( + large.start_ms == 1000 && large.end_ms == 1000 && !large.has_audio_position, + "large model positions cannot overflow exported milliseconds"); +} + +static void +check_result(const Result& result, double received, bool eof) { + require( + result.audio_processed >= 0 && result.audio_processed <= received + 1e-5, + "processed audio exceeds received real samples"); + if (eof) + require( + std::fabs(result.audio_processed - received) < 1e-5, "EOF must report full duration"); + require( + result.audio_extent_ms >= 0 && result.audio_extent_ms <= received * 1000.0 + 1e-6, + "sample-derived display extent exceeds input"); + for (const auto& alt : result.alternatives) { + int32_t previous = 0; + for (const auto& word : alt.words) { + require( + word.start_time >= previous && word.end_time >= word.start_time && + word.end_time <= received * 1000.0 + 1e-6, + "word range outside real audio"); + previous = word.start_time; + if (!word.has_audio_position) + require(word.speaker_tag == 0, "EOS-backfilled words must remain unattributed"); + } + } +} + +static void +check_stream(Recognizer& rec, const std::vector& audio, int rate, size_t packet) { + AsrRequestOptions opts; + opts.enable_word_time_offsets = rec.diar_model() == nullptr; + opts.enable_speaker_diarization = rec.diar_model() != nullptr; + auto stream = rec.streaming_recognize(opts, "auto"); + stream->set_interim_words(true); + float last = 0; + for (size_t off = 0; off < audio.size(); off += packet) { + const size_t n = std::min(packet, audio.size() - off); + stream->push(audio.data() + off, n, rate); + int drains = 0; + while (auto result = stream->next()) { + require(++drains < 100, "stream drain must terminate"); + check_result(*result, double(off + n) / rate, false); + require(result->audio_processed >= last, "decoded progress must be monotonic"); + last = result->audio_processed; + } + } + auto final = stream->finish(); + check_result(final, double(audio.size()) / rate, true); + if (rec.config().streaming.rnnt_right_context == 13 && audio.size() == size_t(rate * 0.64)) { + require( + !final.alternatives.empty() && !final.alternatives.front().words.empty(), + "0.64s JFK/R13 regression must not pass with an empty transcript"); + for (const auto& word : final.alternatives.front().words) + require( + word.end_time > word.start_time && word.end_time <= 640, + "JFK/R13 EOF words must have positive, in-recording intervals"); + } + stream->refresh_speaker_tags(final); + check_result(final, double(audio.size()) / rate, true); + check_result(stream->finish(), double(audio.size()) / rate, true); + bool rejected = false; + const float sample = 0.0f; + try { + stream->push(&sample, 1, rate); + } + catch (const std::invalid_argument&) { + rejected = true; + } + require(rejected, "closed input extent cannot grow after EOF"); +} + +static void +check_model(int argc, char** argv) { + std::vector audio; + int sr = 0; + require(read_wav_mono_16k(argv[2], audio, sr), "cannot read fixture WAV"); + RecognizerConfig base; + base.model.path = argv[1]; + base.backend.gpu = argc >= 4 ? std::stoi(argv[3]) : -1; + base.log_status = false; + base.batching.enabled = false; + if (argc >= 5) + base.diar.model_path = argv[4]; + if (argc >= 6) { + base.vad.model_path = argv[5]; + base.endpointing.enable = true; + base.endpointing.vad_based = true; + } + for (int right : {0, 3, 13}) { + auto cfg = base; + cfg.streaming.rnnt_right_context = right; + Recognizer rec(cfg); + if (right == 13) { + auto* model = static_cast(rec.model()); + CacheStreamRunner runner(model, cfg); + runner.set_prompt_index(model->prompt_index_for_lang("auto")); + AsrRequestOptions opts; + opts.enable_word_time_offsets = true; + runner.set_request_options(opts); + const size_t count = size_t(sr * 0.64); + require(audio.size() >= count, "JFK fixture must contain at least 0.64s"); + runner.feed_audio(audio.data(), count); + runner.step(); + const auto result = runner.finalize(); + require(!result.words.empty(), "raw runner JFK/R13 must preserve tail text"); + const int64_t mels = (count - model->fe().n_fft() / 2) / model->fe().hop_length() + 1; + const int64_t frames = (mels - 1) / model->encoder_config().subsampling_factor + 1; + for (const auto& word : result.words) + require( + word.start_frame >= 0 && word.end_frame > word.start_frame && + word.end_frame <= frames, + "raw runner must bound EOF words before any postprocessing"); + require( + std::any_of( + result.words.begin(), result.words.end(), + [](const auto& w) { return !w.has_audio_position; }), + "JFK/R13 must exercise EOS-backfill provenance"); + runner.reset(); + runner.feed_audio(audio.data(), count); + runner.step(); + const auto repeated = runner.finalize(); + require( + result.transcript_so_far == repeated.transcript_so_far && + result.words.size() == repeated.words.size(), + "reset preserves EOF text"); + for (size_t i = 0; i < result.words.size(); ++i) + require( + result.words[i].start_frame == repeated.words[i].start_frame && + result.words[i].end_frame == repeated.words[i].end_frame && + result.words[i].has_audio_position == repeated.words[i].has_audio_position, + "reset cannot leak timing bounds or provenance"); + } + for (double seconds : {0.0, 0.016, 0.05, 0.16, 0.625, 0.64, 0.66, 2.0, 11.0}) { + const size_t n = std::min(audio.size(), size_t(std::llround(seconds * sr))); + const std::vector clip(audio.begin(), audio.begin() + n); + for (size_t packet : {size_t(1279), size_t(2560), std::max(size_t(1), n)}) + check_stream(rec, clip, sr, packet); + } + if (right == 3) { + const auto resampled = + nemo_speech::audio::resample_audio(audio.data(), audio.size(), sr, 44100); + check_stream(rec, resampled, 44100, 7056); + } + std::printf("[PASS] R=%d SDK bounds/EOF/packets\n", right); + } + auto cfg = base; + cfg.streaming.rnnt_right_context = 3; + cfg.endpointing.enable = true; + cfg.log_status = false; + Recognizer rec(cfg); + auto twice = audio; + twice.insert(twice.end(), size_t(sr) * 2, 0.0f); + twice.insert(twice.end(), audio.begin(), audio.end()); + AsrRequestOptions opts; + opts.enable_word_time_offsets = true; + opts.enable_speaker_diarization = rec.diar_model() != nullptr; + auto stream = rec.streaming_recognize(opts, "auto"); + stream->push(twice.data(), twice.size(), sr); + const double duration = double(twice.size()) / sr; + int finals = 0, interims_after_final = 0, drains = 0; + float last = 0; + while (auto result = stream->next()) { + require(++drains < 100, "large-input drain must terminate"); + check_result(*result, duration, false); + require(result->audio_processed >= last, "EOU progress must not reset"); + if (result->is_final) { + ++finals; + require( + result->audio_processed < duration, + "mid-stream final must not report all queued audio"); + } else if (finals > 0) { + ++interims_after_final; + require( + result->audio_processed > last, + "remaining audio must produce fresh progress without push"); + } + last = result->audio_processed; + } + require( + finals > 0 && interims_after_final > 0, + "exercise finals then a trailing interim from one push"); + check_result(stream->finish(), duration, true); + std::puts("[PASS] EOU drain and result delivery"); +} + +static void +check_itn() { + std::vector words{{"two", 14, 15, 0.8f}, {"kilograms", 21, 22, 0.9f}}; + for (auto& word : words) bound_word_frames(word, 8); + postproc::update_word_timings( + words, "Token:\t0\ttwo kilograms\t0,12\t0,1\nWord:\t0\t2\t0\nWord:\t1\tkg\t0\n"); + require( + words.size() == 2 && words[0].word == "2" && words[1].word == "kg", + "ITN output order must survive equal EOF-clamped timestamps"); + for (const auto& word : words) + require( + !word.has_audio_position && word.start_frame == 7 && word.end_frame == 8, + "ITN exact matching and numeric collapse preserve EOS provenance"); + words = {{"abbr", 7, 8, 1.0f, false}}; + postproc::update_word_timings( + words, "Token:\t0\tabbr\t0,4\t0,1\nWord:\t0\tfirst\t0\nWord:\t1\tsecond\t0\n"); + require( + words.size() == 2 && words[0].start_frame == words[0].end_frame, + "ITN expansion can legitimately produce a zero-width frame span"); + for (const auto& word : words) + require( + !word.has_audio_position && word.start_frame >= 7 && word.end_frame <= 8, + "ITN expansion retains bounds and conservative provenance"); + words = {{"one", 5, 6, 1.0f}, {"two", 6, 7, 1.0f}, {"three", 7, 8, 1.0f, false}}; + postproc::update_word_timings( + words, "Token:\t0\tone two three\t0,13\t0,1\nWord:\t0\talpha\t0\nWord:\t1\tbeta\t0\n"); + require( + words.size() == 2 && words[0].has_audio_position && !words[1].has_audio_position, + "ITN regrouping only invalidates groups that include an EOS-only word"); +} + +int +main(int argc, char** argv) try { + check_projection(); + check_itn(); + if (argc >= 3) + check_model(argc, argv); + else + std::puts("[SKIP] SDK integration: supply model and WAV"); + std::puts("[PASS] result timing"); + return 0; +} +catch (const std::exception& e) { + std::fprintf(stderr, "[FAIL] %s\n", e.what()); + return 1; +} diff --git a/tests/cpp/common/test_subtitles.cpp b/tests/cpp/common/test_subtitles.cpp index 94fc9a15..a3a26383 100644 --- a/tests/cpp/common/test_subtitles.cpp +++ b/tests/cpp/common/test_subtitles.cpp @@ -154,6 +154,36 @@ test_speaker_turns() { require(turns[1].start_ms == 700 && turns[1].end_ms == 1300, "second speaker turn timing"); } +void +test_real_audio_bounds() { + const std::vector padding{{"And", 560, 640}, {"so", 560, 640}}; + const auto fallback = nemo_speech::subtitle::make_cues(padding, "And so", 640); + require( + fallback.size() == 1 && fallback[0].start_ms == 560 && fallback[0].end_ms == 640, + "backfilled EOF words retain their bounded interval"); + require(fallback[0].text == "And so", "padding-only text is not dropped"); + const auto markers = + nemo_speech::subtitle::make_cues({{"And", 640, 640}, {"so", 640, 640}}, "", 640); + require( + markers.size() == 1 && markers[0].text == "And so" && markers[0].end_ms == 640, + "zero-width endpoint markers preserve text"); + const auto partial = nemo_speech::subtitle::make_cues( + {{"Hello", 0, 300}, {"world.", 300, 640}, {"Late", 560, 640}}, "", 640); + std::string text; + for (const auto& cue : partial) { + require( + cue.start_ms >= 0 && cue.start_ms < cue.end_ms && cue.end_ms <= 640, + "subtitle display extensions cannot exceed the audio"); + text += cue.text; + } + require( + text.find("Hello") != std::string::npos && text.find("Late") != std::string::npos, + "mixed timed and late words are retained"); + require( + nemo_speech::subtitle::make_cues({}, "text", 0).empty(), + "empty audio cannot have a positive-duration cue"); +} + } // namespace int @@ -168,6 +198,7 @@ main() { test_duration_and_length_split_are_ordered(); test_fallback_text(); test_speaker_turns(); + test_real_audio_bounds(); std::cout << "subtitle tests passed\n"; return 0; } From d47bfad64b1c54f0a3f6f2f9af3339df4614cfe5 Mon Sep 17 00:00:00 2001 From: Hongyuan Du Date: Sun, 25 Oct 2026 16:05:14 +0800 Subject: [PATCH 2/2] test(asr): narrow follow-up to #46 timing regression coverage Withdraw the independent EOF timestamp, progress-semantic, ITN and subtitle changes from the original #78 revision. Keep only tests for the first/steady causal frontier and absolute RNNT coordinates across EOU and reset. No additional production regression beyond the VAD correction already merged in #46 has been established. Signed-off-by: Hongyuan Du --- app/live_transcript.cpp | 2 - app/transcribe.cpp | 8 +- docs/asr/configuration.md | 31 -- server/http/http_server.cpp | 5 +- src/asr/decoders/decoder.h | 3 - src/asr/postproc/itn_align.cpp | 60 ++-- src/asr/recognizer.cpp | 63 +--- src/asr/recognizer.h | 4 - src/asr/runner.cpp | 30 +- src/asr/runner.h | 5 +- src/asr/types.h | 11 +- src/asr/word_time.h | 66 ---- src/common/subtitles.cpp | 9 +- src/common/subtitles.h | 2 - tests/c/asr_c_smoke.c | 27 +- tests/cpp/asr/CMakeLists.txt | 10 - tests/cpp/asr/test_cache_stream_geometry.cpp | 29 +- tests/cpp/asr/test_decoders.cpp | 58 ++++ tests/cpp/asr/test_live_transcript.cpp | 10 - tests/cpp/asr/test_result_timing.cpp | 316 ------------------- tests/cpp/common/test_subtitles.cpp | 31 -- 21 files changed, 123 insertions(+), 657 deletions(-) delete mode 100644 src/asr/word_time.h delete mode 100644 tests/cpp/asr/test_result_timing.cpp diff --git a/app/live_transcript.cpp b/app/live_transcript.cpp index e9cd55a2..160be857 100644 --- a/app/live_transcript.cpp +++ b/app/live_transcript.cpp @@ -12,8 +12,6 @@ namespace { double speaker_ready_time(const asr::Word& word) { - if (!word.has_audio_position) - return 0.0; // No speaker window can make a padding-only position valid. return std::max(word.end_time / 1000.0, word.start_time / 1000.0 + 0.160); } diff --git a/app/transcribe.cpp b/app/transcribe.cpp index 4fb0ab0a..8ba4f1f8 100644 --- a/app/transcribe.cpp +++ b/app/transcribe.cpp @@ -48,7 +48,6 @@ struct Transcript { std::string target_language; float confidence = 0.0f; float audio_seconds = 0.0f; - int32_t audio_extent_ms = -1; std::vector languages; std::vector words; }; @@ -286,7 +285,6 @@ append_result(Transcript& transcript, const asr::Result& result) { } transcript.confidence = alternative.confidence; transcript.audio_seconds = std::max(transcript.audio_seconds, result.audio_processed); - transcript.audio_extent_ms = std::max(transcript.audio_extent_ms, result.audio_extent_ms); for (const auto& language : alternative.language_codes) if (std::find(transcript.languages.begin(), transcript.languages.end(), language) == transcript.languages.end()) @@ -306,7 +304,6 @@ transcribe_one(asr::Recognizer& recognizer, const Options& options, const fs::pa const asr::AsrRequestOptions request = make_request_options(options); Transcript transcript; - transcript.audio_extent_ms = asr::real_audio_extent_ms(audio.samples.size(), audio.sample_rate); if (!options.stream) { append_result( transcript, recognizer.recognize( @@ -509,8 +506,6 @@ transcribe_live(asr::Recognizer& recognizer, const Options& options) { emit_turns(presenter.finish(), final_audio_processed); transcript.audio_seconds = std::max( transcript.audio_seconds, captured_samples / static_cast(microphone.sample_rate())); - transcript.audio_extent_ms = - asr::real_audio_extent_ms(captured_samples, microphone.sample_rate()); terminal.stopped(transcript.audio_seconds); return transcript; } @@ -568,8 +563,7 @@ render(const Transcript& t, OutputFormat format, const fs::path& source) { if (vtt) output << "WEBVTT\n\n"; const auto cues = nemo_speech::subtitle::make_cues( - t.words, t.text, - t.audio_extent_ms >= 0 ? t.audio_extent_ms : static_cast(t.audio_seconds * 1000)); + t.words, t.text, static_cast(t.audio_seconds * 1000)); for (size_t i = 0; i < cues.size(); ++i) { if (!vtt) output << i + 1 << '\n'; diff --git a/docs/asr/configuration.md b/docs/asr/configuration.md index f1385c57..fb9878f7 100644 --- a/docs/asr/configuration.md +++ b/docs/asr/configuration.md @@ -222,37 +222,6 @@ together. For Riva-compatible gRPC clients, a `runtime_config["force_eou"] = "true"` message finalizes the current utterance once the audio received so far is decoded. -## Time coordinates and word timestamps - -`audio_processed` is the real decoded progress in seconds, not the amount of -audio already queued by the caller. Lookahead and synthetic EOS padding do not -advance it beyond real decoded audio. At end-of-stream it reports the complete -input duration, including when the input was resampled. Publishing an utterance -does not restart the absolute stream clock. Input cannot be appended after -`finish()`. - -Word timestamps are approximate decoder-frame positions, not forced alignment. -They keep the model's nominal frame grid; the causal first-chunk correction for -the decoded frontier must not be subtracted from every word. The cache-aware -runner bounds result copies to the real encoder-frame range before ITN or -diarization; it does not change decoding state or discard tail tokens. The -shared result builder additionally bounds milliseconds to original-rate input -samples, since the last encoder interval can straddle EOF. Subtitle exporters use -the same sample-derived integer-millisecond extent instead of independently -rounding the floating-point progress field. - -EOS padding may be needed to emit a real trailing word. If its nominal position -is entirely outside the real encoder range, its text is retained on the last -real encoder interval. Such backfilled words remain internally marked through -ITN, receive no speaker attribution, and do not wait for future speaker evidence. -This is an approximate display interval, **not** recovered phonetic alignment. -Partly overlapping spans retain their original onset. ITN can split an integer -frame interval into zero-width spans; these remain valid rather than being -artificially extended. Integer-millisecond rounding never extends past the input. - -Subtitle reading-time extensions also remain within the recording. A -zero-duration recording produces no positive-duration subtitle cue. - ## Postprocessing: profanity, ITN, PnC Postprocessing runs on the final transcript in this order: **profanity → ITN → diff --git a/server/http/http_server.cpp b/server/http/http_server.cpp index ce0060f4..c151d3b7 100644 --- a/server/http/http_server.cpp +++ b/server/http/http_server.cpp @@ -280,10 +280,7 @@ transcript_response( return alternative.transcript + "\n"; if (format == "srt" || format == "vtt") { const bool vtt = format == "vtt"; - const int audio_ms = - result.audio_extent_ms >= 0 - ? result.audio_extent_ms - : std::max(0, static_cast(std::round(result.audio_processed * 1000))); + int audio_ms = std::max(1, static_cast(std::round(result.audio_processed * 1000))); std::vector words; words.reserve(alternative.words.size()); for (const auto& word : alternative.words) diff --git a/src/asr/decoders/decoder.h b/src/asr/decoders/decoder.h index 3369959c..f48a239c 100644 --- a/src/asr/decoders/decoder.h +++ b/src/asr/decoders/decoder.h @@ -103,9 +103,6 @@ struct WordTiming { int64_t start_frame = 0; int64_t end_frame = 0; float confidence = 1.0f; - // Internal provenance: false when an EOS-only emission was moved onto - // real audio. Its bounded display interval must not imply speaker evidence. - bool has_audio_position = true; }; // One N-best hypothesis. Times use the same encoder-frame clock as WordTiming. diff --git a/src/asr/postproc/itn_align.cpp b/src/asr/postproc/itn_align.cpp index bb75398b..a6e9d598 100644 --- a/src/asr/postproc/itn_align.cpp +++ b/src/asr/postproc/itn_align.cpp @@ -67,12 +67,6 @@ average_confidence(const std::vector& timings, const std::vector(indices.size()); } -bool -have_audio_positions(const std::vector& timings, const std::vector& indices) { - return std::all_of( - indices.begin(), indices.end(), [&](int i) { return timings[i].has_audio_position; }); -} - std::vector split_ws(const std::string& s) { std::vector out; @@ -147,8 +141,7 @@ update_word_timings(std::vector& timings, const std::string& alignme if (tokens.empty()) return; - std::vector> result; - size_t output_order = 0; + std::vector result; int cursor = 0; const int n = static_cast(timings.size()); @@ -171,8 +164,6 @@ update_word_timings(std::vector& timings, const std::string& alignme continue; // no written words, or no spoken span matched (skip, like riva) const auto& outs = it->second; - const size_t output_base = output_order; - output_order += outs.size(); std::vector spoken_used(spoken.size(), false); std::vector output_used(outs.size(), false); @@ -182,7 +173,7 @@ update_word_timings(std::vector& timings, const std::string& alignme continue; WordTiming exact = timings[spoken[si]]; exact.word = outs[oi]; - result.emplace_back(output_base + oi, std::move(exact)); + result.push_back(std::move(exact)); spoken_used[si] = true; output_used[oi] = true; break; @@ -201,13 +192,10 @@ update_word_timings(std::vector& timings, const std::string& alignme continue; if (remaining_output.size() == 1 && starts_with_digit(outs[remaining_output.front()])) { - result.emplace_back( - output_base + remaining_output.front(), - WordTiming{ - outs[remaining_output.front()], timings[remaining_spoken.front()].start_frame, - timings[remaining_spoken.back()].end_frame, - average_confidence(timings, remaining_spoken), - have_audio_positions(timings, remaining_spoken)}); + result.push_back( + {outs[remaining_output.front()], timings[remaining_spoken.front()].start_frame, + timings[remaining_spoken.back()].end_frame, + average_confidence(timings, remaining_spoken)}); continue; } @@ -219,15 +207,13 @@ update_word_timings(std::vector& timings, const std::string& alignme const int64_t span = end - start; const float confidence = average_confidence(timings, remaining_spoken); for (size_t i = 0; i < remaining_output.size(); ++i) { - result.emplace_back( - output_base + remaining_output[i], - WordTiming{ - outs[remaining_output[i]], - start + span * static_cast(i) / - static_cast(remaining_output.size()), - start + span * static_cast(i + 1) / - static_cast(remaining_output.size()), - confidence, have_audio_positions(timings, remaining_spoken)}); + result.push_back( + {outs[remaining_output[i]], + start + span * static_cast(i) / + static_cast(remaining_output.size()), + start + span * static_cast(i + 1) / + static_cast(remaining_output.size()), + confidence}); } continue; } @@ -239,26 +225,18 @@ update_word_timings(std::vector& timings, const std::string& alignme const size_t take = base + (i < extra ? 1 : 0); std::vector slice( remaining_spoken.begin() + offset, remaining_spoken.begin() + offset + take); - result.emplace_back( - output_base + remaining_output[i], - WordTiming{ - outs[remaining_output[i]], timings[slice.front()].start_frame, - timings[slice.back()].end_frame, average_confidence(timings, slice), - have_audio_positions(timings, slice)}); + result.push_back( + {outs[remaining_output[i]], timings[slice.front()].start_frame, + timings[slice.back()].end_frame, average_confidence(timings, slice)}); offset += take; } } if (!result.empty()) { - // EOS backfill can give distinct words identical frame positions. - // Keep the written order for ties, not the order in which matches were found. - std::sort(result.begin(), result.end(), [](const auto& a, const auto& b) { - if (a.second.start_frame != b.second.start_frame) - return a.second.start_frame < b.second.start_frame; - return a.first < b.first; + std::sort(result.begin(), result.end(), [](const WordTiming& a, const WordTiming& b) { + return a.start_frame < b.start_frame; }); - timings.clear(); - for (auto& entry : result) timings.push_back(std::move(entry.second)); + timings = std::move(result); } } diff --git a/src/asr/recognizer.cpp b/src/asr/recognizer.cpp index 0cfb7d8c..ce9bdb28 100644 --- a/src/asr/recognizer.cpp +++ b/src/asr/recognizer.cpp @@ -355,8 +355,6 @@ RecognitionStream::~RecognitionStream() { void RecognitionStream::push(const float* samples, size_t n, int sample_rate) { - if (input_finished_) - throw std::invalid_argument("cannot push audio after finish"); if (n == 0) return; if (samples == nullptr) @@ -373,7 +371,6 @@ RecognitionStream::push(const float* samples, size_t n, int sample_rate) { } else if (rate != input_sample_rate_) { throw std::invalid_argument("input sample rate cannot change within a stream"); } - input_samples_received_ += n; if (coordinate_ingress_) pending_cohort_target_ = recognizer_->arrive_streaming_ingress(); // The diarizer consumes the same model-rate audio the runner does, so it @@ -404,12 +401,7 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const r.is_final = is_final; r.stability = is_final ? 1.0f : 0.0f; r.channel_tag = 1; - r.audio_extent_ms = real_audio_extent_ms(input_samples_received_, input_sample_rate_); - const double received_sec = - input_sample_rate_ > 0 ? static_cast(input_samples_received_) / input_sample_rate_ - : 0.0; - r.audio_processed = static_cast( - input_finished_ ? received_sec : std::min(u.audio_processed_sec, received_sec)); + r.audio_processed = u.audio_processed_sec; if (!is_final && !interim_words_) { // Interim results contain only the raw top hypothesis. @@ -423,6 +415,7 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const // Detected languages are per-stream (shared by every alternative). const auto langs = runner_->detected_languages(); const bool want_off = opts_.needs_word_timings(); + const double ms = recognizer_->ms_per_enc_frame(); const std::string lang0 = langs.empty() ? std::string() : langs.front(); std::string postproc_language = opts_.language_code; std::string normalized_requested = postproc_language; @@ -454,10 +447,8 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const continue; Word ww; ww.word = w.word; - const auto time = word_time_(w); - ww.start_time = time.start_ms; - ww.end_time = time.end_ms; - ww.has_audio_position = time.has_audio_position; + ww.start_time = static_cast(w.start_frame * ms + 0.5); + ww.end_time = static_cast(w.end_frame * ms + 0.5); ww.confidence = w.confidence; ww.language_code = lang0; alt.words.push_back(std::move(ww)); @@ -488,13 +479,11 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const for (const auto& w : words) { Word ww; ww.word = w.word; - const auto time = word_time_(w); - ww.start_time = time.start_ms; - ww.end_time = time.end_ms; - ww.has_audio_position = time.has_audio_position; + ww.start_time = static_cast(w.start_frame * ms + 0.5); + ww.end_time = static_cast(w.end_frame * ms + 0.5); ww.confidence = w.confidence; ww.language_code = lang0; - if (tag_speakers && diar_ && ww.has_audio_position) { + if (tag_speakers && diar_) { // Transducer punctuation can extend a word timestamp into // the next turn. Anchor attribution to the word onset and // average two diar frames to reject single-frame noise. @@ -520,25 +509,11 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const return r; } -BoundedWordTime -RecognitionStream::word_time_(const WordTiming& word) const { - auto time = bound_word_time( - word.start_frame, word.end_frame, recognizer_->ms_per_enc_frame(), input_samples_received_, - input_sample_rate_); - time.has_audio_position = time.has_audio_position && word.has_audio_position; - return time; -} - void RecognitionStream::flush_diar_deficit_(const StreamingUpdate& u) { if (!diar_ || u.words.empty()) return; - double end_sec = 0.0; - for (const auto& word : u.words) { - const auto time = word_time_(word); - if (time.has_audio_position) - end_sec = std::max(end_sec, time.end_ms / 1000.0); - } + const double end_sec = u.words.back().end_frame * recognizer_->ms_per_enc_frame() / 1000.0; const auto target = static_cast(std::ceil(end_sec / diar_->seconds_per_frame())); if (target > diar_->n_frames()) { const int64_t before = diar_->n_frames(); @@ -563,16 +538,13 @@ RecognitionStream::next() { last_processed_sec_ = std::max(last_processed_sec_, u.audio_processed_sec); auto result = build_result_(u, /*is_final=*/true); if (diar_ && !u.words.empty()) { - // Retain the last observed word's attribution window; a nominal - // padding-only position must not evict real speech history. - for (auto it = u.words.rbegin(); it != u.words.rend(); ++it) { - const auto time = word_time_(*it); - if (time.has_audio_position) { - diar_->hold_frames_from(static_cast( - (time.start_ms / 1000.0) / diar_->seconds_per_frame())); - break; - } - } + // This final's words are tagged and later words start at or after + // its last one. Not audio_processed_sec: the cache runner sets it + // to all pushed audio, which can hold utterances not decoded yet. + const double last_start_sec = + u.words.back().start_frame * recognizer_->ms_per_enc_frame() / 1000.0; + diar_->hold_frames_from( + static_cast(last_start_sec / diar_->seconds_per_frame())); } result.late_punctuation = std::exchange(late_punctuation_, {}); return result; @@ -596,7 +568,6 @@ RecognitionStream::next() { Result RecognitionStream::finish() { - input_finished_ = true; const ScopedBatchCohort cohort_scope( pending_cohort_target_ > 0 ? pending_cohort_target_ : current_batch_cohort_target()); pending_cohort_target_ = 0; @@ -628,10 +599,6 @@ RecognitionStream::refresh_speaker_tags(Result& result) const { if (!diar_ || result.alternatives.empty()) return; for (auto& word : result.alternatives.front().words) { - if (!word.has_audio_position) { - word.speaker_tag = 0; - continue; - } const int speaker = diar_->speaker_for_word_time(word.start_time / 1000.0, word.end_time / 1000.0); word.speaker_tag = speaker >= 0 ? speaker + 1 : 0; diff --git a/src/asr/recognizer.h b/src/asr/recognizer.h index 360dc00b..8b7add09 100644 --- a/src/asr/recognizer.h +++ b/src/asr/recognizer.h @@ -19,7 +19,6 @@ #include "runner.h" #include "runtime.h" #include "types.h" -#include "word_time.h" namespace nemo_speech::asr { @@ -98,7 +97,6 @@ class RecognitionStream { private: Result build_result_(const StreamingUpdate& u, bool is_final) const; - BoundedWordTime word_time_(const WordTiming& word) const; // Extends diarization to cover all words in a final result. void flush_diar_deficit_(const StreamingUpdate& u); @@ -108,11 +106,9 @@ class RecognitionStream { std::unique_ptr diar_; AsrRequestOptions opts_; int input_sample_rate_ = 0; - uint64_t input_samples_received_ = 0; // Original rate; never counts padding. std::unique_ptr resampler_; std::vector resampled_audio_; bool resampler_flushed_ = false; - bool input_finished_ = false; bool coordinate_ingress_ = true; int pending_cohort_target_ = 0; // High-water mark of audio_processed_sec returned by next(). A step that diff --git a/src/asr/runner.cpp b/src/asr/runner.cpp index 7faf3430..74c63ca6 100644 --- a/src/asr/runner.cpp +++ b/src/asr/runner.cpp @@ -16,7 +16,6 @@ #include "greedy_ctc_decoder.h" #include "recognizer.h" -#include "word_time.h" #ifdef NEMO_SPEECH_WITH_FLASHLIGHT #include "flashlight_decoder.h" #endif @@ -973,22 +972,6 @@ CacheStreamRunner::process_one_chunk() { } } -int64_t -CacheStreamRunner::decoded_mel_frames() const { - return std::min( - total_mel_frames_produced_, enc_cfg_.cache_decoded_mel_frames(total_frames_emitted_)); -} - -void -CacheStreamRunner::bound_word_timings(StreamingUpdate& update) const { - const int64_t frames = total_mel_frames_produced_ > 0 - ? (total_mel_frames_produced_ - 1) / enc_cfg_.subsampling_factor + 1 - : 0; - for (auto& word : update.words) bound_word_frames(word, frames); - for (auto& alternative : update.extra_alternatives) - for (auto& word : alternative.words) bound_word_frames(word, frames); -} - StreamingUpdate CacheStreamRunner::step() { StreamingUpdate update; @@ -1048,12 +1031,8 @@ CacheStreamRunner::step() { // per-utterance buffers are already cleared. if (!update.is_final) update.transcript_so_far = transcript_; - // Received audio can include lookahead or whole later utterances. Report - // only real decoded coverage, consistently with BufferedStreamRunner. - const size_t decoded_samples = - std::min(audio_end, static_cast(decoded_mel_frames()) * model_->fe().hop_length()); update.audio_processed_sec = - static_cast(decoded_samples) / static_cast(model_->fe_config().sample_rate); + static_cast(audio_end) / static_cast(model_->fe_config().sample_rate); // A pending force_eou() must fire even on a chunk-less step. if (!update.is_final) poll_endpoint(update, /*after_chunk=*/false); @@ -1061,7 +1040,6 @@ CacheStreamRunner::step() { update.words = head_->word_timings(); if (head_) update.late_punctuation = head_->take_late_punctuation(); - bound_word_timings(update); trim_buffers(); return update; } @@ -1087,7 +1065,7 @@ CacheStreamRunner::poll_endpoint(StreamingUpdate& update, bool after_chunk) { // frames. Causal encoder frame f ends at mel f*sub (inclusive), so // the shorter first chunk shifts the exclusive frontier by sub-1. const double mel_ms = 1000.0 * hop / sample_rate; - const int64_t decoded_mel = decoded_mel_frames(); + const int64_t decoded_mel = enc_cfg_.cache_decoded_mel_frames(total_frames_emitted_); scan_vad_speech(*vad_, decoded_mel, vad_scan_frame_, vad_speech_seen_frame_); now_ms = static_cast(decoded_mel) * mel_ms; last_speech_ms = (vad_speech_seen_frame_ < 0) @@ -1139,7 +1117,7 @@ CacheStreamRunner::trim_buffers() { // masker's neighbourhood scans never cross the trimmed base. const int64_t margin = std::max(256, vad_masker_ ? 2 * vad_masker_->max_scan() : 0); - const int64_t decoded_mel = decoded_mel_frames(); + const int64_t decoded_mel = enc_cfg_.cache_decoded_mel_frames(total_frames_emitted_); const int64_t g = decoded_mel - margin; if (g > 0) vad_->discard_timeline_before(g); @@ -1155,7 +1133,6 @@ CacheStreamRunner::finalize() { static_cast(audio_base_ + audio_buf_.size()) / static_cast(model_->fe_config().sample_rate), opts_.max_alternatives); - bound_word_timings(update); return update; } finalizing_ = true; @@ -1218,7 +1195,6 @@ CacheStreamRunner::finalize() { opts_.max_alternatives); if (head_) update.late_punctuation += head_->take_late_punctuation(/*end_of_stream=*/true); - bound_word_timings(update); return update; } diff --git a/src/asr/runner.h b/src/asr/runner.h index d7df14a3..dc33e224 100644 --- a/src/asr/runner.h +++ b/src/asr/runner.h @@ -61,8 +61,7 @@ struct StreamingUpdate { bool is_final = false; // Mean token posterior for greedy CTC; 1.0 when the decoder has no posterior. float confidence = 1.0f; - // Real decoded progress, excluding lookahead and synthetic EOS padding. - // finalize() reports the complete real input duration. + // Seconds of audio processed. float audio_processed_sec = 0.0f; // Punctuation for the previous final (Decoder::take_late_punctuation); // not part of this update's text. @@ -281,8 +280,6 @@ class CacheStreamRunner final : public AsrRunner { private: int next_chunk_mel_frames() const; - int64_t decoded_mel_frames() const; - void bound_word_timings(StreamingUpdate& update) const; void process_one_chunk(); void upload_attn_mask(); void zero_caches(); diff --git a/src/asr/types.h b/src/asr/types.h index da594f5d..d5597f29 100644 --- a/src/asr/types.h +++ b/src/asr/types.h @@ -50,9 +50,6 @@ struct Word { // proto: WordInfo // proto WordInfo.speaker_tag: 1-based speaker id, set only when // diarization was requested (0 = untagged). int32_t speaker_tag = 0; - // Internal attribution guard. EOS-only emissions can have a bounded - // display interval without evidence for assigning a speaker at that time. - bool has_audio_position = true; }; struct Alternative { // proto: SpeechRecognitionAlternative @@ -67,12 +64,8 @@ struct Result { // proto: Streaming/SpeechRecognitionResult std::vector alternatives; bool is_final = true; float stability = 0.0f; - int32_t channel_tag = 1; // mono channel - // Decoded progress in seconds; the complete input duration at EOF. - float audio_processed = 0.0f; // proto audio_processed - // Internal, sample-derived input extent for display bounds. Unlike decoded - // progress it can include queued audio. -1 means unavailable to a producer. - int32_t audio_extent_ms = -1; + int32_t channel_tag = 1; // mono channel + float audio_processed = 0.0f; // seconds (proto audio_processed) // Streaming only: sentence punctuation emitted after the previous final's // endpoint. Append it to that final; it is not part of this result's text. std::string late_punctuation; diff --git a/src/asr/word_time.h b/src/asr/word_time.h deleted file mode 100644 index 0e608643..00000000 --- a/src/asr/word_time.h +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -#pragma once - -#include -#include -#include -#include - -#include "decoder.h" - -namespace nemo_speech::asr { - -struct BoundedWordTime { - int32_t start_ms = 0; - int32_t end_ms = 0; - bool has_audio_position = false; -}; - -// Project a result copy, never the decoder's frame clock or recurrent state. -// The exclusive frame limit counts causal outputs covering real Mel frames. -inline void -bound_word_frames(WordTiming& word, int64_t frame_limit) { - word.has_audio_position = word.has_audio_position && frame_limit > 0 && - word.start_frame < frame_limit && word.end_frame > 0; - if (frame_limit <= 0) { - word.start_frame = word.end_frame = 0; - return; - } - word.start_frame = std::clamp(word.start_frame, 0, frame_limit - 1); - word.end_frame = std::clamp(word.end_frame, word.start_frame + 1, frame_limit); -} - -inline int32_t -real_audio_extent_ms(uint64_t samples, int rate) { - if (rate <= 0) - return 0; - const long double ms = static_cast(samples) * 1000.0L / rate; - return static_cast( - std::min(ms, static_cast(std::numeric_limits::max()))); -} - -// The final encoder interval can straddle EOF even after frame bounding. -// Bound milliseconds to original-rate input samples too. Preserve legitimate -// zero-width ITN spans; this projection does not invent phonetic alignment. -inline BoundedWordTime -bound_word_time( - int64_t start_frame, int64_t end_frame, double frame_ms, uint64_t input_samples, - int input_rate) { - if (input_rate <= 0 || input_samples == 0 || frame_ms <= 0.0 || !std::isfinite(frame_ms)) - return {}; - const double audio_end = - static_cast(static_cast(input_samples) * 1000.0L / input_rate); - const double limit = real_audio_extent_ms(input_samples, input_rate); - const double start = start_frame * frame_ms; - const double end = end_frame * frame_ms; - const auto project = [limit](double t) { - return static_cast(std::clamp(std::floor(t + 0.5), 0.0, limit)); - }; - const int32_t bounded_start = project(start); - return { - bounded_start, std::max(bounded_start, project(end)), - end_frame >= start_frame && start < audio_end && (end > 0.0 || start >= 0.0)}; -} - -} // namespace nemo_speech::asr diff --git a/src/common/subtitles.cpp b/src/common/subtitles.cpp index ff4617f4..893f9af8 100644 --- a/src/common/subtitles.cpp +++ b/src/common/subtitles.cpp @@ -168,12 +168,10 @@ make_speaker_turns(const std::vector& words) { std::vector make_cues(const std::vector& words, const std::string& fallback_text, int audio_ms) { - if (audio_ms <= 0) - return {}; if (words.empty()) { if (fallback_text.empty()) return {}; - return {{0, audio_ms, fallback_text}}; + return {{0, std::max(1, audio_ms), fallback_text}}; } std::vector groups; @@ -219,9 +217,8 @@ make_cues(const std::vector& words, const std::string& fallback_text, int cues.reserve(groups.size()); for (size_t i = 0; i < groups.size(); ++i) { auto cue = groups[i].cue; - cue.start_ms = std::clamp(cue.start_ms, 0, audio_ms - 1); const int available_end = - i + 1 < groups.size() ? std::min(audio_ms, groups[i + 1].cue.start_ms) : audio_ms; + i + 1 < groups.size() ? groups[i + 1].cue.start_ms : std::max(cue.end_ms, audio_ms); const int words_in_cue = static_cast(groups[i].end - groups[i].begin); const int readable_ms = std::max( kMinimumCueDurationMs, (words_in_cue * 60000 + kWordsPerMinute - 1) / kWordsPerMinute); @@ -230,7 +227,7 @@ make_cues(const std::vector& words, const std::string& fallback_text, int std::max(cue.end_ms, desired_end), std::min(available_end, cue.start_ms + kMaximumCueDurationMs)); if (cue.end_ms <= cue.start_ms) - cue.end_ms = std::min(audio_ms, cue.start_ms + 1); + cue.end_ms = cue.start_ms + 1; cues.push_back(std::move(cue)); } return cues; diff --git a/src/common/subtitles.h b/src/common/subtitles.h index b22e89d9..31c92daa 100644 --- a/src/common/subtitles.h +++ b/src/common/subtitles.h @@ -34,8 +34,6 @@ bool attaches_to_previous(const std::string& text); std::vector make_speaker_turns(const std::vector& words); -// Input word intervals are already bounded by the recognizer. Reading-time -// extensions must stay inside audio_ms too; zero-duration audio yields no cues. std::vector make_cues( const std::vector& words, const std::string& fallback_text, int audio_ms); diff --git a/tests/c/asr_c_smoke.c b/tests/c/asr_c_smoke.c index 5f66182a..965d217a 100644 --- a/tests/c/asr_c_smoke.c +++ b/tests/c/asr_c_smoke.c @@ -45,26 +45,6 @@ read_wav_f32(const char* path, size_t* n_out) { return out; } -static int -check_times(const nemo_speech_asr_result* result, size_t real_samples) { - const double duration_ms = real_samples / 16.0; - const float progress = nemo_speech_asr_result_audio_processed(result); - if (progress < 0 || progress > real_samples / 16000.0 + 1e-5) { - fprintf(stderr, "C API progress exceeds the real input\n"); - return 0; - } - const size_t count = nemo_speech_asr_result_word_count(result, 0); - for (size_t i = 0; i < count; ++i) { - const int32_t start = nemo_speech_asr_result_word_start_time(result, 0, i); - const int32_t end = nemo_speech_asr_result_word_end_time(result, 0, i); - if (start < 0 || end < start || end > duration_ms) { - fprintf(stderr, "C API word timestamp exceeds the real input\n"); - return 0; - } - } - return 1; -} - int main(int argc, char** argv) { printf("nemo_speech_asr_version: %s\n", nemo_speech_asr_version()); @@ -127,7 +107,6 @@ main(int argc, char** argv) { } printf("offline: %s\n", nemo_speech_asr_result_transcript(r, 0)); printf("offline words: %zu\n", nemo_speech_asr_result_word_count(r, 0)); - int timing_ok = check_times(r, n); nemo_speech_asr_result_destroy(r); /* ---- streaming (160 ms chunks) ---- */ @@ -161,7 +140,6 @@ main(int argc, char** argv) { nemo_speech_asr_stream_push_f32(st, audio + pos, cn, 16000); nemo_speech_asr_result* sr = NULL; while (nemo_speech_asr_stream_next(st, &sr) == NEMO_SPEECH_ASR_OK && sr) { - timing_ok &= check_times(sr, pos + cn); if (nemo_speech_asr_result_is_final(sr)) { strncat( streamed, nemo_speech_asr_result_transcript(sr, 0), @@ -174,7 +152,6 @@ main(int argc, char** argv) { nemo_speech_asr_stream_finish(st); nemo_speech_asr_result* sr = NULL; while (nemo_speech_asr_stream_next(st, &sr) == NEMO_SPEECH_ASR_OK && sr) { - timing_ok &= check_times(sr, n); if (nemo_speech_asr_result_is_final(sr)) { strncat( streamed, nemo_speech_asr_result_transcript(sr, 0), @@ -188,6 +165,6 @@ main(int argc, char** argv) { free(audio); nemo_speech_asr_destroy(rec); - printf(timing_ok ? "[C ABI smoke OK]\n" : "[C ABI timing FAILED]\n"); - return timing_ok ? 0 : 1; + printf("[C ABI smoke OK]\n"); + return 0; } diff --git a/tests/cpp/asr/CMakeLists.txt b/tests/cpp/asr/CMakeLists.txt index 6dffddd1..4f072841 100644 --- a/tests/cpp/asr/CMakeLists.txt +++ b/tests/cpp/asr/CMakeLists.txt @@ -40,16 +40,6 @@ add_executable(test_postproc test_postproc.cpp) target_link_libraries(test_postproc PRIVATE nemo_speech_asr Threads::Threads) add_test(NAME asr_postprocessing COMMAND test_postproc) -add_executable(test_result_timing test_result_timing.cpp) -target_link_libraries(test_result_timing PRIVATE nemo_speech_asr) -set(result_timing_args) -if(DEFINED ENV{NEMO_SPEECH_TEST_CACHE_AWARE_RNNT_MODEL} AND - NOT "$ENV{NEMO_SPEECH_TEST_CACHE_AWARE_RNNT_MODEL}" STREQUAL "") - list(APPEND result_timing_args "$ENV{NEMO_SPEECH_TEST_CACHE_AWARE_RNNT_MODEL}" - "${CMAKE_SOURCE_DIR}/test_files/asr/wav/test/jfk.wav") -endif() -add_test(NAME result_timing COMMAND test_result_timing ${result_timing_args}) - add_executable(test_audio_resampler test_audio_resampler.cpp) target_link_libraries(test_audio_resampler PRIVATE nemo_speech_common) diff --git a/tests/cpp/asr/test_cache_stream_geometry.cpp b/tests/cpp/asr/test_cache_stream_geometry.cpp index 1e451bd6..25dd9a41 100644 --- a/tests/cpp/asr/test_cache_stream_geometry.cpp +++ b/tests/cpp/asr/test_cache_stream_geometry.cpp @@ -94,16 +94,10 @@ check_model(const std::string& path, int gpu, int right) { CacheStreamRunner runner(model, cfg); runner.set_prompt_index(prompt); runner.feed_audio(audio.data(), first_samples - 1); - require( - runner.step().audio_processed_sec == 0.0f, - "received lookahead must not count as decoded progress"); + runner.step(); require(runner.chunks_processed() == 0, "first chunk ran before its last stable mel frame"); runner.feed_audio(audio.data() + first_samples - 1, 1); - const auto first_update = runner.step(); - require( - std::fabs(first_update.audio_processed_sec - first_mel * hop / float(fe.sample_rate)) < - 1e-5f, - "first decoded progress must use the causal mel frontier"); + runner.step(); require( runner.chunks_processed() == 1, "first chunk incorrectly waits for steady-state length"); @@ -161,6 +155,13 @@ check_model(const std::string& path, int gpu, int right) { runner.take_encoder_frames(actual, actual_t, dim); require(actual_t == expected_t, "wrong runner encoder frame count"); require(runner.chunks_processed() == step + 1, "wrong chunk consumption"); + const auto emitted = static_cast(runner.rnnt_decode_stats().encoder_frames); + require( + emitted == static_cast(step + 1) * (right + 1), + "first/steady transition changed the absolute decoder frame offset"); + require( + enc.cache_decoded_mel_frames(emitted) == consumed + fresh, + "decoded frontier differs from independently assembled real Mel coverage"); same_encoder(actual, expected); if (step == 0) first_output = actual; @@ -168,6 +169,9 @@ check_model(const std::string& path, int gpu, int right) { cached = std::min(enc.cache_left_ctx, cached + expected_t); } runner.reset(); + require( + runner.rnnt_decode_stats().encoder_frames == 0, + "new stream retained the previous decoder frame clock"); // Preserve FE packet shapes as well: Metal FFT rounding can differ across // packet shapes independently of the encoder's chunk geometry. runner.feed_audio(audio.data(), first_samples - 1); @@ -193,9 +197,6 @@ check_model(const std::string& path, int gpu, int right) { } } const auto final = runner.finalize(); - require( - std::fabs(final.audio_processed_sec - size / float(fe.sample_rate)) < 1e-5f, - "EOS padding must not inflate final audio duration"); ids.insert(ids.end(), final.new_token_ids.begin(), final.new_token_ids.end()); require(final.is_final, "EOF not final"); require(runner.finalize().new_token_ids.empty(), "EOF duplicated tokens"); @@ -223,12 +224,18 @@ check_model(const std::string& path, int gpu, int right) { require(runner.cache_filled_frames() == right + 1, "EOU changed encoder cache"); const int before = runner.chunks_processed(); require(before == 1, "EOU synthesized extra chunks"); + require( + runner.rnnt_decode_stats().encoder_frames == static_cast(right + 1), + "EOU reset the absolute decoder clock after the first chunk"); runner.feed_audio(audio.data() + first_samples, stride_samples - 1); runner.step(); require(runner.chunks_processed() == before, "post-EOU reused first-chunk geometry"); runner.feed_audio(audio.data() + first_samples + stride_samples - 1, 1); runner.step(); require(runner.chunks_processed() == before + 1, "post-EOU steady chunk was not processed"); + require( + runner.rnnt_decode_stats().encoder_frames == static_cast(2 * (right + 1)), + "post-EOU decoder clock did not retain the first chunk offset"); std::vector output; int count = 0, dim = 0; runner.take_encoder_frames(output, count, dim); diff --git a/tests/cpp/asr/test_decoders.cpp b/tests/cpp/asr/test_decoders.cpp index e7fe5282..ad64b1eb 100644 --- a/tests/cpp/asr/test_decoders.cpp +++ b/tests/cpp/asr/test_decoders.cpp @@ -18,6 +18,7 @@ #include #include +#include "fastconformer.h" #include "greedy_ctc_decoder.h" #include "rnnt_greedy_decoder.h" @@ -675,6 +676,62 @@ test_rnnt_punctuation_word_timing() { check(ok, "rnnt: boundary-prefixed terminator stays with timed word"); } +void +test_rnnt_causal_startup_coordinates() { + // #46 shortened the first Mel chunk, not the decoder's emission-frame + // grid. Guard against applying the VAD frontier correction to word times, + // and against restarting the absolute frame offset after an EOU. + for (int right : {0, 3, 6, 13}) { + const auto cfg = make_cache_aware_config(EncoderConfig{}, right); + const int count = right + 1; + const int64_t first_mel = 1 + 8 * right; + MockRnntEngine eng; + RnntGreedyDecoder dec(&eng); + dec.set_compute_timestamps(true); + const int blank = eng.rnnt_config().blank_id; + std::vector last_only(count, blank); + last_only.back() = 0; + eng.rounds = {last_only, {blank}}; + std::vector frames(count * eng.rnnt_config().joint_dim, 0.0f); + const auto first = dec.step(frames.data(), eng.rnnt_config().joint_dim, count, 0); + dec.finalize(); + const auto words = dec.word_timings(); + check( + first == std::vector{0} && words.size() == 1, + "rnnt causal startup: control must emit a timed word"); + if (words.size() != 1) + continue; + check( + words[0].start_frame == right && words[0].end_frame == count, + "rnnt causal startup: first word keeps the emission-frame grid"); + check( + cfg.cache_decoded_mel_frames(count) == first_mel && + words[0].start_frame * 8 == first_mel - 1, + "rnnt causal startup: emission anchor and exclusive Mel frontier differ"); + dec.reset_utterance(); + eng.rounds = {{1}, {blank}}; + const auto next = dec.step(frames.data(), eng.rnnt_config().joint_dim, count, count); + dec.finalize(); + const auto& after_eou = dec.word_timings(); + check( + next == std::vector{1} && after_eou.size() == 1 && + after_eou[0].start_frame == count && after_eou[0].end_frame == count + 1 && + dec.last_speech_frame() == count, + "rnnt causal startup: EOU preserves absolute word and token-silence coordinates"); + check( + cfg.cache_decoded_mel_frames(2 * count) == first_mel + 8 * count, + "rnnt causal startup: second chunk advances by the steady-state stride"); + dec.reset(); + eng.rounds = {{2}, {blank}}; + dec.step(frames.data(), eng.rnnt_config().joint_dim, count, 0); + dec.finalize(); + const auto& reset = dec.word_timings(); + check( + reset.size() == 1 && reset[0].start_frame == 0 && reset[0].end_frame == 1, + "rnnt causal startup: only a new stream restarts word coordinates at zero"); + } +} + void test_rnnt_open_word_needs_finalize() { // The contract CacheStreamRunner::finalize() relies on: an open trailing @@ -818,6 +875,7 @@ main() { test_rnnt_endpoint_punctuation_continuity(); test_rnnt_state_threading_and_reset_utterance(); test_rnnt_word_timings(); + test_rnnt_causal_startup_coordinates(); test_rnnt_punctuation_word_timing(); test_rnnt_open_word_needs_finalize(); test_tdt_duration_and_state_commit(); diff --git a/tests/cpp/asr/test_live_transcript.cpp b/tests/cpp/asr/test_live_transcript.cpp index 2ae25142..bc20eb61 100644 --- a/tests/cpp/asr/test_live_transcript.cpp +++ b/tests/cpp/asr/test_live_transcript.cpp @@ -516,16 +516,6 @@ test_delayed_punctuation_in_revision_buffer() { int main() { - { - nemo_speech::cli::LiveDiarizationBuffer buffer; - auto late = word("Late", 560, 640, 0); - late.has_audio_position = false; - buffer.push(final_result(alternative("Late", {late}), 0.64f)); - const auto ready = buffer.update(0.01, [](Result&) {}); - check( - ready.size() == 1 && ready[0].alternatives[0].words[0].speaker_tag == 0, - "padding-only text does not wait for a nonexistent speaker window"); - } test_canonical_single_speaker_text(); test_silence_final_settles_without_more_speech(); test_punctuation_stays_with_previous_speaker(); diff --git a/tests/cpp/asr/test_result_timing.cpp b/tests/cpp/asr/test_result_timing.cpp deleted file mode 100644 index e3819429..00000000 --- a/tests/cpp/asr/test_result_timing.cpp +++ /dev/null @@ -1,316 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// No arguments: projection policy. Optional MODEL.gguf AUDIO.wav [GPU] [DIAR.gguf] [VAD.gguf] -// exercises actual SDK results, short EOF, input packets, resampling and EOU drain. -#include -#include -#include -#include -#include -#include -#include - -#include "itn_align.h" -#include "recognizer.h" -#include "word_time.h" - -using namespace nemo_speech::asr; - -static void -require(bool ok, const char* message) { - if (!ok) - throw std::runtime_error(message); -} - -static void -check_projection() { - WordTiming late_word{"late", 14, 15, 1.0f}; - bound_word_frames(late_word, 8); - require( - late_word.start_frame == 7 && late_word.end_frame == 8 && !late_word.has_audio_position, - "EOS text gets the last real interval, not speaker evidence"); - bound_word_frames(late_word, 8); - require(!late_word.has_audio_position, "projection must preserve provenance when repeated"); - WordTiming partial_word{"partial", 6, 15, 1.0f}; - bound_word_frames(partial_word, 8); - require( - partial_word.start_frame == 6 && partial_word.end_frame == 8 && - partial_word.has_audio_position, - "partly real word keeps its original onset"); - WordTiming empty_word{"empty", 0, 1, 1.0f}; - bound_word_frames(empty_word, 0); - require( - empty_word.start_frame == 0 && empty_word.end_frame == 0 && !empty_word.has_audio_position, - "empty input has no real frame to borrow"); - for (uint64_t samples = 256; samples <= 32000; samples += 17) { - const int64_t mels = (samples - 256) / 160 + 1; - const int64_t frames = (mels - 1) / 8 + 1; - WordTiming word{"tail", frames + 5, frames + 6, 1.0f}; - bound_word_frames(word, frames); - const auto time = bound_word_time(word.start_frame, word.end_frame, 80, samples, 16000); - require( - time.start_ms < time.end_ms && time.end_ms <= samples * 1000.0 / 16000, - "frame bounding plus sample bounding handles fractional tail frames"); - } - const auto empty = bound_word_time(0, 1, 80.0, 0, 16000); - require( - real_audio_extent_ms(28224, 44100) == 640 && real_audio_extent_ms(1001, 44100) == 22, - "all exporters must use the same sample-derived millisecond bound"); - require(empty.start_ms == 0 && empty.end_ms == 0 && !empty.has_audio_position, "empty bounds"); - const auto normal = bound_word_time(2, 4, 80.0, 16000, 16000); - const auto long_audio = bound_word_time(89999, 90000, 80.0, 7200ULL * 44100, 44100); - require( - long_audio.start_ms == 7199920 && long_audio.end_ms == 7200000, - "long recordings retain absolute time without per-chunk rounding drift"); - require( - normal.start_ms == 160 && normal.end_ms == 320 && normal.has_audio_position, - "valid nominal word times must not receive the decode-frontier offset"); - const auto partial = bound_word_time(7, 15, 80.0, 10240, 16000); - require( - partial.start_ms == 560 && partial.end_ms == 640 && partial.has_audio_position, - "partly observed word keeps its real interval"); - const auto late = bound_word_time(14, 15, 80.0, 10240, 16000); - require( - late.start_ms == 640 && late.end_ms == 640 && !late.has_audio_position, - "millisecond safeguard rejects unbounded coordinates from other producers"); - const auto point = bound_word_time(2, 2, 80.0, 16000, 16000); - require(point.has_audio_position, "valid zero-width ITN spans still have an audio position"); - const auto fraction = bound_word_time(0, 2, 80.0, 1001, 44100); - require(fraction.end_ms == 22, "millisecond rounding must not exceed actual source duration"); - const auto large = bound_word_time( - std::numeric_limits::max() - 1, std::numeric_limits::max(), 80.0, 16000, - 16000); - require( - large.start_ms == 1000 && large.end_ms == 1000 && !large.has_audio_position, - "large model positions cannot overflow exported milliseconds"); -} - -static void -check_result(const Result& result, double received, bool eof) { - require( - result.audio_processed >= 0 && result.audio_processed <= received + 1e-5, - "processed audio exceeds received real samples"); - if (eof) - require( - std::fabs(result.audio_processed - received) < 1e-5, "EOF must report full duration"); - require( - result.audio_extent_ms >= 0 && result.audio_extent_ms <= received * 1000.0 + 1e-6, - "sample-derived display extent exceeds input"); - for (const auto& alt : result.alternatives) { - int32_t previous = 0; - for (const auto& word : alt.words) { - require( - word.start_time >= previous && word.end_time >= word.start_time && - word.end_time <= received * 1000.0 + 1e-6, - "word range outside real audio"); - previous = word.start_time; - if (!word.has_audio_position) - require(word.speaker_tag == 0, "EOS-backfilled words must remain unattributed"); - } - } -} - -static void -check_stream(Recognizer& rec, const std::vector& audio, int rate, size_t packet) { - AsrRequestOptions opts; - opts.enable_word_time_offsets = rec.diar_model() == nullptr; - opts.enable_speaker_diarization = rec.diar_model() != nullptr; - auto stream = rec.streaming_recognize(opts, "auto"); - stream->set_interim_words(true); - float last = 0; - for (size_t off = 0; off < audio.size(); off += packet) { - const size_t n = std::min(packet, audio.size() - off); - stream->push(audio.data() + off, n, rate); - int drains = 0; - while (auto result = stream->next()) { - require(++drains < 100, "stream drain must terminate"); - check_result(*result, double(off + n) / rate, false); - require(result->audio_processed >= last, "decoded progress must be monotonic"); - last = result->audio_processed; - } - } - auto final = stream->finish(); - check_result(final, double(audio.size()) / rate, true); - if (rec.config().streaming.rnnt_right_context == 13 && audio.size() == size_t(rate * 0.64)) { - require( - !final.alternatives.empty() && !final.alternatives.front().words.empty(), - "0.64s JFK/R13 regression must not pass with an empty transcript"); - for (const auto& word : final.alternatives.front().words) - require( - word.end_time > word.start_time && word.end_time <= 640, - "JFK/R13 EOF words must have positive, in-recording intervals"); - } - stream->refresh_speaker_tags(final); - check_result(final, double(audio.size()) / rate, true); - check_result(stream->finish(), double(audio.size()) / rate, true); - bool rejected = false; - const float sample = 0.0f; - try { - stream->push(&sample, 1, rate); - } - catch (const std::invalid_argument&) { - rejected = true; - } - require(rejected, "closed input extent cannot grow after EOF"); -} - -static void -check_model(int argc, char** argv) { - std::vector audio; - int sr = 0; - require(read_wav_mono_16k(argv[2], audio, sr), "cannot read fixture WAV"); - RecognizerConfig base; - base.model.path = argv[1]; - base.backend.gpu = argc >= 4 ? std::stoi(argv[3]) : -1; - base.log_status = false; - base.batching.enabled = false; - if (argc >= 5) - base.diar.model_path = argv[4]; - if (argc >= 6) { - base.vad.model_path = argv[5]; - base.endpointing.enable = true; - base.endpointing.vad_based = true; - } - for (int right : {0, 3, 13}) { - auto cfg = base; - cfg.streaming.rnnt_right_context = right; - Recognizer rec(cfg); - if (right == 13) { - auto* model = static_cast(rec.model()); - CacheStreamRunner runner(model, cfg); - runner.set_prompt_index(model->prompt_index_for_lang("auto")); - AsrRequestOptions opts; - opts.enable_word_time_offsets = true; - runner.set_request_options(opts); - const size_t count = size_t(sr * 0.64); - require(audio.size() >= count, "JFK fixture must contain at least 0.64s"); - runner.feed_audio(audio.data(), count); - runner.step(); - const auto result = runner.finalize(); - require(!result.words.empty(), "raw runner JFK/R13 must preserve tail text"); - const int64_t mels = (count - model->fe().n_fft() / 2) / model->fe().hop_length() + 1; - const int64_t frames = (mels - 1) / model->encoder_config().subsampling_factor + 1; - for (const auto& word : result.words) - require( - word.start_frame >= 0 && word.end_frame > word.start_frame && - word.end_frame <= frames, - "raw runner must bound EOF words before any postprocessing"); - require( - std::any_of( - result.words.begin(), result.words.end(), - [](const auto& w) { return !w.has_audio_position; }), - "JFK/R13 must exercise EOS-backfill provenance"); - runner.reset(); - runner.feed_audio(audio.data(), count); - runner.step(); - const auto repeated = runner.finalize(); - require( - result.transcript_so_far == repeated.transcript_so_far && - result.words.size() == repeated.words.size(), - "reset preserves EOF text"); - for (size_t i = 0; i < result.words.size(); ++i) - require( - result.words[i].start_frame == repeated.words[i].start_frame && - result.words[i].end_frame == repeated.words[i].end_frame && - result.words[i].has_audio_position == repeated.words[i].has_audio_position, - "reset cannot leak timing bounds or provenance"); - } - for (double seconds : {0.0, 0.016, 0.05, 0.16, 0.625, 0.64, 0.66, 2.0, 11.0}) { - const size_t n = std::min(audio.size(), size_t(std::llround(seconds * sr))); - const std::vector clip(audio.begin(), audio.begin() + n); - for (size_t packet : {size_t(1279), size_t(2560), std::max(size_t(1), n)}) - check_stream(rec, clip, sr, packet); - } - if (right == 3) { - const auto resampled = - nemo_speech::audio::resample_audio(audio.data(), audio.size(), sr, 44100); - check_stream(rec, resampled, 44100, 7056); - } - std::printf("[PASS] R=%d SDK bounds/EOF/packets\n", right); - } - auto cfg = base; - cfg.streaming.rnnt_right_context = 3; - cfg.endpointing.enable = true; - cfg.log_status = false; - Recognizer rec(cfg); - auto twice = audio; - twice.insert(twice.end(), size_t(sr) * 2, 0.0f); - twice.insert(twice.end(), audio.begin(), audio.end()); - AsrRequestOptions opts; - opts.enable_word_time_offsets = true; - opts.enable_speaker_diarization = rec.diar_model() != nullptr; - auto stream = rec.streaming_recognize(opts, "auto"); - stream->push(twice.data(), twice.size(), sr); - const double duration = double(twice.size()) / sr; - int finals = 0, interims_after_final = 0, drains = 0; - float last = 0; - while (auto result = stream->next()) { - require(++drains < 100, "large-input drain must terminate"); - check_result(*result, duration, false); - require(result->audio_processed >= last, "EOU progress must not reset"); - if (result->is_final) { - ++finals; - require( - result->audio_processed < duration, - "mid-stream final must not report all queued audio"); - } else if (finals > 0) { - ++interims_after_final; - require( - result->audio_processed > last, - "remaining audio must produce fresh progress without push"); - } - last = result->audio_processed; - } - require( - finals > 0 && interims_after_final > 0, - "exercise finals then a trailing interim from one push"); - check_result(stream->finish(), duration, true); - std::puts("[PASS] EOU drain and result delivery"); -} - -static void -check_itn() { - std::vector words{{"two", 14, 15, 0.8f}, {"kilograms", 21, 22, 0.9f}}; - for (auto& word : words) bound_word_frames(word, 8); - postproc::update_word_timings( - words, "Token:\t0\ttwo kilograms\t0,12\t0,1\nWord:\t0\t2\t0\nWord:\t1\tkg\t0\n"); - require( - words.size() == 2 && words[0].word == "2" && words[1].word == "kg", - "ITN output order must survive equal EOF-clamped timestamps"); - for (const auto& word : words) - require( - !word.has_audio_position && word.start_frame == 7 && word.end_frame == 8, - "ITN exact matching and numeric collapse preserve EOS provenance"); - words = {{"abbr", 7, 8, 1.0f, false}}; - postproc::update_word_timings( - words, "Token:\t0\tabbr\t0,4\t0,1\nWord:\t0\tfirst\t0\nWord:\t1\tsecond\t0\n"); - require( - words.size() == 2 && words[0].start_frame == words[0].end_frame, - "ITN expansion can legitimately produce a zero-width frame span"); - for (const auto& word : words) - require( - !word.has_audio_position && word.start_frame >= 7 && word.end_frame <= 8, - "ITN expansion retains bounds and conservative provenance"); - words = {{"one", 5, 6, 1.0f}, {"two", 6, 7, 1.0f}, {"three", 7, 8, 1.0f, false}}; - postproc::update_word_timings( - words, "Token:\t0\tone two three\t0,13\t0,1\nWord:\t0\talpha\t0\nWord:\t1\tbeta\t0\n"); - require( - words.size() == 2 && words[0].has_audio_position && !words[1].has_audio_position, - "ITN regrouping only invalidates groups that include an EOS-only word"); -} - -int -main(int argc, char** argv) try { - check_projection(); - check_itn(); - if (argc >= 3) - check_model(argc, argv); - else - std::puts("[SKIP] SDK integration: supply model and WAV"); - std::puts("[PASS] result timing"); - return 0; -} -catch (const std::exception& e) { - std::fprintf(stderr, "[FAIL] %s\n", e.what()); - return 1; -} diff --git a/tests/cpp/common/test_subtitles.cpp b/tests/cpp/common/test_subtitles.cpp index a3a26383..94fc9a15 100644 --- a/tests/cpp/common/test_subtitles.cpp +++ b/tests/cpp/common/test_subtitles.cpp @@ -154,36 +154,6 @@ test_speaker_turns() { require(turns[1].start_ms == 700 && turns[1].end_ms == 1300, "second speaker turn timing"); } -void -test_real_audio_bounds() { - const std::vector padding{{"And", 560, 640}, {"so", 560, 640}}; - const auto fallback = nemo_speech::subtitle::make_cues(padding, "And so", 640); - require( - fallback.size() == 1 && fallback[0].start_ms == 560 && fallback[0].end_ms == 640, - "backfilled EOF words retain their bounded interval"); - require(fallback[0].text == "And so", "padding-only text is not dropped"); - const auto markers = - nemo_speech::subtitle::make_cues({{"And", 640, 640}, {"so", 640, 640}}, "", 640); - require( - markers.size() == 1 && markers[0].text == "And so" && markers[0].end_ms == 640, - "zero-width endpoint markers preserve text"); - const auto partial = nemo_speech::subtitle::make_cues( - {{"Hello", 0, 300}, {"world.", 300, 640}, {"Late", 560, 640}}, "", 640); - std::string text; - for (const auto& cue : partial) { - require( - cue.start_ms >= 0 && cue.start_ms < cue.end_ms && cue.end_ms <= 640, - "subtitle display extensions cannot exceed the audio"); - text += cue.text; - } - require( - text.find("Hello") != std::string::npos && text.find("Late") != std::string::npos, - "mixed timed and late words are retained"); - require( - nemo_speech::subtitle::make_cues({}, "text", 0).empty(), - "empty audio cannot have a positive-duration cue"); -} - } // namespace int @@ -198,7 +168,6 @@ main() { test_duration_and_length_split_are_ordered(); test_fallback_text(); test_speaker_turns(); - test_real_audio_bounds(); std::cout << "subtitle tests passed\n"; return 0; }