fix(asr): correct streaming startup and Q8 pointwise precision

conversion/asr.py

@@ -69,7 +69,8 @@
     re.compile(r"^joint\.enc\.weight$"),
     re.compile(r"^joint\.pred\.weight$"),
     re.compile(r"^joint\.joint_net\.2\.weight$"),
-    # ConformerConv pointwise convs are 1×1 — mathematically Linear. We squeeze
+    # ConformerConv pointwise convs are 1×1 — mathematically Linear. Cache-aware
+    # RNNT Q8 exports retain F16 here; see _pick_dtype. We squeeze
     # the k=1 dim in reshape_for_emission so the Q8_0/k-quant kernels see a
     # 2D (out, in) tensor; the runtime detects 2D stored shape and dispatches
     # the matmul through ggml_mul_mat (see Conv1D::build_graph).
@@ -132,18 +133,31 @@ def _q8_0_to_tensor_planar(packed: np.ndarray, n_per_row: int) -> np.ndarray:
 
 
 def _pick_dtype(
-    name: str, shape: tuple, linear_qtype: GGMLQuantizationType, default_qtype: GGMLQuantizationType
+    name: str,
+    shape: tuple,
+    linear_qtype: GGMLQuantizationType,
+    default_qtype: GGMLQuantizationType,
+    *,
+    cache_aware_rnnt: bool = False,
 ) -> tuple[GGMLQuantizationType, Optional[str]]:
     """Return (chosen GGMLQuantizationType, fallback reason or None).
 
     Dispatch by tensor class:
+      * Cache-aware RNNT Q8 pointwise convs → F16; quantizing these kernels
+                                  can cause long non-speech/blank runs in speech.
       * Linear weights         → user-selected `--weight-type` (linear_qtype).
       * Conv weights, embed    → F16 (regardless of --weight-type) — saves
                                   ~50% disk vs F32 with no runtime cost.
       * Everything else        → F32 (default_qtype) — norms, biases,
                                   pos_bias, pe table, etc. The runtime keeps
                                   these in F32 in memory.
     """
+    if (
+        cache_aware_rnnt
+        and linear_qtype == GGMLQuantizationType.Q8_0
+        and _POINTWISE_PATTERN.fullmatch(name)
+    ):
+        return GGMLQuantizationType.F16, "cache-aware RNNT pointwise precision"
     if _is_linear_weight(name):
         inner = int(shape[-1]) if len(shape) >= 1 else 1
         if linear_qtype in _K_QUANTS and inner % _QK_K != 0:
@@ -677,7 +691,11 @@ def convert(
             arr = reshape_for_emission(dst_name, arr)
 
             chosen_qtype, fallback_reason = _pick_dtype(
-                dst_name, arr.shape, linear_qtype, default_qtype
+                dst_name,
+                arr.shape,
+                linear_qtype,
+                default_qtype,
+                cache_aware_rnnt=cache_supported and head_type == "rnnt",
             )
             if fallback_reason is not None:
                 print(f"[convert] kept fp16 ({fallback_reason}): {dst_name}")

docs/model-conversion.md

@@ -61,3 +61,13 @@ option list.
 
 The converters do not import `nemo_toolkit`, and source checkpoints are not
 modified.
+
+### Cache-aware RNNT Q8 precision
+
+Cache-aware RNNT Q8_0 exports keep the Conformer pointwise convolution weights
+in F16. Quantizing these kernels to Q8 can cause speech to be decoded as long
+blank runs; attention, feed-forward and decoder matrix quantization is unchanged.
+Existing GGUFs are not repaired by updating the runtime: regenerate them from
+the original `.nemo` checkpoint with `--outtype q8_0`. Converting already-quantized
+weights to F16 cannot recover the original values. Other model/head types and
+requested weight types keep their previous conversion policy.

src/asr/encoder/cache_aware_encoder.h

@@ -1,6 +1,6 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
-// Cache-aware streaming encoder: one shared, fixed-shape FastConformer encoder
+// Cache-aware streaming encoder: one shared FastConformer encoder
 // Session whose per-layer K/V caches are two planes of one persistent arena
 // and whose convolution cache is a persistent matrix, with one indexed row per
 // stream. A microbatch supplies stream IDs, gathers those rows, and scatters
@@ -65,8 +65,9 @@ class CacheAwareEncoder {
     int chunk_frames();
 
     // One cache-aware encoder step for the stream owning `state`. `mel` is
-    // (n_mels x n_mel_frames) frame-major (= pre_encode_cache_size + sub*(1+R)
-    // frames). The stream's K/V/conv cache lives in `state` (device-resident,
+    // (n_mels x n_mel_frames) frame-major: first chunk = 1+sub*R; subsequent
+    // chunks = pre_encode_cache_size+sub*(1+R). Both produce 1+R encoder frames.
+    // The stream's K/V/conv cache lives in `state` (device-resident,
     // updated in-graph). `enc_out` receives (output_dim x T_enc) flat-packed
     // (d_model without a tail); T_enc = 1 + R. Thread-safe.
     void encode(

src/asr/encoder/fastconformer.cpp

@@ -1410,11 +1410,17 @@ FastConformerEncoder::build_graph(
         pre.set_first_tensor(ggml_runtime::ggml_bf_tensor(x_scaled, x.buft));
     }
 
+    // Only subsequent chunks carry overlap. The shorter first-chunk shape
+    // already produces exactly cache_chunk_frames, with causal padding handled
+    // independently by each subsampling convolution. A shape-based distinction
+    // also keeps concurrent first/steady-state streams in separate graph keys.
+    const bool first_chunk =
+        input_tensors.get_tensor(0).tensor->ne[1] == cfg_.cache_first_chunk_mel_frames();
     // Then drop the leading `cache_drop_extra` frames from the subsampled
     // output so the visible chunk length is exactly cache_chunk_frames
     // (= 1 + R). The dropped frames overlap with the tail of the previous
     // chunk (handled by the K/V cache).
-    if (cfg_.cache_drop_extra > 0) {
+    if (!first_chunk && cfg_.cache_drop_extra > 0) {
         auto x = pre.get_tensor(0);
         auto bf_ctx = session_tensor_container->get_ctx_of_buffer_type(x.buft);
         auto x_view = ggml_view_4d(

src/asr/encoder/fastconformer.h

@@ -14,6 +14,7 @@
 #pragma once
 
 #include <cmath>
+#include <cstdint>
 #include <string>
 #include <vector>
 
@@ -72,6 +73,16 @@ struct EncoderConfig {
     // 2 frames, giving cache_chunk_frames = 1 + R after the drop.
     int cache_drop_extra = 2;
 
+    // Causal subsampling starts at frame zero, without a preceding overlap.
+    // Subsequent-chunk geometry remains owned by the runner.
+    int cache_first_chunk_mel_frames() const { return 1 + subsampling_factor * cache_right_ctx; }
+
+    // Exclusive mel frontier: causal encoder frame f ends at mel f*sub.
+    // The first emitted frame therefore covers one real mel frame, not sub.
+    int64_t cache_decoded_mel_frames(int64_t emitted_frames) const {
+        return emitted_frames > 0 ? 1 + (emitted_frames - 1) * subsampling_factor : 0;
+    }
+
     // Offline attention limits are independent of the streaming cache context.
     // -1 means unlimited; finite values preserve a model's trained attention window.
     int offline_left_ctx = -1;

src/asr/runner.cpp

@@ -874,12 +874,20 @@ CacheStreamRunner::feed_audio(const float* samples, size_t n_samples) {
     audio_buf_.insert(audio_buf_.end(), samples, samples + n_samples);
 }
 
+int
+CacheStreamRunner::next_chunk_mel_frames() const {
+    // Preserve the existing overlap and steady-state formula; only startup
+    // uses the shorter causal chunk.
+    return first_chunk_ ? enc_cfg_.cache_first_chunk_mel_frames()
+                        : pre_encode_cache_size_ +
+                              enc_cfg_.subsampling_factor * (1 + enc_cfg_.cache_right_ctx);
+}
+
 void
 CacheStreamRunner::process_one_chunk() {
-    // NeMo formula: chunk_size_mel = pre_encode_cache_size + sub * (1 + R).
     const int sub = enc_cfg_.subsampling_factor;
     const int R = enc_cfg_.cache_right_ctx;
-    const int chunk_size_mel = pre_encode_cache_size_ + sub * (1 + R);
+    const int chunk_size_mel = next_chunk_mel_frames();
     const int shift_size_mel = sub * (1 + R - cache_drop_size_);
     const int n_mels = model_->fe_config().n_mels;
     const int left_ctx = enc_cfg_.cache_left_ctx;
@@ -945,8 +953,23 @@ CacheStreamRunner::process_one_chunk() {
 
     // Advance mel buffer: drop shift_size_mel; keep overlap_mel_frames at
     // the head so the next chunk's leading frames see the same context.
-    const size_t shift_floats = static_cast<size_t>(shift_size_mel) * n_mels;
-    mel_offset_ = std::min(mel_offset_ + shift_floats, mel_buf_.size());
+    if (first_chunk_) {
+        first_chunk_ = false;
+        if (chunk_size_mel < pre_encode_cache_size_) {
+            // R=0 begins with one real mel frame. Its first subsequent chunk
+            // still needs the existing overlap; only the unavailable negative
+            // history is zero, and only AFTER the first chunk was encoded.
+            compact_mel_buffer();
+            mel_buf_.insert(
+                mel_buf_.begin(),
+                static_cast<size_t>(pre_encode_cache_size_ - chunk_size_mel) * n_mels, 0.0f);
+        } else {
+            mel_offset_ += static_cast<size_t>(chunk_size_mel - pre_encode_cache_size_) * n_mels;
+        }
+    } else {
+        const size_t shift_floats = static_cast<size_t>(shift_size_mel) * n_mels;
+        mel_offset_ = std::min(mel_offset_ + shift_floats, mel_buf_.size());
+    }
 }
 
 StreamingUpdate
@@ -993,24 +1016,12 @@ CacheStreamRunner::step() {
         total_mel_frames_produced_ = i_start + n;
     }
 
-    const int sub = enc_cfg_.subsampling_factor;
-    const int R = enc_cfg_.cache_right_ctx;
-    const int chunk_size_mel = pre_encode_cache_size_ + sub * (1 + R);
-
     // Process every whole chunk available, polling the endpointer per chunk
     // so a silence gap interior to one large feed still fires. An EOU breaks
     // the loop; the next utterance's remaining chunks decode on the next
     // step() call.
     last_step_new_tokens_.clear();
-    while (static_cast<int>((mel_buf_.size() - mel_offset_) / n_mels) >= chunk_size_mel) {
-        if (!stream_zero_padded_) {
-            // First-chunk zero-pad so the encoder's unconditional
-            // cache_drop_extra eats zero-pad, not the leading 160 ms of real audio.
-            compact_mel_buffer();
-            mel_buf_.insert(
-                mel_buf_.begin(), static_cast<size_t>(pre_encode_cache_size_) * n_mels, 0.0f);
-            stream_zero_padded_ = true;
-        }
+    while (static_cast<int>((mel_buf_.size() - mel_offset_) / n_mels) >= next_chunk_mel_frames()) {
         process_one_chunk();
         if (poll_endpoint(update, /*after_chunk=*/true))
             break;
@@ -1051,9 +1062,10 @@ CacheStreamRunner::poll_endpoint(StreamingUpdate& update, bool after_chunk) {
     double now_ms = 0.0, last_speech_ms = 0.0;
     if (vad_ && endpointer_->config().vad_based) {
         // VAD-driven: scan the Silero speech bits over the newly decoded mel
-        // frames (enc frame f covers mel frames [f*sub, (f+1)*sub)).
+        // frames. Causal encoder frame f ends at mel f*sub (inclusive), so
+        // the shorter first chunk shifts the exclusive frontier by sub-1.
         const double mel_ms = 1000.0 * hop / sample_rate;
-        const int64_t decoded_mel = total_frames_emitted_ * enc_cfg_.subsampling_factor;
+        const int64_t decoded_mel = enc_cfg_.cache_decoded_mel_frames(total_frames_emitted_);
         scan_vad_speech(*vad_, decoded_mel, vad_scan_frame_, vad_speech_seen_frame_);
         now_ms = static_cast<double>(decoded_mel) * mel_ms;
         last_speech_ms = (vad_speech_seen_frame_ < 0)
@@ -1105,7 +1117,7 @@ CacheStreamRunner::trim_buffers() {
         // masker's neighbourhood scans never cross the trimmed base.
         const int64_t margin =
             std::max<int64_t>(256, vad_masker_ ? 2 * vad_masker_->max_scan() : 0);
-        const int64_t decoded_mel = total_frames_emitted_ * enc_cfg_.subsampling_factor;
+        const int64_t decoded_mel = enc_cfg_.cache_decoded_mel_frames(total_frames_emitted_);
         const int64_t g = decoded_mel - margin;
         if (g > 0)
             vad_->discard_timeline_before(g);
@@ -1153,23 +1165,15 @@ CacheStreamRunner::finalize() {
     const int shift_size_mel = sub * (1 + R - cache_drop_size_);
     const size_t buffered = (mel_buf_.size() - mel_offset_) / n_mels;
     if (buffered > 0) {
-        // A stream shorter than one chunk never entered step()'s chunk
-        // loop, so the start-of-stream zero-pad hasn't happened yet; apply
-        // it here or the encoder's cache_drop_extra eats the leading
-        // ~160 ms of real audio instead of pad.
-        if (!stream_zero_padded_) {
-            mel_buf_.insert(
-                mel_buf_.begin(), static_cast<size_t>(pre_encode_cache_size_) * n_mels, 0.0f);
-            stream_zero_padded_ = true;
-        }
         // Tail flush: pad enough zero mel for the final real frame to
         // see a full right-context window AND for the RNNT predictor
         // to get at least one all-zero enc frame to commit pending tail tokens.
         const size_t flush_pad_frames = static_cast<size_t>(chunk_size_mel + shift_size_mel);
         mel_buf_.resize(mel_buf_.size() + flush_pad_frames * static_cast<size_t>(n_mels), 0.0f);
         const size_t before = all_tokens_.size();
         last_step_new_tokens_.clear();
-        while (static_cast<int>((mel_buf_.size() - mel_offset_) / n_mels) >= chunk_size_mel) {
+        while (static_cast<int>((mel_buf_.size() - mel_offset_) / n_mels) >=
+               next_chunk_mel_frames()) {
             process_one_chunk();
         }
         for (size_t i = before; i < all_tokens_.size(); i++) {
@@ -1217,7 +1221,7 @@ CacheStreamRunner::reset() {
     last_enc_T_ = 0;
     finalized_ = false;
     finalizing_ = false;
-    stream_zero_padded_ = false;
+    first_chunk_ = true;
     zero_caches();
     std::fill(attn_mask_.begin(), attn_mask_.end(), 0.0f);
     if (head_)

src/asr/runner.h

@@ -243,8 +243,8 @@ class OfflineRunner final : public AsrRunner {
 
 bool exceeds_offline_position_limit(const AsrModel& model, size_t n_samples, int input_sample_rate);
 
-// Drives a fixed-shape encoder graph (chunk_size_mel in, chunk_enc_frames
-// out) with per-layer K/V/conv caches threaded across chunks in a
+// Drives first/steady-state encoder shapes (each producing chunk_enc_frames)
+// with per-layer K/V/conv caches threaded across chunks in a
 // device-resident indexed cache row. Audio is buffered until at least one chunk's
 // worth of mel frames is ready; each step uploads mel + a small validity mask
 // and downloads only the joint-projected encoder output.
@@ -279,6 +279,7 @@ class CacheStreamRunner final : public AsrRunner {
     RnntDecodeStats rnnt_decode_stats() const;
 
    private:
+    int next_chunk_mel_frames() const;
     void process_one_chunk();
     void upload_attn_mask();
     void zero_caches();
@@ -297,7 +298,8 @@ class CacheStreamRunner final : public AsrRunner {
     // GGUF-pinned cache-aware mel-frame geometry. These are NeMo's
     // CacheAwareStreamingConfig chunk_size / shift_size in mel frames
     // (setup_streaming_params, conformer_encoder.py):
-    //   chunk_size_mel = pre_encode_cache_size + subsampling_factor*(1 + R)
+    //   first_chunk_mel = 1 + subsampling_factor*R
+    //   subsequent_chunk_mel = pre_encode_cache_size + subsampling_factor*(1 + R)
     //   shift_size_mel = subsampling_factor*(1 + R - cache_drop_size)
     // These values are fixed by the model.
     int pre_encode_cache_size_ = 9;  // overlap, mel frames
@@ -359,11 +361,9 @@ class CacheStreamRunner final : public AsrRunner {
     bool finalized_ = false;
     // True once finalize() begins, so its internal drain step() doesn't fire EOU.
     bool finalizing_ = false;
-    // Tracks whether we've prepended pre_encode_cache_size zero mel frames at
-    // start-of-stream. NeMo's CacheAwareStreamingAudioBuffer does this once
-    // per stream so the encoder's drop_extra_pre_encoded behaviour throws
-    // away zero-pad rather than the leading ~160 ms of real audio.
-    bool stream_zero_padded_ = false;
+    // Reset at an encoder-state boundary, independently of the absolute stream
+    // frame clock. The first chunk has no pre-encoder overlap to discard.
+    bool first_chunk_ = true;
 
     // Encoder output from most recent chunk (column-major (d_model, T_out)).
     std::vector<float> last_enc_out_;