Skip to content

Commit be0cfcc

Browse files
committed
Add BufferedTdtStream: buffered streaming for offline TDT checkpoints
Productionizes the window loop that lived in test_tdt_context_chunks into a library class. NVIDIA's 10/2/2 schedule as construction parameters, per-window normalization (the frontend runs on each window's audio, as NeMo's buffered script does), and carried TdtDecodeState so every encoder frame is decoded exactly once — the transcript is append-only by construction, with no seam to reconcile. Two corrections to the prototype's arithmetic: framing is derived from model config (sample_rate/hop_length) instead of assuming 100 frames/s, and encoder-frame trimming uses Subsampling::valid_out_len instead of a local ceil(n/8) that silently assumed non-causal downsampling. The retained audio buffer is bounded by left+chunk+right and trimmed every chunk, so memory stays flat over a long stream — unlike the existing stream API's mel_buf, which grows unbounded (upstream issue mudler#63). test_buffered_pcm feeds the same audio at block sizes 7..32000 samples — including 1237 and 999, deliberately not hop multiples — and requires identical tokens, plus token parity against the NeMo streaming fixtures (v2 50/50, v3 44/44, zero edits). It caught one real bug during development: the flush decoded only the first tail chunk and dropped the rest of the stream (41 of 50 tokens). Block-size independence passed throughout — every block size dropped the same tail — only the NeMo comparison exposed it. On the 200s reference-letter fixture this class matches NeMo's own buffered implementation at WER 0.0596 with the identical error breakdown. Assisted-by: Claude:claude-fable-5 [Claude Code]
1 parent dad0972 commit be0cfcc

5 files changed

Lines changed: 393 additions & 1 deletion

File tree

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ set(PARAKEET_SRC
8484
src/streaming_encoder.cpp
8585
src/streaming.cpp
8686
src/transcript_stream.cpp
87+
src/buffered_window.cpp
8788
src/ctc_decoder.cpp
8889
src/prediction.cpp
8990
src/joint.cpp

‎src/buffered_window.cpp‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
#include "buffered_window.hpp"
2+
#include <algorithm>
3+
4+
namespace pk {
5+
6+
bool BufferedTdtStream::supports(const ModelLoader& ml) {
7+
// TDT durations are what this decodes with; a model without them is either
8+
// CTC or plain RNN-T and belongs on a different path.
9+
return !ml.config().tdt_durations.empty();
10+
}
11+
12+
BufferedTdtStream::BufferedTdtStream(const ModelLoader& ml,
13+
double left_secs, double chunk_secs, double right_secs)
14+
: ml_(ml), frontend_(ml), sub_(ml), enc_(ml), pred_(ml), joint_(ml),
15+
state_(tdt_decode_init(pred_)), acc_(ml) {
16+
const ParakeetConfig& cfg = ml.config();
17+
hop_ = (int)(cfg.hop_length ? cfg.hop_length : 160);
18+
blank_id_ = (int)cfg.blank_id;
19+
max_symbols_ = (int)cfg.max_symbols;
20+
durations_ = cfg.tdt_durations;
21+
22+
// Derive the schedule from the model's own framing rather than assuming
23+
// 100 mel frames/s, and round each span to a whole mel frame so every chunk
24+
// boundary lands on a frame boundary.
25+
const double sr = (double)(cfg.sample_rate ? cfg.sample_rate : 16000);
26+
auto to_samples = [&](double secs) -> int64_t {
27+
const int64_t frames = (int64_t)(secs * sr / (double)hop_ + 0.5);
28+
return frames * (int64_t)hop_;
29+
};
30+
left_s_ = to_samples(left_secs);
31+
chunk_s_ = to_samples(chunk_secs);
32+
right_s_ = to_samples(right_secs);
33+
34+
reset();
35+
}
36+
37+
void BufferedTdtStream::reset() {
38+
state_ = tdt_decode_init(pred_);
39+
acc_.reset();
40+
audio_.clear();
41+
audio_origin_ = 0;
42+
total_seen_ = 0;
43+
next_chunk_ = 0;
44+
finished_ = false;
45+
}
46+
47+
double BufferedTdtStream::latency_sec() const {
48+
const double sr = (double)(ml_.config().sample_rate ? ml_.config().sample_rate : 16000);
49+
return (double)(chunk_s_ + right_s_) / sr;
50+
}
51+
52+
std::vector<int32_t> BufferedTdtStream::feed_pcm(const float* pcm, int n_samples, bool is_last) {
53+
std::vector<int32_t> emitted;
54+
if (n_samples < 0 || (!pcm && n_samples > 0)) return emitted;
55+
if (n_samples > 0) {
56+
audio_.insert(audio_.end(), pcm, pcm + n_samples);
57+
total_seen_ += n_samples;
58+
}
59+
drain_ready(is_last, emitted);
60+
if (is_last) finished_ = true;
61+
return emitted;
62+
}
63+
64+
std::string BufferedTdtStream::finalize() {
65+
if (!finished_) {
66+
std::vector<int32_t> emitted;
67+
drain_ready(/*flush=*/true, emitted);
68+
finished_ = true;
69+
}
70+
// No further word-start markers can arrive, so the trailing open word is
71+
// final too.
72+
acc_.regroup_words(/*flush_all=*/true);
73+
return acc_.take_new_text();
74+
}
75+
76+
void BufferedTdtStream::drain_ready(bool flush, std::vector<int32_t>& out) {
77+
for (;;) {
78+
// A chunk is decodable once its own audio AND its right context have
79+
// arrived. On flush the tail is decoded with whatever right context
80+
// exists, which is what makes the last words appear at end of stream.
81+
const bool have_full = total_seen_ >= next_chunk_ + chunk_s_ + right_s_;
82+
const bool have_tail = flush && total_seen_ > next_chunk_;
83+
if (!have_full && !have_tail) return;
84+
85+
const int64_t lo = std::max<int64_t>(0, next_chunk_ - left_s_);
86+
const int64_t commit_hi = std::min<int64_t>(total_seen_, next_chunk_ + chunk_s_);
87+
const int64_t hi = std::min<int64_t>(total_seen_, commit_hi + right_s_);
88+
if (commit_hi <= next_chunk_) return;
89+
90+
// The retained buffer must cover [lo, hi); it always does, because it is
91+
// trimmed to next_chunk_ - left_s_ at the end of each iteration.
92+
const size_t off_lo = (size_t)(lo - audio_origin_);
93+
const size_t off_hi = (size_t)(hi - audio_origin_);
94+
if (off_hi > audio_.size() || off_lo > off_hi) return;
95+
96+
// Frontend on this window's AUDIO: per_feature statistics are the
97+
// window's, matching NeMo's buffered script.
98+
std::vector<float> window_audio(audio_.begin() + off_lo, audio_.begin() + off_hi);
99+
std::vector<float> mel;
100+
int n_mels = 0, window_T = 0;
101+
frontend_.compute(window_audio, mel, n_mels, window_T);
102+
if (window_T <= 0) return;
103+
104+
std::vector<float> enc_cf;
105+
int d_model = 0, Tw = 0;
106+
enc_.forward(mel, n_mels, window_T, enc_cf, d_model, Tw);
107+
if (Tw <= 0 || d_model <= 0) return;
108+
109+
// Encoder frames for this chunk. valid_out_len is the encoder's own
110+
// length recurrence (and handles causal downsampling), rather than a
111+
// local ceil(n/8) that silently assumes the non-causal case.
112+
const int mel_off = (int)((next_chunk_ - lo) / hop_);
113+
const int mel_commit = (int)((commit_hi - lo) / hop_);
114+
const int first = std::min(Tw, sub_.valid_out_len(window_T, mel_off));
115+
const int last = std::min(Tw, sub_.valid_out_len(window_T, mel_commit));
116+
if (last <= first) { next_chunk_ = commit_hi; continue; }
117+
118+
// Channels-first [d_model, Tw] -> time-major [n, d_model].
119+
const int n = last - first;
120+
std::vector<float> frames((size_t)n * d_model);
121+
for (int t = 0; t < n; ++t)
122+
for (int c = 0; c < d_model; ++c)
123+
frames[(size_t)t * d_model + c] = enc_cf[(size_t)c * Tw + (first + t)];
124+
125+
std::vector<TokenInfo> chunk_tokens;
126+
std::vector<int32_t> step =
127+
tdt_decode_frames(pred_, joint_, frames, n, d_model, durations_,
128+
state_, blank_id_, max_symbols_, &chunk_tokens);
129+
130+
// tdt_decode_frames records ABSOLUTE frames (frames_seen + t), so no
131+
// base offset is added here — hence the null local_frames.
132+
acc_.advance_frames(n);
133+
acc_.absorb(step, chunk_tokens, /*local_frames=*/nullptr, /*base_frame=*/0);
134+
acc_.regroup_words(/*flush_all=*/false);
135+
out.insert(out.end(), step.begin(), step.end());
136+
137+
next_chunk_ = commit_hi;
138+
139+
// Drop audio older than the next window's left context. This is what
140+
// keeps memory flat over a long stream instead of growing with it.
141+
const int64_t keep_from = std::max<int64_t>(0, next_chunk_ - left_s_);
142+
if (keep_from > audio_origin_) {
143+
const size_t drop = (size_t)(keep_from - audio_origin_);
144+
if (drop >= audio_.size()) { audio_.clear(); }
145+
else { audio_.erase(audio_.begin(), audio_.begin() + drop); }
146+
audio_origin_ = keep_from;
147+
}
148+
// Keep going: a flush can span several chunks, and stopping after the
149+
// first partial one silently drops the end of the stream. Termination
150+
// is the loop head — have_tail goes false once next_chunk_ reaches
151+
// total_seen_.
152+
}
153+
}
154+
155+
} // namespace pk

‎src/buffered_window.hpp‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#pragma once
2+
#include "encoder.hpp"
3+
#include "joint.hpp"
4+
#include "mel.hpp"
5+
#include "model_loader.hpp"
6+
#include "prediction.hpp"
7+
#include "subsampling.hpp"
8+
#include "tdt.hpp"
9+
#include "transcript_stream.hpp"
10+
#include <cstdint>
11+
#include <string>
12+
#include <vector>
13+
14+
namespace pk {
15+
16+
// Buffered streaming for OFFLINE TDT checkpoints (parakeet-tdt-0.6b-v2/v3).
17+
//
18+
// These models were never trained for cache-aware streaming: there is no
19+
// per-layer conv/attention cache to carry, so the encoder cannot be resumed
20+
// from where it stopped. NVIDIA's answer, which this reproduces, is to
21+
// re-encode a sliding [left | chunk | right] window and decode only the chunk's
22+
// frames once:
23+
//
24+
// 1. Run the offline encoder over [10 s left | 2 s chunk | 2 s right].
25+
// 2. Keep only the encoder frames belonging to the chunk.
26+
// 3. Decode those frames EXACTLY ONCE, carrying prediction-net state, the
27+
// last emitted token and any TDT duration skip into the next chunk.
28+
//
29+
// Because no audio is ever decoded twice, the transcript is append-only by
30+
// construction rather than by reconciliation: there is no second reading of a
31+
// span to disagree with the first.
32+
//
33+
// `TdtDecodeState::pending_skip` is what makes step 3 safe. TDT emits a token
34+
// plus a duration, which may advance several encoder frames; when that skip
35+
// crosses a chunk boundary the frames it consumed belong to the NEXT chunk, and
36+
// decoding that chunk from frame zero would re-consume them.
37+
//
38+
// Normalization follows NeMo's buffered script: the frontend runs on each
39+
// window's AUDIO, so per_feature statistics are the window's. Slicing a
40+
// globally normalized mel is a different computation — it agrees on short clips
41+
// (where every window starts at sample 0) and diverges on long ones — and it
42+
// cannot run live, since whole-utterance statistics need the whole utterance.
43+
//
44+
// Shares TranscriptAccumulator with the cache-aware StreamingSession, so the
45+
// running text, delta marker, event queue and per-word timestamps are one
46+
// implementation rather than two.
47+
class BufferedTdtStream {
48+
public:
49+
// The 10/2/2 schedule is NVIDIA's inference-time recommendation, not model
50+
// metadata — there is no GGUF key for it — so it is a construction
51+
// parameter with those defaults. Each is rounded to a whole mel frame.
52+
BufferedTdtStream(const ModelLoader& ml,
53+
double left_secs = 10.0,
54+
double chunk_secs = 2.0,
55+
double right_secs = 2.0);
56+
57+
// True when the model is an offline TDT checkpoint this can drive.
58+
static bool supports(const ModelLoader& ml);
59+
60+
void reset();
61+
62+
// Feed 16 kHz mono PCM. Decodes every chunk whose right context has now
63+
// arrived and returns the token ids emitted by this call ("" worth of
64+
// tokens is normal until the first chunk + right context is buffered).
65+
// `is_last` flushes the tail, decoding the final partial chunk with
66+
// whatever right context exists.
67+
std::vector<int32_t> feed_pcm(const float* pcm, int n_samples, bool is_last = false);
68+
69+
// Flush the tail without new audio and return the newly finalized text.
70+
std::string finalize();
71+
72+
const std::vector<int32_t>& tokens() const { return state_.hyp; }
73+
const std::string& text() const { return acc_.text(); }
74+
std::string take_new_text() { return acc_.take_new_text(); }
75+
std::vector<Word> drain_words() { return acc_.drain_words(); }
76+
std::vector<EouEvent> drain_events() { return acc_.drain_events(); }
77+
78+
// Seconds between audio arriving and its words being emitted: a chunk must
79+
// fill and its right context must follow. Callers that display text need
80+
// this; it is much larger than a cache-aware session's one-chunk lag.
81+
double latency_sec() const;
82+
83+
// Audio samples retained right now. Bounded by left+chunk+right — the
84+
// buffer never grows with stream length.
85+
size_t buffered_samples() const { return audio_.size(); }
86+
87+
private:
88+
// Decode every chunk whose right context is available. `flush` also decodes
89+
// the trailing partial chunk. Appends emitted ids to `out`.
90+
void drain_ready(bool flush, std::vector<int32_t>& out);
91+
92+
const ModelLoader& ml_;
93+
MelFrontend frontend_;
94+
Subsampling sub_;
95+
Encoder enc_;
96+
PredictionNet pred_;
97+
Joint joint_;
98+
TdtDecodeState state_;
99+
TranscriptAccumulator acc_;
100+
101+
int hop_ = 160;
102+
int blank_id_ = 0;
103+
int max_symbols_ = 10;
104+
std::vector<int32_t> durations_;
105+
106+
int64_t left_s_ = 0, chunk_s_ = 0, right_s_ = 0; // schedule, in samples
107+
108+
std::vector<float> audio_; // retained tail, bounded by the schedule
109+
int64_t audio_origin_ = 0; // absolute sample index of audio_[0]
110+
int64_t total_seen_ = 0; // absolute samples fed so far
111+
int64_t next_chunk_ = 0; // absolute sample index of the next chunk
112+
bool finished_ = false;
113+
};
114+
115+
} // namespace pk

‎tests/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ pk_add_test(test_transducer_core)
4444
pk_add_test(test_tdt_greedy)
4545
pk_add_test(test_tdt_decode_chunking)
4646
pk_add_test(test_tdt_context_chunks)
47+
pk_add_test(test_buffered_pcm)
4748
pk_add_test(test_tdt_beam)
4849
pk_add_test(test_transducer_greedy_batch)
4950
pk_add_test(test_transducer_greedy_batch_rnnt)
@@ -120,7 +121,7 @@ endif()
120121
set_tests_properties(test_model_loader test_mel test_mel_gpu test_subsampling test_subsampling_batch test_subsampling_batch_causal test_relpos_attention test_relpos_attention_batch test_relpos_attention_local_chunked
121122
test_conformer test_conformer_batch test_conv_eou test_encoder test_encoder_batch test_encoder_batch_local test_encoder_eou
122123
test_streaming_encoder test_ctc test_prediction
123-
test_prediction_step test_prediction_step_batch test_tdt_decode_chunking test_tdt_context_chunks
124+
test_prediction_step test_prediction_step_batch test_tdt_decode_chunking test_tdt_context_chunks test_buffered_pcm
124125
test_joint test_joint_step_batch test_prompt_kernel test_transducer_core test_tdt_greedy
125126
test_tdt_beam
126127
test_transducer_greedy_batch test_transducer_greedy_batch_rnnt

0 commit comments

Comments
 (0)