From 8ee15ceef6bea34c97d579c2f0cf7e0a88298b3a Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Mon, 7 Sep 2026 04:14:53 -0400 Subject: [PATCH 1/4] Tighten BreezeTTS streaming path --- include/engine/models/breeze_tts/generator.h | 8 + include/engine/models/breeze_tts/session.h | 8 + .../engine/models/breeze_tts/speech_decoder.h | 12 + model_specs/breeze_tts.json | 23 ++ src/models/breeze_tts/generator.cpp | 266 ++++++++++++++++++ src/models/breeze_tts/session.cpp | 88 +++++- src/models/breeze_tts/speech_decoder.cpp | 163 ++++++++--- 7 files changed, 525 insertions(+), 43 deletions(-) diff --git a/include/engine/models/breeze_tts/generator.h b/include/engine/models/breeze_tts/generator.h index 7156ea6b5..76d9272a7 100644 --- a/include/engine/models/breeze_tts/generator.h +++ b/include/engine/models/breeze_tts/generator.h @@ -29,6 +29,11 @@ struct BreezeGenerationRequest { uint64_t seed = 0; }; +struct BreezeStreamEvent { + engine::runtime::AudioBuffer audio; + bool done = false; +}; + class BreezeGeneratorRuntime { public: BreezeGeneratorRuntime( @@ -42,6 +47,9 @@ class BreezeGeneratorRuntime { engine::runtime::AudioBuffer generate(const BreezeGenerationRequest & request); BreezeSpeechCodes encode_reference(const engine::runtime::AudioBuffer & audio) const; + void begin_stream(const BreezeGenerationRequest & request); + BreezeStreamEvent next_stream_audio(size_t max_new_frames, int64_t lookahead_margin); + void end_stream(); private: struct Impl; diff --git a/include/engine/models/breeze_tts/session.h b/include/engine/models/breeze_tts/session.h index 8c839da0a..4b9939b04 100644 --- a/include/engine/models/breeze_tts/session.h +++ b/include/engine/models/breeze_tts/session.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -67,6 +68,7 @@ class BreezeTTSSession final const engine::runtime::TaskRequest & request, const std::optional & reference_codes, size_t chunk_index) const; + std::optional next_subchunk_event(); engine::runtime::TaskSpec task_; std::shared_ptr assets_; @@ -77,7 +79,13 @@ class BreezeTTSSession final std::vector stream_chunk_requests_; std::optional stream_reference_codes_; engine::runtime::AudioBuffer stream_merged_audio_; + std::chrono::steady_clock::time_point stream_started_at_; size_t stream_chunk_index_ = 0; + bool stream_subchunk_ = false; + size_t stream_frames_per_event_ = 32; + int64_t stream_lookahead_margin_ = 12; + bool stream_chunk_active_ = false; + size_t stream_event_seq_ = 0; bool stream_started_ = false; }; diff --git a/include/engine/models/breeze_tts/speech_decoder.h b/include/engine/models/breeze_tts/speech_decoder.h index a5c269c6d..0fcb58da0 100644 --- a/include/engine/models/breeze_tts/speech_decoder.h +++ b/include/engine/models/breeze_tts/speech_decoder.h @@ -42,12 +42,22 @@ class BreezeSpeechDecoderRuntime { ~BreezeSpeechDecoderRuntime(); runtime::AudioBuffer decode(const BreezeSpeechCodes & codec_codes) const; + void reset_streaming_state() const; + runtime::AudioBuffer decode_streaming_step( + const BreezeSpeechCodes & codec_codes, + int64_t lookahead_margin, + bool final) const; runtime::AudioBuffer decode_and_trim_reference( const BreezeSpeechCodes & reference_codes, const BreezeSpeechCodes & generated_codes) const; void release_runtime_graphs() const; private: + std::vector decode_window_samples( + const std::vector & chunk, + int64_t chunk_frames, + int64_t context_frames) const; + std::shared_ptr assets_; core::ExecutionContext * execution_context_ = nullptr; std::shared_ptr weights_; @@ -55,6 +65,8 @@ class BreezeSpeechDecoderRuntime { bool allow_flash_attention_ = true; std::unique_ptr constants_; mutable std::unique_ptr graph_; + struct StreamingState; + mutable std::unique_ptr streaming_state_; // Always present to keep this public class layout identical when the private // Strix Halo compile definition differs between translation units. mutable std::array, 2> optimized_graphs_; diff --git a/model_specs/breeze_tts.json b/model_specs/breeze_tts.json index d94e62cbf..569335fe0 100644 --- a/model_specs/breeze_tts.json +++ b/model_specs/breeze_tts.json @@ -127,6 +127,29 @@ "required": false, "min": 0, "default": 0 + }, + { + "name": "stream_subchunk", + "type": "bool", + "description": "Emit multiple streaming audio events per text chunk.", + "required": false, + "default": false + }, + { + "name": "stream_frames_per_event", + "type": "int", + "description": "Generated codec frames per streaming audio event.", + "required": false, + "min": 1, + "default": 32 + }, + { + "name": "stream_lookahead_margin", + "type": "int", + "description": "Trailing codec frames held before emission to reduce streaming boundary artifacts.", + "required": false, + "min": 0, + "default": 12 } ], "session": [ diff --git a/src/models/breeze_tts/generator.cpp b/src/models/breeze_tts/generator.cpp index 54b5e360b..4304940a5 100644 --- a/src/models/breeze_tts/generator.cpp +++ b/src/models/breeze_tts/generator.cpp @@ -967,6 +967,260 @@ struct BreezeGeneratorRuntime::Impl { return frame; } + struct StreamState { + BreezeGenerationRequest request; + modules::QwenCausalPrefillResult cond; + std::optional uncond; + sampling::HfSamplerScratch scratch; + std::mt19937 fallback_rng; + sampling::HfSamplingOptions first_options; + std::vector first_codebook_history; + std::vector codes; + int64_t steps_taken = 0; + bool done = false; + bool use_cfg = false; + double ar_total_ms = 0.0; + double codec_decode_ms = 0.0; + double backbone_cond_prefill_ms = 0.0; + double backbone_uncond_prefill_ms = 0.0; + double backbone_cond_decode_ms = 0.0; + double backbone_uncond_decode_ms = 0.0; + uint64_t sample_call_index = 0; + uint64_t offset_blocks = 0; + }; + + std::unique_ptr stream_; + + void begin_stream(const BreezeGenerationRequest & request) { + if (stream_ != nullptr) { + throw std::runtime_error("BreezeTTS stream is already active"); + } + if (request.text.empty()) { + throw std::runtime_error("BreezeTTS requires text"); + } + const auto & config = assets->config; + BreezeSpeechCodes reference; + if (request.reference_codes.has_value()) { + reference = *request.reference_codes; + } else if (request.reference_audio.has_value()) { + reference = speech_encoder->encode(*request.reference_audio); + speech_encoder->release_runtime_graphs(); + } + std::vector reference_codes; + int64_t reference_frames = 0; + if (!reference.codes.empty()) { + if (reference.frames < 0 || reference.code_groups <= 0) { + throw std::runtime_error("BreezeTTS speech codes have invalid shape"); + } + if (static_cast(reference.codes.size()) != reference.frames * reference.code_groups) { + throw std::runtime_error("BreezeTTS speech code count does not match shape"); + } + reference_codes = reference.codes; + reference_frames = static_cast(reference_codes.size()) / config.num_codebooks; + } + + BreezePromptBranch cond_branch; + BreezePromptBranch uncond_branch; + std::vector cond_embeddings; + std::vector uncond_embeddings; + int64_t cond_steps = 0; + int64_t uncond_steps = 0; + const bool use_cfg = request.guidance_scale != 1.0F; + const double prompt_ms = engine::debug::measure_ms([&] { + if (!reference_codes.empty()) { + if (request.reference_text.empty()) { + throw std::runtime_error("BreezeTTS clone requires reference_text"); + } + cond_branch = tokenizer.build_clone(request.text, request.instruction, request.reference_text, reference_frames); + if (use_cfg) { + uncond_branch = tokenizer.build_clone_negative(request.text, request.reference_text, reference_frames); + } + } else { + cond_branch = tokenizer.build_tts_instruction(request.text, request.instruction); + if (use_cfg) { + uncond_branch = tokenizer.build_tts_plain(request.text); + } + } + cond_embeddings = merge_prompt(cond_branch, reference_codes); + cond_steps = static_cast(cond_branch.input_ids.size()); + if (use_cfg) { + uncond_embeddings = merge_prompt(uncond_branch, reference_codes); + uncond_steps = static_cast(uncond_branch.input_ids.size()); + } + }); + engine::debug::timing_log_scalar("breeze_tts.generate.prompt_ms", prompt_ms); + text_encoder.release_runtime_graphs(); + + auto state = std::make_unique(); + state->request = request; + state->use_cfg = use_cfg; + state->codes.reserve(static_cast(request.max_tokens * config.num_codebooks)); + state->scratch.reserve_vocab(static_cast(config.lm_head_size)); + state->fallback_rng = std::mt19937(static_cast(request.seed)); + state->first_options.do_sample = true; + state->first_options.temperature = request.temperature; + state->first_options.top_k = request.top_k; + state->first_options.top_p = request.top_p; + state->first_options.repetition_penalty = kRepetitionPenalty; + state->first_options.min_tokens_to_keep = 1; + speech_decoder->reset_streaming_state(); + state->ar_total_ms += engine::debug::measure_ms([&] { + state->backbone_cond_prefill_ms = engine::debug::measure_ms([&] { + state->cond = backbone_cond->prefill_embeddings(cond_embeddings, cond_steps); + }); + if (use_cfg) { + state->uncond.emplace(); + state->backbone_uncond_prefill_ms = engine::debug::measure_ms([&] { + *state->uncond = backbone_uncond->prefill_embeddings(uncond_embeddings, uncond_steps); + }); + } + backbone_cond->start_decode_embeddings(state->cond.state, cond_steps + request.max_tokens); + if (use_cfg) { + backbone_uncond->start_decode_embeddings(state->uncond->state, uncond_steps + request.max_tokens); + } + }); + stream_ = std::move(state); + } + + int step_frame_once() { + if (stream_ == nullptr) { + throw std::runtime_error("BreezeTTS stream has not been started"); + } + auto & state = *stream_; + const auto & request = state.request; + const auto & config = assets->config; + if (state.done || state.steps_taken >= request.max_tokens) { + state.done = true; + return 2; + } + ++state.steps_taken; + if (state.use_cfg) { + if (!state.uncond.has_value() || state.cond.logits.size() != state.uncond->logits.size()) { + throw std::runtime_error("BreezeTTS CFG logits shape mismatch"); + } + } + std::vector logits; + if (state.use_cfg) { + logits.resize(state.cond.logits.size()); + for (size_t i = 0; i < logits.size(); ++i) { + logits[i] = + state.uncond->logits[i] + request.guidance_scale * (state.cond.logits[i] - state.uncond->logits[i]); + } + } else { + logits = state.cond.logits; + } + suppress_reserved(logits, kCodecCodebookSize, config.vocab_size); + const int32_t first_token = sample_logits( + std::move(logits), + state.first_codebook_history, + state.first_options, + state.scratch, + state.fallback_rng, + sampling_policy.cuda_fast_path ? &sampling_policy : nullptr, + request.seed, + state.sample_call_index, + state.offset_blocks, + "BreezeTTS semantic sampler"); + if (first_token == config.vocab_size) { + state.done = true; + return 2; + } + if (first_token == config.codebook_pad_token_id) { + return 1; + } + const auto frame = generate_frame( + state.cond.hidden, + state.use_cfg ? state.uncond->hidden : state.cond.hidden, + first_token, + request, + state.scratch, + state.fallback_rng, + state.sample_call_index, + state.offset_blocks); + state.first_codebook_history.push_back(first_token); + state.codes.insert(state.codes.end(), frame.begin(), frame.end()); + const auto embedded = frame_embedding( + weights->audio_embedding, + config.num_codebooks * config.vocab_size, + config.hidden_size, + config.vocab_size, + frame); + modules::QwenCausalDecodeStepResult cond_step; + state.backbone_cond_decode_ms += engine::debug::measure_ms([&] { + cond_step = backbone_cond->decode_embedding(embedded); + }); + state.cond.logits = cond_step.logits; + state.cond.hidden = cond_step.hidden; + if (state.use_cfg) { + modules::QwenCausalDecodeStepResult uncond_step; + state.backbone_uncond_decode_ms += engine::debug::measure_ms([&] { + uncond_step = backbone_uncond->decode_embedding(embedded); + }); + state.uncond->logits = uncond_step.logits; + state.uncond->hidden = uncond_step.hidden; + } + return 0; + } + + BreezeStreamEvent next_stream_audio(size_t max_new_frames, int64_t lookahead_margin) { + if (max_new_frames == 0) { + throw std::runtime_error("BreezeTTS stream step size must be positive"); + } + if (stream_ == nullptr) { + throw std::runtime_error("BreezeTTS stream has not been started"); + } + const auto & config = assets->config; + BreezeStreamEvent out; + int64_t new_frames = 0; + const size_t code_begin = stream_->codes.size(); + stream_->ar_total_ms += engine::debug::measure_ms([&] { + while (new_frames < static_cast(max_new_frames)) { + const int status = step_frame_once(); + if (status == 0) { + ++new_frames; + } else if (status == 2) { + out.done = true; + break; + } + } + }); + BreezeSpeechCodes speech_codes; + speech_codes.codes.assign( + stream_->codes.begin() + static_cast(code_begin), + stream_->codes.end()); + speech_codes.code_groups = config.num_codebooks; + speech_codes.frames = new_frames; + if (new_frames * config.num_codebooks != static_cast(speech_codes.codes.size())) { + throw std::runtime_error("BreezeTTS stream generated code shape mismatch"); + } + if (stream_->done) { + out.done = true; + } + stream_->codec_decode_ms += engine::debug::measure_ms([&] { + out.audio = speech_decoder->decode_streaming_step(speech_codes, lookahead_margin, out.done); + }); + for (float & sample : out.audio.samples) { + sample = std::clamp(sample, -1.0F, 1.0F); + } + return out; + } + + void end_stream() { + if (stream_ == nullptr) { + return; + } + engine::debug::timing_log_scalar("breeze_tts.ar.total_ms", stream_->ar_total_ms); + engine::debug::timing_log_scalar("breeze_tts.ar.backbone_cond_prefill_ms", stream_->backbone_cond_prefill_ms); + engine::debug::timing_log_scalar("breeze_tts.ar.backbone_uncond_prefill_ms", stream_->backbone_uncond_prefill_ms); + engine::debug::timing_log_scalar("breeze_tts.ar.backbone_cond_decode_ms", stream_->backbone_cond_decode_ms); + engine::debug::timing_log_scalar("breeze_tts.ar.backbone_uncond_decode_ms", stream_->backbone_uncond_decode_ms); + engine::debug::timing_log_scalar("breeze_tts.speech_decoder.streaming_total_ms", stream_->codec_decode_ms); + stream_.reset(); + backbone_cond->release_runtime_graphs(); + backbone_uncond->release_runtime_graphs(); + depth_pair->release_runtime_graphs(); + } + runtime::AudioBuffer generate(const BreezeGenerationRequest & request) { if (request.text.empty()) { throw std::runtime_error("BreezeTTS requires text"); @@ -1200,4 +1454,16 @@ BreezeSpeechCodes BreezeGeneratorRuntime::encode_reference(const engine::runtime return impl_->speech_encoder->encode(audio); } +void BreezeGeneratorRuntime::begin_stream(const BreezeGenerationRequest & request) { + impl_->begin_stream(request); +} + +BreezeStreamEvent BreezeGeneratorRuntime::next_stream_audio(size_t max_new_frames, int64_t lookahead_margin) { + return impl_->next_stream_audio(max_new_frames, lookahead_margin); +} + +void BreezeGeneratorRuntime::end_stream() { + impl_->end_stream(); +} + } // namespace engine::models::breeze_tts diff --git a/src/models/breeze_tts/session.cpp b/src/models/breeze_tts/session.cpp index dd400d5a1..46ace5094 100644 --- a/src/models/breeze_tts/session.cpp +++ b/src/models/breeze_tts/session.cpp @@ -7,12 +7,14 @@ #include "engine/framework/text/chunking.h" #include "engine/models/breeze_tts/generator.h" +#include #include #include #include #include #include #include +#include #include namespace engine::models::breeze_tts { @@ -80,6 +82,18 @@ void validate_session_options( runtime::validate_spec_backed_session_options(validation_options, contract, kFamily, kModelName); } +void validate_request_options( + const std::unordered_map & options, + const engine::model_spec::ModelContract & contract) { + auto validation_options = options; + for (const char * key : {"stream_subchunk", "stream_frames_per_event", "stream_lookahead_margin"}) { + if (contract.request_option_keys.find(key) == contract.request_option_keys.end()) { + validation_options.erase(key); + } + } + runtime::validate_spec_backed_request_options(validation_options, contract, kModelName); +} + std::size_t reference_cache_slots_from_options(const runtime::SessionOptions & options) { const int64_t slots = runtime::parse_i64_option( options.options, @@ -183,7 +197,7 @@ runtime::RunMode BreezeTTSSession::run_mode() const { } void BreezeTTSSession::prepare(const runtime::SessionPreparationRequest & request) { - runtime::validate_spec_backed_request_options(request.options, *contract_, kModelName); + validate_request_options(request.options, *contract_); mark_prepared(); } @@ -225,7 +239,7 @@ BreezeSpeechCodes BreezeTTSSession::resolve_reference_codes(const runtime::Audio runtime::TaskResult BreezeTTSSession::run(const runtime::TaskRequest & request) { const auto wall_start = std::chrono::steady_clock::now(); - runtime::validate_spec_backed_request_options(request.options, *contract_, kModelName); + validate_request_options(request.options, *contract_); require_prepared("BreezeTTS run"); if (task_.mode != runtime::RunMode::Offline) { throw std::runtime_error("BreezeTTS run requires an offline session"); @@ -265,7 +279,7 @@ runtime::StreamingPolicy BreezeTTSSession::streaming_policy() const { } void BreezeTTSSession::start_stream(const runtime::TaskRequest & request) { - runtime::validate_spec_backed_request_options(request.options, *contract_, kModelName); + validate_request_options(request.options, *contract_); require_prepared("BreezeTTS streaming"); if (task_.mode != runtime::RunMode::Streaming) { throw std::runtime_error("BreezeTTS start_stream requires a streaming session"); @@ -287,6 +301,27 @@ void BreezeTTSSession::start_stream(const runtime::TaskRequest & request) { request.voice->speaker->audio.has_value()) { stream_reference_codes_ = resolve_reference_codes(*request.voice->speaker->audio); } + stream_subchunk_ = false; + if (const auto subchunk = runtime::find_option(request.options, {"stream_subchunk"})) { + stream_subchunk_ = runtime::parse_bool_option(*subchunk, "stream_subchunk"); + } + const auto frames_per_event = runtime::parse_i64_option(request.options, {"stream_frames_per_event"}) + .value_or(static_cast(stream_frames_per_event_)); + if (frames_per_event <= 0) { + throw std::runtime_error("BreezeTTS stream_frames_per_event must be positive"); + } + stream_frames_per_event_ = static_cast(frames_per_event); + stream_lookahead_margin_ = runtime::parse_i64_option(request.options, {"stream_lookahead_margin"}) + .value_or(stream_lookahead_margin_); + if (stream_lookahead_margin_ < 0) { + throw std::runtime_error("BreezeTTS stream_lookahead_margin must be non-negative"); + } + stream_merged_audio_ = runtime::AudioBuffer{24000, 1, {}}; + stream_started_at_ = std::chrono::steady_clock::now(); + engine::debug::trace_log_scalar("breeze_tts.streaming.subchunk", stream_subchunk_ ? 1 : 0); + engine::debug::trace_log_scalar( + "breeze_tts.streaming.frames_per_event", static_cast(stream_frames_per_event_)); + engine::debug::trace_log_scalar("breeze_tts.streaming.lookahead_margin", stream_lookahead_margin_); stream_started_ = true; } @@ -297,6 +332,9 @@ std::optional BreezeTTSSession::next_stream_event() { if (stream_chunk_index_ >= stream_chunk_requests_.size()) { return std::nullopt; } + if (stream_subchunk_) { + return next_subchunk_event(); + } const size_t chunk_index = stream_chunk_index_++; auto chunk_audio = generator_->generate( build_generation_request(stream_chunk_requests_[chunk_index], stream_reference_codes_, chunk_index)); @@ -310,6 +348,41 @@ std::optional BreezeTTSSession::next_stream_event() { return event; } +std::optional BreezeTTSSession::next_subchunk_event() { + while (true) { + if (stream_chunk_index_ >= stream_chunk_requests_.size()) { + return std::nullopt; + } + const size_t chunk_index = stream_chunk_index_; + if (!stream_chunk_active_) { + generator_->begin_stream(build_generation_request( + stream_chunk_requests_[chunk_index], + stream_reference_codes_, + chunk_index)); + stream_chunk_active_ = true; + } + + auto event_audio = generator_->next_stream_audio(stream_frames_per_event_, stream_lookahead_margin_); + if (event_audio.done) { + generator_->end_stream(); + stream_chunk_active_ = false; + ++stream_chunk_index_; + } + if (event_audio.audio.samples.empty()) { + continue; + } + runtime::append_audio_buffer(stream_merged_audio_, event_audio.audio); + runtime::StreamEvent event; + event.named_audio_outputs.push_back({ + "chunk_" + std::to_string(chunk_index) + "_part_" + std::to_string(stream_event_seq_++), + std::move(event_audio.audio), + {}, + }); + engine::debug::trace_log_scalar("breeze_tts.streaming.event_index", static_cast(stream_event_seq_)); + return event; + } +} + void BreezeTTSSession::set_stream_event_sink(runtime::StreamEventCallback sink) { (void)sink; } @@ -322,15 +395,24 @@ runtime::TaskResult BreezeTTSSession::finish_stream() { } runtime::TaskResult result; result.audio_output = std::move(stream_merged_audio_); + engine::debug::timing_log_scalar("session.wall_ms", engine::debug::elapsed_ms(stream_started_at_)); reset(); return result; } void BreezeTTSSession::reset() { + if (stream_chunk_active_) { + generator_->end_stream(); + stream_chunk_active_ = false; + } stream_chunk_requests_.clear(); stream_reference_codes_.reset(); stream_merged_audio_ = runtime::AudioBuffer{}; stream_chunk_index_ = 0; + stream_subchunk_ = false; + stream_frames_per_event_ = 32; + stream_lookahead_margin_ = 12; + stream_event_seq_ = 0; stream_started_ = false; } diff --git a/src/models/breeze_tts/speech_decoder.cpp b/src/models/breeze_tts/speech_decoder.cpp index 1edf0c1c4..7197340f9 100644 --- a/src/models/breeze_tts/speech_decoder.cpp +++ b/src/models/breeze_tts/speech_decoder.cpp @@ -1079,6 +1079,57 @@ BreezeSpeechDecoderRuntime::BreezeSpeechDecoderRuntime( BreezeSpeechDecoderRuntime::~BreezeSpeechDecoderRuntime() = default; +std::vector BreezeSpeechDecoderRuntime::decode_window_samples( + const std::vector & chunk, + int64_t chunk_frames, + int64_t context_frames) const { + if (chunk_frames <= 0 || context_frames < 0 || context_frames > chunk_frames) { + throw std::runtime_error("Breeze speech decoder received invalid chunk shape"); + } + if (static_cast(chunk.size()) != chunk_frames * weights_->config.num_quantizers) { + throw std::runtime_error("Breeze speech decoder chunk payload size mismatch"); + } + const int threads = std::max(1, execution_context_->config().threads); + auto * graph_slot = &graph_; +#if defined(ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS) + const bool optimized_cache_enabled = + kStrixHaloGraphCacheEnabled && execution_context_->backend_type() == core::BackendType::Hip; + if (optimized_cache_enabled) { + for (size_t index = 0; index < kStrixHaloCachedChunkFrames.size(); ++index) { + if (chunk_frames == kStrixHaloCachedChunkFrames[index]) { + graph_slot = &optimized_graphs_[index]; + break; + } + } + } +#endif + auto & graph = *graph_slot; + const bool graph_rebuilt = + graph == nullptr || !graph->matches(*weights_, chunk_frames, execution_context_->backend(), threads); + if (graph_rebuilt) { + graph.reset(); + graph = std::make_unique( + weights_, + chunk_frames, + *execution_context_, + *constants_, + graph_arena_bytes_, + allow_flash_attention_); + } + auto decoded = graph->run(chunk.data(), chunk.size()); + const int64_t drop = context_frames * kDecodeSamplesPerCode; + if (drop > static_cast(decoded.size())) { + throw std::runtime_error("Breeze speech decoder chunk context exceeds decoded waveform"); + } + const int64_t valid_samples = chunk_frames * kDecodeSamplesPerCode; + if (valid_samples < drop || valid_samples > static_cast(decoded.size())) { + throw std::runtime_error("Breeze speech decoder valid sample range exceeds decoded waveform"); + } + return std::vector( + decoded.begin() + static_cast(drop), + decoded.begin() + static_cast(valid_samples)); +} + runtime::AudioBuffer BreezeSpeechDecoderRuntime::decode(const BreezeSpeechCodes & codec_codes) const { const auto total_start = Clock::now(); if (codec_codes.frames <= 0 || codec_codes.code_groups != weights_->config.num_quantizers) { @@ -1101,51 +1152,83 @@ runtime::AudioBuffer BreezeSpeechDecoderRuntime::decode(const BreezeSpeechCodes const auto dst = chunk.begin() + static_cast(frame * codec_codes.code_groups); std::copy(src, src + codec_codes.code_groups, dst); } - const int threads = std::max(1, execution_context_->config().threads); - auto * graph_slot = &graph_; -#if defined(ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS) - const bool optimized_cache_enabled = - kStrixHaloGraphCacheEnabled && execution_context_->backend_type() == core::BackendType::Hip; - if (optimized_cache_enabled) { - for (size_t index = 0; index < kStrixHaloCachedChunkFrames.size(); ++index) { - if (chunk_frames == kStrixHaloCachedChunkFrames[index]) { - graph_slot = &optimized_graphs_[index]; - break; - } - } - } -#endif - auto & graph = *graph_slot; - const bool graph_rebuilt = - graph == nullptr || !graph->matches(*weights_, chunk_frames, execution_context_->backend(), threads); - if (graph_rebuilt) { - auto replacement = std::make_unique( - weights_, - chunk_frames, - *execution_context_, - *constants_, - graph_arena_bytes_, - allow_flash_attention_); - graph = std::move(replacement); - } - auto decoded = graph->run(chunk.data(), chunk.size()); - const int64_t drop = context * kDecodeSamplesPerCode; - if (drop > static_cast(decoded.size())) { - throw std::runtime_error("Breeze speech decoder chunk context exceeds decoded waveform"); - } - const int64_t valid_samples = chunk_frames * kDecodeSamplesPerCode; - if (valid_samples < drop || valid_samples > static_cast(decoded.size())) { - throw std::runtime_error("Breeze speech decoder valid sample range exceeds decoded waveform"); - } - samples.insert( - samples.end(), - decoded.begin() + static_cast(drop), - decoded.begin() + static_cast(valid_samples)); + auto decoded = decode_window_samples(chunk, chunk_frames, context); + samples.insert(samples.end(), decoded.begin(), decoded.end()); } debug::timing_log_scalar("breeze_tts.speech_decoder.total_ms", engine::debug::elapsed_ms(total_start, Clock::now())); return runtime::AudioBuffer{kSampleRate, 1, std::move(samples)}; } +struct BreezeSpeechDecoderRuntime::StreamingState { + std::vector left_context_codes; + std::vector pending_codes; +}; + +void BreezeSpeechDecoderRuntime::reset_streaming_state() const { + streaming_state_ = std::make_unique(); +} + +runtime::AudioBuffer BreezeSpeechDecoderRuntime::decode_streaming_step( + const BreezeSpeechCodes & codec_codes, + int64_t lookahead_margin, + bool final) const { + const auto total_start = Clock::now(); + if (lookahead_margin < 0) { + throw std::runtime_error("Breeze speech decoder streaming lookahead must be non-negative"); + } + if (codec_codes.frames < 0 || codec_codes.code_groups != weights_->config.num_quantizers) { + throw std::runtime_error("Breeze speech decoder received invalid streaming codec shape"); + } + if (static_cast(codec_codes.codes.size()) != codec_codes.frames * codec_codes.code_groups) { + throw std::runtime_error("Breeze speech decoder streaming codec payload size mismatch"); + } + if (!streaming_state_) { + reset_streaming_state(); + } + auto & state = *streaming_state_; + state.pending_codes.insert(state.pending_codes.end(), codec_codes.codes.begin(), codec_codes.codes.end()); + const int64_t groups = weights_->config.num_quantizers; + int64_t pending_frames = static_cast(state.pending_codes.size()) / groups; + int64_t frames_to_emit = final ? pending_frames : pending_frames - lookahead_margin; + if (frames_to_emit <= 0) { + return runtime::AudioBuffer{kSampleRate, 1, {}}; + } + std::vector samples; + while (frames_to_emit > 0) { + const int64_t emit_frames = std::min(frames_to_emit, kChunkCodes); + const int64_t context_frames = static_cast(state.left_context_codes.size()) / groups; + const int64_t chunk_frames = context_frames + emit_frames; + std::vector chunk; + chunk.reserve(static_cast(chunk_frames * groups)); + chunk.insert(chunk.end(), state.left_context_codes.begin(), state.left_context_codes.end()); + chunk.insert( + chunk.end(), + state.pending_codes.begin(), + state.pending_codes.begin() + static_cast(emit_frames * groups)); + auto decoded = decode_window_samples(chunk, chunk_frames, context_frames); + samples.insert(samples.end(), decoded.begin(), decoded.end()); + + std::vector next_context; + const int64_t available_context_frames = context_frames + emit_frames; + const int64_t keep_context_frames = std::min(available_context_frames, kLeftContextCodes); + next_context.reserve(static_cast(keep_context_frames * groups)); + const int64_t skip_frames = available_context_frames - keep_context_frames; + next_context.insert( + next_context.end(), + chunk.begin() + static_cast(skip_frames * groups), + chunk.end()); + state.left_context_codes = std::move(next_context); + state.pending_codes.erase( + state.pending_codes.begin(), + state.pending_codes.begin() + static_cast(emit_frames * groups)); + pending_frames -= emit_frames; + frames_to_emit -= emit_frames; + } + engine::debug::trace_log_scalar("breeze_tts.speech_decoder.streaming.pending_frames", pending_frames); + engine::debug::timing_log_scalar("breeze_tts.speech_decoder.streaming_ms", engine::debug::elapsed_ms(total_start, Clock::now())); + return runtime::AudioBuffer{kSampleRate, 1, std::move(samples)}; +} + runtime::AudioBuffer BreezeSpeechDecoderRuntime::decode_and_trim_reference( const BreezeSpeechCodes & reference_codes, const BreezeSpeechCodes & generated_codes) const { From e8b98fa548efb7c42327e95c97100060b4d4a47e Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Mon, 7 Sep 2026 04:19:52 -0400 Subject: [PATCH 2/4] Make BreezeTTS streaming incremental by default --- include/engine/models/breeze_tts/session.h | 2 -- model_specs/breeze_tts.json | 7 ------ src/models/breeze_tts/session.cpp | 25 +--------------------- 3 files changed, 1 insertion(+), 33 deletions(-) diff --git a/include/engine/models/breeze_tts/session.h b/include/engine/models/breeze_tts/session.h index 4b9939b04..0278bd773 100644 --- a/include/engine/models/breeze_tts/session.h +++ b/include/engine/models/breeze_tts/session.h @@ -68,7 +68,6 @@ class BreezeTTSSession final const engine::runtime::TaskRequest & request, const std::optional & reference_codes, size_t chunk_index) const; - std::optional next_subchunk_event(); engine::runtime::TaskSpec task_; std::shared_ptr assets_; @@ -81,7 +80,6 @@ class BreezeTTSSession final engine::runtime::AudioBuffer stream_merged_audio_; std::chrono::steady_clock::time_point stream_started_at_; size_t stream_chunk_index_ = 0; - bool stream_subchunk_ = false; size_t stream_frames_per_event_ = 32; int64_t stream_lookahead_margin_ = 12; bool stream_chunk_active_ = false; diff --git a/model_specs/breeze_tts.json b/model_specs/breeze_tts.json index 569335fe0..480c31c96 100644 --- a/model_specs/breeze_tts.json +++ b/model_specs/breeze_tts.json @@ -128,13 +128,6 @@ "min": 0, "default": 0 }, - { - "name": "stream_subchunk", - "type": "bool", - "description": "Emit multiple streaming audio events per text chunk.", - "required": false, - "default": false - }, { "name": "stream_frames_per_event", "type": "int", diff --git a/src/models/breeze_tts/session.cpp b/src/models/breeze_tts/session.cpp index 46ace5094..0ad9efe3e 100644 --- a/src/models/breeze_tts/session.cpp +++ b/src/models/breeze_tts/session.cpp @@ -86,7 +86,7 @@ void validate_request_options( const std::unordered_map & options, const engine::model_spec::ModelContract & contract) { auto validation_options = options; - for (const char * key : {"stream_subchunk", "stream_frames_per_event", "stream_lookahead_margin"}) { + for (const char * key : {"stream_frames_per_event", "stream_lookahead_margin"}) { if (contract.request_option_keys.find(key) == contract.request_option_keys.end()) { validation_options.erase(key); } @@ -301,10 +301,6 @@ void BreezeTTSSession::start_stream(const runtime::TaskRequest & request) { request.voice->speaker->audio.has_value()) { stream_reference_codes_ = resolve_reference_codes(*request.voice->speaker->audio); } - stream_subchunk_ = false; - if (const auto subchunk = runtime::find_option(request.options, {"stream_subchunk"})) { - stream_subchunk_ = runtime::parse_bool_option(*subchunk, "stream_subchunk"); - } const auto frames_per_event = runtime::parse_i64_option(request.options, {"stream_frames_per_event"}) .value_or(static_cast(stream_frames_per_event_)); if (frames_per_event <= 0) { @@ -318,7 +314,6 @@ void BreezeTTSSession::start_stream(const runtime::TaskRequest & request) { } stream_merged_audio_ = runtime::AudioBuffer{24000, 1, {}}; stream_started_at_ = std::chrono::steady_clock::now(); - engine::debug::trace_log_scalar("breeze_tts.streaming.subchunk", stream_subchunk_ ? 1 : 0); engine::debug::trace_log_scalar( "breeze_tts.streaming.frames_per_event", static_cast(stream_frames_per_event_)); engine::debug::trace_log_scalar("breeze_tts.streaming.lookahead_margin", stream_lookahead_margin_); @@ -332,23 +327,6 @@ std::optional BreezeTTSSession::next_stream_event() { if (stream_chunk_index_ >= stream_chunk_requests_.size()) { return std::nullopt; } - if (stream_subchunk_) { - return next_subchunk_event(); - } - const size_t chunk_index = stream_chunk_index_++; - auto chunk_audio = generator_->generate( - build_generation_request(stream_chunk_requests_[chunk_index], stream_reference_codes_, chunk_index)); - runtime::append_audio_buffer(stream_merged_audio_, chunk_audio); - runtime::StreamEvent event; - event.named_audio_outputs.push_back({ - "chunk_" + std::to_string(chunk_index), - std::move(chunk_audio), - {}, - }); - return event; -} - -std::optional BreezeTTSSession::next_subchunk_event() { while (true) { if (stream_chunk_index_ >= stream_chunk_requests_.size()) { return std::nullopt; @@ -409,7 +387,6 @@ void BreezeTTSSession::reset() { stream_reference_codes_.reset(); stream_merged_audio_ = runtime::AudioBuffer{}; stream_chunk_index_ = 0; - stream_subchunk_ = false; stream_frames_per_event_ = 32; stream_lookahead_margin_ = 12; stream_event_seq_ = 0; From d2fe76ea45d8c178aa4aa3d86b05d4d1bfd41db2 Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Mon, 7 Sep 2026 04:28:54 -0400 Subject: [PATCH 3/4] Document BreezeTTS incremental streaming controls --- docs/models/breeze_tts.md | 40 +++++++++++++++++++++- include/engine/models/breeze_tts/session.h | 2 +- model_specs/breeze_tts.json | 2 +- src/models/breeze_tts/session.cpp | 2 +- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/docs/models/breeze_tts.md b/docs/models/breeze_tts.md index 584713bb8..849895732 100644 --- a/docs/models/breeze_tts.md +++ b/docs/models/breeze_tts.md @@ -39,6 +39,22 @@ audiocpp_cli \ --out breeze_tts_design.wav ``` +Streaming: + +```bash +audiocpp_cli \ + --task tts \ + --mode streaming \ + --family breeze_tts \ + --model models/Breeze-TTS-2-GGUF/breeze-tts-2-q8_0.gguf \ + --backend cuda \ + --text "Welcome to the BreezeTTS 2 streaming demo." \ + --request-option instruction="A confident product demo narrator with steady pacing." \ + --request-option stream_frames_per_event=16 \ + --out breeze_tts_stream.wav \ + --out-dir breeze_tts_stream_chunks +``` + ## Model | Field | Value | @@ -46,7 +62,7 @@ audiocpp_cli \ | Family | `breeze_tts` | | Default GGUF | `models/Breeze-TTS-2-GGUF/breeze-tts-2-q8_0.gguf` | | Tasks | `tts`, `clon` | -| Modes | `offline` | +| Modes | `offline`, `streaming` | | Languages | `zh`, `en` | | Voice input | Optional for `tts`; required for `clon` | @@ -66,10 +82,32 @@ audiocpp_cli \ | `--request-option top_k=` | integer >= 0 | `50` | Top-k sampling limit; `0` disables top-k filtering. | | `--request-option top_p=` | `0..1` | `1.0` | Top-p sampling limit. | | `--request-option seed=` | integer >= 0 | `0` | Generation seed. | +| `--request-option stream_frames_per_event=` | integer > 0 | `16` | Streaming codec frames per emitted audio event. Smaller values can reduce TTFT but increase event/decoder overhead. | +| `--request-option stream_lookahead_margin=` | integer >= 0 | `12` | Trailing codec frames held before emission to reduce streaming boundary artifacts. | | `--session-option breeze_tts.reference_cache_slots=` | integer >= 0 | `1` | Prepared reference-audio cache slots. | | `--session-option breeze_tts.attention=` | `auto`, `flash`, `eager` | `auto` | Attention kernel. `auto` uses flash except on Volta/Turing GPUs (e.g. V100), where it falls back to eager to avoid missing MMA kernels. | | `--session-option weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0`, `q4_0`, `q4_k` | `native` | Weight storage type; quantized types convert at load time from the BF16 package. | +BreezeTTS streaming is incremental by default. It emits audio events from the +generated codec-frame stream instead of waiting for a whole text chunk. For the +OpenAI-compatible speech endpoint, pass streaming options inside the request +`options` object: + +```json +{ + "model": "breeze-stream", + "input": "Welcome to the BreezeTTS 2 streaming demo.", + "stream": true, + "stream_format": "sse", + "response_format": "pcm", + "options": { + "instruction": "A confident product demo narrator with steady pacing.", + "stream_frames_per_event": "16", + "stream_lookahead_margin": "12" + } +} +``` + Quantized weight storage is the largest measured speedup and applies to CUDA and HIP alike: `q8_0` cut the fixed 100-token regression case from RTF ~1.5 to ~0.95 on gfx1151 and from ~0.77 to ~0.56 on an RTX 2080 Ti, and `q4_k` reached diff --git a/include/engine/models/breeze_tts/session.h b/include/engine/models/breeze_tts/session.h index 0278bd773..97ad2416f 100644 --- a/include/engine/models/breeze_tts/session.h +++ b/include/engine/models/breeze_tts/session.h @@ -80,7 +80,7 @@ class BreezeTTSSession final engine::runtime::AudioBuffer stream_merged_audio_; std::chrono::steady_clock::time_point stream_started_at_; size_t stream_chunk_index_ = 0; - size_t stream_frames_per_event_ = 32; + size_t stream_frames_per_event_ = 16; int64_t stream_lookahead_margin_ = 12; bool stream_chunk_active_ = false; size_t stream_event_seq_ = 0; diff --git a/model_specs/breeze_tts.json b/model_specs/breeze_tts.json index 480c31c96..5e39c7569 100644 --- a/model_specs/breeze_tts.json +++ b/model_specs/breeze_tts.json @@ -134,7 +134,7 @@ "description": "Generated codec frames per streaming audio event.", "required": false, "min": 1, - "default": 32 + "default": 16 }, { "name": "stream_lookahead_margin", diff --git a/src/models/breeze_tts/session.cpp b/src/models/breeze_tts/session.cpp index 0ad9efe3e..a24b1666a 100644 --- a/src/models/breeze_tts/session.cpp +++ b/src/models/breeze_tts/session.cpp @@ -387,7 +387,7 @@ void BreezeTTSSession::reset() { stream_reference_codes_.reset(); stream_merged_audio_ = runtime::AudioBuffer{}; stream_chunk_index_ = 0; - stream_frames_per_event_ = 32; + stream_frames_per_event_ = 16; stream_lookahead_margin_ = 12; stream_event_seq_ = 0; stream_started_ = false; From cf547f01551e5e51a40f6d8f657bfbfc8e4f20db Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Mon, 7 Sep 2026 14:24:38 -0400 Subject: [PATCH 4/4] Clean up Breeze streaming errors --- src/models/breeze_tts/session.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/models/breeze_tts/session.cpp b/src/models/breeze_tts/session.cpp index a24b1666a..6fe4e983b 100644 --- a/src/models/breeze_tts/session.cpp +++ b/src/models/breeze_tts/session.cpp @@ -340,7 +340,14 @@ std::optional BreezeTTSSession::next_stream_event() { stream_chunk_active_ = true; } - auto event_audio = generator_->next_stream_audio(stream_frames_per_event_, stream_lookahead_margin_); + BreezeStreamEvent event_audio; + try { + event_audio = generator_->next_stream_audio(stream_frames_per_event_, stream_lookahead_margin_); + } catch (...) { + generator_->end_stream(); + stream_chunk_active_ = false; + throw; + } if (event_audio.done) { generator_->end_stream(); stream_chunk_active_ = false;