diff --git a/CMakeLists.txt b/CMakeLists.txt index f85c1a94b..e653af20c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1467,6 +1467,23 @@ audiocpp_add_model(ace_step LOADERS engine::models::ace_step::make_ace_step_loader ) +audiocpp_add_model(sopro_tts + SOURCES + src/community_models/sopro_tts/acoustic.cpp + src/community_models/sopro_tts/assets.cpp + src/community_models/sopro_tts/reference.cpp + src/community_models/sopro_tts/semantic_encoder.cpp + src/community_models/sopro_tts/semantic_lm.cpp + src/community_models/sopro_tts/session.cpp + src/community_models/sopro_tts/speaker_encoder.cpp + src/community_models/sopro_tts/text_tokenizer.cpp + src/community_models/sopro_tts/vocoder.cpp + INCLUDES + engine/community_models/sopro_tts/session.h + LOADERS + engine::community_models::sopro_tts::make_sopro_tts_loader +) + audiocpp_add_model(soprano_tts SOURCES src/community_models/soprano_tts/assets.cpp @@ -1943,6 +1960,7 @@ if (ENGINE_BUILD_WARMBENCH) target_compile_definitions(soprano_warm_bench PRIVATE ENGINE_REPO_ROOT="${CMAKE_CURRENT_SOURCE_DIR}" ) + add_engine_warmbench(sopro_probe tests/sopro_tts/sopro_probe.cpp) add_engine_warmbench(sortformer_diar_warm_bench tests/sortformer_diar/sortformer_diar_warm_bench.cpp) add_engine_warmbench(supertonic_warm_bench tests/supertonic/supertonic_warm_bench.cpp) add_engine_warmbench(vevo2_warm_bench tests/vevo2/vevo2_warm_bench.cpp) @@ -2363,6 +2381,22 @@ if (ENGINE_BUILD_TESTS) target_link_libraries(test_granite5asr_golden_transcription PRIVATE OpenMP::OpenMP_CXX) endif() + if (sopro_tts IN_LIST AUDIOCPP_LINKED_MODELS) + add_executable(test_sopro_tts_audio_ops + tests/sopro_tts/test_sopro_tts_audio_ops.cpp + ) + target_link_libraries(test_sopro_tts_audio_ops PRIVATE engine_runtime ggml) + target_include_directories(test_sopro_tts_audio_ops PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests) + if (ENGINE_ENABLE_OPENMP) + target_link_libraries(test_sopro_tts_audio_ops PRIVATE OpenMP::OpenMP_CXX) + endif() + + add_test( + NAME test_sopro_tts_audio_ops + COMMAND test_sopro_tts_audio_ops + ) + endif() + if (audio8_asr IN_LIST AUDIOCPP_LINKED_MODELS) add_executable(test_audio8_asr_units tests/audio8_asr/test_audio8_asr_units.cpp diff --git a/README.md b/README.md index c24e0951a..97fec33c7 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ Community model ports live under `community_models` to make the ownership bounda | **outetts** | TTS, Clone | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | GGUF | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | GGUF F32/16/Q8, Stream | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](docs/community_models/parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | GGUF Q8, Stream | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](docs/community_models/sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | +| **sopro_tts** | TTS, Clone | en, pt, fr, de | Safetensors, GGUF, Stream | Community | [Sopro V2 Turbo](docs/community_models/sopro_tts.md) 120M zero-shot voice cloning: style-prefix semantic LM over FSQ tokens, rectified-flow acoustic DiT, Vocos ISTFT vocoder at 24 kHz | | **soprano_tts** | TTS | en | GGUF Q8, Stream | [@WalkingCat](https://github.com/WalkingCat) | [Soprano-1.1-80M](https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF) ultra-lightweight TTS with Qwen3 LM + Vocos decoder | | **vietneu_tts** | TTS, Clone | vi, en | GGUF | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](docs/community_models/vietneu_tts.md) TTS and voice cloning support | diff --git a/docs/community_models/models.md b/docs/community_models/models.md index fc6bc07cd..62821e4d7 100644 --- a/docs/community_models/models.md +++ b/docs/community_models/models.md @@ -28,6 +28,7 @@ Practical expectations: | **minimax_music3** | Music | auto | [@0xShug0](https://github.com/0xShug0) | [MiniMax Music 3](minimax_music3.md) text-to-music generation with lyrics conditioning | | **moss_tts_local** | TTS, voice cloning | auto, optional language hint | [@justinjohn0306](https://github.com/justinjohn0306) | [MOSS-TTS-Local Transformer v1.5](../models/moss_tts.md) support in the core model tree | | **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Mirek [@mirek190](https://github.com/mirek190) | [Llama-OuteTTS-1.0-1B](outetts.md) TTS and voice cloning support | +| **sopro_tts** | TTS, voice cloning | en, pt, fr, de | Community | [Sopro V2 Turbo](sopro_tts.md) 120M zero-shot cloning — SentencePiece text, style-prefix semantic LM over FSQ tokens, two-step rectified-flow acoustic DiT, Vocos ISTFT vocoder at 24 kHz; offline plus segment-level streaming | | **voxcpm1** | TTS, voice cloning | zh, en, ja, ko | Community | [VoxCPM1](voxcpm1.md) tokenizer-free 0.5B TTS with 16 kHz output, streaming, and continuation-mode voice cloning | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | diff --git a/docs/community_models/sopro_tts.md b/docs/community_models/sopro_tts.md new file mode 100644 index 000000000..857ba2abe --- /dev/null +++ b/docs/community_models/sopro_tts.md @@ -0,0 +1,247 @@ +# Sopro V2 Turbo (`sopro_tts`) + +[samuel-vitorino/sopro-v2-turbo](https://huggingface.co/samuel-vitorino/sopro-v2-turbo) is a +120M-parameter zero-shot voice-cloning TTS covering English, European Portuguese, French and +German, released under Apache-2.0. It clones from 5–20 s of reference audio and outputs +24 kHz mono. + +> **Not the same model as `soprano_tts`.** audio.cpp's existing `soprano_tts` family is +> [ekwek/Soprano-1.1-80M](https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF), an unrelated +> project with a Qwen3 backbone and a different decoder. Sopro V2 Turbo shares nothing with it +> beyond a similar name, so it ships as its own family. The `--family` hints `sopro`, +> `sopro_v2` and `sopro_v2_turbo` all resolve to `sopro_tts`. + +## Installation + +The upstream safetensors checkpoint runs directly — no conversion step: + +```bash +python3 tools/model_manager_v2.py install sopro_v2_turbo_safetensors +# -> models/sopro-v2-turbo/{config.json,tokenizer.model,*.safetensors} +``` + +To run from GGUF instead, pack the four stages into one file. `audiocpp_gguf` takes one +namespaced input per stage, and `--root` makes it embed `config.json` and `tokenizer.model` as +sidecars, so the resulting `.gguf` is self-contained: + +```bash +build/bin/audiocpp_gguf \ + --input model=models/sopro-v2-turbo/model.safetensors \ + --input semantic_encoder=models/sopro-v2-turbo/semantic_encoder.safetensors \ + --input speaker_encoder=models/sopro-v2-turbo/speaker_encoder.safetensors \ + --input vocoder=models/sopro-v2-turbo/vocoder.safetensors \ + --family sopro_tts --root models/sopro-v2-turbo \ + --output models/sopro-v2-turbo-GGUF/sopro-v2-turbo-f16.gguf --type f16 +``` + +No public audio.cpp GGUF build of this family is published yet, so the spec's default +`sopro_v2_turbo_f16` package has `download.kind = "unsupported"` and expects the file above to +be produced locally. + +## Build + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=sopro_tts +cmake --build build --target audiocpp_cli -j"$(nproc)" +``` + +## Run + +Zero-shot cloning always needs a reference clip: + +```bash +build/bin/audiocpp_cli \ + --task tts --family sopro_tts \ + --model models/sopro-v2-turbo \ + --backend cpu --threads 8 \ + --text "Sopro is a lightweight text-to-speech model that runs on device." \ + --voice-ref ref.wav \ + --request-option language=en \ + --out out.wav --metrics +``` + +`--task clon` works the same way. The reference is resampled to 24 kHz, cropped at a pause near +`ref_seconds`, and level-normalised before the speaker and semantic encoders see it. That +normalisation is boost-only and peak-guarded (sopro 2.1): a reference already at or above the +−19.8 dB prompt level is passed through untouched, and a boost is never large enough to push +the peak past 0.95. The level the reference ends up at is what the output gain falls back to +when the generated audio is too short to measure. + +## Streaming + +`--mode streaming` emits one pull event per text segment instead of one buffer at the end: + +```bash +build/bin/audiocpp_cli \ + --task tts --family sopro_tts \ + --model models/sopro-v2-turbo \ + --backend cpu --threads 8 --mode streaming \ + --text "$(cat article.txt)" \ + --voice-ref ref.wav --language en \ + --text-chunk-size 120 \ + --out stream.wav --out-dir segments/ +``` + +Each event carries a `segment_` named audio buffer that is already levelled, trimmed and +faded, so a consumer can play events back to back; `--out` still writes the whole utterance, +and it is exactly the concatenation of the events. The reference voice is encoded once in +`start_stream`, so every event after the first costs only its own LM, solver and vocoder pass. + +**Granularity is one text segment, not one frame.** The acoustic DiT and the Vocos vocoder both +see a whole span at once, and this checkpoint ships no causal vocoder, so a segment is the +smallest unit that can leave without boundary artefacts. `text_chunk_size` is the latency dial: +on a 16-core CPU build at 8 threads, 8 solver steps and a 14 s reference, `text_chunk_size=120` +put the first audio out at ~3.1 s for a 6.7 s segment, against ~9.9 s for the same text offline. + +Two things to know before turning it down further: + +- Every segment re-solves the *whole* reference mel prompt alongside its own frames, so the + per-segment cost has a floor of roughly `ref_seconds` worth of DiT work. Segments shorter + than about 1 s take longer to generate than to play, even though the stream as a whole stays + ahead of real time (`text_chunk_size=40` measured 0.75 RTF overall, with the shortest + segment at 2.1). Lowering `ref_seconds` shrinks that floor at some cost to cloning fidelity. +- Streaming reproduces the offline waveform for the same `seed`, sample count included, with + one deliberate exception: offline level-matches over the finished utterance, which a stream + cannot see, so the first segment fixes the gain for the rest. The measured difference is a + constant scale factor (1.15x, +1.2 dB, on the clip above) with a −47 dB residual. + +## Options + +| Request option | Type | Default | Meaning | +|---|---|---|---| +| `language` | string | *(empty)* | Prepends `<\|lang_xx\|>`; one of `en`, `pt`, `fr`, `de`. Optional, helps on ambiguous text | +| `temperature` | float | 0.8 | Semantic LM sampling temperature; `0` selects arg-max | +| `top_p` | float | 0.9 | Nucleus threshold, applied after top-k renormalisation | +| `top_k` | int | 25 | Top-k truncation; `0` disables | +| `num_inference_steps` | int | 2 | Acoustic rectified-flow Euler steps | +| `max_seconds` | float | 30.0 | Audio cap per segment; long text is split, so total length is unbounded | +| `min_seconds` | float | 0.4 | Minimum audio before the semantic LM may emit EOS | +| `ref_seconds` | float | 10.0 | Reference window used for cloning | +| `text_chunk_size` | int | 300 | Max codepoints per synthesis segment | +| `seed` | int | *(random)* | Seeds semantic sampling and the acoustic noise prior | + +| Session option | Default | Meaning | +|---|---|---| +| `sopro_tts.language` | *(empty)* | Default language tag for requests that do not set one | + +| Load option | Default | Meaning | +|---|---|---| +| `sopro_tts.matmul_weight_type` | `f32` | Storage type for matmul weights (`native`, `f32`, `f16`, `bf16`, `q8_0`) | +| `sopro_tts.conv_weight_type` | `f32` | Storage type for convolution weights (`native`, `f32`, `f16`) | + +Every default comes from the checkpoint's `config.json` `generation` block, so a retrained +variant picks up its own values without a code change. + +## Architecture notes + +Five stages run per request, mirroring `sopro/` upstream: + +1. **Text** (`text_tokenizer.cpp`) — SentencePiece unigram, 8192 pieces, plus the reference's + punctuation clean-up and sentence/clause/word segmentation. No phonemiser. +2. **Speaker encoder** (`speaker_encoder.cpp`, ~11M) — 16 kHz log-mel into a three-stage gated + depthwise ResNet with squeeze-excite, then attentive-statistics pooling for identity and + multi-scale mean/std pooling for style. The convolution trunk runs on the backend; the two + pooling heads and their MLPs run on the host, where they cost a few hundred kFLOP. +3. **Semantic encoder** (`semantic_encoder.cpp`, ~82M) — a Whisper-style front end and six + non-causal transformer layers, resampled to one frame per 1024 output samples and quantised + by an FSQ head with levels `[7,5,5,5,5]` (4375 codes). +4. **Semantic LM** (`semantic_lm.cpp`) — 12 pre-norm blocks, dim 512, QK RMS-norm, SwiGLU, + half-rotation RoPE. The prompt is `[style prefix | text | carried tokens | BOS]`. Because the + only structural difference from a Qwen3 decoder is a LayerScale vector on each residual + branch, and those branches end in a bias-free projection, the scale is folded into that + projection's rows at load time and the shared `QwenCausalDecodeRuntime` runs the stack + unmodified. The eight-query style prefix cross-attention runs on the host. +5. **Acoustic head + vocoder** (`acoustic.cpp`, `vocoder.cpp`) — an 8-block adaptive-layer-norm + DiT solving a rectified flow in two Euler steps on a sway-sampled time grid, with the prompt + mel re-pinned after every step; then a 14-layer Vocos ConvNeXt backbone and one centred + ISTFT. The ISTFT head is band-limited: bins at or above `vocoder.band_limit_hz` (10900 Hz by + default, as in sopro 2.1) are zeroed before the inverse transform, which removes the + high-frequency hiss the unlimited head produced. `mu` (the upsampled semantic conditioning) + is built in its own graph because it is constant across solver steps. + +Two implementation details worth knowing: + +- **Front-end buffers come from the checkpoint.** torchaudio stores its analysis window and mel + filterbank as persistent buffers, and all three front ends load those rather than rebuilding + the filterbank, which removes the usual mel-parity risk. A checkpoint exported without them + fails at load with a message naming the missing tensor. +- **Grouped convolutions are split.** The DiT's causal positional embedding uses + `Conv1d(512, 512, k=31, groups=16)`; ggml has no grouped conv1d, so the weight is split into + 16 independent convolutions at load time. +- **The velocity graph re-uploads every leaf per Euler step.** `ggml_gallocr` exempts only + `GGML_TENSOR_FLAG_OUTPUT` tensors from being freed and reused + (`ggml_gallocr_free_node` in `ggml-alloc.c`); an *input* leaf's arena space is handed to a + later intermediate once its last consumer has run. That is correct for a one-shot graph, but + the solver replays the velocity graph once per step, so staging `mu`, `cond_mel`, `cond_mask`, + `spk` and the RoPE positions once would leave the second and later steps reading whatever + overwrote them. `SoproAcousticGraphs::upload_constants` re-uploads all of them before every + compute; it costs a few hundred kB per step against a multi-GFLOP DiT pass. + +## Known limitations + +- **Streaming is segment-level, not frame-level.** The upstream frame-level path (chunked DiT + attention plus the causal vocoder, `vocoder_streaming.safetensors`) is not implemented, and + that vocoder is not part of the published checkpoint this family loads. What ships is one + pull event per text segment; see [Streaming](#streaming) for the latency it actually buys. +- **Sampling RNG is not torch-bit-exact.** `sample_next_token` reproduces the reference's + masking, temperature, top-k and top-p arithmetic exactly, but draws from a seeded + `std::mt19937_64` rather than torch's generator, so a given `seed` will not reproduce the + Python output sample-for-sample. The same `seed` is reproducible within audio.cpp. +- **No `int8` AR path.** The upstream `--int8` CPU option has no equivalent; use + `sopro_tts.matmul_weight_type=q8_0` instead. +- The text front end is deliberately minimal upstream: prefer words to symbols (`one plus two`, + not `1 + 2`), and avoid mixing languages inside one sentence. + +## Validation status + +Verified against the real checkpoint on a 16-core x86-64 CPU build, 8 threads. Every stage was +reimplemented independently in numpy, driven from the checkpoint's own weights, and diffed +against the C++. + +| Stage | Check | Result | +|---|---|---| +| Tensor inventory | 762 names + shapes vs. the four real files | exact match | +| Vocoder mel front end | vs. numpy STFT + checkpoint filterbank | max diff 1.9e-3 | +| Vocos backbone + ISTFT head | vs. numpy, all 14 blocks | max diff 1.0e-5 (measured before the band limit; the numpy reference does not zero the bins above 10900 Hz) | +| Semantic encoder mel | vs. numpy | max diff 1.9e-5 | +| Semantic encoder transformer | vs. numpy, all 6 layers | max diff 6.0e-5 | +| FSQ token ids | vs. numpy | 188/188 identical | +| Speaker encoder mel / trunk / heads | vs. numpy | max diff 3.7e-5 | +| Acoustic `mu`, `spk`, time embedding | vs. numpy | max diff 1.3e-5 | +| Acoustic velocity field, every Euler step | vs. numpy | max diff 5.8e-3 | +| Acoustic self-reconstruction | NMSE vs. the reference's own mel | 0.38 | +| Fixed `seed` reproducibility | byte-identical WAV across runs | pass | +| Streaming vs. offline, same `seed` | 22.5 s clip, 4 segments | identical sample count; a constant 1.15x gain, −47 dB residual | +| Streaming segment sum | segments vs. `--out` | exact | +| Long-form, 6026 chars | 371.6 s of audio, 48 segments | offline and streaming both complete; peak RSS 1.083 vs 1.086 GB | +| `matmul_weight_type` f16 / bf16 / q8_0 | runs clean | pass | +| Single-file GGUF | end to end | pass | + +Numeric parity against the upstream PyTorch implementation has still not been measured +directly; the numpy references above are independent reimplementations from the same source, +which catches implementation bugs but not a shared misreading of the architecture. + +### Debugging + +`tests/sopro_tts/sopro_probe.cpp` (built with `-DENGINE_BUILD_WARMBENCH=ON`) exercises each +stage in isolation against a reference clip: + +```bash +build/bin/sopro_probe models/sopro-v2-turbo reference.wav /tmp/soproprobe +``` + +It reports the `crop_on_pause` decision, a mel round trip through the vocoder (which is +phase-invariant and so the meaningful vocoder check), the FSQ token histogram, the speaker +embedding statistics, and an acoustic self-reconstruction NMSE. It also writes +`probe_vocoder_roundtrip.wav` — the reference passed through mel then the vocoder; if that +sounds like the speaker, the whole back half of the pipeline is fine. + +Setting `SOPRO_DUMP_DIR=` additionally dumps the encoder and solver intermediates as raw +f32 for diffing against a reference implementation. + +## References + +- Model card: +- Reference implementation: +- Blog post: diff --git a/docs/gguf.md b/docs/gguf.md index fd7ec4908..1cd8932f4 100644 --- a/docs/gguf.md +++ b/docs/gguf.md @@ -97,6 +97,7 @@ Status labels: | `qwen3_tts` voice design | Done | Pass | --- | Pass (ASR match, drift) | Pass (ASR match, drift) | | `rvc` | Done | --- | --- | Pass | --- | | `seed_vc` | Done | Pass | --- | Pass (drift) | Pass (drift) | +| `sopro_tts` | Done | Pass | --- | Pass | Pass | | `soprano_tts` | Done | Pass | --- | Pass | Pass (drift) | | `silero_vad` | Skip (tiny model) | --- | --- | --- | --- | | `sortformer_diar` | Done | Pass | --- | Pass | Pass | diff --git a/include/engine/community_models/sopro_tts/acoustic.h b/include/engine/community_models/sopro_tts/acoustic.h new file mode 100644 index 000000000..44510c202 --- /dev/null +++ b/include/engine/community_models/sopro_tts/acoustic.h @@ -0,0 +1,69 @@ +#pragma once + +#include "engine/community_models/sopro_tts/assets.h" + +#include +#include +#include +#include + +namespace engine::core { +class ExecutionContext; +} +namespace engine::assets { +enum class TensorStorageType; +} + +namespace engine::community_models::sopro_tts { + +struct SoproAcousticWeights; +struct SoproAcousticGraphs; + +struct SoproAcousticRequest { + // Reference tokens followed by the generated ones; the acoustic head sees + // the whole span so the prompt mel and the new audio stay phase-coherent. + std::vector semantic_tokens; + std::vector cond_vec; // [cond_hidden_dim] + std::vector prompt_mel; // [n_mels, prompt_frames], normalised + int64_t prompt_frames = 0; + int64_t total_frames = 0; + int64_t steps = 2; + uint64_t seed = 0; +}; + +// sopro/nn/acoustic.py AcousticHead.solve, offline (unchunked) path: a +// rectified-flow DiT with adaptive layer norm conditioning, solved with the +// Euler steps of a sway-sampled time grid while the prompt frames are pinned +// to the reference mel at every step. +class SoproAcousticRuntime final { +public: + SoproAcousticRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type); + ~SoproAcousticRuntime(); + + SoproAcousticRuntime(const SoproAcousticRuntime &) = delete; + SoproAcousticRuntime & operator=(const SoproAcousticRuntime &) = delete; + + // Returns the normalised mel [n_mels, total_frames], channel-major. + std::vector solve(const SoproAcousticRequest & request) const; + +private: + const SoproModelConfig & config_; + engine::core::ExecutionContext & execution_context_; + size_t graph_context_bytes_ = 0; + std::shared_ptr weights_; + mutable std::unique_ptr graphs_; +}; + +// build_time_grid: linspace(0, 1, steps + 1) warped by the sway coefficient. +std::vector build_time_grid(int64_t steps, float sway_coefficient); + +// sinusoidal_time_embedding(t, dim, scale=1000). +std::vector sinusoidal_time_embedding(float t, int64_t dim, float scale = 1000.0F); + +} // namespace engine::community_models::sopro_tts diff --git a/include/engine/community_models/sopro_tts/assets.h b/include/engine/community_models/sopro_tts/assets.h new file mode 100644 index 000000000..57b222734 --- /dev/null +++ b/include/engine/community_models/sopro_tts/assets.h @@ -0,0 +1,196 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { + +// samuel-vitorino/sopro-v2-turbo. Four stages run per request: +// speaker encoder (16 kHz reference -> id/style/style-ctrl embeddings) +// semantic encoder (24 kHz reference -> FSQ semantic token ids) +// semantic LM (text + style prefix + prompt tokens -> semantic tokens) +// acoustic head (semantic tokens -> mel, rectified-flow Euler solve) +// vocoder (mel -> 24 kHz waveform, Vocos ConvNeXt + ISTFT head) +// Every field below mirrors one key of the checkpoint's config.json; nothing +// about the architecture is hardcoded so a retrained variant loads unchanged. + +// config.json -> "model" +struct SoproModelConfig { + int64_t latent_dim = 1280; + int64_t semantic_vocab_size = 4375; + int64_t text_vocab_size = 8192; + int64_t max_text_len = 2048; + int64_t cond_in_dim = 328; // id_emb + style_emb + style_ctrl + int64_t cond_hidden_dim = 512; + + int64_t ar_model_dim = 512; + int64_t ar_blocks = 12; + int64_t ar_heads = 8; + int64_t ar_kv_heads = 8; // defaults to ar_heads when absent + float ar_ffn_mult = 4.0F; + bool ar_qk_rms_norm = true; + int64_t style_prefix_tokens = 8; + + int64_t acoustic_time_embed_dim = 256; + float acoustic_sway_sampling_coef = -1.0F; + int64_t acoustic_upsampler_kernel_size = 3; + int64_t acoustic_dit_dim = 512; + int64_t acoustic_dit_depth = 8; + int64_t acoustic_dit_heads = 8; + int64_t acoustic_dit_dim_head = 64; + float acoustic_dit_ff_mult = 2.0F; + int64_t acoustic_mu_dim = 100; // defaults to acoustic_mel_n_mels + int64_t acoustic_spk_dim = 80; + int64_t acoustic_pre_lookahead_frames = 3; + int64_t acoustic_pos_kernel_size = 31; + float acoustic_sigma_min = 1.0e-6F; + int64_t acoustic_num_left_chunks = -1; + int64_t acoustic_mel_n_mels = 100; + int64_t acoustic_mel_hop_length = 256; + std::vector acoustic_mel_mean; + std::vector acoustic_mel_std; + + // BOS/EOS live just past the FSQ codebook; the AR head is + // Linear(dim -> semantic_vocab_size + 2). + int64_t semantic_bos_id() const noexcept { return semantic_vocab_size; } + int64_t semantic_eos_id() const noexcept { return semantic_vocab_size + 1; } + int64_t ar_head_dim() const noexcept { return ar_model_dim / ar_heads; } + int64_t ar_ffn_dim() const noexcept; + int64_t acoustic_dit_ff_dim() const noexcept; +}; + +// config.json -> "semantic_encoder" +struct SoproSemanticEncoderConfig { + int64_t n_mels = 80; + int64_t d_model = 512; + int64_t layers = 6; + int64_t heads = 8; + int64_t ffn_dim = 2048; + int64_t max_positions = 1500; + std::vector fsq_levels{7, 5, 5, 5, 5}; + int64_t sample_rate = 16000; + int64_t n_fft = 400; + int64_t hop_length = 160; + int64_t token_samples_24k = 1024; + + int64_t head_dim() const noexcept { return d_model / heads; } + int64_t digit_dim() const noexcept; // sum(fsq_levels) + int64_t codebook_size() const noexcept; // prod(fsq_levels) +}; + +// config.json -> "speaker_encoder" +struct SoproSpeakerEncoderConfig { + int64_t sample_rate = 16000; + int64_t n_mels = 80; + int64_t n_fft = 1024; + int64_t win_length = 400; + int64_t hop_length = 160; + float f_min = 20.0F; + float f_max = 7600.0F; + float mel_log_floor = 1.0e-5F; + int64_t stem_channels = 128; + std::vector stage_channels{160, 192, 224}; + std::vector blocks_per_stage{4, 4, 4}; + std::vector dilation_cycle{1, 2, 4, 8}; + int64_t depthwise_kernel_size = 5; + int64_t se_reduction = 8; + int64_t id_emb_dim = 192; + int64_t style_emb_dim = 128; + int64_t style_ctrl_dim = 8; + int64_t id_head_hidden = 256; + int64_t style_head_hidden = 256; + int64_t attn_hidden = 128; +}; + +// config.json -> "vocoder" / "vocoder_streaming" +struct SoproVocoderConfig { + int64_t sample_rate = 24000; + int64_t n_fft = 1024; + int64_t hop_length = 256; + int64_t n_mels = 100; + int64_t dim = 512; + int64_t intermediate_dim = 1536; + int64_t num_layers = 14; + float max_magnitude = 100.0F; + // sopro/config.py VocoderConfig.band_limit_hz. Zero (or a negative value) + // disables the cut; the published checkpoints do not carry the key, so the + // default has to match the reference dataclass. + float band_limit_hz = 10900.0F; + bool causal = false; + int64_t lookahead_frames = 0; + std::vector block_lookaheads; +}; + +// config.json -> "generation" +struct SoproGenerationConfig { + float temperature = 0.8F; + float top_p = 0.9F; + int64_t top_k = 25; + int64_t steps = 2; + float max_seconds = 30.0F; + float min_seconds = 0.4F; + int64_t max_segment_chars = 300; + float ref_seconds = 10.0F; + int64_t style_tokens = 160; + int64_t prompt_tokens = 120; + int64_t stream_chunk_frames = 64; +}; + +struct SoproTTSConfig { + int64_t sample_rate = 24000; + SoproModelConfig model; + SoproSemanticEncoderConfig semantic_encoder; + SoproSpeakerEncoderConfig speaker_encoder; + SoproVocoderConfig vocoder; + SoproVocoderConfig vocoder_streaming; + SoproGenerationConfig generation; + + // Mel frames produced per semantic token (token_samples_24k / mel hop). + int64_t hop_ratio() const noexcept; +}; + +struct SoproTTSAssets { + assets::ResourceBundle resources; + SoproTTSConfig config; + std::shared_ptr model_weights; + std::shared_ptr semantic_encoder_weights; + std::shared_ptr speaker_encoder_weights; + std::shared_ptr vocoder_weights; + std::filesystem::path tokenizer_path; +}; + +// Per-request knobs; defaults come from config.json "generation". +struct SoproRequestOptions { + std::string language; // "", en, pt, fr, de + float temperature = 0.8F; + float top_p = 0.9F; + int64_t top_k = 25; + int64_t steps = 2; + float max_seconds = 30.0F; + float min_seconds = 0.4F; + int64_t max_segment_chars = 300; + float ref_seconds = 10.0F; + uint64_t seed = 0; + bool has_seed = false; +}; + +std::shared_ptr load_sopro_tts_assets( + const std::filesystem::path & model_path); + +// The front ends reuse the analysis window and mel filterbank that torchaudio +// stores as persistent buffers instead of rebuilding them, which is what keeps +// them comparable with the reference pipeline. Fail early and say why when a +// checkpoint was exported without them. +void require_frontend_buffers( + const assets::TensorSource & source, + const char * stage, + std::initializer_list tensor_names); + +} // namespace engine::community_models::sopro_tts diff --git a/include/engine/community_models/sopro_tts/reference.h b/include/engine/community_models/sopro_tts/reference.h new file mode 100644 index 000000000..4e75195e3 --- /dev/null +++ b/include/engine/community_models/sopro_tts/reference.h @@ -0,0 +1,105 @@ +#pragma once + +#include "engine/community_models/sopro_tts/assets.h" + +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { + +class SoproSpeakerEncoderRuntime; +class SoproSemanticEncoderRuntime; +class SoproVocoderRuntime; + +// Ports of sopro/audio.py. All of them operate on mono float waveforms. +namespace audio_ops { + +constexpr float kPromptLevelDb = -19.8F; +constexpr float kOutputLevelDb = -23.0F; +constexpr float kLimiterKnee = 0.9F; +constexpr float kLeadInSeconds = 0.08F; +constexpr float kSegmentLeadSeconds = 0.30F; +constexpr float kSegmentSkipSeconds = 0.10F; +constexpr float kTrailSeconds = 0.30F; +constexpr float kJoinFadeSeconds = 0.01F; +constexpr float kFinalFadeSeconds = 0.08F; + +struct SpeechLevel { + float level_db = 0.0F; + float active_seconds = 0.0F; +}; + +// normalize_reference returns the boosted waveform together with the speech +// level it ends up at, so the output gain can be derived from the reference +// the model actually heard. +struct NormalizedReference { + std::vector wav; + float level_db = kPromptLevelDb; +}; + +std::vector crop_on_pause( + const std::vector & wav, float target_seconds, int sample_rate, std::mt19937_64 & rng); +SpeechLevel speech_level_db(const std::vector & wav, int sample_rate); +// Boost-only and peak-guarded: a reference already at or above the prompt level +// is left alone, and the boost never pushes the peak past 0.95. +NormalizedReference normalize_reference(const std::vector & wav, int sample_rate); +float output_gain(float prompt_level_db = kPromptLevelDb); +float match_gain( + const std::vector & wav, int sample_rate, float target_db = kOutputLevelDb, + float prompt_level_db = kPromptLevelDb); +void soft_limit(std::vector & wav, float knee = kLimiterKnee); +std::optional speech_onset(const std::vector & wav, int sample_rate); +std::vector trim_lead( + const std::vector & wav, int sample_rate, + float lead = kLeadInSeconds, float skip = 0.0F); +std::vector trim_trail( + const std::vector & wav, int sample_rate, float trail = kTrailSeconds); +void fade_edges( + std::vector & wav, int sample_rate, bool fade_in, bool fade_out, + float fade_seconds = kJoinFadeSeconds); +std::vector join_segments(std::vector> parts, int sample_rate); + +} // namespace audio_ops + +// The per-voice state the semantic LM and the acoustic head both condition on. +struct SoproReference { + std::vector cond_vec; // [cond_hidden_dim] + std::vector semantic_tokens; // one id per 1024 reference samples + std::vector mel; // [n_mels, mel_frames], normalised + int64_t mel_frames = 0; + // Speech level of the normalised reference, i.e. what the output gain is + // derived from (Reference.level_db). + float level_db = audio_ops::kPromptLevelDb; +}; + +// SoproTTS.prepare_reference: crop on a pause, level-normalise, then run the +// speaker encoder, the semantic encoder and the analysis mel in one pass. +class SoproReferenceBuilder final { +public: + SoproReferenceBuilder( + const SoproTTSAssets & assets, + const SoproSpeakerEncoderRuntime & speaker_encoder, + const SoproSemanticEncoderRuntime & semantic_encoder, + const SoproVocoderRuntime & vocoder); + + // audio24: mono 24 kHz reference waveform. + SoproReference build( + const std::vector & audio24, float ref_seconds, std::mt19937_64 & rng) const; + +private: + const SoproTTSConfig & config_; + const SoproSpeakerEncoderRuntime & speaker_encoder_; + const SoproSemanticEncoderRuntime & semantic_encoder_; + const SoproVocoderRuntime & vocoder_; + // SoproModel.cond_proj = Sequential(Linear, SiLU, Identity, Linear). + std::vector cond_proj_w0; + std::vector cond_proj_b0; + std::vector cond_proj_w3; + std::vector cond_proj_b3; + std::vector mel_mean_; + std::vector mel_std_; +}; + +} // namespace engine::community_models::sopro_tts diff --git a/include/engine/community_models/sopro_tts/semantic_encoder.h b/include/engine/community_models/sopro_tts/semantic_encoder.h new file mode 100644 index 000000000..4a386ebda --- /dev/null +++ b/include/engine/community_models/sopro_tts/semantic_encoder.h @@ -0,0 +1,54 @@ +#pragma once + +#include "engine/community_models/sopro_tts/assets.h" + +#include +#include +#include +#include + +namespace engine::core { +class ExecutionContext; +} +namespace engine::assets { +enum class TensorStorageType; +} + +namespace engine::community_models::sopro_tts { + +struct SoproSemanticEncoderWeights; +struct SoproSemanticEncoderGraph; + +// sopro/encoders/semantic.py. A Whisper-style log-mel front end and two +// striding convolutions feed six non-causal transformer layers; the result is +// resampled to one frame per 1024 output samples and quantised by a finite +// scalar quantiser whose per-level arg-maxes are packed into a single id. +class SoproSemanticEncoderRuntime final { +public: + SoproSemanticEncoderRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type); + ~SoproSemanticEncoderRuntime(); + + SoproSemanticEncoderRuntime(const SoproSemanticEncoderRuntime &) = delete; + SoproSemanticEncoderRuntime & operator=(const SoproSemanticEncoderRuntime &) = delete; + + // audio24: mono 24 kHz reference waveform. Returns ceil(n / 1024) token ids. + std::vector encode(const std::vector & audio24) const; + +private: + const SoproSemanticEncoderConfig & config_; + // Rate the caller's reference waveform arrives at (config.json sample_rate), + // kept so the resample and its pinned length are not tied to 24 kHz. + int64_t source_sample_rate_ = 0; + engine::core::ExecutionContext & execution_context_; + size_t graph_context_bytes_ = 0; + std::shared_ptr weights_; + mutable std::unique_ptr graph_; +}; + +} // namespace engine::community_models::sopro_tts diff --git a/include/engine/community_models/sopro_tts/semantic_lm.h b/include/engine/community_models/sopro_tts/semantic_lm.h new file mode 100644 index 000000000..fb91a3ad0 --- /dev/null +++ b/include/engine/community_models/sopro_tts/semantic_lm.h @@ -0,0 +1,75 @@ +#pragma once + +#include "engine/community_models/sopro_tts/assets.h" + +#include +#include +#include +#include +#include + +namespace engine::core { +class ExecutionContext; +} +namespace engine::assets { +enum class TensorStorageType; +} + +namespace engine::community_models::sopro_tts { + +struct SoproSemanticLMOptions { + int64_t max_steps = 1; + int64_t min_steps = 1; + float temperature = 0.8F; + float top_p = 0.9F; + int64_t top_k = 25; +}; + +// sopro/nn/ar.py + SoproModel.stream_semantic_tokens. The prompt is +// [style prefix | text | carried semantic tokens | BOS] and the model +// autoregresses semantic ids until EOS or the step budget. +// +// The stack is a standard pre-norm transformer with QK RMS-norm, SwiGLU and +// half-rotation RoPE, so it maps onto the shared Qwen decoder runtime once the +// per-branch LayerScale vectors are folded into the output projections. +class SoproSemanticLMRuntime final { +public: + SoproSemanticLMRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + engine::assets::TensorStorageType weight_storage_type); + ~SoproSemanticLMRuntime(); + + SoproSemanticLMRuntime(const SoproSemanticLMRuntime &) = delete; + SoproSemanticLMRuntime & operator=(const SoproSemanticLMRuntime &) = delete; + + std::vector generate( + const std::vector & text_ids, + const std::vector & style_tokens, + const std::vector & prompt_tokens, + const SoproSemanticLMOptions & options, + std::mt19937_64 & rng) const; + + void release_runtime_graphs(); + +private: + class Impl; + std::unique_ptr impl_; +}; + +// sopro/sampling.py sample_next_token, exposed for testing. `logits` is +// modified in place. +int32_t sample_next_token( + std::vector & logits, + float temperature, + float top_p, + int64_t top_k, + int32_t bos_id, + int32_t eos_id, + bool allow_eos, + std::mt19937_64 & rng); + +} // namespace engine::community_models::sopro_tts diff --git a/include/engine/community_models/sopro_tts/session.h b/include/engine/community_models/sopro_tts/session.h new file mode 100644 index 000000000..44f5a5bc4 --- /dev/null +++ b/include/engine/community_models/sopro_tts/session.h @@ -0,0 +1,83 @@ +#pragma once + +#include "engine/community_models/sopro_tts/assets.h" +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/model.h" +#include "engine/framework/runtime/session_base.h" + +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { + +std::shared_ptr make_sopro_tts_loader(); + +class SoproAcousticRuntime; +class SoproReferenceBuilder; +class SoproSemanticEncoderRuntime; +class SoproSemanticLMRuntime; +class SoproSpeakerEncoderRuntime; +class SoproTextTokenizer; +class SoproVocoderRuntime; + +// Everything one synthesis run carries between its text segments: the parsed +// options, the encoded reference voice, the LM carry-over prompt and the RNG. +// Offline builds one and drains it in a loop; streaming keeps it alive across +// next_stream_event calls, so both paths consume the seed in the same order. +struct SoproSynthesisState; + +class SoproTTSSession final : public runtime::RuntimeSessionBase, + public runtime::IOfflineVoiceTaskSession, + public runtime::IStreamingVoiceTaskSession { +public: + SoproTTSSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract); + ~SoproTTSSession() override; + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest & request) override; + runtime::TaskResult run(const runtime::TaskRequest & request) override; + + runtime::StreamingPolicy streaming_policy() const override; + void start_stream(const runtime::TaskRequest & request) override; + std::optional next_stream_event() override; + void set_stream_event_sink(runtime::StreamEventCallback sink) override; + runtime::TaskResult finish_stream() override; + void reset() override; + runtime::StreamEvent process_audio_chunk(const runtime::AudioChunk & chunk) override; + runtime::TaskResult finalize() override; + +private: + SoproRequestOptions parse_options(const runtime::TaskRequest & request) const; + // Validates the request, encodes the reference voice and splits the text. + std::unique_ptr begin_synthesis(const runtime::TaskRequest & request); + // Runs one text segment through the LM, the acoustic head and the vocoder. + // Returns the raw 24 kHz waveform before any levelling, or an empty vector + // when the segment generated nothing. + std::vector synthesize_segment(SoproSynthesisState & state); + + runtime::TaskSpec task_; + std::shared_ptr assets_; + std::shared_ptr contract_; + std::string default_language_; + + std::unique_ptr tokenizer_; + std::unique_ptr speaker_encoder_; + std::unique_ptr semantic_encoder_; + std::unique_ptr vocoder_; + std::unique_ptr semantic_lm_; + std::unique_ptr acoustic_; + std::unique_ptr reference_builder_; + + std::unique_ptr stream_state_; + std::vector stream_chunks_; +}; + +} // namespace engine::community_models::sopro_tts diff --git a/include/engine/community_models/sopro_tts/speaker_encoder.h b/include/engine/community_models/sopro_tts/speaker_encoder.h new file mode 100644 index 000000000..07ba07346 --- /dev/null +++ b/include/engine/community_models/sopro_tts/speaker_encoder.h @@ -0,0 +1,63 @@ +#pragma once + +#include "engine/community_models/sopro_tts/assets.h" + +#include +#include +#include +#include + +namespace engine::core { +class ExecutionContext; +} +namespace engine::assets { +enum class TensorStorageType; +} + +namespace engine::community_models::sopro_tts { + +struct SoproSpeakerWeights; +struct SoproSpeakerGraph; + +struct SoproSpeakerEmbeddings { + std::vector id_emb; // id_emb_dim, L2-normalised + std::vector style_emb; // style_emb_dim + std::vector style_ctrl; // style_ctrl_dim +}; + +// sopro/encoders/speaker.py. A log-mel front end feeding a three-stage +// gated depthwise ResNet with squeeze-excite, then two pooling heads: +// attentive statistics for speaker identity and multi-scale mean/std for +// style. The convolution trunk runs on the backend; the pooling heads and +// their small MLPs run on the host, where they cost a few hundred kFLOP. +class SoproSpeakerEncoderRuntime final { +public: + SoproSpeakerEncoderRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type); + ~SoproSpeakerEncoderRuntime(); + + SoproSpeakerEncoderRuntime(const SoproSpeakerEncoderRuntime &) = delete; + SoproSpeakerEncoderRuntime & operator=(const SoproSpeakerEncoderRuntime &) = delete; + + // audio16: mono 16 kHz reference waveform. + SoproSpeakerEmbeddings encode(const std::vector & audio16) const; + + int64_t sample_rate() const noexcept; + +private: + // Returns the fused trunk features [channels, frames], channel-major. + std::vector trunk(const std::vector & log_mel, int64_t frames, int64_t & out_frames) const; + + const SoproSpeakerEncoderConfig & config_; + engine::core::ExecutionContext & execution_context_; + size_t graph_context_bytes_ = 0; + std::shared_ptr weights_; + mutable std::unique_ptr graph_; +}; + +} // namespace engine::community_models::sopro_tts diff --git a/include/engine/community_models/sopro_tts/text_tokenizer.h b/include/engine/community_models/sopro_tts/text_tokenizer.h new file mode 100644 index 000000000..e270588f1 --- /dev/null +++ b/include/engine/community_models/sopro_tts/text_tokenizer.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { + +// Mirrors sopro/text.py. The reference pipeline is deliberately minimal: no +// grapheme-to-phoneme stage, just light punctuation clean-up, an optional +// language tag and a SentencePiece unigram model with 8192 pieces. + +// sopro.text.split_text: sentence -> clause -> word packing, codepoint budget. +std::vector split_text(const std::string & text, int64_t max_chars); + +// sopro.text.normalize_text. +std::string normalize_text(const std::string & text); + +// sopro.text.language_tag; throws for anything outside {en, pt, fr, de}. +// An empty language yields an empty tag. +std::string language_tag(const std::string & language); + +class SoproTextTokenizer { +public: + explicit SoproTextTokenizer(const std::filesystem::path & model_path, int64_t max_length = 512); + ~SoproTextTokenizer(); + + SoproTextTokenizer(const SoproTextTokenizer &) = delete; + SoproTextTokenizer & operator=(const SoproTextTokenizer &) = delete; + + // [bos] + pieces(normalize_text(tag + text)) + [eos], truncated to + // max_length; never empty (falls back to the unk id). + std::vector encode(const std::string & text, const std::string & language) const; + + int32_t bos_id() const noexcept; + int32_t eos_id() const noexcept; + int32_t unk_id() const noexcept; + int64_t vocab_size() const noexcept; + +private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::community_models::sopro_tts diff --git a/include/engine/community_models/sopro_tts/vocoder.h b/include/engine/community_models/sopro_tts/vocoder.h new file mode 100644 index 000000000..91b0e33d6 --- /dev/null +++ b/include/engine/community_models/sopro_tts/vocoder.h @@ -0,0 +1,70 @@ +#pragma once + +#include "engine/community_models/sopro_tts/assets.h" + +#include +#include +#include +#include + +namespace engine::core { +class ExecutionContext; +} +namespace engine::assets { +enum class TensorStorageType; +} + +namespace engine::community_models::sopro_tts { + +struct SoproVocoderWeights; +struct SoproVocoderGraph; + +// ISTFTHead band limit (sopro/vocoder.py band_limit_bin): the first FFT bin the +// head zeroes, i.e. how many of the n_fft/2 + 1 bins it actually synthesises. +// A band_limit_hz of zero or less keeps every bin. +int64_t band_limit_bin(const SoproVocoderConfig & config); + +// sopro/vocoder.py, offline path. A Vocos backbone (Conv1d embed, 14 ConvNeXt +// blocks with per-channel gamma, final LayerNorm) feeding one ISTFT head: +// Linear(dim -> n_fft + 2) split into log-magnitude and phase, then a single +// centred inverse STFT with the checkpoint's Hann window. +// +// The same object also owns the analysis mel filterbank, because the acoustic +// stage conditions on the reference mel produced by exactly this extractor. +class SoproVocoderRuntime final { +public: + SoproVocoderRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type); + ~SoproVocoderRuntime(); + + SoproVocoderRuntime(const SoproVocoderRuntime &) = delete; + SoproVocoderRuntime & operator=(const SoproVocoderRuntime &) = delete; + + // mel: [n_mels, frames], channel-major (mel[c * frames + t]). + // Returns (frames - 1) * hop_length mono samples at config.sample_rate. + std::vector decode(const std::vector & mel, int64_t frames) const; + + // MelFeatures.forward: log(clamp(|STFT|, min=1e-7)) with the torchaudio + // MelSpectrogram buffers stored in the checkpoint (power=1, centred). + // Returns [n_mels, frames] channel-major. + std::vector log_mel(const std::vector & audio) const; + + int64_t mel_frames(int64_t samples) const noexcept; + int64_t hop_length() const noexcept; + int64_t n_mels() const noexcept; + int sample_rate() const noexcept; + +private: + const SoproVocoderConfig & config_; + engine::core::ExecutionContext & execution_context_; + size_t graph_context_bytes_ = 0; + std::shared_ptr weights_; + mutable std::unique_ptr graph_; +}; + +} // namespace engine::community_models::sopro_tts diff --git a/model_specs/sopro_tts.json b/model_specs/sopro_tts.json new file mode 100644 index 000000000..3b37853a9 --- /dev/null +++ b/model_specs/sopro_tts.json @@ -0,0 +1,257 @@ +{ + "schema_version": 1, + "family": "sopro_tts", + "display_name": "Sopro V2 Turbo", + "description": "Community Sopro V2 Turbo (samuel-vitorino/sopro-v2-turbo): a 120M zero-shot voice-cloning TTS. SentencePiece text tokenizer, style-prefix conditioned autoregressive semantic LM over FSQ tokens, rectified-flow acoustic DiT and a Vocos ISTFT vocoder at 24 kHz.", + "category": "tts", + "status": "community", + "tasks": [ + "tts", + "clone" + ], + "modes": [ + "offline", + "streaming" + ], + "languages": [ + "en", + "pt", + "fr", + "de" + ], + "runtime": { + "tags": [ + "server" + ] + }, + "capabilities": { + "clone": [ + "speaker_reference" + ] + }, + "options": { + "request": [ + { + "name": "language", + "type": "string", + "description": "Language tag prepended to the prompt (en, pt, fr, de). Optional; helps pronunciation on ambiguous text.", + "required": false, + "default": "" + }, + { + "name": "temperature", + "type": "float", + "description": "Semantic LM sampling temperature; default 0.8.", + "required": false, + "min": 0.0, + "max": 2.0, + "default": 0.8 + }, + { + "name": "top_p", + "type": "float", + "description": "Nucleus sampling threshold for the semantic LM; default 0.9.", + "required": false, + "min": 0.0, + "max": 1.0, + "default": 0.9 + }, + { + "name": "top_k", + "type": "int", + "description": "Top-k truncation for the semantic LM; 0 disables. Default 25.", + "required": false, + "min": 0, + "default": 25 + }, + { + "name": "num_inference_steps", + "type": "int", + "description": "Acoustic rectified-flow Euler steps; default 2.", + "required": false, + "min": 1, + "max": 32, + "default": 2 + }, + { + "name": "max_seconds", + "type": "float", + "description": "Cap on generated audio per segment; long text is split into segments so total length is unbounded. Default 30.", + "required": false, + "min": 1.0, + "max": 60.0, + "default": 30.0 + }, + { + "name": "min_seconds", + "type": "float", + "description": "Minimum audio per segment before the semantic LM may emit EOS; default 0.4.", + "required": false, + "min": 0.0, + "max": 10.0, + "default": 0.4 + }, + { + "name": "ref_seconds", + "type": "float", + "description": "Reference audio window used for cloning; default 10.", + "required": false, + "min": 1.0, + "max": 30.0, + "default": 10.0 + }, + { + "name": "text_chunk_size", + "type": "int", + "description": "Maximum codepoints per synthesis segment; default 300 (config.json generation.max_segment_chars).", + "required": false, + "min": 20, + "max": 2000, + "default": 300 + }, + { + "name": "seed", + "type": "int", + "description": "Non-negative seed for semantic sampling and the acoustic noise prior; omit for a random seed.", + "required": false, + "min": 0 + } + ], + "session": [ + { + "name": "language", + "type": "string", + "description": "Default language tag for requests that do not set one.", + "required": false, + "default": "" + } + ], + "load": [ + { + "name": "matmul_weight_type", + "type": "enum", + "preset": "weight_type_full", + "description": "Storage type for the matmul weights of every stage (semantic LM, acoustic DiT, encoders, vocoder head).", + "required": false, + "default": "f32" + }, + { + "name": "conv_weight_type", + "type": "enum", + "preset": "weight_type_conv", + "description": "Storage type for convolution weights (speaker/semantic encoders and the Vocos backbone).", + "required": false, + "default": "f32" + } + ] + }, + "package_defaults": { + "download": { + "kind": "unsupported", + "reason": "No audio.cpp GGUF build of sopro-v2-turbo is published yet: install the sopro_v2_turbo_safetensors package and run from safetensors, or pack one locally with audiocpp_gguf (one --input namespace per stage: model, semantic_encoder, speaker_encoder, vocoder)." + } + }, + "packages": [ + { + "id": "sopro_v2_turbo_f16", + "display_name": "Sopro V2 Turbo F16 GGUF", + "description": "Locally packed GGUF holding all four stages plus the embedded config and tokenizer sidecars. Produced with audiocpp_gguf --family sopro_tts.", + "default": true, + "format": "gguf", + "precision": "f16", + "target_directory": "sopro-v2-turbo-GGUF", + "files": [ + "sopro-v2-turbo-GGUF/sopro-v2-turbo-f16.gguf" + ], + "strip_prefix": "sopro-v2-turbo-GGUF" + }, + { + "id": "sopro_v2_turbo_safetensors", + "display_name": "Sopro V2 Turbo (upstream safetensors)", + "description": "Upstream checkpoint from samuel-vitorino/sopro-v2-turbo: config.json, tokenizer.model and the four safetensors stages. Runs directly, no conversion needed.", + "format": "safetensors", + "precision": "orig", + "target_directory": "sopro-v2-turbo", + "files": [ + "config.json", + "tokenizer.model", + "model.safetensors", + "semantic_encoder.safetensors", + "speaker_encoder.safetensors", + "vocoder.safetensors" + ], + "download": { + "kind": "huggingface_snapshot", + "repo": "samuel-vitorino/sopro-v2-turbo", + "revision": "main", + "gated": false + } + } + ], + "dependencies": [], + "ui": { + "recommended_package": "sopro_v2_turbo_safetensors", + "tags": [ + "TTS", + "Clone" + ], + "docs": [ + "docs/community_models/sopro_tts.md" + ] + }, + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "config": "model:config.json", + "tokenizer": "model:tokenizer.model" + }, + "tensors": { + "model": { + "source": "weights:", + "prefix": "model" + }, + "semantic_encoder": { + "source": "weights:", + "prefix": "semantic_encoder" + }, + "speaker_encoder": { + "source": "weights:", + "prefix": "speaker_encoder" + }, + "vocoder": { + "source": "weights:", + "prefix": "vocoder" + } + } + }, + { + "format": "safetensors", + "roots": { + "model": "." + }, + "files": { + "config": "model:config.json", + "tokenizer": "model:tokenizer.model" + }, + "tensors": { + "model": { + "source": "model:model.safetensors" + }, + "semantic_encoder": { + "source": "model:semantic_encoder.safetensors" + }, + "speaker_encoder": { + "source": "model:speaker_encoder.safetensors" + }, + "vocoder": { + "source": "model:vocoder.safetensors" + } + } + } + ] +} diff --git a/src/community_models/sopro_tts/acoustic.cpp b/src/community_models/sopro_tts/acoustic.cpp new file mode 100644 index 000000000..6abf24ac9 --- /dev/null +++ b/src/community_models/sopro_tts/acoustic.cpp @@ -0,0 +1,903 @@ +#include "engine/community_models/sopro_tts/acoustic.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/attention/scaled_dot_product_attention.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/positional_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { + +struct SoproDiTBlockWeights { + engine::modules::LinearWeights modulation; // attn_norm.mod.1: dim -> 6 * dim + engine::modules::LinearWeights to_q; + engine::modules::LinearWeights to_k; + engine::modules::LinearWeights to_v; + engine::modules::LinearWeights to_out; + engine::modules::LinearWeights ff_in; + engine::modules::LinearWeights ff_out; +}; + +// A Conv1d with `groups` > 1 and groups != channels, expressed as one +// independent convolution per group (ggml has no grouped conv1d primitive). +struct SoproGroupedConvWeights { + std::vector groups; + int64_t group_in_channels = 0; + int64_t group_out_channels = 0; + int64_t kernel_size = 0; +}; + +struct SoproAcousticWeights { + std::shared_ptr store; + engine::core::TensorValue semantic_token_emb; // [semantic_vocab, latent_dim] + engine::modules::Conv1dWeights prelook_conv1; + engine::modules::Conv1dWeights prelook_conv2; + engine::modules::Conv1dWeights upsampler_in; + engine::modules::Conv1dWeights upsampler_mix; + engine::modules::Conv1dWeights upsampler_out; + engine::modules::Conv1dWeights mu_proj; + engine::modules::LinearWeights input_proj; + engine::modules::LinearWeights cond_mask_proj; + SoproGroupedConvWeights pos_conv1; + SoproGroupedConvWeights pos_conv2; + std::vector blocks; + engine::modules::LinearWeights out_modulation; // out_norm.mod.1: dim -> 2 * dim + engine::modules::LinearWeights out_proj; + // Host-side conditioning projections. + std::vector time_mlp_w0; // [dim, time_embed_dim] + std::vector time_mlp_b0; + std::vector time_mlp_w2; // [dim, dim] + std::vector time_mlp_b2; + std::vector spk_proj_w; // [spk_dim, cond_hidden_dim] + std::vector spk_proj_b; +}; + +namespace { + +// MSVC does not define M_PI; use our own constant, as f5_tts does for the +// same sway-time grid. +constexpr float kPi = 3.14159265358979323846F; + +namespace binding = engine::modules::binding; +namespace mod = engine::modules; + +constexpr float kDiTLayerNormEps = 1.0e-6F; + +// Set SOPRO_DUMP_DIR to write the solver's intermediates as raw f32 for +// stage-by-stage comparison against the reference implementation. +void dump(const std::string & name, const std::vector & values) { + const char * dir = std::getenv("SOPRO_DUMP_DIR"); + if (dir == nullptr) { + return; + } + const std::string path = std::string(dir) + "/" + name + ".f32"; + std::FILE * fh = std::fopen(path.c_str(), "wb"); + if (fh != nullptr) { + std::fwrite(values.data(), sizeof(float), values.size(), fh); + std::fclose(fh); + } +} + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +engine::core::TensorValue dense( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & value) { + return engine::core::wrap_tensor(ggml_cont(ctx.ggml, value.tensor), value.shape, GGML_TYPE_F32); +} + +mod::TransposeConfig swap_channel_time() { + return mod::TransposeConfig{{0, 2, 1, 3}, 3}; +} + +// Zero-pad the time axis of a [B, C, T] tensor. +engine::core::TensorValue pad_time( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & value, + int left, + int right) { + if (left == 0 && right == 0) { + return value; + } + auto contiguous = engine::core::ensure_backend_addressable_layout(ctx, value); + auto shape = contiguous.shape; + shape.dims[2] += left + right; + return engine::core::wrap_tensor( + ggml_pad_ext(ctx.ggml, contiguous.tensor, left, right, 0, 0, 0, 0, 0, 0), + shape, + GGML_TYPE_F32); +} + +// mish(x) = x * tanh(softplus(x)) +engine::core::TensorValue mish( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & value) { + auto * gate = ggml_tanh(ctx.ggml, ggml_softplus(ctx.ggml, value.tensor)); + return engine::core::wrap_tensor( + ggml_mul(ctx.ggml, value.tensor, gate), value.shape, GGML_TYPE_F32); +} + +// x * (1 + scale) + shift, with scale/shift broadcast over the time axis. +engine::core::TensorValue modulate( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & x, + const engine::core::TensorValue & scale, + const engine::core::TensorValue & shift) { + auto scaled = mod::MulModule{}.build(ctx, x, mod::RepeatModule({x.shape}).build(ctx, scale)); + auto sum = mod::AddModule{}.build(ctx, x, scaled); + return mod::AddModule{}.build(ctx, sum, mod::RepeatModule({x.shape}).build(ctx, shift)); +} + +SoproGroupedConvWeights load_grouped_conv( + engine::core::BackendWeightStore & store, + const engine::assets::TensorSource & source, + const std::string & prefix, + engine::assets::TensorStorageType storage_type, + int64_t channels, + int64_t kernel_size, + int64_t groups) { + if (channels % groups != 0) { + throw std::runtime_error(prefix + ": channel count is not divisible by the group count"); + } + SoproGroupedConvWeights out; + out.group_in_channels = channels / groups; + out.group_out_channels = channels / groups; + out.kernel_size = kernel_size; + const auto weight = source.require_f32( + prefix + ".weight", {channels, out.group_in_channels, kernel_size}); + const auto bias = source.require_f32(prefix + ".bias", {channels}); + const auto group_weight_elements = + static_cast(out.group_out_channels * out.group_in_channels * kernel_size); + out.groups.reserve(static_cast(groups)); + for (int64_t group = 0; group < groups; ++group) { + const size_t weight_offset = static_cast(group) * group_weight_elements; + std::vector group_weight( + weight.begin() + static_cast(weight_offset), + weight.begin() + static_cast(weight_offset + group_weight_elements)); + const size_t bias_offset = static_cast(group * out.group_out_channels); + std::vector group_bias( + bias.begin() + static_cast(bias_offset), + bias.begin() + static_cast(bias_offset + out.group_out_channels)); + engine::modules::Conv1dWeights conv; + conv.weight = store.make_from_f32( + engine::core::TensorShape::from_dims( + {out.group_out_channels, out.group_in_channels, kernel_size}), + storage_type, + std::move(group_weight)); + conv.bias = store.make_from_f32( + engine::core::TensorShape::from_dims({out.group_out_channels}), + engine::assets::TensorStorageType::F32, + std::move(group_bias)); + out.groups.push_back(std::move(conv)); + } + return out; +} + +engine::core::TensorValue build_grouped_conv( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & input, + const SoproGroupedConvWeights & weights) { + engine::core::TensorValue result; + for (size_t group = 0; group < weights.groups.size(); ++group) { + auto slice = mod::SliceModule({ + 1, + static_cast(group) * weights.group_in_channels, + weights.group_in_channels, + }).build(ctx, input); + auto convolved = mod::Conv1dModule({ + weights.group_in_channels, weights.group_out_channels, weights.kernel_size, + 1, 0, 1, true, + }).build(ctx, dense(ctx, slice), weights.groups[group]); + result = group == 0 ? convolved : mod::ConcatModule({1}).build(ctx, result, convolved); + } + return result; +} + +std::shared_ptr load_acoustic_weights( + ggml_backend_t backend, + engine::core::BackendType backend_type, + const engine::assets::TensorSource & source, + const SoproModelConfig & config, + size_t weight_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) { + auto weights = std::make_shared(); + weights->store = std::make_shared( + backend, backend_type, "sopro_tts.acoustic.weights", weight_context_bytes); + auto & store = *weights->store; + const std::string root = "acoustic_head."; + const int64_t latent = config.latent_dim; + const int64_t dim = config.acoustic_dit_dim; + const int64_t mel = config.acoustic_mel_n_mels; + + weights->semantic_token_emb = store.load_f32_tensor( + source, root + "semantic_token_emb.weight", {config.semantic_vocab_size, latent}); + weights->prelook_conv1 = binding::conv1d_from_source( + store, source, root + "semantic_prelook.conv1", conv_storage_type, + latent, latent, config.acoustic_pre_lookahead_frames + 1, true); + weights->prelook_conv2 = binding::conv1d_from_source( + store, source, root + "semantic_prelook.conv2", conv_storage_type, latent, latent, 3, true); + // LearnedCausalUpsampler hidden width is max(8, channels). + const int64_t upsampler_hidden = std::max(8, latent); + weights->upsampler_in = binding::conv1d_from_source( + store, source, root + "semantic_upsampler.in_proj", conv_storage_type, + upsampler_hidden, latent, 1, true); + weights->upsampler_mix = binding::conv1d_from_source( + store, source, root + "semantic_upsampler.mix.conv", conv_storage_type, + upsampler_hidden, upsampler_hidden, config.acoustic_upsampler_kernel_size, true); + weights->upsampler_out = binding::conv1d_from_source( + store, source, root + "semantic_upsampler.out_proj", conv_storage_type, + latent, upsampler_hidden, 1, true); + weights->mu_proj = binding::conv1d_from_source( + store, source, root + "mu_proj", conv_storage_type, config.acoustic_mu_dim, latent, 1, true); + + const int64_t proj_in = mel * 2 + config.acoustic_mu_dim + config.acoustic_spk_dim; + weights->input_proj = binding::linear_from_source( + store, source, root + "input_embed.proj", matmul_storage_type, dim, proj_in, true); + weights->cond_mask_proj = binding::linear_from_source( + store, source, root + "input_embed.cond_mask_proj", matmul_storage_type, dim, 1, false); + weights->pos_conv1 = load_grouped_conv( + store, source, root + "input_embed.pos.conv1", conv_storage_type, + dim, config.acoustic_pos_kernel_size, 16); + weights->pos_conv2 = load_grouped_conv( + store, source, root + "input_embed.pos.conv2", conv_storage_type, + dim, config.acoustic_pos_kernel_size, 16); + + const int64_t inner = config.acoustic_dit_heads * config.acoustic_dit_dim_head; + const int64_t ff_dim = config.acoustic_dit_ff_dim(); + weights->blocks.reserve(static_cast(config.acoustic_dit_depth)); + for (int64_t index = 0; index < config.acoustic_dit_depth; ++index) { + const std::string prefix = root + "blocks." + std::to_string(index); + SoproDiTBlockWeights block; + block.modulation = binding::linear_from_source( + store, source, prefix + ".attn_norm.mod.1", matmul_storage_type, dim * 6, dim, true); + block.to_q = binding::linear_from_source( + store, source, prefix + ".attn.to_q", matmul_storage_type, inner, dim, true); + block.to_k = binding::linear_from_source( + store, source, prefix + ".attn.to_k", matmul_storage_type, inner, dim, true); + block.to_v = binding::linear_from_source( + store, source, prefix + ".attn.to_v", matmul_storage_type, inner, dim, true); + block.to_out = binding::linear_from_source( + store, source, prefix + ".attn.to_out.0", matmul_storage_type, dim, inner, true); + block.ff_in = binding::linear_from_source( + store, source, prefix + ".ff.0", matmul_storage_type, ff_dim, dim, true); + block.ff_out = binding::linear_from_source( + store, source, prefix + ".ff.3", matmul_storage_type, dim, ff_dim, true); + weights->blocks.push_back(std::move(block)); + } + weights->out_modulation = binding::linear_from_source( + store, source, root + "out_norm.mod.1", matmul_storage_type, dim * 2, dim, true); + weights->out_proj = binding::linear_from_source( + store, source, root + "out_proj", matmul_storage_type, mel, dim, true); + + weights->time_mlp_w0 = source.require_f32( + root + "time_mlp.0.weight", {dim, config.acoustic_time_embed_dim}); + weights->time_mlp_b0 = source.require_f32(root + "time_mlp.0.bias", {dim}); + weights->time_mlp_w2 = source.require_f32(root + "time_mlp.2.weight", {dim, dim}); + weights->time_mlp_b2 = source.require_f32(root + "time_mlp.2.bias", {dim}); + weights->spk_proj_w = source.require_f32( + root + "spk_proj.weight", {config.acoustic_spk_dim, config.cond_hidden_dim}); + weights->spk_proj_b = source.require_f32(root + "spk_proj.bias", {config.acoustic_spk_dim}); + + store.upload(); + return weights; +} + +std::vector affine( + const std::vector & weight, + const std::vector & bias, + const std::vector & input, + int64_t in_dim, + int64_t out_dim) { + std::vector out(static_cast(out_dim), 0.0F); + for (int64_t o = 0; o < out_dim; ++o) { + const float * row = weight.data() + static_cast(o * in_dim); + double sum = bias[static_cast(o)]; + for (int64_t i = 0; i < in_dim; ++i) { + sum += static_cast(row[i]) * static_cast(input[static_cast(i)]); + } + out[static_cast(o)] = static_cast(sum); + } + return out; +} + +} // namespace + +std::vector build_time_grid(int64_t steps, float sway_coefficient) { + if (steps < 1) { + throw std::runtime_error("Sopro acoustic solver requires at least one step"); + } + std::vector times(static_cast(steps + 1), 0.0F); + for (int64_t i = 0; i <= steps; ++i) { + times[static_cast(i)] = static_cast(i) / static_cast(steps); + } + if (std::fabs(sway_coefficient) <= 1.0e-8F) { + return times; + } + for (auto & value : times) { + value += sway_coefficient * + (std::cos(0.5F * kPi * value) - 1.0F + value); + } + return times; +} + +std::vector sinusoidal_time_embedding(float t, int64_t dim, float scale) { + const int64_t half = std::max(1, dim / 2); + const int64_t denominator = std::max(1, half - 1); + std::vector out(static_cast(half * 2), 0.0F); + for (int64_t i = 0; i < half; ++i) { + const float frequency = std::exp( + static_cast(i) * (-(std::log(10000.0F) / static_cast(denominator)))); + const float argument = scale * t * frequency; + out[static_cast(i)] = std::sin(argument); + out[static_cast(half + i)] = std::cos(argument); + } + return out; +} + +// One graph produces mu (constant across solver steps); the other evaluates the +// DiT velocity field and is replayed once per Euler step. +struct SoproAcousticGraphs { + SoproAcousticGraphs( + ggml_backend_t backend_in, + engine::core::BackendType backend_type, + size_t graph_context_bytes, + const SoproModelConfig & config_in, + std::shared_ptr weights_in, + int64_t token_count, + int64_t frame_count) + : backend(backend_in), + weights(std::move(weights_in)), + tokens(token_count), + frames(frame_count), + config(&config_in) { + if (backend == nullptr || weights == nullptr) { + throw std::runtime_error("Sopro acoustic graphs require a backend and weights"); + } + if (tokens <= 0 || frames <= 0) { + throw std::runtime_error("Sopro acoustic graphs require positive lengths"); + } + build_conditioning(backend_type, graph_context_bytes, config_in); + build_velocity(backend_type, graph_context_bytes, config_in); + } + + ~SoproAcousticGraphs() { + if (conditioning_allocr != nullptr) { + ggml_gallocr_free(conditioning_allocr); + } + if (velocity_allocr != nullptr) { + ggml_gallocr_free(velocity_allocr); + } + } + + bool matches(const SoproAcousticWeights & other, int64_t token_count, int64_t frame_count) const noexcept { + return weights.get() == &other && tokens == token_count && frames == frame_count; + } + + void build_conditioning( + engine::core::BackendType backend_type, + size_t graph_context_bytes, + const SoproModelConfig & config_in) { + ggml_init_params params{graph_context_bytes, nullptr, true}; + conditioning_ctx.reset(ggml_init(params)); + if (conditioning_ctx == nullptr) { + throw std::runtime_error("failed to initialize the Sopro conditioning graph context"); + } + engine::core::ModuleBuildContext ctx{ + conditioning_ctx.get(), "sopro_tts.acoustic.conditioning", backend_type}; + const int64_t latent = config_in.latent_dim; + const int64_t upsampler_hidden = std::max(8, latent); + + token_input = ggml_new_tensor_1d(ctx.ggml, GGML_TYPE_I32, tokens); + ggml_set_input(token_input); + expand_index = ggml_new_tensor_1d(ctx.ggml, GGML_TYPE_I32, frames); + ggml_set_input(expand_index); + + // semantic_latents: [1, latent, tokens] + auto * rows = ggml_get_rows(ctx.ggml, weights->semantic_token_emb.tensor, token_input); + auto latents = engine::core::wrap_tensor( + rows, engine::core::TensorShape::from_dims({1, tokens, latent}), GGML_TYPE_F32); + latents = dense(ctx, mod::TransposeModule(swap_channel_time()).build(ctx, latents)); + + // PreLookahead: right-pad the lookahead, conv, then a causal 3-tap conv. + { + const auto lookahead = static_cast(config_in.acoustic_pre_lookahead_frames); + auto y = pad_time(ctx, latents, 0, lookahead); + y = mod::Conv1dModule({latent, latent, lookahead + 1, 1, 0, 1, true}) + .build(ctx, y, weights->prelook_conv1); + y = mod::LeakyReluModule({0.1F}).build(ctx, y); + y = pad_time(ctx, y, 2, 0); + y = mod::Conv1dModule({latent, latent, 3, 1, 0, 1, true}) + .build(ctx, y, weights->prelook_conv2); + latents = mod::AddModule{}.build(ctx, latents, y); + } + + // LearnedCausalUpsampler: nearest-index expansion onto the mel grid, + // then a residual causal mixer. + engine::core::TensorValue expanded; + { + auto btc = dense(ctx, mod::TransposeModule(swap_channel_time()).build(ctx, latents)); + auto * gathered = ggml_get_rows( + ctx.ggml, + ggml_reshape_2d(ctx.ggml, btc.tensor, latent, tokens), + expand_index); + expanded = dense(ctx, mod::TransposeModule(swap_channel_time()).build( + ctx, + engine::core::wrap_tensor( + gathered, engine::core::TensorShape::from_dims({1, frames, latent}), + GGML_TYPE_F32))); + } + const auto mix_kernel = static_cast(config_in.acoustic_upsampler_kernel_size); + auto hidden = mod::Conv1dModule({latent, upsampler_hidden, 1, 1, 0, 1, true}) + .build(ctx, expanded, weights->upsampler_in); + hidden = mod::SiluModule{}.build(ctx, hidden); + hidden = pad_time(ctx, hidden, mix_kernel - 1, 0); + hidden = mod::Conv1dModule({upsampler_hidden, upsampler_hidden, mix_kernel, 1, 0, 1, true}) + .build(ctx, hidden, weights->upsampler_mix); + hidden = mod::SiluModule{}.build(ctx, hidden); + hidden = mod::Conv1dModule({upsampler_hidden, latent, 1, 1, 0, 1, true}) + .build(ctx, hidden, weights->upsampler_out); + hidden = mod::AddModule{}.build(ctx, expanded, hidden); + hidden = mod::Conv1dModule({latent, config_in.acoustic_mu_dim, 1, 1, 0, 1, true}) + .build(ctx, hidden, weights->mu_proj); + hidden = engine::core::ensure_backend_addressable_layout(ctx, hidden); + mu_output = hidden.tensor; + ggml_set_output(mu_output); + conditioning_graph = ggml_new_graph_custom(conditioning_ctx.get(), 65536, false); + ggml_build_forward_expand(conditioning_graph, mu_output); + conditioning_allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (conditioning_allocr == nullptr || + !ggml_gallocr_reserve(conditioning_allocr, conditioning_graph) || + !ggml_gallocr_alloc_graph(conditioning_allocr, conditioning_graph)) { + throw std::runtime_error("failed to allocate the Sopro conditioning graph"); + } + } + + void build_velocity( + engine::core::BackendType backend_type, + size_t graph_context_bytes, + const SoproModelConfig & config_in) { + ggml_init_params params{graph_context_bytes, nullptr, true}; + velocity_ctx.reset(ggml_init(params)); + if (velocity_ctx == nullptr) { + throw std::runtime_error("failed to initialize the Sopro velocity graph context"); + } + engine::core::ModuleBuildContext ctx{ + velocity_ctx.get(), "sopro_tts.acoustic.velocity", backend_type}; + const int64_t mel = config_in.acoustic_mel_n_mels; + const int64_t mu_dim = config_in.acoustic_mu_dim; + const int64_t spk_dim = config_in.acoustic_spk_dim; + const int64_t dim = config_in.acoustic_dit_dim; + const int64_t heads = config_in.acoustic_dit_heads; + const int64_t head_dim = config_in.acoustic_dit_dim_head; + const int64_t inner = heads * head_dim; + const int64_t ff_dim = config_in.acoustic_dit_ff_dim(); + + const auto mel_shape = engine::core::TensorShape::from_dims({1, mel, frames}); + const auto mu_shape = engine::core::TensorShape::from_dims({1, mu_dim, frames}); + const auto mask_shape = engine::core::TensorShape::from_dims({1, 1, frames}); + x_input = engine::core::make_tensor(ctx, GGML_TYPE_F32, mel_shape).tensor; + cond_mel_input = engine::core::make_tensor(ctx, GGML_TYPE_F32, mel_shape).tensor; + cond_mask_input = engine::core::make_tensor(ctx, GGML_TYPE_F32, mask_shape).tensor; + mu_input = engine::core::make_tensor(ctx, GGML_TYPE_F32, mu_shape).tensor; + spk_input = engine::core::make_tensor( + ctx, GGML_TYPE_F32, engine::core::TensorShape::from_dims({1, 1, spk_dim})).tensor; + emb_input = engine::core::make_tensor( + ctx, GGML_TYPE_F32, engine::core::TensorShape::from_dims({1, 1, dim})).tensor; + // The solver replays this graph once per Euler step and only re-uploads + // x_t and the time embedding, so every leaf must survive the allocator. + for (ggml_tensor * leaf : + {x_input, cond_mel_input, cond_mask_input, mu_input, spk_input, emb_input}) { + ggml_set_input(leaf); + } + positions = ggml_new_tensor_1d(ctx.ggml, GGML_TYPE_I32, frames); + ggml_set_input(positions); + + auto to_btc = [&](ggml_tensor * tensor, int64_t channels) { + auto value = engine::core::wrap_tensor( + tensor, engine::core::TensorShape::from_dims({1, channels, frames}), GGML_TYPE_F32); + return dense(ctx, mod::TransposeModule(swap_channel_time()).build(ctx, value)); + }; + auto emb = engine::core::wrap_tensor( + emb_input, engine::core::TensorShape::from_dims({1, 1, dim}), GGML_TYPE_F32); + + // InputEmbedding.proj over [x_t | cond_mel | mu | spk]. + auto features = mod::ConcatModule({2}).build(ctx, to_btc(x_input, mel), to_btc(cond_mel_input, mel)); + features = mod::ConcatModule({2}).build(ctx, features, to_btc(mu_input, mu_dim)); + { + auto spk = engine::core::wrap_tensor( + spk_input, engine::core::TensorShape::from_dims({1, 1, spk_dim}), GGML_TYPE_F32); + auto broadcast = mod::RepeatModule({ + engine::core::TensorShape::from_dims({1, frames, spk_dim})}).build(ctx, spk); + features = mod::ConcatModule({2}).build(ctx, features, broadcast); + } + const int64_t proj_in = mel * 2 + mu_dim + spk_dim; + auto hidden = mod::LinearModule({proj_in, dim, true, GGML_PREC_F32}) + .build(ctx, dense(ctx, features), weights->input_proj); + hidden = mod::AddModule{}.build( + ctx, hidden, + mod::LinearModule({1, dim, false, GGML_PREC_F32}) + .build(ctx, to_btc(cond_mask_input, 1), weights->cond_mask_proj)); + { + // CausalConvPositionEmbedding: two causal grouped convolutions. + const auto kernel = static_cast(config_in.acoustic_pos_kernel_size); + auto y = dense(ctx, mod::TransposeModule(swap_channel_time()).build(ctx, hidden)); + y = build_grouped_conv(ctx, pad_time(ctx, y, kernel - 1, 0), weights->pos_conv1); + y = mish(ctx, y); + y = build_grouped_conv(ctx, pad_time(ctx, y, kernel - 1, 0), weights->pos_conv2); + y = mish(ctx, y); + y = dense(ctx, mod::TransposeModule(swap_channel_time()).build(ctx, y)); + hidden = mod::AddModule{}.build(ctx, hidden, y); + } + + auto silu_emb = mod::SiluModule{}.build(ctx, emb); + for (const auto & block : weights->blocks) { + auto modulation = mod::LinearModule({dim, dim * 6, true, GGML_PREC_F32}) + .build(ctx, silu_emb, block.modulation); + auto chunk = [&](int64_t index) { + return mod::SliceModule({2, index * dim, dim}).build(ctx, modulation); + }; + const auto shift_msa = chunk(0); + const auto scale_msa = chunk(1); + const auto gate_msa = chunk(2); + const auto shift_mlp = chunk(3); + const auto scale_mlp = chunk(4); + const auto gate_mlp = chunk(5); + + auto norm = modulate( + ctx, + mod::LayerNormModule({dim, kDiTLayerNormEps, false, false}) + .build(ctx, hidden, mod::NormWeights{}), + scale_msa, shift_msa); + auto q = mod::LinearModule({dim, inner, true, GGML_PREC_F32}).build(ctx, norm, block.to_q); + auto k = mod::LinearModule({dim, inner, true, GGML_PREC_F32}).build(ctx, norm, block.to_k); + auto v = mod::LinearModule({dim, inner, true, GGML_PREC_F32}).build(ctx, norm, block.to_v); + const auto head_shape = engine::core::TensorShape::from_dims({1, frames, heads, head_dim}); + auto reshape_heads = [&](const engine::core::TensorValue & value) { + return dense(ctx, engine::core::reshape_tensor( + ctx, engine::core::ensure_backend_addressable_layout(ctx, value), head_shape)); + }; + // Half-rotation RoPE over the frame index, applied per head. + auto rope = [&](engine::core::TensorValue value) { + return mod::RoPEModule({head_dim, GGML_ROPE_TYPE_NEOX, 10000.0F}) + .build(ctx, value, engine::core::wrap_tensor( + positions, engine::core::TensorShape::from_dims({frames}), GGML_TYPE_I32)); + }; + auto to_flash = [&](const engine::core::TensorValue & value) { + return dense(ctx, mod::TransposeModule({{0, 2, 1, 3}, 4}).build(ctx, value)); + }; + auto q_heads = to_flash(rope(reshape_heads(q))); + auto k_heads = to_flash(rope(reshape_heads(k))); + auto v_heads = to_flash(reshape_heads(v)); + auto attention = mod::ScaledDotProductAttentionModule({ + head_dim, + mod::ScaledDotProductAttentionLowering::Flash, + GGML_PREC_F32, + mod::AttentionCausality::NonCausal, + }).build(ctx, q_heads, k_heads, v_heads); + auto flat = engine::core::reshape_tensor( + ctx, engine::core::ensure_backend_addressable_layout(ctx, attention), + engine::core::TensorShape::from_dims({1, frames, inner})); + auto projected = mod::LinearModule({inner, dim, true, GGML_PREC_F32}) + .build(ctx, flat, block.to_out); + hidden = mod::AddModule{}.build( + ctx, hidden, + mod::MulModule{}.build( + ctx, projected, mod::RepeatModule({projected.shape}).build(ctx, gate_msa))); + + auto feed = modulate( + ctx, + mod::LayerNormModule({dim, kDiTLayerNormEps, false, false}) + .build(ctx, hidden, mod::NormWeights{}), + scale_mlp, shift_mlp); + feed = mod::LinearModule({dim, ff_dim, true, GGML_PREC_F32}).build(ctx, feed, block.ff_in); + feed = mod::GeluModule({mod::GeluApproximation::Tanh}).build(ctx, feed); + feed = mod::LinearModule({ff_dim, dim, true, GGML_PREC_F32}).build(ctx, feed, block.ff_out); + hidden = mod::AddModule{}.build( + ctx, hidden, + mod::MulModule{}.build( + ctx, feed, mod::RepeatModule({feed.shape}).build(ctx, gate_mlp))); + } + + { + // AdaLayerNormFinal emits (scale, shift) in that order. + auto modulation = mod::LinearModule({dim, dim * 2, true, GGML_PREC_F32}) + .build(ctx, silu_emb, weights->out_modulation); + const auto scale = mod::SliceModule({2, 0, dim}).build(ctx, modulation); + const auto shift = mod::SliceModule({2, dim, dim}).build(ctx, modulation); + hidden = modulate( + ctx, + mod::LayerNormModule({dim, kDiTLayerNormEps, false, false}) + .build(ctx, hidden, mod::NormWeights{}), + scale, shift); + } + hidden = mod::LinearModule({dim, mel, true, GGML_PREC_F32}).build(ctx, hidden, weights->out_proj); + // Back to [1, mel, frames] so the Euler update sees the solver layout. + hidden = dense(ctx, mod::TransposeModule(swap_channel_time()).build(ctx, hidden)); + velocity_output = hidden.tensor; + ggml_set_output(velocity_output); + velocity_graph = ggml_new_graph_custom(velocity_ctx.get(), 262144, false); + ggml_build_forward_expand(velocity_graph, velocity_output); + velocity_allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (velocity_allocr == nullptr || + !ggml_gallocr_reserve(velocity_allocr, velocity_graph) || + !ggml_gallocr_alloc_graph(velocity_allocr, velocity_graph)) { + throw std::runtime_error("failed to allocate the Sopro velocity graph"); + } + } + + std::vector run_conditioning(const std::vector & token_ids) { + std::vector index(static_cast(frames), 0); + for (int64_t frame = 0; frame < frames; ++frame) { + index[static_cast(frame)] = static_cast( + std::min(frame * tokens / frames, tokens - 1)); + } + ggml_backend_tensor_set(token_input, token_ids.data(), 0, token_ids.size() * sizeof(int32_t)); + ggml_backend_tensor_set(expand_index, index.data(), 0, index.size() * sizeof(int32_t)); + const ggml_status status = engine::core::compute_backend_graph(backend, conditioning_graph); + ggml_backend_synchronize(backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Sopro acoustic conditioning graph compute failed"); + } + std::vector mu(static_cast(config->acoustic_mu_dim * frames), 0.0F); + ggml_backend_tensor_get(mu_output, mu.data(), 0, mu.size() * sizeof(float)); + return mu; + } + + // ggml_gallocr only exempts GGML_TENSOR_FLAG_OUTPUT tensors from being + // freed and reused (ggml-alloc.c, ggml_gallocr_free_node); an input leaf's + // arena space is handed to a later intermediate once its last consumer has + // run. That is fine for a one-shot graph, but the solver replays this one + // per Euler step, so every leaf has to be re-uploaded before each compute + // rather than staged once. + void set_constants( + std::vector cond_mel, + std::vector cond_mask, + std::vector mu, + std::vector spk) { + cond_mel_host = std::move(cond_mel); + cond_mask_host = std::move(cond_mask); + mu_host = std::move(mu); + spk_host = std::move(spk); + position_host.assign(static_cast(frames), 0); + for (int64_t frame = 0; frame < frames; ++frame) { + position_host[static_cast(frame)] = static_cast(frame); + } + } + + void upload_constants() { + ggml_backend_tensor_set(positions, position_host.data(), 0, + position_host.size() * sizeof(int32_t)); + ggml_backend_tensor_set(cond_mel_input, cond_mel_host.data(), 0, + cond_mel_host.size() * sizeof(float)); + ggml_backend_tensor_set(cond_mask_input, cond_mask_host.data(), 0, + cond_mask_host.size() * sizeof(float)); + ggml_backend_tensor_set(mu_input, mu_host.data(), 0, mu_host.size() * sizeof(float)); + ggml_backend_tensor_set(spk_input, spk_host.data(), 0, spk_host.size() * sizeof(float)); + } + + std::vector run_velocity(const std::vector & x, const std::vector & emb) { + upload_constants(); + ggml_backend_tensor_set(x_input, x.data(), 0, x.size() * sizeof(float)); + ggml_backend_tensor_set(emb_input, emb.data(), 0, emb.size() * sizeof(float)); + const ggml_status status = engine::core::compute_backend_graph(backend, velocity_graph); + ggml_backend_synchronize(backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Sopro acoustic velocity graph compute failed"); + } + std::vector velocity(x.size(), 0.0F); + ggml_backend_tensor_get(velocity_output, velocity.data(), 0, velocity.size() * sizeof(float)); + return velocity; + } + + ggml_backend_t backend = nullptr; + std::shared_ptr weights; + int64_t tokens = 0; + int64_t frames = 0; + const SoproModelConfig * config = nullptr; + + std::unique_ptr conditioning_ctx; + ggml_tensor * token_input = nullptr; + ggml_tensor * expand_index = nullptr; + ggml_tensor * mu_output = nullptr; + ggml_cgraph * conditioning_graph = nullptr; + ggml_gallocr_t conditioning_allocr = nullptr; + + std::unique_ptr velocity_ctx; + ggml_tensor * x_input = nullptr; + ggml_tensor * cond_mel_input = nullptr; + ggml_tensor * cond_mask_input = nullptr; + ggml_tensor * mu_input = nullptr; + ggml_tensor * spk_input = nullptr; + ggml_tensor * emb_input = nullptr; + ggml_tensor * positions = nullptr; + ggml_tensor * velocity_output = nullptr; + std::vector cond_mel_host; + std::vector cond_mask_host; + std::vector mu_host; + std::vector spk_host; + std::vector position_host; + ggml_cgraph * velocity_graph = nullptr; + ggml_gallocr_t velocity_allocr = nullptr; +}; + +SoproAcousticRuntime::SoproAcousticRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) + : config_(assets.config.model), + execution_context_(execution_context), + graph_context_bytes_(graph_context_bytes), + weights_(load_acoustic_weights( + execution_context.backend(), + execution_context.backend_type(), + *assets.model_weights, + assets.config.model, + weight_context_bytes, + matmul_storage_type, + conv_storage_type)) {} + +SoproAcousticRuntime::~SoproAcousticRuntime() = default; + +std::vector SoproAcousticRuntime::solve(const SoproAcousticRequest & request) const { + const int64_t mel = config_.acoustic_mel_n_mels; + const int64_t frames = request.total_frames; + const auto tokens = static_cast(request.semantic_tokens.size()); + if (frames <= 0 || tokens <= 0) { + throw std::runtime_error("Sopro acoustic solve requires tokens and frames"); + } + if (request.prompt_frames < 0 || request.prompt_frames > frames) { + throw std::runtime_error("Sopro acoustic prompt frame count is out of range"); + } + if (static_cast(request.prompt_mel.size()) != mel * request.prompt_frames) { + throw std::runtime_error("Sopro acoustic prompt mel shape mismatch"); + } + if (static_cast(request.cond_vec.size()) != config_.cond_hidden_dim) { + throw std::runtime_error("Sopro acoustic conditioning vector shape mismatch"); + } + const int64_t steps = std::max(1, request.steps); + + if (graphs_ == nullptr || !graphs_->matches(*weights_, tokens, frames)) { + // Free the previous arena first; otherwise both are resident while the + // replacement is allocated, and every segment rebuilds this graph. + graphs_.reset(); + graphs_ = std::make_unique( + execution_context_.backend(), + execution_context_.backend_type(), + graph_context_bytes_, + config_, + weights_, + tokens, + frames); + } + + // _row_has_signal: an all-zero conditioning vector disables the speaker + // branch entirely (the reference uses it for unconditional batches). + float cond_peak = 0.0F; + for (const float value : request.cond_vec) { + cond_peak = std::max(cond_peak, std::fabs(value)); + } + std::vector spk(static_cast(config_.acoustic_spk_dim), 0.0F); + if (cond_peak > 0.0F) { + double norm = 0.0; + for (const float value : request.cond_vec) { + norm += static_cast(value) * static_cast(value); + } + // F.normalize uses an epsilon floor rather than a plain division. + const auto inv = static_cast(1.0 / std::max(std::sqrt(norm), 1.0e-12)); + std::vector normalised(request.cond_vec); + for (auto & value : normalised) { + value *= inv; + } + spk = affine( + weights_->spk_proj_w, weights_->spk_proj_b, normalised, + config_.cond_hidden_dim, config_.acoustic_spk_dim); + } + + auto mu = graphs_->run_conditioning(request.semantic_tokens); + dump("mu", mu); + dump("spk", spk); + + std::vector cond_mel(static_cast(mel * frames), 0.0F); + std::vector cond_mask(static_cast(frames), 0.0F); + for (int64_t c = 0; c < mel; ++c) { + std::copy( + request.prompt_mel.begin() + static_cast(c * request.prompt_frames), + request.prompt_mel.begin() + static_cast((c + 1) * request.prompt_frames), + cond_mel.begin() + static_cast(c * frames)); + } + std::fill(cond_mask.begin(), cond_mask.begin() + static_cast(request.prompt_frames), 1.0F); + graphs_->set_constants(cond_mel, cond_mask, mu, spk); + + std::mt19937_64 rng(request.seed); + std::normal_distribution normal(0.0F, 1.0F); + std::vector x_init(static_cast(mel * frames), 0.0F); + for (auto & value : x_init) { + value = normal(rng); + } + std::vector x(x_init); + dump("x_init", x_init); + dump("cond_mel", cond_mel); + + const auto grid = build_time_grid(steps, config_.acoustic_sway_sampling_coef); + const float sigma_min = config_.acoustic_sigma_min; + for (int64_t step = 0; step < steps; ++step) { + const float t0 = grid[static_cast(step)]; + const float t1 = grid[static_cast(step + 1)]; + const auto raw = sinusoidal_time_embedding(t0, config_.acoustic_time_embed_dim); + auto emb = affine( + weights_->time_mlp_w0, weights_->time_mlp_b0, raw, + config_.acoustic_time_embed_dim, config_.acoustic_dit_dim); + for (auto & value : emb) { + value = value / (1.0F + std::exp(-value)); // SiLU + } + emb = affine( + weights_->time_mlp_w2, weights_->time_mlp_b2, emb, + config_.acoustic_dit_dim, config_.acoustic_dit_dim); + + const std::string tag = std::to_string(step); + if (step == 0) { + dump("emb0", emb); + dump("x_step0", x); + } + dump(("traj_x_" + tag).c_str(), x); + const auto velocity = graphs_->run_velocity(x, emb); + if (step == 0) { + dump("velocity0", velocity); + } + dump(("traj_v_" + tag).c_str(), velocity); + const float dt = t1 - t0; + for (size_t i = 0; i < x.size(); ++i) { + x[i] += dt * velocity[i]; + } + // Re-pin the prompt span to the reference mel's flow at t1. + const float prompt_scale = 1.0F - (1.0F - sigma_min) * t1; + for (int64_t c = 0; c < mel; ++c) { + const size_t base = static_cast(c * frames); + for (int64_t t = 0; t < request.prompt_frames; ++t) { + const size_t index = base + static_cast(t); + x[index] = prompt_scale * x_init[index] + t1 * cond_mel[index]; + } + } + } + for (int64_t c = 0; c < mel; ++c) { + const size_t base = static_cast(c * frames); + for (int64_t t = 0; t < request.prompt_frames; ++t) { + x[base + static_cast(t)] = cond_mel[base + static_cast(t)]; + } + } + dump("solved", x); + return x; +} + +} // namespace engine::community_models::sopro_tts diff --git a/src/community_models/sopro_tts/assets.cpp b/src/community_models/sopro_tts/assets.cpp new file mode 100644 index 000000000..b0500b4d2 --- /dev/null +++ b/src/community_models/sopro_tts/assets.cpp @@ -0,0 +1,263 @@ +#include "engine/community_models/sopro_tts/assets.h" + +#include "engine/framework/io/json.h" +#include "engine/framework/model_spec/package.h" + +#include +#include +#include + +namespace engine::community_models::sopro_tts { +namespace { + +namespace json = engine::io::json; + +constexpr const char * kFamily = "sopro_tts"; + +SoproModelConfig parse_model(const json::Value & root) { + SoproModelConfig out; + const auto * node = root.find("model"); + if (node == nullptr) { + throw std::runtime_error("Sopro config.json is missing the \"model\" section"); + } + const auto & model = *node; + out.latent_dim = json::optional_i64(model, "latent_dim", out.latent_dim); + out.semantic_vocab_size = json::optional_i64(model, "semantic_vocab_size", out.semantic_vocab_size); + out.text_vocab_size = json::optional_i64(model, "text_vocab_size", out.text_vocab_size); + out.max_text_len = json::optional_i64(model, "max_text_len", out.max_text_len); + out.cond_in_dim = json::optional_i64(model, "cond_in_dim", out.cond_in_dim); + out.cond_hidden_dim = json::optional_i64(model, "cond_hidden_dim", out.cond_hidden_dim); + out.ar_model_dim = json::optional_i64(model, "ar_model_dim", out.ar_model_dim); + out.ar_blocks = json::optional_i64(model, "ar_blocks", out.ar_blocks); + out.ar_heads = json::optional_i64(model, "ar_heads", out.ar_heads); + // ModelConfig.__post_init__: ar_kv_heads defaults to ar_heads. + out.ar_kv_heads = json::optional_nullable_i64(model, "ar_kv_heads", out.ar_heads); + out.ar_ffn_mult = json::optional_f32(model, "ar_ffn_mult", out.ar_ffn_mult); + out.ar_qk_rms_norm = json::optional_bool(model, "ar_qk_rms_norm", out.ar_qk_rms_norm); + out.style_prefix_tokens = json::optional_i64(model, "style_prefix_tokens", out.style_prefix_tokens); + out.acoustic_time_embed_dim = json::optional_i64(model, "acoustic_time_embed_dim", out.acoustic_time_embed_dim); + out.acoustic_sway_sampling_coef = json::optional_f32(model, "acoustic_sway_sampling_coef", out.acoustic_sway_sampling_coef); + out.acoustic_upsampler_kernel_size = json::optional_i64(model, "acoustic_upsampler_kernel_size", out.acoustic_upsampler_kernel_size); + out.acoustic_dit_dim = json::optional_i64(model, "acoustic_dit_dim", out.acoustic_dit_dim); + out.acoustic_dit_depth = json::optional_i64(model, "acoustic_dit_depth", out.acoustic_dit_depth); + out.acoustic_dit_heads = json::optional_i64(model, "acoustic_dit_heads", out.acoustic_dit_heads); + out.acoustic_dit_dim_head = json::optional_i64(model, "acoustic_dit_dim_head", out.acoustic_dit_dim_head); + out.acoustic_dit_ff_mult = json::optional_f32(model, "acoustic_dit_ff_mult", out.acoustic_dit_ff_mult); + out.acoustic_spk_dim = json::optional_i64(model, "acoustic_spk_dim", out.acoustic_spk_dim); + out.acoustic_pre_lookahead_frames = json::optional_i64(model, "acoustic_pre_lookahead_frames", out.acoustic_pre_lookahead_frames); + out.acoustic_pos_kernel_size = json::optional_i64(model, "acoustic_pos_kernel_size", out.acoustic_pos_kernel_size); + out.acoustic_sigma_min = json::optional_f32(model, "acoustic_sigma_min", out.acoustic_sigma_min); + out.acoustic_num_left_chunks = json::optional_i64(model, "acoustic_num_left_chunks", out.acoustic_num_left_chunks); + out.acoustic_mel_n_mels = json::optional_i64(model, "acoustic_mel_n_mels", out.acoustic_mel_n_mels); + out.acoustic_mel_hop_length = json::optional_i64(model, "acoustic_mel_hop_length", out.acoustic_mel_hop_length); + // ModelConfig.__post_init__: acoustic_mu_dim defaults to acoustic_mel_n_mels. + out.acoustic_mu_dim = json::optional_nullable_i64(model, "acoustic_mu_dim", out.acoustic_mel_n_mels); + out.acoustic_mel_mean = json::optional_f32_array(model, "acoustic_mel_mean"); + out.acoustic_mel_std = json::optional_f32_array(model, "acoustic_mel_std"); + if (static_cast(out.acoustic_mel_mean.size()) != out.acoustic_mel_n_mels || + static_cast(out.acoustic_mel_std.size()) != out.acoustic_mel_n_mels) { + throw std::runtime_error( + "Sopro config.json must provide acoustic_mel_mean/acoustic_mel_std with " + "acoustic_mel_n_mels entries"); + } + if (out.ar_model_dim % out.ar_heads != 0) { + throw std::runtime_error("Sopro ar_model_dim must be divisible by ar_heads"); + } + return out; +} + +SoproSemanticEncoderConfig parse_semantic_encoder(const json::Value & root) { + SoproSemanticEncoderConfig out; + const auto * node = root.find("semantic_encoder"); + if (node == nullptr) { + return out; + } + const auto & cfg = *node; + out.n_mels = json::optional_i64(cfg, "n_mels", out.n_mels); + out.d_model = json::optional_i64(cfg, "d_model", out.d_model); + out.layers = json::optional_i64(cfg, "layers", out.layers); + out.heads = json::optional_i64(cfg, "heads", out.heads); + out.ffn_dim = json::optional_i64(cfg, "ffn_dim", out.ffn_dim); + out.max_positions = json::optional_i64(cfg, "max_positions", out.max_positions); + out.fsq_levels = json::optional_i64_array(cfg, "fsq_levels", out.fsq_levels); + out.sample_rate = json::optional_i64(cfg, "sample_rate", out.sample_rate); + out.n_fft = json::optional_i64(cfg, "n_fft", out.n_fft); + out.hop_length = json::optional_i64(cfg, "hop_length", out.hop_length); + out.token_samples_24k = json::optional_i64(cfg, "token_samples_24k", out.token_samples_24k); + if (out.fsq_levels.empty()) { + throw std::runtime_error("Sopro semantic_encoder.fsq_levels must not be empty"); + } + return out; +} + +SoproSpeakerEncoderConfig parse_speaker_encoder(const json::Value & root) { + SoproSpeakerEncoderConfig out; + const auto * node = root.find("speaker_encoder"); + if (node == nullptr) { + return out; + } + const auto & cfg = *node; + out.sample_rate = json::optional_i64(cfg, "sample_rate", out.sample_rate); + out.n_mels = json::optional_i64(cfg, "n_mels", out.n_mels); + out.n_fft = json::optional_i64(cfg, "n_fft", out.n_fft); + out.win_length = json::optional_i64(cfg, "win_length", out.win_length); + out.hop_length = json::optional_i64(cfg, "hop_length", out.hop_length); + out.f_min = json::optional_f32(cfg, "f_min", out.f_min); + out.f_max = json::optional_f32(cfg, "f_max", out.f_max); + out.mel_log_floor = json::optional_f32(cfg, "mel_log_floor", out.mel_log_floor); + out.stem_channels = json::optional_i64(cfg, "stem_channels", out.stem_channels); + out.stage_channels = json::optional_i64_array(cfg, "stage_channels", out.stage_channels); + out.blocks_per_stage = json::optional_i64_array(cfg, "blocks_per_stage", out.blocks_per_stage); + out.dilation_cycle = json::optional_i64_array(cfg, "dilation_cycle", out.dilation_cycle); + out.depthwise_kernel_size = json::optional_i64(cfg, "depthwise_kernel_size", out.depthwise_kernel_size); + out.se_reduction = json::optional_i64(cfg, "se_reduction", out.se_reduction); + out.id_emb_dim = json::optional_i64(cfg, "id_emb_dim", out.id_emb_dim); + out.style_emb_dim = json::optional_i64(cfg, "style_emb_dim", out.style_emb_dim); + out.style_ctrl_dim = json::optional_i64(cfg, "style_ctrl_dim", out.style_ctrl_dim); + out.id_head_hidden = json::optional_i64(cfg, "id_head_hidden", out.id_head_hidden); + out.style_head_hidden = json::optional_i64(cfg, "style_head_hidden", out.style_head_hidden); + out.attn_hidden = json::optional_i64(cfg, "attn_hidden", out.attn_hidden); + if (out.stage_channels.size() != out.blocks_per_stage.size() || out.stage_channels.empty()) { + throw std::runtime_error( + "Sopro speaker_encoder.stage_channels and blocks_per_stage must be " + "non-empty and the same length"); + } + if (out.dilation_cycle.empty()) { + throw std::runtime_error("Sopro speaker_encoder.dilation_cycle must not be empty"); + } + return out; +} + +SoproVocoderConfig parse_vocoder(const json::Value & root, const std::string & key) { + SoproVocoderConfig out; + const auto * node = root.find(key); + if (node == nullptr) { + return out; + } + const auto & cfg = *node; + out.sample_rate = json::optional_i64(cfg, "sample_rate", out.sample_rate); + out.n_fft = json::optional_i64(cfg, "n_fft", out.n_fft); + out.hop_length = json::optional_i64(cfg, "hop_length", out.hop_length); + out.n_mels = json::optional_i64(cfg, "n_mels", out.n_mels); + out.dim = json::optional_i64(cfg, "dim", out.dim); + out.intermediate_dim = json::optional_i64(cfg, "intermediate_dim", out.intermediate_dim); + out.num_layers = json::optional_i64(cfg, "num_layers", out.num_layers); + out.max_magnitude = json::optional_f32(cfg, "max_magnitude", out.max_magnitude); + out.band_limit_hz = json::optional_f32(cfg, "band_limit_hz", out.band_limit_hz); + out.causal = json::optional_bool(cfg, "causal", out.causal); + out.lookahead_frames = json::optional_nullable_i64(cfg, "lookahead_frames", out.lookahead_frames); + out.block_lookaheads = json::optional_i64_array(cfg, "block_lookaheads"); + return out; +} + +SoproGenerationConfig parse_generation(const json::Value & root) { + SoproGenerationConfig out; + const auto * node = root.find("generation"); + if (node == nullptr) { + return out; + } + const auto & cfg = *node; + out.temperature = json::optional_f32(cfg, "temperature", out.temperature); + out.top_p = json::optional_f32(cfg, "top_p", out.top_p); + out.top_k = json::optional_i64(cfg, "top_k", out.top_k); + out.steps = json::optional_i64(cfg, "steps", out.steps); + out.max_seconds = json::optional_f32(cfg, "max_seconds", out.max_seconds); + out.min_seconds = json::optional_f32(cfg, "min_seconds", out.min_seconds); + out.max_segment_chars = json::optional_i64(cfg, "max_segment_chars", out.max_segment_chars); + out.ref_seconds = json::optional_f32(cfg, "ref_seconds", out.ref_seconds); + out.style_tokens = json::optional_i64(cfg, "style_tokens", out.style_tokens); + out.prompt_tokens = json::optional_i64(cfg, "prompt_tokens", out.prompt_tokens); + out.stream_chunk_frames = json::optional_i64(cfg, "stream_chunk_frames", out.stream_chunk_frames); + return out; +} + +SoproTTSConfig parse_config(const assets::ResourceBundle & resources) { + const auto root = resources.parse_json("config"); + SoproTTSConfig out; + out.sample_rate = json::optional_i64(root, "sample_rate", out.sample_rate); + out.model = parse_model(root); + out.semantic_encoder = parse_semantic_encoder(root); + out.speaker_encoder = parse_speaker_encoder(root); + out.vocoder = parse_vocoder(root, "vocoder"); + out.vocoder_streaming = parse_vocoder(root, "vocoder_streaming"); + out.generation = parse_generation(root); + const int64_t codebook = out.semantic_encoder.codebook_size(); + if (codebook != out.model.semantic_vocab_size) { + throw std::runtime_error( + "Sopro config mismatch: prod(semantic_encoder.fsq_levels) = " + + std::to_string(codebook) + " but model.semantic_vocab_size = " + + std::to_string(out.model.semantic_vocab_size)); + } + if (out.semantic_encoder.token_samples_24k % out.model.acoustic_mel_hop_length != 0) { + throw std::runtime_error( + "Sopro config mismatch: semantic_encoder.token_samples_24k must be a " + "multiple of model.acoustic_mel_hop_length"); + } + return out; +} + +} // namespace + +int64_t SoproModelConfig::ar_ffn_dim() const noexcept { + // SwiGLUFeedForward: hidden = max(1, round(mult * dim)). + const auto hidden = static_cast( + std::llround(static_cast(ar_ffn_mult) * static_cast(ar_model_dim))); + return hidden < 1 ? 1 : hidden; +} + +int64_t SoproModelConfig::acoustic_dit_ff_dim() const noexcept { + const auto hidden = static_cast( + std::llround(static_cast(acoustic_dit_ff_mult) * static_cast(acoustic_dit_dim))); + return hidden < 1 ? 1 : hidden; +} + +int64_t SoproSemanticEncoderConfig::digit_dim() const noexcept { + int64_t sum = 0; + for (const int64_t level : fsq_levels) { + sum += level; + } + return sum; +} + +int64_t SoproSemanticEncoderConfig::codebook_size() const noexcept { + int64_t product = 1; + for (const int64_t level : fsq_levels) { + product *= level; + } + return product; +} + +int64_t SoproTTSConfig::hop_ratio() const noexcept { + return semantic_encoder.token_samples_24k / model.acoustic_mel_hop_length; +} + +void require_frontend_buffers( + const assets::TensorSource & source, + const char * stage, + std::initializer_list tensor_names) { + for (const char * name : tensor_names) { + if (!source.has_tensor(name)) { + throw std::runtime_error( + std::string("Sopro ") + stage + " checkpoint is missing '" + name + + "'. audio.cpp reuses torchaudio's stored analysis window and mel " + "filterbank rather than rebuilding them; re-export the checkpoint " + "with persistent buffers."); + } + } +} + +std::shared_ptr load_sopro_tts_assets( + const std::filesystem::path & model_path) { + auto assets = std::make_shared(); + assets->resources = engine::model_spec::load_resource_bundle( + model_path, engine::model_spec::default_spec_path(kFamily)); + assets->config = parse_config(assets->resources); + assets->model_weights = assets->resources.open_tensor_source("model"); + assets->semantic_encoder_weights = assets->resources.open_tensor_source("semantic_encoder"); + assets->speaker_encoder_weights = assets->resources.open_tensor_source("speaker_encoder"); + assets->vocoder_weights = assets->resources.open_tensor_source("vocoder"); + assets->tokenizer_path = assets->resources.require_file("tokenizer"); + return assets; +} + +} // namespace engine::community_models::sopro_tts diff --git a/src/community_models/sopro_tts/reference.cpp b/src/community_models/sopro_tts/reference.cpp new file mode 100644 index 000000000..b0d74e59f --- /dev/null +++ b/src/community_models/sopro_tts/reference.cpp @@ -0,0 +1,490 @@ +#include "engine/community_models/sopro_tts/reference.h" + +#include "engine/community_models/sopro_tts/semantic_encoder.h" +#include "engine/community_models/sopro_tts/speaker_encoder.h" +#include "engine/community_models/sopro_tts/vocoder.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/audio/resampling.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { +namespace audio_ops { +namespace { + +constexpr float kRefGainLimitDb = 30.0F; +constexpr float kRefPeakCeiling = 0.95F; +constexpr float kPauseMinSeconds = 0.10F; +constexpr float kPauseKeepSeconds = 0.15F; +constexpr float kCropForwardSeconds = 5.0F; +constexpr float kCropBackwardSeconds = 5.0F; +constexpr float kMinKeepFraction = 0.75F; +constexpr float kRoomToneSeconds = 0.25F; +constexpr float kFadeSeconds = 0.02F; +constexpr float kOnsetThresholdDb = -45.0F; +constexpr float kOnsetOverFloorDb = 15.0F; +constexpr int64_t kOnsetWindowFrames = 6; +constexpr int64_t kOnsetMinFrames = 5; +constexpr float kMinActiveSeconds = 0.4F; + +// torch.unfold + RMS: floor((n - win) / hop) + 1 frames, floored at 1e-6. +std::vector frame_rms( + const std::vector & wav, int64_t window, int64_t hop) { + const auto total = static_cast(wav.size()); + std::vector out; + if (window <= 0 || hop <= 0 || total < window) { + return out; + } + const int64_t frames = (total - window) / hop + 1; + out.resize(static_cast(frames), 0.0F); + for (int64_t frame = 0; frame < frames; ++frame) { + const float * values = wav.data() + static_cast(frame * hop); + double sum = 0.0; + for (int64_t i = 0; i < window; ++i) { + sum += static_cast(values[i]) * static_cast(values[i]); + } + out[static_cast(frame)] = std::max( + static_cast(std::sqrt(sum / static_cast(window))), 1.0e-6F); + } + return out; +} + +// torch.quantile's default "linear" interpolation. +float quantile(std::vector values, float q) { + if (values.empty()) { + return 0.0F; + } + std::sort(values.begin(), values.end()); + const double position = static_cast(q) * static_cast(values.size() - 1); + const auto lower = static_cast(std::floor(position)); + const size_t upper = std::min(lower + 1, values.size() - 1); + const double weight = position - static_cast(lower); + return static_cast(values[lower] * (1.0 - weight) + values[upper] * weight); +} + +// torch.median returns the lower of the two middle elements for even counts. +float lower_median(std::vector values) { + if (values.empty()) { + return 0.0F; + } + const size_t index = (values.size() - 1) / 2; + std::nth_element(values.begin(), values.begin() + static_cast(index), values.end()); + return values[index]; +} + +struct PauseRun { + int64_t begin = 0; + int64_t end = 0; +}; + +std::vector pause_runs(const std::vector & quiet, int64_t min_run) { + std::vector runs; + const auto count = static_cast(quiet.size()); + int64_t index = 0; + while (index < count) { + if (quiet[static_cast(index)] == 0) { + ++index; + continue; + } + int64_t end = index; + while (end < count && quiet[static_cast(end)] != 0) { + ++end; + } + if (end - index >= min_run) { + runs.push_back({index, end}); + } + index = end; + } + return runs; +} + +float onset_threshold(const std::vector & rms) { + float threshold = std::pow(10.0F, kOnsetThresholdDb / 20.0F); + if (rms.size() >= 30) { + threshold = std::max( + threshold, quantile(rms, 0.1F) * std::pow(10.0F, kOnsetOverFloorDb / 20.0F)); + } + return threshold; +} + +// x[:(n // win) * win].view(-1, win) RMS, i.e. non-overlapping windows. +std::vector block_rms(const std::vector & wav, int64_t window) { + std::vector out; + if (window <= 0) { + return out; + } + const int64_t frames = static_cast(wav.size()) / window; + out.resize(static_cast(frames), 0.0F); + for (int64_t frame = 0; frame < frames; ++frame) { + const float * values = wav.data() + static_cast(frame * window); + double sum = 0.0; + for (int64_t i = 0; i < window; ++i) { + sum += static_cast(values[i]) * static_cast(values[i]); + } + out[static_cast(frame)] = static_cast(std::sqrt(sum / static_cast(window))); + } + return out; +} + +// First index where `min_hits` of the next `window` frames are above threshold. +std::optional first_sustained_hit( + const std::vector & rms, float threshold, int64_t window, int64_t min_hits) { + const auto count = static_cast(rms.size()); + if (count < window) { + return std::nullopt; + } + for (int64_t start = 0; start + window <= count; ++start) { + int64_t hits = 0; + for (int64_t i = 0; i < window; ++i) { + if (rms[static_cast(start + i)] > threshold) { + ++hits; + } + } + if (hits >= min_hits) { + return start; + } + } + return std::nullopt; +} + +std::vector finish_with_room_tone( + const std::vector & wav, int sample_rate, float floor_level, std::mt19937_64 & rng) { + const auto fade = static_cast(kFadeSeconds * static_cast(sample_rate)); + std::vector out(wav); + if (fade > 0 && static_cast(out.size()) >= fade) { + const auto offset = static_cast(out.size()) - fade; + for (int64_t i = 0; i < fade; ++i) { + const float ramp = fade == 1 ? 1.0F + : 1.0F - static_cast(i) / static_cast(fade - 1); + out[static_cast(offset + i)] *= ramp; + } + } + const auto tone_samples = static_cast(kRoomToneSeconds * static_cast(sample_rate)); + std::normal_distribution normal(0.0F, 1.0F); + for (int64_t i = 0; i < tone_samples; ++i) { + out.push_back(normal(rng) * floor_level); + } + return out; +} + +} // namespace + +std::vector crop_on_pause( + const std::vector & wav, float target_seconds, int sample_rate, std::mt19937_64 & rng) { + const auto rate = static_cast(sample_rate); + const auto window = static_cast(rate * 0.025F); + const auto hop = static_cast(rate * 0.010F); + const auto total = static_cast(wav.size()); + if (window <= 0 || hop <= 0 || total < 4 * window) { + return wav; + } + const auto rms = frame_rms(wav, window, hop); + if (rms.empty()) { + return wav; + } + const float floor_level = quantile(rms, 0.1F); + std::vector quiet(rms.size(), 0); + for (size_t i = 0; i < rms.size(); ++i) { + quiet[i] = rms[i] < floor_level * 4.0F ? 1 : 0; + } + const auto runs = pause_runs( + quiet, std::max(1, std::lround(kPauseMinSeconds / 0.010F))); + const auto keep = static_cast(std::lround(kPauseKeepSeconds / 0.010F)); + const auto target = static_cast(std::lround(target_seconds * rate)); + + const auto cut_at = [&](const PauseRun & run) { + const int64_t end = (run.begin + std::min(run.end - run.begin, keep)) * hop + window; + return std::vector( + wav.begin(), wav.begin() + static_cast(std::min(end, total))); + }; + + if (total <= target) { + if (!runs.empty() && runs.back().end * hop + window >= total - hop) { + return wav; + } + const auto boundary = static_cast(static_cast(total) * kMinKeepFraction); + for (auto it = runs.rbegin(); it != runs.rend(); ++it) { + if (it->begin * hop >= boundary) { + return cut_at(*it); + } + } + return finish_with_room_tone(wav, sample_rate, floor_level, rng); + } + const auto forward_limit = target + static_cast(kCropForwardSeconds * rate); + for (const auto & run : runs) { + if (run.begin * hop >= target && run.begin * hop <= forward_limit) { + return cut_at(run); + } + } + const auto backward_limit = target - static_cast(kCropBackwardSeconds * rate); + for (auto it = runs.rbegin(); it != runs.rend(); ++it) { + if (it->begin * hop < target && it->end * hop >= backward_limit) { + return cut_at(*it); + } + } + std::vector truncated( + wav.begin(), wav.begin() + static_cast(std::min(target, total))); + return finish_with_room_tone(truncated, sample_rate, floor_level, rng); +} + +SpeechLevel speech_level_db(const std::vector & wav, int sample_rate) { + const auto rate = static_cast(sample_rate); + const auto window = static_cast(rate * 0.025F); + const auto hop = static_cast(rate * 0.010F); + SpeechLevel out; + if (window <= 0 || static_cast(wav.size()) < window) { + double sum = 0.0; + for (const float value : wav) { + sum += static_cast(value) * static_cast(value); + } + const double rms = wav.empty() ? 0.0 : std::sqrt(sum / static_cast(wav.size())); + out.level_db = 20.0F * std::log10(std::max(static_cast(rms), 1.0e-6F)); + return out; + } + const auto rms = frame_rms(wav, window, hop); + const float threshold = quantile(rms, 0.2F) * 1.5F; + std::vector active; + active.reserve(rms.size()); + for (const float value : rms) { + if (value > threshold) { + active.push_back(value); + } + } + if (active.empty()) { + active = rms; + } + out.level_db = 20.0F * std::log10(lower_median(active)); + out.active_seconds = static_cast(active.size()) * static_cast(hop) / rate; + return out; +} + +NormalizedReference normalize_reference(const std::vector & wav, int sample_rate) { + const auto level = speech_level_db(wav, sample_rate); + // Boost only: a reference that is already hotter than the prompt level is + // passed through untouched instead of being pulled down. + float gain_db = std::min(std::max(kPromptLevelDb - level.level_db, 0.0F), kRefGainLimitDb); + float peak = 0.0F; + for (const float value : wav) { + peak = std::max(peak, std::fabs(value)); + } + if (peak > 0.0F) { + // Never let the boost clip: cap it at the headroom left below 0.95. + gain_db = std::min(gain_db, std::max(0.0F, 20.0F * std::log10(kRefPeakCeiling / peak))); + } + const float gain = std::pow(10.0F, gain_db / 20.0F); + NormalizedReference out; + out.wav = wav; + for (auto & value : out.wav) { + value *= gain; + } + out.level_db = level.level_db + gain_db; + return out; +} + +float output_gain(float prompt_level_db) { + return std::pow(10.0F, (kOutputLevelDb - prompt_level_db) / 20.0F); +} + +float match_gain( + const std::vector & wav, int sample_rate, float target_db, float prompt_level_db) { + const auto level = speech_level_db(wav, sample_rate); + if (level.active_seconds < kMinActiveSeconds) { + return output_gain(prompt_level_db); + } + return std::pow(10.0F, (target_db - level.level_db) / 20.0F); +} + +void soft_limit(std::vector & wav, float knee) { + const float span = 1.0F - knee; + if (span <= 0.0F) { + return; + } + for (auto & value : wav) { + const float magnitude = std::fabs(value); + if (magnitude > knee) { + const float limited = knee + span * std::tanh((magnitude - knee) / span); + value = value < 0.0F ? -limited : limited; + } + } +} + +std::optional speech_onset(const std::vector & wav, int sample_rate) { + const auto window = static_cast(static_cast(sample_rate) * 0.010F); + if (window <= 0 || static_cast(wav.size()) < window * kOnsetWindowFrames) { + return std::nullopt; + } + const auto rms = block_rms(wav, window); + const auto hit = first_sustained_hit( + rms, onset_threshold(rms), kOnsetWindowFrames, kOnsetMinFrames); + if (!hit.has_value()) { + return std::nullopt; + } + return *hit * window; +} + +std::vector trim_lead( + const std::vector & wav, int sample_rate, float lead, float skip) { + const auto onset = speech_onset(wav, sample_rate); + if (!onset.has_value()) { + return wav; + } + const auto rate = static_cast(sample_rate); + int64_t cut = std::max(*onset - static_cast(lead * rate), + static_cast(skip * rate)); + cut = std::min(cut, std::max(0, *onset - static_cast(0.02F * rate))); + cut = std::min(cut, static_cast(wav.size())); + return std::vector(wav.begin() + static_cast(cut), wav.end()); +} + +std::vector trim_trail(const std::vector & wav, int sample_rate, float trail) { + const auto window = static_cast(static_cast(sample_rate) * 0.010F); + if (window <= 0 || static_cast(wav.size()) < window) { + return wav; + } + const auto rms = block_rms(wav, window); + const float threshold = onset_threshold(rms); + int64_t last = -1; + for (int64_t i = 0; i < static_cast(rms.size()); ++i) { + if (rms[static_cast(i)] > threshold) { + last = i; + } + } + if (last < 0) { + return wav; + } + const int64_t end = std::min( + static_cast(wav.size()), + (last + 1) * window + static_cast(trail * static_cast(sample_rate))); + return std::vector(wav.begin(), wav.begin() + static_cast(end)); +} + +void fade_edges( + std::vector & wav, int sample_rate, bool fade_in, bool fade_out, float fade_seconds) { + const auto fade = static_cast(fade_seconds * static_cast(sample_rate)); + if (fade <= 1 || static_cast(wav.size()) <= 2 * fade) { + return; + } + for (int64_t i = 0; i < fade; ++i) { + const float ramp = static_cast(i) / static_cast(fade - 1); + if (fade_in) { + wav[static_cast(i)] *= ramp; + } + if (fade_out) { + wav[wav.size() - static_cast(fade) + static_cast(i)] *= 1.0F - ramp; + } + } +} + +std::vector join_segments(std::vector> parts, int sample_rate) { + std::vector out; + const auto count = static_cast(parts.size()); + for (int64_t i = 0; i < count; ++i) { + fade_edges(parts[static_cast(i)], sample_rate, i > 0, i + 1 < count); + out.insert( + out.end(), + parts[static_cast(i)].begin(), + parts[static_cast(i)].end()); + } + return out; +} + +} // namespace audio_ops + +SoproReferenceBuilder::SoproReferenceBuilder( + const SoproTTSAssets & assets, + const SoproSpeakerEncoderRuntime & speaker_encoder, + const SoproSemanticEncoderRuntime & semantic_encoder, + const SoproVocoderRuntime & vocoder) + : config_(assets.config), + speaker_encoder_(speaker_encoder), + semantic_encoder_(semantic_encoder), + vocoder_(vocoder), + mel_mean_(assets.config.model.acoustic_mel_mean), + mel_std_(assets.config.model.acoustic_mel_std) { + const auto & source = *assets.model_weights; + const int64_t in_dim = config_.model.cond_in_dim; + const int64_t hidden = config_.model.cond_hidden_dim; + cond_proj_w0 = source.require_f32("cond_proj.0.weight", {hidden, in_dim}); + cond_proj_b0 = source.require_f32("cond_proj.0.bias", {hidden}); + cond_proj_w3 = source.require_f32("cond_proj.3.weight", {hidden, hidden}); + cond_proj_b3 = source.require_f32("cond_proj.3.bias", {hidden}); +} + +SoproReference SoproReferenceBuilder::build( + const std::vector & audio24, + float ref_seconds, + std::mt19937_64 & rng) const { + if (audio24.empty()) { + throw std::runtime_error("Sopro requires non-empty reference audio"); + } + const auto sample_rate = static_cast(config_.sample_rate); + auto cropped = audio_ops::crop_on_pause(audio24, ref_seconds, sample_rate, rng); + auto normalized = audio_ops::normalize_reference(cropped, sample_rate); + const float reference_level_db = normalized.level_db; + auto wav = std::move(normalized.wav); + + const auto speaker_rate = static_cast(config_.speaker_encoder.sample_rate); + const auto wav16 = engine::audio::resample_mono_torchaudio_sinc_hann( + wav, sample_rate, speaker_rate); + const auto embeddings = speaker_encoder_.encode(wav16); + + const int64_t in_dim = config_.model.cond_in_dim; + const int64_t hidden = config_.model.cond_hidden_dim; + std::vector conditioning; + conditioning.reserve(static_cast(in_dim)); + conditioning.insert(conditioning.end(), embeddings.id_emb.begin(), embeddings.id_emb.end()); + conditioning.insert(conditioning.end(), embeddings.style_emb.begin(), embeddings.style_emb.end()); + conditioning.insert(conditioning.end(), embeddings.style_ctrl.begin(), embeddings.style_ctrl.end()); + if (static_cast(conditioning.size()) != in_dim) { + throw std::runtime_error( + "Sopro speaker embeddings do not add up to model.cond_in_dim; check config.json"); + } + std::vector projected(static_cast(hidden), 0.0F); + for (int64_t o = 0; o < hidden; ++o) { + const float * row = cond_proj_w0.data() + static_cast(o * in_dim); + double sum = cond_proj_b0[static_cast(o)]; + for (int64_t i = 0; i < in_dim; ++i) { + sum += static_cast(row[i]) * static_cast(conditioning[static_cast(i)]); + } + const auto value = static_cast(sum); + projected[static_cast(o)] = value / (1.0F + std::exp(-value)); // SiLU + } + SoproReference out; + out.cond_vec.assign(static_cast(hidden), 0.0F); + for (int64_t o = 0; o < hidden; ++o) { + const float * row = cond_proj_w3.data() + static_cast(o * hidden); + double sum = cond_proj_b3[static_cast(o)]; + for (int64_t i = 0; i < hidden; ++i) { + sum += static_cast(row[i]) * static_cast(projected[static_cast(i)]); + } + out.cond_vec[static_cast(o)] = static_cast(sum); + } + + out.semantic_tokens = semantic_encoder_.encode(wav); + + auto mel = vocoder_.log_mel(wav); + const int64_t n_mels = vocoder_.n_mels(); + if (n_mels <= 0 || mel.size() % static_cast(n_mels) != 0) { + throw std::runtime_error("Sopro reference mel has an unexpected shape"); + } + out.mel_frames = static_cast(mel.size()) / n_mels; + for (int64_t c = 0; c < n_mels; ++c) { + const float mean = mel_mean_[static_cast(c)]; + const float scale = mel_std_[static_cast(c)]; + float * row = mel.data() + static_cast(c * out.mel_frames); + for (int64_t t = 0; t < out.mel_frames; ++t) { + row[t] = (row[t] - mean) / scale; + } + } + out.mel = std::move(mel); + out.level_db = reference_level_db; + return out; +} + +} // namespace engine::community_models::sopro_tts diff --git a/src/community_models/sopro_tts/semantic_encoder.cpp b/src/community_models/sopro_tts/semantic_encoder.cpp new file mode 100644 index 000000000..a1f6161c5 --- /dev/null +++ b/src/community_models/sopro_tts/semantic_encoder.cpp @@ -0,0 +1,488 @@ +#include "engine/community_models/sopro_tts/semantic_encoder.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/audio/dsp.h" +#include "engine/framework/audio/resampling.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/attention/scaled_dot_product_attention.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { + +struct SoproSemanticLayerWeights { + engine::modules::NormWeights attn_norm; + engine::modules::LinearWeights q_proj; + engine::modules::LinearWeights k_proj; // bias=False upstream + engine::modules::LinearWeights v_proj; + engine::modules::LinearWeights out_proj; + engine::modules::NormWeights ffn_norm; + engine::modules::LinearWeights fc1; + engine::modules::LinearWeights fc2; +}; + +struct SoproSemanticEncoderWeights { + std::shared_ptr store; + engine::modules::Conv1dWeights conv1; + engine::modules::Conv1dWeights conv2; + engine::core::TensorValue pos_emb; + std::vector layers; + engine::modules::NormWeights final_norm; + // Host-side quantiser head. + std::vector pre_head_norm_weight; + std::vector pre_head_norm_bias; + std::vector digit_head_weight; // [digit_dim, d_model] + std::vector digit_head_bias; + // torchaudio MelSpectrogram buffers. + std::vector analysis_window; + std::vector mel_filterbank; +}; + +namespace { + +namespace binding = engine::modules::binding; + +constexpr float kLayerNormEps = 1.0e-5F; // torch.nn.LayerNorm default +constexpr int64_t kConvRightContextFrames = 2; // semantic.CONV_RIGHT_CONTEXT_FRAMES + +// SOPRO_DUMP_DIR: raw f32/i32 dumps for stage comparison against the reference. +template +void dump(const std::string & name, const std::vector & values) { + const char * dir = std::getenv("SOPRO_DUMP_DIR"); + if (dir == nullptr) { + return; + } + std::FILE * fh = std::fopen((std::string(dir) + "/" + name).c_str(), "wb"); + if (fh != nullptr) { + std::fwrite(values.data(), sizeof(T), values.size(), fh); + std::fclose(fh); + } +} + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +engine::core::TensorValue dense( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & value) { + return engine::core::wrap_tensor(ggml_cont(ctx.ggml, value.tensor), value.shape, GGML_TYPE_F32); +} + +std::shared_ptr load_weights( + ggml_backend_t backend, + engine::core::BackendType backend_type, + const engine::assets::TensorSource & source, + const SoproSemanticEncoderConfig & config, + size_t weight_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) { + auto weights = std::make_shared(); + require_frontend_buffers( + source, "semantic encoder", + {"frontend.mel.spectrogram.window", + "frontend.mel.mel_scale.fb"}); + weights->store = std::make_shared( + backend, backend_type, "sopro_tts.semantic_encoder.weights", weight_context_bytes); + auto & store = *weights->store; + weights->conv1 = binding::conv1d_from_source( + store, source, "conv1", conv_storage_type, config.d_model, config.n_mels, 3, true); + weights->conv2 = binding::conv1d_from_source( + store, source, "conv2", conv_storage_type, config.d_model, config.d_model, 3, true); + weights->pos_emb = store.load_f32_tensor( + source, "pos_emb", {config.max_positions, config.d_model}); + weights->layers.reserve(static_cast(config.layers)); + for (int64_t layer = 0; layer < config.layers; ++layer) { + const std::string prefix = "layers." + std::to_string(layer); + SoproSemanticLayerWeights out; + out.attn_norm = binding::norm_from_source( + store, source, prefix + ".self_attn_layer_norm", config.d_model); + out.q_proj = binding::linear_from_source( + store, source, prefix + ".self_attn.q_proj", matmul_storage_type, + config.d_model, config.d_model, true); + out.k_proj = binding::linear_from_source( + store, source, prefix + ".self_attn.k_proj", matmul_storage_type, + config.d_model, config.d_model, false); + out.v_proj = binding::linear_from_source( + store, source, prefix + ".self_attn.v_proj", matmul_storage_type, + config.d_model, config.d_model, true); + out.out_proj = binding::linear_from_source( + store, source, prefix + ".self_attn.out_proj", matmul_storage_type, + config.d_model, config.d_model, true); + out.ffn_norm = binding::norm_from_source( + store, source, prefix + ".final_layer_norm", config.d_model); + out.fc1 = binding::linear_from_source( + store, source, prefix + ".fc1", matmul_storage_type, config.ffn_dim, config.d_model, true); + out.fc2 = binding::linear_from_source( + store, source, prefix + ".fc2", matmul_storage_type, config.d_model, config.ffn_dim, true); + weights->layers.push_back(std::move(out)); + } + weights->final_norm = binding::norm_from_source(store, source, "final_norm", config.d_model); + weights->pre_head_norm_weight = source.require_f32("pre_head_norm.weight", {config.d_model}); + weights->pre_head_norm_bias = source.require_f32("pre_head_norm.bias", {config.d_model}); + weights->digit_head_weight = source.require_f32( + "digit_head.weight", {config.digit_dim(), config.d_model}); + weights->digit_head_bias = source.require_f32("digit_head.bias", {config.digit_dim()}); + weights->analysis_window = source.require_f32( + "frontend.mel.spectrogram.window", {config.n_fft}); + weights->mel_filterbank = source.require_f32( + "frontend.mel.mel_scale.fb", {config.n_fft / 2 + 1, config.n_mels}); + store.upload(); + return weights; +} + +} // namespace + +struct SoproSemanticEncoderGraph { + SoproSemanticEncoderGraph( + ggml_backend_t backend_in, + engine::core::BackendType backend_type, + size_t graph_context_bytes, + const SoproSemanticEncoderConfig & config, + std::shared_ptr weights_in, + int64_t mel_frames_in, + int64_t keep_steps) + : backend(backend_in), + weights(std::move(weights_in)), + mel_frames(mel_frames_in), + steps(keep_steps), + d_model(config.d_model) { + if (backend == nullptr || weights == nullptr) { + throw std::runtime_error("Sopro semantic encoder graph requires a backend and weights"); + } + if (mel_frames <= 0 || steps <= 0) { + throw std::runtime_error("Sopro semantic encoder graph requires positive lengths"); + } + ggml_init_params params{graph_context_bytes, nullptr, true}; + ctx.reset(ggml_init(params)); + if (ctx == nullptr) { + throw std::runtime_error("failed to initialize the Sopro semantic encoder graph context"); + } + engine::core::ModuleBuildContext build_ctx{ctx.get(), "sopro_tts.semantic_encoder", backend_type}; + namespace mod = engine::modules; + const auto shape = engine::core::TensorShape::from_dims({1, config.n_mels, mel_frames}); + input = engine::core::make_tensor(build_ctx, GGML_TYPE_F32, shape).tensor; + ggml_set_input(input); + + auto hidden = mod::Conv1dModule({config.n_mels, d_model, 3, 1, 1, 1, true}) + .build(build_ctx, engine::core::wrap_tensor(input, shape, GGML_TYPE_F32), + weights->conv1); + hidden = mod::GeluModule({mod::GeluApproximation::ExactErf}).build(build_ctx, hidden); + hidden = mod::Conv1dModule({d_model, d_model, 3, 2, 1, 1, true}) + .build(build_ctx, hidden, weights->conv2); + hidden = mod::GeluModule({mod::GeluApproximation::ExactErf}).build(build_ctx, hidden); + // [1, C, T] -> [1, T, C] + hidden = mod::TransposeModule({{0, 2, 1, 3}, 3}).build(build_ctx, hidden); + const int64_t conv_steps = hidden.shape.dims[1]; + if (conv_steps < steps) { + throw std::runtime_error("Sopro semantic encoder produced fewer frames than expected"); + } + { + auto positions = mod::SliceModule({0, 0, conv_steps}).build(build_ctx, weights->pos_emb); + positions = engine::core::reshape_tensor( + build_ctx, engine::core::ensure_backend_addressable_layout(build_ctx, positions), + engine::core::TensorShape::from_dims({1, conv_steps, d_model})); + hidden = mod::AddModule{}.build(build_ctx, hidden, positions); + } + hidden = mod::SliceModule({1, 0, steps}).build(build_ctx, hidden); + hidden = dense(build_ctx, hidden); + + const int64_t heads = config.heads; + const int64_t head_dim = config.head_dim(); + for (const auto & layer : weights->layers) { + auto norm = mod::LayerNormModule({d_model, kLayerNormEps, true, true}) + .build(build_ctx, hidden, layer.attn_norm); + auto q = mod::LinearModule({d_model, d_model, true, GGML_PREC_F32}) + .build(build_ctx, norm, layer.q_proj); + auto k = mod::LinearModule({d_model, d_model, false, GGML_PREC_F32}) + .build(build_ctx, norm, layer.k_proj); + auto v = mod::LinearModule({d_model, d_model, true, GGML_PREC_F32}) + .build(build_ctx, norm, layer.v_proj); + const auto head_shape = engine::core::TensorShape::from_dims({1, steps, heads, head_dim}); + auto to_heads = [&](const engine::core::TensorValue & value) { + auto reshaped = engine::core::reshape_tensor( + build_ctx, engine::core::ensure_backend_addressable_layout(build_ctx, value), + head_shape); + // Flash attention needs dense [1, H, T, DH] operands. + return dense(build_ctx, mod::TransposeModule({{0, 2, 1, 3}, 4}).build(build_ctx, reshaped)); + }; + auto attn = mod::ScaledDotProductAttentionModule({ + head_dim, + mod::ScaledDotProductAttentionLowering::Flash, + GGML_PREC_F32, + mod::AttentionCausality::NonCausal, + }).build(build_ctx, to_heads(q), to_heads(k), to_heads(v)); + auto flat = engine::core::reshape_tensor( + build_ctx, engine::core::ensure_backend_addressable_layout(build_ctx, attn), + engine::core::TensorShape::from_dims({1, steps, d_model})); + auto projected = mod::LinearModule({d_model, d_model, true, GGML_PREC_F32}) + .build(build_ctx, flat, layer.out_proj); + hidden = mod::AddModule{}.build(build_ctx, hidden, projected); + + auto ffn = mod::LayerNormModule({d_model, kLayerNormEps, true, true}) + .build(build_ctx, hidden, layer.ffn_norm); + ffn = mod::LinearModule({d_model, config.ffn_dim, true, GGML_PREC_F32}) + .build(build_ctx, ffn, layer.fc1); + ffn = mod::GeluModule({mod::GeluApproximation::ExactErf}).build(build_ctx, ffn); + ffn = mod::LinearModule({config.ffn_dim, d_model, true, GGML_PREC_F32}) + .build(build_ctx, ffn, layer.fc2); + hidden = mod::AddModule{}.build(build_ctx, hidden, ffn); + } + hidden = mod::LayerNormModule({d_model, kLayerNormEps, true, true}) + .build(build_ctx, hidden, weights->final_norm); + hidden = engine::core::ensure_backend_addressable_layout(build_ctx, hidden); + output = hidden.tensor; + ggml_set_output(output); + graph = ggml_new_graph_custom(ctx.get(), 65536, false); + ggml_build_forward_expand(graph, output); + gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (gallocr == nullptr || !ggml_gallocr_reserve(gallocr, graph) || + !ggml_gallocr_alloc_graph(gallocr, graph)) { + throw std::runtime_error("failed to allocate the Sopro semantic encoder graph"); + } + } + + ~SoproSemanticEncoderGraph() { + if (gallocr != nullptr) { + ggml_gallocr_free(gallocr); + gallocr = nullptr; + } + } + + bool matches(const SoproSemanticEncoderWeights & other, int64_t frames, int64_t keep) const noexcept { + return weights.get() == &other && mel_frames == frames && steps == keep; + } + + std::vector run(const std::vector & log_mel) { + ggml_backend_tensor_set(input, log_mel.data(), 0, log_mel.size() * sizeof(float)); + const ggml_status status = engine::core::compute_backend_graph(backend, graph); + ggml_backend_synchronize(backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Sopro semantic encoder graph compute failed"); + } + std::vector out(static_cast(steps * d_model), 0.0F); + ggml_backend_tensor_get(output, out.data(), 0, out.size() * sizeof(float)); + return out; + } + + ggml_backend_t backend = nullptr; + std::shared_ptr weights; + int64_t mel_frames = 0; + int64_t steps = 0; + int64_t d_model = 0; + std::unique_ptr ctx; + ggml_tensor * input = nullptr; + ggml_tensor * output = nullptr; + ggml_cgraph * graph = nullptr; + ggml_gallocr_t gallocr = nullptr; +}; + +SoproSemanticEncoderRuntime::SoproSemanticEncoderRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) + : config_(assets.config.semantic_encoder), + source_sample_rate_(assets.config.sample_rate), + execution_context_(execution_context), + graph_context_bytes_(graph_context_bytes), + weights_(load_weights( + execution_context.backend(), + execution_context.backend_type(), + *assets.semantic_encoder_weights, + assets.config.semantic_encoder, + weight_context_bytes, + matmul_storage_type, + conv_storage_type)) {} + +SoproSemanticEncoderRuntime::~SoproSemanticEncoderRuntime() = default; + +std::vector SoproSemanticEncoderRuntime::encode(const std::vector & audio24) const { + if (audio24.empty()) { + throw std::runtime_error("Sopro semantic encoder requires a non-empty reference waveform"); + } + const auto n24 = static_cast(audio24.size()); + const int64_t token_samples = config_.token_samples_24k; + const int64_t n_tokens = (n24 + token_samples - 1) / token_samples; + + if (source_sample_rate_ <= 0 || config_.sample_rate <= 0) { + throw std::runtime_error("Sopro semantic encoder sample rates must be positive"); + } + auto audio16 = engine::audio::resample_mono_torchaudio_sinc_hann( + audio24, static_cast(source_sample_rate_), static_cast(config_.sample_rate)); + // SemanticEncoder.encode pins the resampled length so the token grid is + // exactly reproducible regardless of the resampler's tail behaviour. + const int64_t n16 = + (n24 * config_.sample_rate + source_sample_rate_ - 1) / source_sample_rate_; + audio16.resize(static_cast(n16), 0.0F); + + dump("sem_wav16.f32", audio16); + const int64_t frames = (n16 + config_.hop_length - 1) / config_.hop_length; + const int64_t mel_frames = frames + kConvRightContextFrames; + // WhisperMelFrontend right-pads n_fft zeros before the centred STFT. + audio16.resize(static_cast(n16 + config_.n_fft), 0.0F); + + const int64_t freq_bins = config_.n_fft / 2 + 1; + engine::audio::STFTConfig stft; + stft.n_fft = config_.n_fft; + stft.hop_length = config_.hop_length; + stft.win_length = config_.n_fft; + stft.center = true; + stft.pad_mode = engine::audio::STFTPadMode::Reflect; + const auto magnitude = engine::audio::STFT{}.compute_magnitude( + audio16, weights_->analysis_window, 1, static_cast(audio16.size()), stft, + static_cast(execution_context_.config().threads)); + if (magnitude.shape.size() != 3 || magnitude.shape[1] != freq_bins) { + throw std::runtime_error("Sopro semantic encoder STFT produced an unexpected layout"); + } + const int64_t stft_frames = magnitude.shape[2]; + if (stft_frames < mel_frames) { + throw std::runtime_error("Sopro semantic encoder reference audio is too short"); + } + + std::vector mel(static_cast(config_.n_mels * mel_frames), 0.0F); + for (int64_t f = 0; f < freq_bins; ++f) { + const float * fb_row = weights_->mel_filterbank.data() + static_cast(f * config_.n_mels); + const float * spec_row = magnitude.values.data() + static_cast(f * stft_frames); + for (int64_t m = 0; m < config_.n_mels; ++m) { + const float weight = fb_row[m]; + if (weight == 0.0F) { + continue; + } + float * out_row = mel.data() + static_cast(m * mel_frames); + for (int64_t t = 0; t < mel_frames; ++t) { + out_row[t] += weight * spec_row[t] * spec_row[t]; // power = 2.0 + } + } + } + float peak = -std::numeric_limits::infinity(); + for (auto & value : mel) { + value = std::log10(std::max(value, 1.0e-10F)); + peak = std::max(peak, value); + } + const float floor_value = peak - 8.0F; + for (auto & value : mel) { + value = (std::max(value, floor_value) + 4.0F) / 4.0F; + } + + dump("sem_mel.f32", mel); + const int64_t steps = (frames + 1) / 2; // n50 + if (steps <= 0) { + throw std::runtime_error("Sopro semantic encoder reference audio is too short"); + } + if (steps > config_.max_positions) { + throw std::runtime_error( + "Sopro semantic encoder reference audio exceeds the positional embedding table"); + } + if (graph_ == nullptr || !graph_->matches(*weights_, mel_frames, steps)) { + // Free the previous arena first; otherwise both are resident while the + // replacement is allocated, and every segment rebuilds this graph. + graph_.reset(); + graph_ = std::make_unique( + execution_context_.backend(), + execution_context_.backend_type(), + graph_context_bytes_, + config_, + weights_, + mel_frames, + steps); + } + const auto hidden = graph_->run(mel); + dump("sem_hidden.f32", hidden); + + // SemanticEncoder._interpolate: half-pixel aligned linear resampling from + // `steps` encoder frames onto the `n_tokens` output grid. + const int64_t d_model = config_.d_model; + const int64_t digit_dim = config_.digit_dim(); + std::vector tokens(static_cast(n_tokens), 0); + std::vector frame(static_cast(d_model), 0.0F); + std::vector logits(static_cast(digit_dim), 0.0F); + const float ratio = static_cast(steps) / static_cast(n_tokens); + for (int64_t t = 0; t < n_tokens; ++t) { + float source = (static_cast(t) + 0.5F) * ratio - 0.5F; + source = std::min(std::max(source, 0.0F), static_cast(steps - 1)); + const auto left = static_cast(std::floor(source)); + const int64_t right = std::min(left + 1, steps - 1); + const float weight = source - static_cast(left); + const float * left_row = hidden.data() + static_cast(left * d_model); + const float * right_row = hidden.data() + static_cast(right * d_model); + for (int64_t c = 0; c < d_model; ++c) { + frame[static_cast(c)] = left_row[c] * (1.0F - weight) + right_row[c] * weight; + } + // pre_head_norm + double sum = 0.0; + for (int64_t c = 0; c < d_model; ++c) { + sum += frame[static_cast(c)]; + } + const double mean = sum / static_cast(d_model); + double variance = 0.0; + for (int64_t c = 0; c < d_model; ++c) { + const double centred = frame[static_cast(c)] - mean; + variance += centred * centred; + } + variance /= static_cast(d_model); + const double inv_std = 1.0 / std::sqrt(variance + kLayerNormEps); + for (int64_t c = 0; c < d_model; ++c) { + frame[static_cast(c)] = static_cast( + (frame[static_cast(c)] - mean) * inv_std * + weights_->pre_head_norm_weight[static_cast(c)] + + weights_->pre_head_norm_bias[static_cast(c)]); + } + for (int64_t d = 0; d < digit_dim; ++d) { + double value = weights_->digit_head_bias[static_cast(d)]; + const float * row = weights_->digit_head_weight.data() + static_cast(d * d_model); + for (int64_t c = 0; c < d_model; ++c) { + value += static_cast(row[c]) * static_cast(frame[static_cast(c)]); + } + logits[static_cast(d)] = static_cast(value); + } + // Finite scalar quantiser: per-level arg-max, packed with mixed radix. + int64_t token = 0; + int64_t base = 1; + int64_t offset = 0; + for (const int64_t level : config_.fsq_levels) { + int64_t best = 0; + float best_value = logits[static_cast(offset)]; + for (int64_t i = 1; i < level; ++i) { + const float value = logits[static_cast(offset + i)]; + if (value > best_value) { + best_value = value; + best = i; + } + } + token += best * base; + base *= level; + offset += level; + } + tokens[static_cast(t)] = static_cast(token); + } + dump("sem_tokens.i32", tokens); + return tokens; +} + +} // namespace engine::community_models::sopro_tts diff --git a/src/community_models/sopro_tts/semantic_lm.cpp b/src/community_models/sopro_tts/semantic_lm.cpp new file mode 100644 index 000000000..940e9012e --- /dev/null +++ b/src/community_models/sopro_tts/semantic_lm.cpp @@ -0,0 +1,597 @@ +#include "engine/community_models/sopro_tts/semantic_lm.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/transformers/qwen_causal_decode_runtime.h" +#include "engine/framework/modules/transformers/qwen_causal_decoder.h" +#include "engine/framework/modules/weight_binding.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { +namespace { + +namespace binding = engine::modules::binding; + +constexpr float kRmsNormEps = 1.0e-6F; // sopro.nn.layers.RMSNorm default +constexpr float kMaskedLogit = -1.0e9F; // sampling.sample_next_token + +// StylePrefixEncoder runs on the host: eight learned queries cross-attend to at +// most a few hundred reference frames, which is under 0.1 GFLOP. +struct SoproStylePrefixWeights { + std::vector queries; // [tokens, dim] + std::vector kv_norm; // [dim] + std::vector q_proj; // [dim, dim] + std::vector k_proj; + std::vector v_proj; + std::vector out_proj; + std::vector out_norm; // [dim] +}; + +struct SoproSemanticLMHostWeights { + std::vector text_embedding; // [text_vocab, dim] + std::vector semantic_embedding; // [semantic_vocab + 2, dim], fused + SoproStylePrefixWeights style_prefix; +}; + +struct SoproSemanticLMBackendWeights { + std::shared_ptr store; + engine::core::TensorValue token_embedding; + engine::modules::QwenDecoderStackWeights stack; + engine::modules::NormWeights final_norm; + engine::modules::LinearWeights token_head; +}; + +void rms_norm(std::vector & values, const std::vector & weight) { + const size_t dim = weight.size(); + double sum = 0.0; + for (size_t i = 0; i < dim; ++i) { + sum += static_cast(values[i]) * static_cast(values[i]); + } + const double inv = 1.0 / std::sqrt(sum / static_cast(dim) + kRmsNormEps); + for (size_t i = 0; i < dim; ++i) { + values[i] = static_cast(values[i] * inv * weight[i]); + } +} + +// out[row] = weight @ in[row]; weight is [out_dim, in_dim] row-major. +void matmul_rows( + const std::vector & weight, + const float * input, + int64_t rows, + int64_t in_dim, + int64_t out_dim, + float * output) { +#ifdef _OPENMP +#pragma omp parallel for if (rows > 8) +#endif + for (int64_t row = 0; row < rows; ++row) { + const float * source = input + row * in_dim; + float * target = output + row * out_dim; + for (int64_t o = 0; o < out_dim; ++o) { + const float * w = weight.data() + static_cast(o * in_dim); + double sum = 0.0; + for (int64_t i = 0; i < in_dim; ++i) { + sum += static_cast(w[i]) * static_cast(source[i]); + } + target[o] = static_cast(sum); + } + } +} + +// Fuse LayerScale into the projection that feeds the residual: the branch is +// scale * W @ y with no bias, so scaling the rows of W is exact. +std::vector scale_rows(std::vector weight, const std::vector & scale, int64_t in_dim) { + for (size_t row = 0; row < scale.size(); ++row) { + float * values = weight.data() + static_cast(static_cast(row) * in_dim); + const float factor = scale[row]; + for (int64_t i = 0; i < in_dim; ++i) { + values[i] *= factor; + } + } + return weight; +} + +engine::modules::QwenCausalDecoderConfig make_decoder_config( + const SoproModelConfig & config, + engine::core::BackendType backend_type) { + engine::modules::QwenCausalDecoderConfig out; + out.stack.hidden_size = config.ar_model_dim; + out.stack.num_attention_heads = config.ar_heads; + out.stack.num_key_value_heads = config.ar_kv_heads; + out.stack.head_dim = config.ar_head_dim(); + out.stack.intermediate_size = config.ar_ffn_dim(); + out.stack.layers = config.ar_blocks; + out.stack.rms_norm_eps = kRmsNormEps; + out.stack.rope_theta = 10000.0F; + // sopro.nn.layers.rotate_half splits the head in halves, which is ggml's + // NEOX rotary layout. + out.stack.rope_type = GGML_ROPE_TYPE_NEOX; + out.stack.attention_precision = GGML_PREC_DEFAULT; + out.stack.projection_precision = GGML_PREC_DEFAULT; + out.stack.use_qk_norm = config.ar_qk_rms_norm; + out.stack.qkv_layout = engine::modules::QwenDecoderQKVLayout::PackedQKV; + out.stack.runtime.mlp.mode = engine::modules::QwenDecoderMLPMode::PackedGateUp; + out.stack.runtime.attention.prefill_mode = engine::modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.attention.static_mode = engine::modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.static_cache.update_mode = engine::modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows; + out.logits_size = config.semantic_vocab_size + 2; + out.logits_mode = engine::modules::QwenCausalDecoderLogitsMode::LastStep; + out.use_lm_head_bias = true; // SemanticLM.token_head is a biased Linear + out.lm_head_precision = GGML_PREC_DEFAULT; + (void) backend_type; + return out; +} + +engine::modules::QwenDecoderLayerWeights load_layer( + engine::core::BackendWeightStore & store, + const engine::assets::TensorSource & source, + const SoproModelConfig & config, + engine::assets::TensorStorageType storage_type, + int64_t layer) { + const std::string prefix = "ar_prior.temporal.layers." + std::to_string(layer); + const int64_t dim = config.ar_model_dim; + const int64_t head_dim = config.ar_head_dim(); + const int64_t q_out = config.ar_heads * head_dim; + const int64_t kv_out = config.ar_kv_heads * head_dim; + const int64_t ffn = config.ar_ffn_dim(); + + engine::modules::QwenDecoderLayerWeights out; + out.input_norm = binding::norm_weight_from_source(store, source, prefix + ".attn_norm", dim); + auto qkv = source.require_f32(prefix + ".attn.q_proj.weight", {q_out, dim}); + const auto k_rows = source.require_f32(prefix + ".attn.k_proj.weight", {kv_out, dim}); + const auto v_rows = source.require_f32(prefix + ".attn.v_proj.weight", {kv_out, dim}); + qkv.insert(qkv.end(), k_rows.begin(), k_rows.end()); + qkv.insert(qkv.end(), v_rows.begin(), v_rows.end()); + out.self_attention.qkv_weight = store.make_from_f32( + engine::core::TensorShape::from_dims({q_out + kv_out * 2, dim}), storage_type, std::move(qkv)); + out.self_attention.out_weight = store.make_from_f32( + engine::core::TensorShape::from_dims({dim, q_out}), + storage_type, + scale_rows( + source.require_f32(prefix + ".attn.out_proj.weight", {dim, q_out}), + source.require_f32(prefix + ".attn_scale.scale", {dim}), + q_out)); + if (config.ar_qk_rms_norm) { + out.q_norm = binding::norm_weight_from_source(store, source, prefix + ".attn.q_norm", head_dim); + out.k_norm = binding::norm_weight_from_source(store, source, prefix + ".attn.k_norm", head_dim); + } + out.post_norm = binding::norm_weight_from_source(store, source, prefix + ".ffn_norm", dim); + auto gate_up = source.require_f32(prefix + ".ffn.gate_proj.weight", {ffn, dim}); + const auto up_rows = source.require_f32(prefix + ".ffn.up_proj.weight", {ffn, dim}); + gate_up.insert(gate_up.end(), up_rows.begin(), up_rows.end()); + out.mlp.gate_up_proj = engine::modules::LinearWeights{ + store.make_from_f32( + engine::core::TensorShape::from_dims({ffn * 2, dim}), storage_type, std::move(gate_up)), + std::nullopt}; + out.mlp.down_proj = engine::modules::LinearWeights{ + store.make_from_f32( + engine::core::TensorShape::from_dims({dim, ffn}), + storage_type, + scale_rows( + source.require_f32(prefix + ".ffn.down_proj.weight", {dim, ffn}), + source.require_f32(prefix + ".ffn_scale.scale", {dim}), + ffn)), + std::nullopt}; + return out; +} + +SoproSemanticLMHostWeights load_host_weights( + const engine::assets::TensorSource & source, + const SoproModelConfig & config) { + SoproSemanticLMHostWeights out; + const int64_t dim = config.ar_model_dim; + const int64_t latent = config.latent_dim; + const int64_t semantic_rows = config.semantic_vocab_size + 2; + + out.text_embedding = source.require_f32("text_tok_emb.weight", {config.text_vocab_size, dim}); + + // embed_semantic = sem_in_proj(semantic_tok_emb(id)); fold the projection + // into the table so every later lookup is a plain row gather. + const auto table = source.require_f32("semantic_tok_emb.weight", {semantic_rows, latent}); + const auto projection = source.require_f32("sem_in_proj.weight", {dim, latent}); + const auto bias = source.require_f32("sem_in_proj.bias", {dim}); + out.semantic_embedding.assign(static_cast(semantic_rows * dim), 0.0F); +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int64_t row = 0; row < semantic_rows; ++row) { + const float * source_row = table.data() + static_cast(row * latent); + float * target = out.semantic_embedding.data() + static_cast(row * dim); + for (int64_t o = 0; o < dim; ++o) { + const float * w = projection.data() + static_cast(o * latent); + double sum = bias[static_cast(o)]; + for (int64_t i = 0; i < latent; ++i) { + sum += static_cast(w[i]) * static_cast(source_row[i]); + } + target[o] = static_cast(sum); + } + } + + auto & style = out.style_prefix; + style.queries = source.require_f32("style_prefix.queries", {config.style_prefix_tokens, dim}); + style.kv_norm = source.require_f32("style_prefix.kv_norm.weight", {dim}); + style.q_proj = source.require_f32("style_prefix.q_proj.weight", {dim, dim}); + style.k_proj = source.require_f32("style_prefix.k_proj.weight", {dim, dim}); + style.v_proj = source.require_f32("style_prefix.v_proj.weight", {dim, dim}); + style.out_proj = source.require_f32("style_prefix.out_proj.weight", {dim, dim}); + style.out_norm = source.require_f32("style_prefix.out_norm.weight", {dim}); + return out; +} + +// StylePrefixEncoder.forward for a single sequence. +std::vector build_style_prefix( + const SoproStylePrefixWeights & weights, + const std::vector & reference, // [steps, dim] + int64_t steps, + int64_t tokens, + int64_t dim, + int64_t heads) { + const int64_t head_dim = dim / heads; + std::vector out(static_cast(tokens * dim), 0.0F); + if (steps == 0) { + // The reference path returns out_norm(queries) when there is nothing + // to attend to. + for (int64_t t = 0; t < tokens; ++t) { + std::vector row( + weights.queries.begin() + static_cast(t * dim), + weights.queries.begin() + static_cast((t + 1) * dim)); + rms_norm(row, weights.out_norm); + std::copy(row.begin(), row.end(), out.begin() + static_cast(t * dim)); + } + return out; + } + + std::vector kv(static_cast(steps * dim), 0.0F); + for (int64_t s = 0; s < steps; ++s) { + std::vector row( + reference.begin() + static_cast(s * dim), + reference.begin() + static_cast((s + 1) * dim)); + rms_norm(row, weights.kv_norm); + std::copy(row.begin(), row.end(), kv.begin() + static_cast(s * dim)); + } + + std::vector q(static_cast(tokens * dim), 0.0F); + std::vector k(static_cast(steps * dim), 0.0F); + std::vector v(static_cast(steps * dim), 0.0F); + matmul_rows(weights.q_proj, weights.queries.data(), tokens, dim, dim, q.data()); + matmul_rows(weights.k_proj, kv.data(), steps, dim, dim, k.data()); + matmul_rows(weights.v_proj, kv.data(), steps, dim, dim, v.data()); + + const auto scale = static_cast(1.0 / std::sqrt(static_cast(head_dim))); + std::vector context(static_cast(tokens * dim), 0.0F); + std::vector scores(static_cast(steps), 0.0F); + for (int64_t t = 0; t < tokens; ++t) { + for (int64_t h = 0; h < heads; ++h) { + const float * q_head = q.data() + static_cast(t * dim + h * head_dim); + float max_score = -std::numeric_limits::infinity(); + for (int64_t s = 0; s < steps; ++s) { + const float * k_head = k.data() + static_cast(s * dim + h * head_dim); + double sum = 0.0; + for (int64_t d = 0; d < head_dim; ++d) { + sum += static_cast(q_head[d]) * static_cast(k_head[d]); + } + const auto value = static_cast(sum) * scale; + scores[static_cast(s)] = value; + max_score = std::max(max_score, value); + } + double total = 0.0; + for (auto & score : scores) { + score = std::exp(score - max_score); + total += score; + } + float * target = context.data() + static_cast(t * dim + h * head_dim); + for (int64_t s = 0; s < steps; ++s) { + const float weight = static_cast(scores[static_cast(s)] / total); + const float * v_head = v.data() + static_cast(s * dim + h * head_dim); + for (int64_t d = 0; d < head_dim; ++d) { + target[d] += weight * v_head[d]; + } + } + } + } + + std::vector projected(static_cast(tokens * dim), 0.0F); + matmul_rows(weights.out_proj, context.data(), tokens, dim, dim, projected.data()); + for (int64_t t = 0; t < tokens; ++t) { + std::vector row(static_cast(dim), 0.0F); + for (int64_t d = 0; d < dim; ++d) { + row[static_cast(d)] = + weights.queries[static_cast(t * dim + d)] + + projected[static_cast(t * dim + d)]; + } + rms_norm(row, weights.out_norm); + std::copy(row.begin(), row.end(), out.begin() + static_cast(t * dim)); + } + return out; +} + +} // namespace + +int32_t sample_next_token( + std::vector & logits, + float temperature, + float top_p, + int64_t top_k, + int32_t bos_id, + int32_t eos_id, + bool allow_eos, + std::mt19937_64 & rng) { + const auto vocab = static_cast(logits.size()); + if (vocab <= 0) { + throw std::runtime_error("Sopro semantic LM produced empty logits"); + } + if (bos_id < 0 || bos_id >= vocab || eos_id < 0 || eos_id >= vocab) { + throw std::runtime_error("Sopro semantic LM bos/eos id is outside the logit range"); + } + logits[static_cast(bos_id)] = kMaskedLogit; + if (!allow_eos) { + logits[static_cast(eos_id)] = kMaskedLogit; + } + if (temperature <= 0.0F) { + return static_cast( + std::distance(logits.begin(), std::max_element(logits.begin(), logits.end()))); + } + const float inv_temperature = 1.0F / std::max(1.0e-5F, temperature); + float max_logit = -std::numeric_limits::infinity(); + for (auto & value : logits) { + value *= inv_temperature; + max_logit = std::max(max_logit, value); + } + std::vector probs(logits.size(), 0.0F); + double total = 0.0; + for (size_t i = 0; i < logits.size(); ++i) { + probs[i] = std::exp(logits[i] - max_logit); + total += probs[i]; + } + for (auto & value : probs) { + value = static_cast(value / total); + } + + if (top_k > 0 && top_k < vocab) { + std::vector sorted(probs); + std::nth_element( + sorted.begin(), sorted.begin() + static_cast(top_k - 1), sorted.end(), + std::greater()); + const float kth = sorted[static_cast(top_k - 1)]; + double sum = 0.0; + for (auto & value : probs) { + if (value < kth) { + value = 0.0F; + } + sum += value; + } + const auto inv = static_cast(1.0 / std::max(sum, 1.0e-8)); + for (auto & value : probs) { + value *= inv; + } + } + + if (top_p < 1.0F) { + const float threshold = std::min(std::max(top_p, 0.0F), 1.0F); + std::vector order(static_cast(vocab)); + std::iota(order.begin(), order.end(), 0); + std::stable_sort(order.begin(), order.end(), [&](int32_t a, int32_t b) { + return probs[static_cast(a)] > probs[static_cast(b)]; + }); + // remove[i] = cdf[i - 1] > p: the first token above the threshold is + // kept, everything past it is dropped. + std::vector nucleus(probs.size(), 0.0F); + double cumulative = 0.0; + double sum = 0.0; + for (size_t rank = 0; rank < order.size(); ++rank) { + const auto index = static_cast(order[rank]); + if (rank == 0 || cumulative <= threshold) { + nucleus[index] = probs[index]; + sum += probs[index]; + } + cumulative += probs[index]; + } + const auto inv = static_cast(1.0 / std::max(sum, 1.0e-8)); + for (size_t i = 0; i < probs.size(); ++i) { + probs[i] = nucleus[i] * inv; + } + } + + std::uniform_real_distribution uniform(0.0, 1.0); + const double draw = uniform(rng); + double cumulative = 0.0; + for (size_t i = 0; i < probs.size(); ++i) { + cumulative += probs[i]; + if (draw < cumulative) { + return static_cast(i); + } + } + for (int64_t i = vocab - 1; i >= 0; --i) { + if (probs[static_cast(i)] > 0.0F) { + return static_cast(i); + } + } + throw std::runtime_error("Sopro semantic LM sampling found no candidate token"); +} + +class SoproSemanticLMRuntime::Impl { +public: + Impl( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + engine::assets::TensorStorageType weight_storage_type) + : config_(assets.config.model) { + const auto & source = *assets.model_weights; + host_ = load_host_weights(source, config_); + + backend_.store = std::make_shared( + execution.backend(), execution.backend_type(), "sopro_tts.semantic_lm.weights", + weight_context_bytes); + auto & store = *backend_.store; + backend_.token_embedding = store.load_tensor( + source, "text_tok_emb.weight", weight_storage_type, + {config_.text_vocab_size, config_.ar_model_dim}); + backend_.stack.layers.reserve(static_cast(config_.ar_blocks)); + for (int64_t layer = 0; layer < config_.ar_blocks; ++layer) { + backend_.stack.layers.push_back( + load_layer(store, source, config_, weight_storage_type, layer)); + } + backend_.final_norm = binding::norm_weight_from_source( + store, source, "ar_prior.out_norm", config_.ar_model_dim); + backend_.token_head = binding::linear_from_source( + store, source, "ar_prior.token_head", weight_storage_type, + config_.semantic_vocab_size + 2, config_.ar_model_dim, true); + store.upload(); + + engine::modules::QwenCausalDecodeRuntimeConfig runtime_config; + runtime_config.trace_name = "sopro_tts.semantic_lm"; + runtime_config.decoder = make_decoder_config(config_, execution.backend_type()); + runtime_config.prefill_graph_arena_bytes = prefill_graph_arena_bytes; + runtime_config.decode_graph_arena_bytes = decode_graph_arena_bytes; + runtime_config.output_mode = engine::modules::QwenCausalDecodeOutputMode::Logits; + runtime_config.return_hidden = false; + + engine::modules::QwenCausalDecodeRuntimeWeights runtime_weights; + runtime_weights.token_embedding = backend_.token_embedding; + runtime_weights.stack = backend_.stack; + runtime_weights.final_norm = backend_.final_norm; + runtime_weights.lm_head = backend_.token_head; + decoder_ = std::make_unique( + execution, std::move(runtime_config), std::move(runtime_weights)); + } + + std::vector generate( + const std::vector & text_ids, + const std::vector & style_tokens, + const std::vector & prompt_tokens, + const SoproSemanticLMOptions & options, + std::mt19937_64 & rng) { + const int64_t dim = config_.ar_model_dim; + const auto text_steps = std::min( + static_cast(text_ids.size()), config_.max_text_len); + const int64_t style_steps = config_.style_prefix_tokens; + const auto prompt_steps = static_cast(prompt_tokens.size()); + const int64_t steps = style_steps + text_steps + prompt_steps + 1; + + std::vector prefix(static_cast(steps * dim), 0.0F); + { + std::vector style_reference( + static_cast(static_cast(style_tokens.size()) * dim), 0.0F); + for (size_t i = 0; i < style_tokens.size(); ++i) { + copy_semantic_row(style_tokens[i], style_reference.data() + i * static_cast(dim)); + } + const auto style = build_style_prefix( + host_.style_prefix, style_reference, static_cast(style_tokens.size()), + style_steps, dim, config_.ar_heads); + std::copy(style.begin(), style.end(), prefix.begin()); + } + int64_t offset = style_steps; + for (int64_t i = 0; i < text_steps; ++i) { + const int32_t id = text_ids[static_cast(i)]; + if (id < 0 || id >= config_.text_vocab_size) { + throw std::runtime_error("Sopro text token id is out of range"); + } + const float * row = host_.text_embedding.data() + static_cast(id * dim); + std::copy(row, row + dim, prefix.begin() + static_cast((offset + i) * dim)); + } + offset += text_steps; + for (int64_t i = 0; i < prompt_steps; ++i) { + copy_semantic_row( + prompt_tokens[static_cast(i)], + prefix.data() + static_cast((offset + i) * dim)); + } + offset += prompt_steps; + copy_semantic_row( + static_cast(config_.semantic_bos_id()), + prefix.data() + static_cast(offset * dim)); + + const int64_t max_steps = std::max(1, options.max_steps); + auto prefill = decoder_->prefill_embeddings(prefix, steps); + decoder_->start_decode_embeddings(prefill.state, steps + max_steps); + + const auto bos_id = static_cast(config_.semantic_bos_id()); + const auto eos_id = static_cast(config_.semantic_eos_id()); + const int64_t min_steps = std::max(1, options.min_steps); + std::vector tokens; + tokens.reserve(static_cast(max_steps)); + std::vector logits = std::move(prefill.logits); + std::vector embedding(static_cast(dim), 0.0F); + for (int64_t step = 0; step < max_steps; ++step) { + const bool allow_eos = (step + 1) >= min_steps; + int32_t token = sample_next_token( + logits, options.temperature, options.top_p, options.top_k, + bos_id, eos_id, allow_eos, rng); + if (allow_eos && token == eos_id) { + break; + } + token = std::min( + std::max(token, 0), static_cast(config_.semantic_vocab_size - 1)); + tokens.push_back(token); + if (step + 1 >= max_steps) { + break; + } + copy_semantic_row(token, embedding.data()); + logits = std::move(decoder_->decode_embedding(embedding).logits); + } + return tokens; + } + + void release_runtime_graphs() { + decoder_->release_runtime_graphs(); + } + +private: + void copy_semantic_row(int32_t id, float * target) const { + if (id < 0 || id >= config_.semantic_vocab_size + 2) { + throw std::runtime_error("Sopro semantic token id is out of range"); + } + const float * row = + host_.semantic_embedding.data() + static_cast(id * config_.ar_model_dim); + std::copy(row, row + config_.ar_model_dim, target); + } + + const SoproModelConfig & config_; + SoproSemanticLMHostWeights host_; + SoproSemanticLMBackendWeights backend_; + std::unique_ptr decoder_; +}; + +SoproSemanticLMRuntime::SoproSemanticLMRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + engine::assets::TensorStorageType weight_storage_type) + : impl_(std::make_unique( + assets, execution_context, prefill_graph_arena_bytes, decode_graph_arena_bytes, + weight_context_bytes, weight_storage_type)) {} + +SoproSemanticLMRuntime::~SoproSemanticLMRuntime() = default; + +std::vector SoproSemanticLMRuntime::generate( + const std::vector & text_ids, + const std::vector & style_tokens, + const std::vector & prompt_tokens, + const SoproSemanticLMOptions & options, + std::mt19937_64 & rng) const { + return impl_->generate(text_ids, style_tokens, prompt_tokens, options, rng); +} + +void SoproSemanticLMRuntime::release_runtime_graphs() { + impl_->release_runtime_graphs(); +} + +} // namespace engine::community_models::sopro_tts diff --git a/src/community_models/sopro_tts/session.cpp b/src/community_models/sopro_tts/session.cpp new file mode 100644 index 000000000..1a9921196 --- /dev/null +++ b/src/community_models/sopro_tts/session.cpp @@ -0,0 +1,591 @@ +#include "engine/community_models/sopro_tts/session.h" + +#include "engine/community_models/sopro_tts/acoustic.h" +#include "engine/community_models/sopro_tts/reference.h" +#include "engine/community_models/sopro_tts/semantic_encoder.h" +#include "engine/community_models/sopro_tts/semantic_lm.h" +#include "engine/community_models/sopro_tts/speaker_encoder.h" +#include "engine/community_models/sopro_tts/text_tokenizer.h" +#include "engine/community_models/sopro_tts/vocoder.h" + +#include "engine/framework/audio/conversion.h" +#include "engine/framework/audio/resampling.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/debug/trace.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { +namespace { + +constexpr const char * kFamily = "sopro_tts"; +constexpr size_t kWeightContextBytes = 512ull * 1024ull * 1024ull; +constexpr size_t kGraphArenaBytes = 1024ull * 1024ull * 1024ull; +// SoproTTS.DECODE_CONTEXT_FRAMES: mel frames of prompt fed to the vocoder so +// its convolutions start warm, then dropped from the output. +constexpr int64_t kDecodeContextFrames = 32; + +std::shared_ptr require_assets(std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("Sopro session requires assets"); + } + return assets; +} + +std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("Sopro session requires a model contract"); + } + return contract; +} + +const runtime::AudioBuffer * reference_audio(const runtime::TaskRequest & request) { + if (request.voice.has_value() && request.voice->speaker.has_value() && + request.voice->speaker->audio.has_value()) { + return &*request.voice->speaker->audio; + } + return request.audio_input.has_value() ? &*request.audio_input : nullptr; +} + +std::vector to_mono_24k(const runtime::AudioBuffer & audio, int target_rate) { + if (audio.samples.empty()) { + throw std::runtime_error("Sopro reference audio is empty"); + } + const int channels = std::max(1, audio.channels); + std::vector mono = channels == 1 + ? audio.samples + : engine::audio::mixdown_interleaved_to_mono_average(audio.samples, channels); + if (audio.sample_rate > 0 && audio.sample_rate != target_rate) { + mono = engine::audio::resample_mono_torchaudio_sinc_hann(mono, audio.sample_rate, target_rate); + } + // sopro.audio.to_mono_resampled clamps before anything else touches it. + for (auto & value : mono) { + value = std::min(1.0F, std::max(-1.0F, value)); + } + return mono; +} + +std::unique_ptr create_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique(task, options, std::move(assets), std::move(contract)); +} + +} // namespace + +SoproTTSSession::SoproTTSSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(options), + task_(task), + assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))) { + runtime::validate_spec_backed_session_options(options, *contract_, kFamily, "Sopro"); + if (const auto value = runtime::find_option(options.options, {"language"})) { + default_language_ = *value; + } + const auto matmul_storage = runtime::parse_tensor_storage_option( + options.options, + "matmul_weight_type", + assets::TensorStorageType::F32, + {assets::TensorStorageType::Native, + assets::TensorStorageType::F32, + assets::TensorStorageType::F16, + assets::TensorStorageType::BF16, + assets::TensorStorageType::Q8_0}); + const auto conv_storage = runtime::parse_tensor_storage_option( + options.options, + "conv_weight_type", + assets::TensorStorageType::F32, + {assets::TensorStorageType::Native, + assets::TensorStorageType::F32, + assets::TensorStorageType::F16}); + + core::ExecutionContext & execution = execution_context(); + tokenizer_ = std::make_unique( + assets_->tokenizer_path, assets_->config.model.max_text_len); + speaker_encoder_ = std::make_unique( + *assets_, execution, kWeightContextBytes, kGraphArenaBytes, matmul_storage, conv_storage); + semantic_encoder_ = std::make_unique( + *assets_, execution, kWeightContextBytes, kGraphArenaBytes, matmul_storage, conv_storage); + vocoder_ = std::make_unique( + *assets_, execution, kWeightContextBytes, kGraphArenaBytes, matmul_storage, conv_storage); + semantic_lm_ = std::make_unique( + *assets_, execution, kGraphArenaBytes, kGraphArenaBytes, kWeightContextBytes, matmul_storage); + acoustic_ = std::make_unique( + *assets_, execution, kWeightContextBytes, kGraphArenaBytes, matmul_storage, conv_storage); + reference_builder_ = std::make_unique( + *assets_, *speaker_encoder_, *semantic_encoder_, *vocoder_); +} + +SoproTTSSession::~SoproTTSSession() = default; + +std::string SoproTTSSession::family() const { + return kFamily; +} + +runtime::VoiceTaskKind SoproTTSSession::task_kind() const { + return runtime::VoiceTaskKind::Tts; +} + +runtime::RunMode SoproTTSSession::run_mode() const { + return task_.mode; +} + +void SoproTTSSession::prepare(const runtime::SessionPreparationRequest & request) { + runtime::validate_spec_backed_request_options(request.options, *contract_, "Sopro"); + mark_prepared(); +} + +SoproRequestOptions SoproTTSSession::parse_options(const runtime::TaskRequest & request) const { + const auto & defaults = assets_->config.generation; + SoproRequestOptions out; + out.language = default_language_; + out.temperature = defaults.temperature; + out.top_p = defaults.top_p; + out.top_k = defaults.top_k; + out.steps = defaults.steps; + out.max_seconds = defaults.max_seconds; + out.min_seconds = defaults.min_seconds; + out.max_segment_chars = defaults.max_segment_chars; + out.ref_seconds = defaults.ref_seconds; + + if (const auto value = runtime::find_option(request.options, {"language"})) { + out.language = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"temperature"})) { + out.temperature = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"top_p"})) { + out.top_p = *value; + } + if (const auto value = runtime::parse_i64_option(request.options, {"top_k"})) { + out.top_k = *value; + } + if (const auto value = runtime::parse_i64_option(request.options, {"num_inference_steps"})) { + out.steps = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"max_seconds"})) { + out.max_seconds = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"min_seconds"})) { + out.min_seconds = *value; + } + if (const auto value = runtime::parse_i64_option(request.options, {"text_chunk_size"})) { + out.max_segment_chars = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"ref_seconds"})) { + out.ref_seconds = *value; + } + if (const auto value = runtime::parse_u64_option(request.options, {"seed"})) { + out.seed = *value; + out.has_seed = true; + } + if (!out.has_seed) { + out.seed = runtime::random_u64_seed(); + } + if (out.steps < 1) { + throw std::runtime_error("Sopro num_inference_steps must be positive"); + } + if (out.max_segment_chars < 1) { + throw std::runtime_error("Sopro text_chunk_size must be positive"); + } + if (out.max_seconds <= 0.0F) { + throw std::runtime_error("Sopro max_seconds must be positive"); + } + if (out.ref_seconds <= 0.0F) { + throw std::runtime_error("Sopro ref_seconds must be positive"); + } + if (out.min_seconds < 0.0F) { + throw std::runtime_error("Sopro min_seconds must not be negative"); + } + // A min above max leaves min_steps > max_steps, which never lets the LM + // emit EOS: every segment would run the full budget and be cut mid-word. + if (out.min_seconds > out.max_seconds) { + throw std::runtime_error("Sopro min_seconds must not exceed max_seconds"); + } + // language_tag() rejects anything outside the four supported languages, so + // fail before any weights are touched. + (void) language_tag(out.language); + return out; +} + +// The per-run state that every text segment of one synthesis shares. Offline +// drains it in a loop; streaming keeps it alive between next_stream_event +// calls, so both paths draw from the seeded RNG in the same order and a given +// seed produces the same audio either way. +struct SoproSynthesisState { + SoproRequestOptions options; + SoproReference voice; + SoproSemanticLMOptions lm_options; + std::vector style_tokens; + std::vector carry; + std::vector segments; + std::mt19937_64 rng; + size_t index = 0; // next segment to synthesize + size_t emitted = 0; // segments that produced audio so far + int sample_rate = 0; + float gain = 0.0F; + bool gain_ready = false; +}; + +std::unique_ptr SoproTTSSession::begin_synthesis( + const runtime::TaskRequest & request) { + runtime::validate_spec_backed_request_options(request.options, *contract_, "Sopro"); + if (!request.text_input.has_value() || request.text_input->text.empty()) { + throw std::runtime_error("Sopro requires non-empty text input"); + } + const runtime::AudioBuffer * reference = reference_audio(request); + if (reference == nullptr || reference->samples.empty()) { + throw std::runtime_error( + "Sopro requires reference voice audio (voice preset or voice_ref) for zero-shot cloning"); + } + + auto state = std::make_unique(); + state->options = parse_options(request); + const auto & config = assets_->config; + state->sample_rate = static_cast(config.sample_rate); + state->rng.seed(state->options.seed); + + const auto reference_audio24 = to_mono_24k(*reference, state->sample_rate); + const auto reference_start = std::chrono::steady_clock::now(); + state->voice = reference_builder_->build( + reference_audio24, state->options.ref_seconds, state->rng); + engine::debug::timing_log_scalar( + "sopro_tts.reference.prepare_ms", + engine::debug::elapsed_ms(reference_start, std::chrono::steady_clock::now())); + if (state->voice.semantic_tokens.empty() || state->voice.mel_frames <= 0) { + throw std::runtime_error("Sopro reference audio produced no semantic tokens"); + } + + // _steps(): one semantic token per token_samples output samples. + const int64_t token_samples = config.semantic_encoder.token_samples_24k; + const auto steps_for = [&](float seconds) { + return std::max( + 1, + static_cast(std::ceil( + static_cast(seconds) * static_cast(state->sample_rate) / + static_cast(token_samples)))); + }; + + const auto style_count = std::max( + 0, + std::min( + config.generation.style_tokens, + static_cast(state->voice.semantic_tokens.size()))); + state->style_tokens.assign( + state->voice.semantic_tokens.begin(), + state->voice.semantic_tokens.begin() + static_cast(style_count)); + if (config.generation.prompt_tokens > 0) { + const auto count = std::min( + config.generation.prompt_tokens, + static_cast(state->voice.semantic_tokens.size())); + // Continue from the *end* of the reference. synthesize_segment places + // this segment's tokens after the whole reference, and every later + // segment carries the tail of its predecessor, so anchoring the first + // one at the head would continue from the wrong point in the clip. + state->carry.assign( + state->voice.semantic_tokens.end() - static_cast(count), + state->voice.semantic_tokens.end()); + } + + state->lm_options.max_steps = steps_for(state->options.max_seconds); + state->lm_options.min_steps = steps_for(state->options.min_seconds); + state->lm_options.temperature = state->options.temperature; + state->lm_options.top_p = state->options.top_p; + state->lm_options.top_k = state->options.top_k; + + state->segments = split_text(request.text_input->text, state->options.max_segment_chars); + engine::debug::trace_log_scalar( + "sopro_tts.text.segments", static_cast(state->segments.size())); + return state; +} + +std::vector SoproTTSSession::synthesize_segment(SoproSynthesisState & state) { + if (state.index >= state.segments.size()) { + return {}; + } + const std::string & segment = state.segments[state.index++]; + const auto & config = assets_->config; + const int64_t token_samples = config.semantic_encoder.token_samples_24k; + const int64_t hop_ratio = config.hop_ratio(); + const int64_t n_mels = config.model.acoustic_mel_n_mels; + const int64_t vocoder_hop = vocoder_->hop_length(); + const auto prompt_budget = config.generation.prompt_tokens; + + const auto text_ids = tokenizer_->encode(segment, state.options.language); + const auto lm_start = std::chrono::steady_clock::now(); + const auto tokens = semantic_lm_->generate( + text_ids, state.style_tokens, state.carry, state.lm_options, state.rng); + engine::debug::timing_log_scalar( + "sopro_tts.semantic_lm.generate_ms", + engine::debug::elapsed_ms(lm_start, std::chrono::steady_clock::now())); + engine::debug::trace_log_scalar( + "sopro_tts.semantic_lm.tokens", static_cast(tokens.size())); + if (tokens.empty()) { + return {}; + } + if (prompt_budget > 0) { + const auto count = std::min(prompt_budget, static_cast(tokens.size())); + state.carry.assign(tokens.end() - static_cast(count), tokens.end()); + } + + SoproAcousticRequest acoustic; + acoustic.semantic_tokens = state.voice.semantic_tokens; + acoustic.semantic_tokens.insert(acoustic.semantic_tokens.end(), tokens.begin(), tokens.end()); + acoustic.cond_vec = state.voice.cond_vec; + acoustic.prompt_mel = state.voice.mel; + acoustic.prompt_frames = state.voice.mel_frames; + acoustic.total_frames = + state.voice.mel_frames + static_cast(tokens.size()) * hop_ratio; + acoustic.steps = state.options.steps; + acoustic.seed = state.rng(); + const auto acoustic_start = std::chrono::steady_clock::now(); + const auto mel = acoustic_->solve(acoustic); + engine::debug::timing_log_scalar( + "sopro_tts.acoustic.solve_ms", + engine::debug::elapsed_ms(acoustic_start, std::chrono::steady_clock::now())); + + // Denormalise and hand the vocoder a short prompt run-up so its + // convolution state matches the reference, then drop that run-up. + const int64_t context = std::min(kDecodeContextFrames, state.voice.mel_frames); + const int64_t begin = state.voice.mel_frames - context; + const int64_t decode_frames = acoustic.total_frames - begin; + std::vector decode_mel(static_cast(n_mels * decode_frames), 0.0F); + for (int64_t c = 0; c < n_mels; ++c) { + const float mean = config.model.acoustic_mel_mean[static_cast(c)]; + const float scale = config.model.acoustic_mel_std[static_cast(c)]; + const float * source = mel.data() + static_cast(c * acoustic.total_frames + begin); + float * target = decode_mel.data() + static_cast(c * decode_frames); + for (int64_t t = 0; t < decode_frames; ++t) { + target[t] = source[t] * scale + mean; + } + } + auto wav = vocoder_->decode(decode_mel, decode_frames); + const int64_t skip = context * vocoder_hop; + const int64_t target_length = static_cast(tokens.size()) * token_samples; + if (static_cast(wav.size()) <= skip) { + return {}; + } + const int64_t end = std::min(static_cast(wav.size()), skip + target_length); + return std::vector( + wav.begin() + static_cast(skip), wav.begin() + static_cast(end)); +} + +runtime::TaskResult SoproTTSSession::run(const runtime::TaskRequest & request) { + require_prepared("Sopro run"); + if (task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("Sopro run requires an offline session"); + } + auto state = begin_synthesis(request); + + std::vector> parts; + while (state->index < state->segments.size()) { + auto part = synthesize_segment(*state); + if (!part.empty()) { + parts.push_back(std::move(part)); + } + } + + const int sample_rate = state->sample_rate; + runtime::TaskResult result; + runtime::AudioBuffer audio; + audio.sample_rate = sample_rate; + audio.channels = 1; + if (parts.empty()) { + audio.samples.assign( + static_cast(assets_->config.semantic_encoder.token_samples_24k), 0.0F); + result.audio_output = std::move(audio); + return result; + } + + // Level-match once over the whole utterance, then trim and cross-fade the + // segment joins (SoproTTS.synthesize). + std::vector concatenated; + for (const auto & part : parts) { + concatenated.insert(concatenated.end(), part.begin(), part.end()); + } + const float gain = audio_ops::match_gain( + concatenated, sample_rate, audio_ops::kOutputLevelDb, state->voice.level_db); + std::vector> trimmed; + trimmed.reserve(parts.size()); + for (size_t index = 0; index < parts.size(); ++index) { + auto part = parts[index]; + for (auto & value : part) { + value *= gain; + } + part = index == 0 + ? audio_ops::trim_lead(part, sample_rate) + : audio_ops::trim_lead( + part, sample_rate, audio_ops::kSegmentLeadSeconds, audio_ops::kSegmentSkipSeconds); + trimmed.push_back(audio_ops::trim_trail(part, sample_rate)); + } + auto out = audio_ops::join_segments(std::move(trimmed), sample_rate); + audio_ops::soft_limit(out); + audio_ops::fade_edges(out, sample_rate, false, true, audio_ops::kFinalFadeSeconds); + audio.samples = std::move(out); + result.audio_output = std::move(audio); + return result; +} + +// --------------------------------------------------------------------------- // +// Streaming interface +// --------------------------------------------------------------------------- // +runtime::StreamingPolicy SoproTTSSession::streaming_policy() const { + // The acoustic head and the vocoder both look at a whole span at once, and + // this checkpoint ships no causal vocoder, so one text segment is the + // smallest unit that can leave without boundary artefacts. text_chunk_size + // is what trades first-audio latency against segment length. + runtime::StreamingPolicy policy; + policy.input = runtime::StreamingInputKind::None; + policy.output = runtime::StreamingOutputKind::PullEvents; + return policy; +} + +void SoproTTSSession::start_stream(const runtime::TaskRequest & request) { + require_prepared("Sopro start_stream"); + if (task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error("Sopro start_stream requires a streaming session"); + } + reset(); + // The reference voice is encoded once here rather than per event, so every + // event after the first costs only its own LM, solver and vocoder passes. + stream_state_ = begin_synthesis(request); + if (stream_state_->segments.empty()) { + throw std::runtime_error("Sopro streaming text chunking produced no segments"); + } +} + +std::optional SoproTTSSession::next_stream_event() { + if (stream_state_ == nullptr) { + throw std::runtime_error("Sopro streaming has not been started"); + } + SoproSynthesisState & state = *stream_state_; + const auto event_start = std::chrono::steady_clock::now(); + std::vector part; + while (part.empty() && state.index < state.segments.size()) { + part = synthesize_segment(state); + } + if (part.empty()) { + return std::nullopt; + } + engine::debug::timing_log_scalar( + "sopro_tts.streaming.event.synthesize_ms", + engine::debug::elapsed_ms(event_start, std::chrono::steady_clock::now())); + + // Offline levels the whole utterance at once. A stream cannot see the + // segments it has not generated yet, so the first one fixes the gain for + // all of them; that keeps their relative loudness instead of pushing every + // segment onto the target level on its own. + const int sample_rate = state.sample_rate; + if (!state.gain_ready) { + state.gain = audio_ops::match_gain( + part, sample_rate, audio_ops::kOutputLevelDb, state.voice.level_db); + state.gain_ready = true; + engine::debug::trace_log_scalar( + "sopro_tts.streaming.gain", static_cast(state.gain)); + } + for (auto & value : part) { + value *= state.gain; + } + part = state.emitted == 0 + ? audio_ops::trim_lead(part, sample_rate) + : audio_ops::trim_lead( + part, sample_rate, audio_ops::kSegmentLeadSeconds, audio_ops::kSegmentSkipSeconds); + part = audio_ops::trim_trail(part, sample_rate); + // Same order as the offline tail: join fade, limiter, then the final fade + // on whichever segment turns out to be the last one. + const bool has_more = state.index < state.segments.size(); + audio_ops::fade_edges(part, sample_rate, state.emitted > 0, has_more); + audio_ops::soft_limit(part); + if (!has_more) { + audio_ops::fade_edges(part, sample_rate, false, true, audio_ops::kFinalFadeSeconds); + } + + runtime::AudioBuffer audio; + audio.sample_rate = sample_rate; + audio.channels = 1; + audio.samples = std::move(part); + const size_t chunk_index = state.emitted++; + stream_chunks_.push_back(audio); + + runtime::StreamEvent event; + event.named_audio_outputs.push_back({ + "segment_" + std::to_string(chunk_index), + std::move(audio), + {}, + }); + return event; +} + +void SoproTTSSession::set_stream_event_sink(runtime::StreamEventCallback sink) { + // Every driver of a PullEvents session (app/streaming/streaming.cpp, and the + // server through it) forwards whatever next_stream_event returns to its own + // sink, so pushing here as well would deliver each segment twice. + (void) sink; +} + +runtime::TaskResult SoproTTSSession::finish_stream() { + if (stream_state_ == nullptr) { + throw std::runtime_error("Sopro streaming has not been started"); + } + // Each event is already levelled, trimmed and faded, so the utterance is a + // plain concatenation of what the consumer has already heard. + runtime::AudioBuffer merged; + merged.sample_rate = stream_state_->sample_rate; + merged.channels = 1; + if (stream_chunks_.empty()) { + merged.samples.assign( + static_cast(assets_->config.semantic_encoder.token_samples_24k), 0.0F); + } + for (const auto & chunk : stream_chunks_) { + runtime::append_audio_buffer(merged, chunk); + } + runtime::TaskResult result; + result.audio_output = std::move(merged); + reset(); + return result; +} + +void SoproTTSSession::reset() { + stream_state_.reset(); + stream_chunks_.clear(); +} + +runtime::StreamEvent SoproTTSSession::process_audio_chunk(const runtime::AudioChunk & chunk) { + (void) chunk; + throw std::runtime_error("Sopro is a TTS model and does not accept streamed audio input"); +} + +runtime::TaskResult SoproTTSSession::finalize() { + return runtime::TaskResult{}; +} + +std::shared_ptr make_sopro_tts_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + // The upstream repo and the model card both call the family "sopro"; keep + // the short spelling working as a --family hint. + config.aliases = {"sopro", "sopro_v2", "sopro_v2_turbo"}; + config.load_assets = load_sopro_tts_assets; + config.create_session = create_session; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::community_models::sopro_tts diff --git a/src/community_models/sopro_tts/speaker_encoder.cpp b/src/community_models/sopro_tts/speaker_encoder.cpp new file mode 100644 index 000000000..9c79583b6 --- /dev/null +++ b/src/community_models/sopro_tts/speaker_encoder.cpp @@ -0,0 +1,637 @@ +#include "engine/community_models/sopro_tts/speaker_encoder.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/audio/dsp.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/streaming_conv_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { + +// A two-layer MLP with SiLU between, i.e. nn.Sequential(Linear, SiLU, +// Identity, Linear) -> keys ".0" and ".3". +struct SoproSpeakerHeadWeights { + std::vector fc1_weight; // [hidden, in] + std::vector fc1_bias; + std::vector fc2_weight; // [out, hidden] + std::vector fc2_bias; + int64_t in_features = 0; + int64_t hidden = 0; + int64_t out_features = 0; +}; + +struct SoproSpeakerResBlockWeights { + engine::modules::NormWeights norm1; + engine::modules::Conv1dWeights pw_in; + engine::modules::DepthwiseConv1dWeights dw; + engine::modules::NormWeights norm2; + engine::modules::Conv1dWeights se_reduce; + engine::modules::Conv1dWeights se_expand; + engine::modules::Conv1dWeights pw_out; + int64_t channels = 0; + int64_t se_hidden = 0; + int dilation = 1; +}; + +struct SoproSpeakerStageWeights { + engine::modules::Conv1dWeights transition_conv; + engine::modules::NormWeights transition_norm; + int transition_stride = 1; + int64_t out_channels = 0; + std::vector blocks; +}; + +struct SoproSpeakerWeights { + std::shared_ptr store; + engine::modules::Conv1dWeights stem_conv; + engine::modules::NormWeights stem_norm; + std::vector stages; + engine::modules::Conv1dWeights fuse_conv; + engine::modules::NormWeights fuse_norm; + // Host-side pooling heads. + std::vector attn_conv1_weight; // [attn_hidden, channels] + std::vector attn_conv1_bias; + std::vector attn_conv2_weight; // [1, attn_hidden] + std::vector attn_conv2_bias; + SoproSpeakerHeadWeights id_head; + SoproSpeakerHeadWeights style_head; + SoproSpeakerHeadWeights style_ctrl_head; + // torchaudio MelSpectrogram buffers. + std::vector analysis_window; // win_length taps + std::vector mel_filterbank; // [freq_bins, n_mels] +}; + +namespace { + +namespace binding = engine::modules::binding; + +constexpr float kGroupNormEps = 1.0e-5F; // torch.nn.GroupNorm default +constexpr float kLayerNormEps = 1.0e-5F; // torch.nn.functional.layer_norm default + +// SOPRO_DUMP_DIR: raw f32 dumps for stage comparison against the reference. +void dump(const std::string & name, const std::vector & values) { + const char * dir = std::getenv("SOPRO_DUMP_DIR"); + if (dir == nullptr) { + return; + } + std::FILE * fh = std::fopen((std::string(dir) + "/" + name).c_str(), "wb"); + if (fh != nullptr) { + std::fwrite(values.data(), sizeof(float), values.size(), fh); + std::fclose(fh); + } +} + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +engine::modules::NormWeights group_norm( + engine::core::BackendWeightStore & store, + const engine::assets::TensorSource & source, + const std::string & prefix, + int64_t channels) { + return binding::norm_from_source(store, source, prefix, channels); +} + +SoproSpeakerHeadWeights load_head( + const engine::assets::TensorSource & source, + const std::string & prefix, + int64_t in_features, + int64_t hidden, + int64_t out_features) { + SoproSpeakerHeadWeights out; + out.in_features = in_features; + out.hidden = hidden; + out.out_features = out_features; + out.fc1_weight = source.require_f32(prefix + ".0.weight", {hidden, in_features}); + out.fc1_bias = source.require_f32(prefix + ".0.bias", {hidden}); + out.fc2_weight = source.require_f32(prefix + ".3.weight", {out_features, hidden}); + out.fc2_bias = source.require_f32(prefix + ".3.bias", {out_features}); + return out; +} + +std::vector apply_head(const SoproSpeakerHeadWeights & head, const std::vector & input) { + if (static_cast(input.size()) != head.in_features) { + throw std::runtime_error("Sopro speaker head input size mismatch"); + } + std::vector hidden(static_cast(head.hidden), 0.0F); + for (int64_t h = 0; h < head.hidden; ++h) { + double sum = head.fc1_bias[static_cast(h)]; + const float * row = head.fc1_weight.data() + static_cast(h * head.in_features); + for (int64_t i = 0; i < head.in_features; ++i) { + sum += static_cast(row[i]) * static_cast(input[static_cast(i)]); + } + const auto value = static_cast(sum); + hidden[static_cast(h)] = value / (1.0F + std::exp(-value)); // SiLU + } + std::vector out(static_cast(head.out_features), 0.0F); + for (int64_t o = 0; o < head.out_features; ++o) { + double sum = head.fc2_bias[static_cast(o)]; + const float * row = head.fc2_weight.data() + static_cast(o * head.hidden); + for (int64_t h = 0; h < head.hidden; ++h) { + sum += static_cast(row[h]) * static_cast(hidden[static_cast(h)]); + } + out[static_cast(o)] = static_cast(sum); + } + return out; +} + +std::shared_ptr load_speaker_weights( + ggml_backend_t backend, + engine::core::BackendType backend_type, + const engine::assets::TensorSource & source, + const SoproSpeakerEncoderConfig & config, + size_t weight_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) { + (void) matmul_storage_type; + auto weights = std::make_shared(); + require_frontend_buffers( + source, "speaker encoder", + {"frontend.mel.spectrogram.window", + "frontend.mel.mel_scale.fb"}); + weights->store = std::make_shared( + backend, backend_type, "sopro_tts.speaker_encoder.weights", weight_context_bytes); + auto & store = *weights->store; + + weights->stem_conv = binding::conv1d_from_source( + store, source, "stem.0.conv", conv_storage_type, + config.stem_channels, config.n_mels, 5, true); + weights->stem_norm = group_norm(store, source, "stem.1", config.stem_channels); + + int64_t in_channels = config.stem_channels; + int64_t fused_in = 0; + weights->stages.reserve(config.stage_channels.size()); + for (size_t stage = 0; stage < config.stage_channels.size(); ++stage) { + SoproSpeakerStageWeights out; + out.out_channels = config.stage_channels[stage]; + out.transition_stride = stage == 0 ? 2 : 1; + const std::string transition = "transitions." + std::to_string(stage); + out.transition_conv = binding::conv1d_from_source( + store, source, transition + ".conv", conv_storage_type, + out.out_channels, in_channels, 3, true); + out.transition_norm = group_norm(store, source, transition + ".norm", out.out_channels); + const int64_t blocks = config.blocks_per_stage[stage]; + out.blocks.reserve(static_cast(blocks)); + for (int64_t block = 0; block < blocks; ++block) { + const std::string prefix = + "stages." + std::to_string(stage) + "." + std::to_string(block); + SoproSpeakerResBlockWeights weights_block; + weights_block.channels = out.out_channels; + weights_block.dilation = static_cast( + config.dilation_cycle[static_cast(block) % config.dilation_cycle.size()]); + weights_block.se_hidden = std::max(8, out.out_channels / config.se_reduction); + weights_block.norm1 = group_norm(store, source, prefix + ".norm1", out.out_channels); + weights_block.pw_in = binding::conv1d_from_source( + store, source, prefix + ".pw_in", conv_storage_type, + out.out_channels * 2, out.out_channels, 1, true); + weights_block.dw = binding::depthwise_conv1d_from_source( + store, source, prefix + ".dw.conv", conv_storage_type, + out.out_channels, config.depthwise_kernel_size, true); + weights_block.norm2 = group_norm(store, source, prefix + ".norm2", out.out_channels); + weights_block.se_reduce = binding::conv1d_from_source( + store, source, prefix + ".se.net.1", conv_storage_type, + weights_block.se_hidden, out.out_channels, 1, true); + weights_block.se_expand = binding::conv1d_from_source( + store, source, prefix + ".se.net.3", conv_storage_type, + out.out_channels, weights_block.se_hidden, 1, true); + weights_block.pw_out = binding::conv1d_from_source( + store, source, prefix + ".pw_out", conv_storage_type, + out.out_channels, out.out_channels, 1, true); + out.blocks.push_back(std::move(weights_block)); + } + fused_in += out.out_channels; + in_channels = out.out_channels; + weights->stages.push_back(std::move(out)); + } + + const int64_t head_channels = config.stage_channels.back(); + weights->fuse_conv = binding::conv1d_from_source( + store, source, "fuse.0", conv_storage_type, head_channels, fused_in, 1, true); + weights->fuse_norm = group_norm(store, source, "fuse.1", head_channels); + + weights->attn_conv1_weight = source.require_f32( + "id_pool.attn.0.weight", {config.attn_hidden, head_channels, 1}); + weights->attn_conv1_bias = source.require_f32("id_pool.attn.0.bias", {config.attn_hidden}); + weights->attn_conv2_weight = source.require_f32( + "id_pool.attn.2.weight", {1, config.attn_hidden, 1}); + weights->attn_conv2_bias = source.require_f32("id_pool.attn.2.bias", {1}); + weights->id_head = load_head( + source, "id_head", head_channels * 2, config.id_head_hidden, config.id_emb_dim); + weights->style_head = load_head( + source, "style_head", head_channels * 4, config.style_head_hidden, config.style_emb_dim); + weights->style_ctrl_head = load_head( + source, "style_ctrl_head", head_channels * 4, config.style_head_hidden, config.style_ctrl_dim); + + weights->analysis_window = source.require_f32( + "frontend.mel.spectrogram.window", {config.win_length}); + weights->mel_filterbank = source.require_f32( + "frontend.mel.mel_scale.fb", {config.n_fft / 2 + 1, config.n_mels}); + + store.upload(); + return weights; +} + +engine::core::TensorValue build_res_block( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & input, + const SoproSpeakerResBlockWeights & weights, + int64_t kernel_size, + int64_t frames) { + const int64_t channels = weights.channels; + auto hidden = engine::modules::GroupNormModule({channels, 1, kGroupNormEps, true, true}) + .build(ctx, input, weights.norm1); + hidden = engine::modules::Conv1dModule({channels, channels * 2, 1, 1, 0, 1, true}) + .build(ctx, hidden, weights.pw_in); + // chunk(2, dim=1): the gate multiplies the first half by sigmoid(second). + auto gate_a = engine::modules::SliceModule({1, 0, channels}).build(ctx, hidden); + auto gate_b = engine::modules::SliceModule({1, channels, channels}).build(ctx, hidden); + gate_b = engine::modules::SigmoidModule{}.build(ctx, gate_b); + hidden = engine::modules::MulModule{}.build(ctx, gate_a, gate_b); + hidden = engine::modules::DepthwiseConv1dModule({ + channels, kernel_size, 1, + static_cast(weights.dilation * (kernel_size - 1) / 2), weights.dilation, true, + }).build(ctx, hidden, weights.dw); + hidden = engine::modules::GroupNormModule({channels, 1, kGroupNormEps, true, true}) + .build(ctx, hidden, weights.norm2); + hidden = engine::modules::SiluModule{}.build(ctx, hidden); + // SqueezeExcite1d: global average pool -> 1x1 bottleneck -> sigmoid gate. + auto pooled = engine::modules::ReduceMeanModule({2}).build(ctx, hidden); + pooled = engine::modules::Conv1dModule({channels, weights.se_hidden, 1, 1, 0, 1, true}) + .build(ctx, pooled, weights.se_reduce); + pooled = engine::modules::SiluModule{}.build(ctx, pooled); + pooled = engine::modules::Conv1dModule({weights.se_hidden, channels, 1, 1, 0, 1, true}) + .build(ctx, pooled, weights.se_expand); + pooled = engine::modules::SigmoidModule{}.build(ctx, pooled); + auto scale = engine::modules::RepeatModule({ + engine::core::TensorShape::from_dims({1, channels, frames})}).build(ctx, pooled); + hidden = engine::modules::MulModule{}.build(ctx, hidden, scale); + hidden = engine::modules::Conv1dModule({channels, channels, 1, 1, 0, 1, true}) + .build(ctx, hidden, weights.pw_out); + return engine::modules::AddModule{}.build(ctx, input, hidden); +} + +} // namespace + +struct SoproSpeakerGraph { + SoproSpeakerGraph( + ggml_backend_t backend_in, + engine::core::BackendType backend_type, + size_t graph_context_bytes, + const SoproSpeakerEncoderConfig & config, + std::shared_ptr weights_in, + int64_t frames_in) + : backend(backend_in), + weights(std::move(weights_in)), + frames(frames_in), + mel_bins(config.n_mels) { + if (backend == nullptr || weights == nullptr) { + throw std::runtime_error("Sopro speaker encoder graph requires a backend and weights"); + } + if (frames <= 0) { + throw std::runtime_error("Sopro speaker encoder graph requires a positive frame count"); + } + ggml_init_params params{graph_context_bytes, nullptr, true}; + ctx.reset(ggml_init(params)); + if (ctx == nullptr) { + throw std::runtime_error("failed to initialize the Sopro speaker encoder graph context"); + } + engine::core::ModuleBuildContext build_ctx{ctx.get(), "sopro_tts.speaker_encoder", backend_type}; + const auto shape = engine::core::TensorShape::from_dims({1, mel_bins, frames}); + input = engine::core::make_tensor(build_ctx, GGML_TYPE_F32, shape).tensor; + ggml_set_input(input); + + auto hidden = engine::modules::Conv1dModule({ + mel_bins, config.stem_channels, 5, 1, 2, 1, true, + }).build(build_ctx, engine::core::wrap_tensor(input, shape, GGML_TYPE_F32), weights->stem_conv); + hidden = engine::modules::GroupNormModule({config.stem_channels, 1, kGroupNormEps, true, true}) + .build(build_ctx, hidden, weights->stem_norm); + hidden = engine::modules::SiluModule{}.build(build_ctx, hidden); + + int64_t in_channels = config.stem_channels; + int64_t stage_frames = frames; + std::vector stage_outputs; + for (const auto & stage : weights->stages) { + if (stage.transition_stride == 2) { + // F.pad(x, (1, 1)) then Conv1d(kernel 3, stride 2). + stage_frames = (stage_frames + 2 - 3) / 2 + 1; + } + hidden = engine::modules::Conv1dModule({ + in_channels, stage.out_channels, 3, stage.transition_stride, 1, 1, true, + }).build(build_ctx, hidden, stage.transition_conv); + hidden = engine::modules::GroupNormModule({stage.out_channels, 1, kGroupNormEps, true, true}) + .build(build_ctx, hidden, stage.transition_norm); + hidden = engine::modules::SiluModule{}.build(build_ctx, hidden); + for (const auto & block : stage.blocks) { + hidden = build_res_block( + build_ctx, hidden, block, config.depthwise_kernel_size, stage_frames); + } + stage_outputs.push_back(hidden); + in_channels = stage.out_channels; + } + trunk_frames = stage_frames; + + auto fused = stage_outputs.front(); + for (size_t i = 1; i < stage_outputs.size(); ++i) { + fused = engine::modules::ConcatModule({1}).build(build_ctx, fused, stage_outputs[i]); + } + int64_t fused_in = 0; + for (const auto & stage : weights->stages) { + fused_in += stage.out_channels; + } + channels = config.stage_channels.back(); + fused = engine::modules::Conv1dModule({fused_in, channels, 1, 1, 0, 1, true}) + .build(build_ctx, fused, weights->fuse_conv); + fused = engine::modules::GroupNormModule({channels, 1, kGroupNormEps, true, true}) + .build(build_ctx, fused, weights->fuse_norm); + fused = engine::modules::SiluModule{}.build(build_ctx, fused); + fused = engine::core::ensure_backend_addressable_layout(build_ctx, fused); + output = fused.tensor; + ggml_set_output(output); + graph = ggml_new_graph_custom(ctx.get(), 65536, false); + ggml_build_forward_expand(graph, output); + gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (gallocr == nullptr || !ggml_gallocr_reserve(gallocr, graph) || + !ggml_gallocr_alloc_graph(gallocr, graph)) { + throw std::runtime_error("failed to allocate the Sopro speaker encoder graph"); + } + } + + ~SoproSpeakerGraph() { + if (gallocr != nullptr) { + ggml_gallocr_free(gallocr); + gallocr = nullptr; + } + } + + bool matches(const SoproSpeakerWeights & other, int64_t other_frames) const noexcept { + return weights.get() == &other && frames == other_frames; + } + + std::vector run(const std::vector & log_mel) { + ggml_backend_tensor_set(input, log_mel.data(), 0, log_mel.size() * sizeof(float)); + const ggml_status status = engine::core::compute_backend_graph(backend, graph); + ggml_backend_synchronize(backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Sopro speaker encoder graph compute failed"); + } + std::vector out(static_cast(channels * trunk_frames), 0.0F); + ggml_backend_tensor_get(output, out.data(), 0, out.size() * sizeof(float)); + return out; + } + + ggml_backend_t backend = nullptr; + std::shared_ptr weights; + int64_t frames = 0; + int64_t mel_bins = 0; + int64_t trunk_frames = 0; + int64_t channels = 0; + std::unique_ptr ctx; + ggml_tensor * input = nullptr; + ggml_tensor * output = nullptr; + ggml_cgraph * graph = nullptr; + ggml_gallocr_t gallocr = nullptr; +}; + +SoproSpeakerEncoderRuntime::SoproSpeakerEncoderRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) + : config_(assets.config.speaker_encoder), + execution_context_(execution_context), + graph_context_bytes_(graph_context_bytes), + weights_(load_speaker_weights( + execution_context.backend(), + execution_context.backend_type(), + *assets.speaker_encoder_weights, + assets.config.speaker_encoder, + weight_context_bytes, + matmul_storage_type, + conv_storage_type)) {} + +SoproSpeakerEncoderRuntime::~SoproSpeakerEncoderRuntime() = default; + +int64_t SoproSpeakerEncoderRuntime::sample_rate() const noexcept { + return config_.sample_rate; +} + +std::vector SoproSpeakerEncoderRuntime::trunk( + const std::vector & log_mel, + int64_t frames, + int64_t & out_frames) const { + if (graph_ == nullptr || !graph_->matches(*weights_, frames)) { + // Free the previous arena first; otherwise both are resident while the + // replacement is allocated, and every segment rebuilds this graph. + graph_.reset(); + graph_ = std::make_unique( + execution_context_.backend(), + execution_context_.backend_type(), + graph_context_bytes_, + config_, + weights_, + frames); + } + out_frames = graph_->trunk_frames; + return graph_->run(log_mel); +} + +SoproSpeakerEmbeddings SoproSpeakerEncoderRuntime::encode(const std::vector & audio16) const { + if (audio16.empty()) { + throw std::runtime_error("Sopro speaker encoder requires a non-empty reference waveform"); + } + // LogMelFrontend: power-2 mel, log with a floor, then a per-frame LayerNorm + // across the mel axis (no affine parameters). + const int64_t freq_bins = config_.n_fft / 2 + 1; + engine::audio::STFTConfig stft; + stft.n_fft = config_.n_fft; + stft.hop_length = config_.hop_length; + stft.win_length = config_.win_length; + stft.center = true; + stft.pad_mode = engine::audio::STFTPadMode::Reflect; + const auto magnitude = engine::audio::STFT{}.compute_magnitude( + audio16, weights_->analysis_window, 1, static_cast(audio16.size()), stft, + static_cast(execution_context_.config().threads)); + if (magnitude.shape.size() != 3 || magnitude.shape[1] != freq_bins) { + throw std::runtime_error("Sopro speaker encoder STFT produced an unexpected layout"); + } + const int64_t frames = magnitude.shape[2]; + if (frames < 3) { + throw std::runtime_error("Sopro speaker encoder reference audio is too short"); + } + std::vector mel(static_cast(config_.n_mels * frames), 0.0F); + for (int64_t f = 0; f < freq_bins; ++f) { + const float * fb_row = weights_->mel_filterbank.data() + static_cast(f * config_.n_mels); + const float * spec_row = magnitude.values.data() + static_cast(f * frames); + for (int64_t m = 0; m < config_.n_mels; ++m) { + const float weight = fb_row[m]; + if (weight == 0.0F) { + continue; + } + float * out_row = mel.data() + static_cast(m * frames); + for (int64_t t = 0; t < frames; ++t) { + // power=2.0: the mel filters see squared magnitudes. + out_row[t] += weight * spec_row[t] * spec_row[t]; + } + } + } + for (auto & value : mel) { + value = std::log(std::max(value, config_.mel_log_floor)); + } + for (int64_t t = 0; t < frames; ++t) { + double sum = 0.0; + for (int64_t m = 0; m < config_.n_mels; ++m) { + sum += mel[static_cast(m * frames + t)]; + } + const double mean = sum / static_cast(config_.n_mels); + double variance = 0.0; + for (int64_t m = 0; m < config_.n_mels; ++m) { + const double centred = mel[static_cast(m * frames + t)] - mean; + variance += centred * centred; + } + variance /= static_cast(config_.n_mels); + const double inv_std = 1.0 / std::sqrt(variance + kLayerNormEps); + for (int64_t m = 0; m < config_.n_mels; ++m) { + auto & value = mel[static_cast(m * frames + t)]; + value = static_cast((value - mean) * inv_std); + } + } + + dump("spk_wav16.f32", audio16); + dump("spk_mel.f32", mel); + int64_t trunk_frames = 0; + const auto features = trunk(mel, frames, trunk_frames); + dump("spk_trunk.f32", features); + const int64_t channels = config_.stage_channels.back(); + if (trunk_frames <= 0) { + throw std::runtime_error("Sopro speaker encoder produced no trunk frames"); + } + + // AttentiveStatsPool: softmax attention over time, then weighted mean/std. + std::vector scores(static_cast(trunk_frames), 0.0F); + std::vector attn_hidden(static_cast(config_.attn_hidden), 0.0F); + for (int64_t t = 0; t < trunk_frames; ++t) { + for (int64_t h = 0; h < config_.attn_hidden; ++h) { + double sum = weights_->attn_conv1_bias[static_cast(h)]; + const float * row = weights_->attn_conv1_weight.data() + static_cast(h * channels); + for (int64_t c = 0; c < channels; ++c) { + sum += static_cast(row[c]) * + static_cast(features[static_cast(c * trunk_frames + t)]); + } + attn_hidden[static_cast(h)] = std::tanh(static_cast(sum)); + } + double sum = weights_->attn_conv2_bias[0]; + for (int64_t h = 0; h < config_.attn_hidden; ++h) { + sum += static_cast(weights_->attn_conv2_weight[static_cast(h)]) * + static_cast(attn_hidden[static_cast(h)]); + } + scores[static_cast(t)] = static_cast(sum); + } + const float max_score = *std::max_element(scores.begin(), scores.end()); + double score_sum = 0.0; + for (auto & score : scores) { + score = std::exp(score - max_score); + score_sum += score; + } + for (auto & score : scores) { + score = static_cast(score / score_sum); + } + + std::vector id_input(static_cast(channels * 2), 0.0F); + for (int64_t c = 0; c < channels; ++c) { + const float * row = features.data() + static_cast(c * trunk_frames); + double mean = 0.0; + for (int64_t t = 0; t < trunk_frames; ++t) { + mean += static_cast(scores[static_cast(t)]) * static_cast(row[t]); + } + double variance = 0.0; + for (int64_t t = 0; t < trunk_frames; ++t) { + const double centred = static_cast(row[t]) - mean; + variance += static_cast(scores[static_cast(t)]) * centred * centred; + } + id_input[static_cast(c)] = static_cast(mean); + id_input[static_cast(channels + c)] = + static_cast(std::sqrt(std::max(variance, 1.0e-6))); + } + + // MultiScaleStylePool: mean/std of the trunk features and of a length-5 + // moving average of them (AvgPool1d(kernel 5, stride 1, padding 2), which + // divides by the kernel size including the zero padding). + std::vector style_input(static_cast(channels * 4), 0.0F); + std::vector smoothed(static_cast(trunk_frames), 0.0F); + const auto denominator = static_cast(trunk_frames); + for (int64_t c = 0; c < channels; ++c) { + const float * row = features.data() + static_cast(c * trunk_frames); + for (int64_t t = 0; t < trunk_frames; ++t) { + double sum = 0.0; + for (int64_t k = -2; k <= 2; ++k) { + const int64_t index = t + k; + if (index >= 0 && index < trunk_frames) { + sum += static_cast(row[index]); + } + } + smoothed[static_cast(t)] = static_cast(sum / 5.0); + } + const float * scales[2] = {row, smoothed.data()}; + for (int scale = 0; scale < 2; ++scale) { + const float * values = scales[scale]; + double sum = 0.0; + for (int64_t t = 0; t < trunk_frames; ++t) { + sum += static_cast(values[t]); + } + const double mean = sum / denominator; + double variance = 0.0; + for (int64_t t = 0; t < trunk_frames; ++t) { + const double centred = static_cast(values[t]) - mean; + variance += centred * centred; + } + variance /= denominator; + style_input[static_cast(scale * 2 * channels + c)] = static_cast(mean); + style_input[static_cast((scale * 2 + 1) * channels + c)] = + static_cast(std::sqrt(std::max(variance, 1.0e-6))); + } + } + + SoproSpeakerEmbeddings out; + out.id_emb = apply_head(weights_->id_head, id_input); + double norm = 0.0; + for (const float value : out.id_emb) { + norm += static_cast(value) * static_cast(value); + } + const auto inv_norm = static_cast(1.0 / std::max(std::sqrt(norm), 1.0e-12)); + for (auto & value : out.id_emb) { + value *= inv_norm; + } + out.style_emb = apply_head(weights_->style_head, style_input); + out.style_ctrl = apply_head(weights_->style_ctrl_head, style_input); + dump("spk_id_emb.f32", out.id_emb); + dump("spk_style_emb.f32", out.style_emb); + dump("spk_style_ctrl.f32", out.style_ctrl); + return out; +} + +} // namespace engine::community_models::sopro_tts diff --git a/src/community_models/sopro_tts/text_tokenizer.cpp b/src/community_models/sopro_tts/text_tokenizer.cpp new file mode 100644 index 000000000..af9b4c651 --- /dev/null +++ b/src/community_models/sopro_tts/text_tokenizer.cpp @@ -0,0 +1,433 @@ +#include "engine/community_models/sopro_tts/text_tokenizer.h" + +#include "sentencepiece_processor.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { +namespace { + +bool is_ascii_space(char c) noexcept { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +// Python str.split() on whitespace followed by " ".join(...). +std::string collapse_whitespace(const std::string & text) { + std::string out; + out.reserve(text.size()); + size_t index = 0; + bool first = true; + while (index < text.size()) { + while (index < text.size() && is_ascii_space(text[index])) { + ++index; + } + const size_t start = index; + while (index < text.size() && !is_ascii_space(text[index])) { + ++index; + } + if (index > start) { + if (!first) { + out.push_back(' '); + } + out.append(text, start, index - start); + first = false; + } + } + return out; +} + +std::string trim(const std::string & text) { + size_t begin = 0; + size_t end = text.size(); + while (begin < end && is_ascii_space(text[begin])) { + ++begin; + } + while (end > begin && is_ascii_space(text[end - 1])) { + --end; + } + return text.substr(begin, end - begin); +} + +size_t utf8_sequence_length(unsigned char lead) noexcept { + if (lead < 0x80U) { + return 1; + } + if ((lead & 0xE0U) == 0xC0U) { + return 2; + } + if ((lead & 0xF0U) == 0xE0U) { + return 3; + } + if ((lead & 0xF8U) == 0xF0U) { + return 4; + } + return 1; // lone continuation byte: treat as one unit so we never stall +} + +size_t codepoint_length(const std::string & text) noexcept { + size_t count = 0; + size_t index = 0; + while (index < text.size()) { + index += std::min(utf8_sequence_length(static_cast(text[index])), + text.size() - index); + ++count; + } + return count; +} + +uint32_t decode_utf8(const std::string & text, size_t offset, size_t & length) noexcept { + const auto lead = static_cast(text[offset]); + length = std::min(utf8_sequence_length(lead), text.size() - offset); + if (length == 1) { + return lead; + } + static constexpr std::array kLeadMask = {0, 0x7FU, 0x1FU, 0x0FU, 0x07U}; + uint32_t value = lead & kLeadMask[length]; + for (size_t i = 1; i < length; ++i) { + value = (value << 6U) | (static_cast(text[offset + i]) & 0x3FU); + } + return value; +} + +void append_utf8(std::string & out, uint32_t codepoint) { + if (codepoint < 0x80U) { + out.push_back(static_cast(codepoint)); + } else if (codepoint < 0x800U) { + out.push_back(static_cast(0xC0U | (codepoint >> 6U))); + out.push_back(static_cast(0x80U | (codepoint & 0x3FU))); + } else if (codepoint < 0x10000U) { + out.push_back(static_cast(0xE0U | (codepoint >> 12U))); + out.push_back(static_cast(0x80U | ((codepoint >> 6U) & 0x3FU))); + out.push_back(static_cast(0x80U | (codepoint & 0x3FU))); + } else { + out.push_back(static_cast(0xF0U | (codepoint >> 18U))); + out.push_back(static_cast(0x80U | ((codepoint >> 12U) & 0x3FU))); + out.push_back(static_cast(0x80U | ((codepoint >> 6U) & 0x3FU))); + out.push_back(static_cast(0x80U | (codepoint & 0x3FU))); + } +} + +// str.islower()/str.upper() for the Latin ranges the four supported languages +// use (ASCII, Latin-1 supplement, Latin Extended-A). Anything else is left as +// it is, which matches Python for scripts without case. +bool codepoint_is_lower(uint32_t cp) noexcept { + if (cp >= 'a' && cp <= 'z') { + return true; + } + if (cp == 0xDFU) { // sharp s has no single-codepoint uppercase + return false; + } + if (cp >= 0xE0U && cp <= 0xFEU && cp != 0xF7U) { + return true; + } + if (cp >= 0x100U && cp <= 0x17FU) { + return (cp % 2U) == 1U; // Latin Extended-A alternates upper/lower + } + return false; +} + +uint32_t codepoint_to_upper(uint32_t cp) noexcept { + if (cp >= 'a' && cp <= 'z') { + return cp - 32U; + } + if (cp >= 0xE0U && cp <= 0xFEU && cp != 0xF7U) { + return cp - 32U; + } + if (cp >= 0x100U && cp <= 0x17FU && (cp % 2U) == 1U) { + return cp - 1U; + } + return cp; +} + +std::string capitalize_first(const std::string & text) { + if (text.empty()) { + return text; + } + size_t length = 0; + const uint32_t cp = decode_utf8(text, 0, length); + if (!codepoint_is_lower(cp)) { + return text; + } + std::string out; + out.reserve(text.size() + 1); + append_utf8(out, codepoint_to_upper(cp)); + out.append(text, length, std::string::npos); + return out; +} + +void replace_all(std::string & text, const std::string & from, const std::string & to) { + if (from.empty()) { + return; + } + size_t position = 0; + while ((position = text.find(from, position)) != std::string::npos) { + text.replace(position, from.size(), to); + position += to.size(); + } +} + +// sopro.text._TERMINALS +bool ends_with_terminal(const std::string & text) noexcept { + if (text.empty()) { + return false; + } + const char last = text.back(); + return last == '.' || last == '!' || last == '?' || last == '-' || + last == ',' || last == ';' || last == ':'; +} + +// sopro.text._pack: greedily join parts while they fit the codepoint budget. +std::vector pack(const std::vector & parts, int64_t max_chars) { + std::vector out; + std::string current; + for (const auto & part : parts) { + if (current.empty()) { + current = part; + } else if (static_cast(codepoint_length(current) + 1 + codepoint_length(part)) <= + max_chars) { + current += " "; + current += part; + } else { + out.push_back(current); + current = part; + } + } + if (!current.empty()) { + out.push_back(current); + } + return out; +} + +// Split on runs of whitespace that follow one of `delimiters`, i.e. the +// Python lookbehind patterns (?<=[.!?…])\s+ and (?<=[,;:])\s+. +std::vector split_after( + const std::string & text, + const std::vector & delimiters) { + std::vector out; + size_t start = 0; + size_t index = 0; + while (index < text.size()) { + if (!is_ascii_space(text[index])) { + ++index; + continue; + } + bool preceded = false; + for (const auto & delimiter : delimiters) { + if (index >= delimiter.size() && + text.compare(index - delimiter.size(), delimiter.size(), delimiter) == 0) { + preceded = true; + break; + } + } + size_t run_end = index; + while (run_end < text.size() && is_ascii_space(text[run_end])) { + ++run_end; + } + if (preceded) { + out.push_back(text.substr(start, index - start)); + start = run_end; + } + index = run_end; + } + out.push_back(text.substr(start)); + return out; +} + +std::vector split_on_spaces(const std::string & text) { + std::vector out; + size_t start = 0; + for (size_t index = 0; index <= text.size(); ++index) { + if (index == text.size() || text[index] == ' ') { + out.push_back(text.substr(start, index - start)); + start = index + 1; + } + } + return out; +} + +// Matches ^((?:<\|[^|\s]+?\|>\s*)+)(.*)$ — a run of leading <|...|> markers. +size_t special_prefix_end(const std::string & text) noexcept { + size_t index = 0; + size_t last_match_end = 0; + while (index + 2 <= text.size() && text.compare(index, 2, "<|") == 0) { + const size_t close = text.find("|>", index + 2); + if (close == std::string::npos) { + break; + } + bool valid = close > index + 2; + for (size_t i = index + 2; i < close && valid; ++i) { + if (text[i] == '|' || is_ascii_space(text[i])) { + valid = false; + } + } + if (!valid) { + break; + } + index = close + 2; + last_match_end = index; + while (index < text.size() && is_ascii_space(text[index])) { + ++index; + } + } + return last_match_end == 0 ? 0 : index; +} + +} // namespace + +std::string language_tag(const std::string & language) { + if (language.empty()) { + return {}; + } + std::string key = trim(language); + std::transform(key.begin(), key.end(), key.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + if (key.empty()) { + return {}; + } + if (key == "en" || key == "pt" || key == "fr" || key == "de") { + return "<|lang_" + key + "|>"; + } + throw std::runtime_error( + "Sopro: unsupported language '" + language + "'; expected one of de, en, fr, pt"); +} + +std::string normalize_text(const std::string & text) { + std::string body = trim(text); + if (body.empty()) { + return "You need to add some text for me to talk."; + } + const size_t prefix_end = special_prefix_end(body); + if (prefix_end > 0) { + const std::string prefix = collapse_whitespace(body.substr(0, prefix_end)); + const std::string rest = trim(body.substr(prefix_end)); + return rest.empty() ? prefix : prefix + " " + normalize_text(rest); + } + body = capitalize_first(body); + body = collapse_whitespace(body); + replace_all(body, "\xE2\x80\xA6", "..."); // U+2026 HORIZONTAL ELLIPSIS + static const std::pair kReplacements[] = { + {" ,", ","}, {" .", "."}, {" !", "!"}, {" ?", "?"}, {" ;", ";"}, {" :", ":"}, + {"\xE2\x80\x9C", "\""}, // U+201C + {"\xE2\x80\x9D", "\""}, // U+201D + {"\xE2\x80\x98", "'"}, // U+2018 + {"\xE2\x80\x99", "'"}, // U+2019 + }; + for (const auto & [from, to] : kReplacements) { + replace_all(body, from, to); + } + body = trim(collapse_whitespace(body)); + if (!ends_with_terminal(body)) { + body += "."; + } + return body; +} + +std::vector split_text(const std::string & text, int64_t max_chars) { + const std::string flat = collapse_whitespace(text); + if (max_chars < 1) { + throw std::runtime_error("Sopro max_segment_chars must be positive"); + } + if (static_cast(codepoint_length(flat)) <= max_chars) { + return flat.empty() ? std::vector{} : std::vector{flat}; + } + static const std::vector kSentenceEnd = {".", "!", "?", "\xE2\x80\xA6"}; + static const std::vector kClauseEnd = {",", ";", ":"}; + std::vector segments; + for (const auto & sentence : pack(split_after(flat, kSentenceEnd), max_chars)) { + if (static_cast(codepoint_length(sentence)) <= max_chars) { + segments.push_back(sentence); + continue; + } + for (const auto & clause : pack(split_after(sentence, kClauseEnd), max_chars)) { + if (static_cast(codepoint_length(clause)) <= max_chars) { + segments.push_back(clause); + } else { + for (auto & piece : pack(split_on_spaces(clause), max_chars)) { + segments.push_back(std::move(piece)); + } + } + } + } + return segments; +} + +class SoproTextTokenizer::Impl { +public: + Impl(const std::filesystem::path & model_path, int64_t max_length) + : max_length_(max_length) { + const auto status = processor_.Load(model_path.string()); + if (!status.ok()) { + throw std::runtime_error( + "Sopro: failed to load SentencePiece model '" + model_path.string() + + "': " + status.ToString()); + } + bos_id_ = processor_.bos_id() >= 0 ? processor_.bos_id() : 1; + eos_id_ = processor_.eos_id() >= 0 ? processor_.eos_id() : 2; + unk_id_ = processor_.unk_id() >= 0 ? processor_.unk_id() : 0; + vocab_size_ = processor_.GetPieceSize(); + if (max_length_ < 2) { + throw std::runtime_error("Sopro tokenizer max_length must be at least 2"); + } + } + + std::vector encode(const std::string & text, const std::string & language) const { + const std::string tag = language_tag(language); + const std::string normalized = normalize_text(tag.empty() ? text : tag + " " + text); + std::vector pieces; + const auto status = processor_.Encode(normalized, &pieces); + if (!status.ok()) { + throw std::runtime_error("Sopro: SentencePiece encode failed: " + status.ToString()); + } + std::vector ids; + ids.reserve(pieces.size() + 2); + ids.push_back(bos_id_); + for (const int piece : pieces) { + ids.push_back(static_cast(piece)); + } + ids.push_back(eos_id_); + if (static_cast(ids.size()) > max_length_) { + // Drop overflowing text ids, not the EOS marker the LM keys on. + // The constructor guarantees max_length_ >= 2. + ids.resize(static_cast(max_length_ - 1)); + ids.push_back(eos_id_); + } + if (ids.empty()) { + ids.push_back(unk_id_); + } + return ids; + } + + int32_t bos_id_ = 1; + int32_t eos_id_ = 2; + int32_t unk_id_ = 0; + int64_t vocab_size_ = 0; + +private: + sentencepiece::SentencePieceProcessor processor_; + int64_t max_length_ = 512; +}; + +SoproTextTokenizer::SoproTextTokenizer(const std::filesystem::path & model_path, int64_t max_length) + : impl_(std::make_unique(model_path, max_length)) {} + +SoproTextTokenizer::~SoproTextTokenizer() = default; + +std::vector SoproTextTokenizer::encode( + const std::string & text, + const std::string & language) const { + return impl_->encode(text, language); +} + +int32_t SoproTextTokenizer::bos_id() const noexcept { return impl_->bos_id_; } +int32_t SoproTextTokenizer::eos_id() const noexcept { return impl_->eos_id_; } +int32_t SoproTextTokenizer::unk_id() const noexcept { return impl_->unk_id_; } +int64_t SoproTextTokenizer::vocab_size() const noexcept { return impl_->vocab_size_; } + +} // namespace engine::community_models::sopro_tts diff --git a/src/community_models/sopro_tts/vocoder.cpp b/src/community_models/sopro_tts/vocoder.cpp new file mode 100644 index 000000000..40774c736 --- /dev/null +++ b/src/community_models/sopro_tts/vocoder.cpp @@ -0,0 +1,458 @@ +#include "engine/community_models/sopro_tts/vocoder.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/audio/dsp.h" +#include "engine/framework/audio/fft.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/streaming_conv_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sopro_tts { + +struct SoproVocoderConvNeXtWeights { + engine::modules::DepthwiseConv1dWeights dwconv; + engine::modules::NormWeights norm; + engine::modules::LinearWeights pwconv1; + engine::modules::LinearWeights pwconv2; + engine::core::TensorValue gamma; +}; + +struct SoproVocoderWeights { + std::shared_ptr store; + engine::modules::Conv1dWeights embed; + engine::modules::NormWeights norm; + std::vector convnext; + engine::modules::NormWeights final_norm; + engine::modules::LinearWeights head_out; + std::vector istft_window; // head.istft.window, n_fft taps + std::vector analysis_window; // MelSpectrogram STFT window + std::vector mel_filterbank; // [freq_bins, n_mels], row-major +}; + +namespace { + +namespace binding = engine::modules::binding; + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +// Multiply the last (channel) dimension of a channel-last tensor by a vector. +engine::core::TensorValue scale_last_dim( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & input, + const engine::core::TensorValue & scale) { + const auto view = engine::core::reshape_tensor( + ctx, scale, engine::core::TensorShape::from_dims({1, 1, scale.shape.dims[0]})); + const auto repeated = engine::modules::RepeatModule({input.shape}).build(ctx, view); + return engine::modules::MulModule{}.build(ctx, input, repeated); +} + +engine::modules::TransposeConfig swap_channel_time() { + return engine::modules::TransposeConfig{{0, 2, 1, 3}, 3}; +} + +std::shared_ptr load_vocoder_weights( + ggml_backend_t backend, + engine::core::BackendType backend_type, + const engine::assets::TensorSource & source, + const SoproVocoderConfig & config, + size_t weight_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) { + auto weights = std::make_shared(); + require_frontend_buffers( + source, "vocoder", + {"feature_extractor.mel_spec.spectrogram.window", + "feature_extractor.mel_spec.mel_scale.fb", + "head.istft.window"}); + weights->store = std::make_shared( + backend, backend_type, "sopro_tts.vocoder.weights", weight_context_bytes); + weights->embed = binding::conv1d_from_source( + *weights->store, source, "backbone.embed", conv_storage_type, + config.dim, config.n_mels, 7, true); + weights->norm = binding::norm_from_source( + *weights->store, source, "backbone.norm", config.dim); + weights->convnext.reserve(static_cast(config.num_layers)); + for (int64_t layer = 0; layer < config.num_layers; ++layer) { + const std::string prefix = "backbone.convnext." + std::to_string(layer); + SoproVocoderConvNeXtWeights block; + block.dwconv = binding::depthwise_conv1d_from_source( + *weights->store, source, prefix + ".dwconv", conv_storage_type, config.dim, 7, true); + block.norm = binding::norm_from_source( + *weights->store, source, prefix + ".norm", config.dim); + block.pwconv1 = binding::linear_from_source( + *weights->store, source, prefix + ".pwconv1", matmul_storage_type, + config.intermediate_dim, config.dim, true); + block.pwconv2 = binding::linear_from_source( + *weights->store, source, prefix + ".pwconv2", matmul_storage_type, + config.dim, config.intermediate_dim, true); + block.gamma = weights->store->load_f32_tensor(source, prefix + ".gamma", {config.dim}); + weights->convnext.push_back(std::move(block)); + } + weights->final_norm = binding::norm_from_source( + *weights->store, source, "backbone.final_layer_norm", config.dim); + weights->head_out = binding::linear_from_source( + *weights->store, source, "head.out", matmul_storage_type, + config.n_fft + 2, config.dim, true); + weights->istft_window = source.require_f32("head.istft.window", {config.n_fft}); + // torchaudio MelSpectrogram keeps both of these as persistent buffers, so + // the analysis filterbank is byte-identical to the reference pipeline's. + weights->analysis_window = source.require_f32( + "feature_extractor.mel_spec.spectrogram.window", {config.n_fft}); + weights->mel_filterbank = source.require_f32( + "feature_extractor.mel_spec.mel_scale.fb", {config.n_fft / 2 + 1, config.n_mels}); + weights->store->upload(); + return weights; +} + +engine::core::TensorValue build_convnext_block( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & input_bct, + const SoproVocoderConvNeXtWeights & weights, + const SoproVocoderConfig & config) { + auto hidden = engine::modules::DepthwiseConv1dModule({ + config.dim, 7, 1, 3, 1, weights.dwconv.bias.has_value(), + }).build(ctx, input_bct, weights.dwconv); + hidden = engine::modules::TransposeModule(swap_channel_time()).build(ctx, hidden); + hidden = engine::modules::LayerNormModule({config.dim, 1.0e-6F, true, true}) + .build(ctx, hidden, weights.norm); + hidden = engine::modules::LinearModule({config.dim, config.intermediate_dim, true, GGML_PREC_F32}) + .build(ctx, hidden, weights.pwconv1); + hidden = engine::modules::GeluModule({engine::modules::GeluApproximation::ExactErf}).build(ctx, hidden); + hidden = engine::modules::LinearModule({config.intermediate_dim, config.dim, true, GGML_PREC_F32}) + .build(ctx, hidden, weights.pwconv2); + hidden = scale_last_dim(ctx, hidden, weights.gamma); + hidden = engine::modules::TransposeModule(swap_channel_time()).build(ctx, hidden); + return engine::modules::AddModule{}.build(ctx, input_bct, hidden); +} + +engine::core::TensorValue build_vocoder_head( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & mel_bct, + const SoproVocoderWeights & weights, + const SoproVocoderConfig & config) { + auto hidden = engine::modules::Conv1dModule({ + config.n_mels, config.dim, 7, 1, 3, 1, weights.embed.bias.has_value(), + }).build(ctx, mel_bct, weights.embed); + hidden = engine::modules::TransposeModule(swap_channel_time()).build(ctx, hidden); + hidden = engine::modules::LayerNormModule({config.dim, 1.0e-6F, true, true}) + .build(ctx, hidden, weights.norm); + hidden = engine::modules::TransposeModule(swap_channel_time()).build(ctx, hidden); + for (const auto & block : weights.convnext) { + hidden = build_convnext_block(ctx, hidden, block, config); + } + hidden = engine::modules::TransposeModule(swap_channel_time()).build(ctx, hidden); + hidden = engine::modules::LayerNormModule({config.dim, 1.0e-6F, true, true}) + .build(ctx, hidden, weights.final_norm); + return engine::modules::LinearModule({config.dim, config.n_fft + 2, true, GGML_PREC_F32}) + .build(ctx, hidden, weights.head_out); +} + +} // namespace + +int64_t band_limit_bin(const SoproVocoderConfig & config) { + const int64_t freq_bins = config.n_fft / 2 + 1; + if (config.band_limit_hz <= 0.0F) { + return freq_bins; + } + const auto cut = static_cast(std::ceil( + static_cast(config.band_limit_hz) * static_cast(config.n_fft) / + static_cast(config.sample_rate))); + return std::clamp(cut, 0, freq_bins); +} + +namespace { + +// ISTFTHead.spectrogram + ISTFT.forward. torch.fft.irfft(norm="backward") +// scales by 1/n_fft; the overlap-add envelope is fold(window^2) and, unlike +// the streaming path, the offline path divides by it without clamping. +std::vector istft_from_head( + const std::vector & head, + int64_t frames, + const SoproVocoderConfig & config, + const std::vector & window, + size_t threads) { + const int64_t freq_bins = config.n_fft / 2 + 1; + const int64_t out_dim = config.n_fft + 2; + if (static_cast(head.size()) != frames * out_dim) { + throw std::runtime_error("Sopro vocoder head output shape mismatch"); + } + if (static_cast(window.size()) != config.n_fft) { + throw std::runtime_error("Sopro vocoder ISTFT window shape mismatch"); + } + if (frames < 2) { + throw std::runtime_error("Sopro vocoder requires at least two mel frames"); + } + const float log_max_magnitude = std::log(config.max_magnitude); + // Everything at or above band_limit_hz is zeroed before the inverse + // transform, which kills the vocoder's high-frequency hiss. spectrum is + // value-initialised, so the loop below simply stops at the cut instead of + // writing zeros over the tail. + const int64_t synthesised_bins = band_limit_bin(config); + std::vector> spectrum(static_cast(frames * freq_bins)); + const int omp_threads = static_cast(std::max(1, threads)); +#ifdef _OPENMP +#pragma omp parallel for num_threads(omp_threads) if (frames >= 8) +#endif + for (int64_t frame = 0; frame < frames; ++frame) { + const float * row = head.data() + static_cast(frame * out_dim); + for (int64_t freq = 0; freq < synthesised_bins; ++freq) { + const float magnitude = std::exp(std::min(row[freq], log_max_magnitude)); + const float phase = row[freq_bins + freq]; + spectrum[static_cast(frame * freq_bins + freq)] = { + magnitude * std::cos(phase), magnitude * std::sin(phase)}; + } + } + + std::vector framed(static_cast(frames * config.n_fft), 0.0F); + engine::audio::real_fft_inverse( + {static_cast(frames), static_cast(config.n_fft)}, + { + static_cast(freq_bins * static_cast(sizeof(std::complex))), + static_cast(sizeof(std::complex)), + }, + { + static_cast(config.n_fft * static_cast(sizeof(float))), + static_cast(sizeof(float)), + }, + 1, spectrum.data(), framed.data(), + 1.0F / static_cast(config.n_fft), threads); + + const int64_t output_size = (frames - 1) * config.hop_length + config.n_fft; + std::vector folded(static_cast(output_size), 0.0F); + std::vector envelope(static_cast(output_size), 0.0F); + { + // Blocked over the output axis so each sample is accumulated in the + // same frame order as the serial loop. + const int64_t block = 4096; + const int64_t blocks = (output_size + block - 1) / block; +#ifdef _OPENMP +#pragma omp parallel for num_threads(omp_threads) if (blocks > 1) +#endif + for (int64_t b = 0; b < blocks; ++b) { + const int64_t begin = b * block; + const int64_t end = std::min(output_size, begin + block); + int64_t first = (begin - config.n_fft) / config.hop_length + 1; + first = std::max(first, 0); + int64_t last = std::min((end - 1) / config.hop_length, frames - 1); + for (int64_t frame = first; frame <= last; ++frame) { + const int64_t start = frame * config.hop_length; + const int64_t i0 = std::max(begin - start, 0); + const int64_t i1 = std::min(end - start, config.n_fft); + const float * src = framed.data() + static_cast(frame * config.n_fft); + for (int64_t i = i0; i < i1; ++i) { + const float w = window[static_cast(i)]; + folded[static_cast(start + i)] += src[i] * w; + envelope[static_cast(start + i)] += w * w; + } + } + } + } + + const int64_t pad = config.n_fft / 2; + const int64_t samples = output_size - 2 * pad; + if (samples <= 0) { + throw std::runtime_error("Sopro vocoder ISTFT produced no samples after trimming"); + } + std::vector audio(static_cast(samples), 0.0F); + for (int64_t i = 0; i < samples; ++i) { + const size_t src = static_cast(i + pad); + const float denominator = envelope[src]; + audio[static_cast(i)] = denominator != 0.0F ? folded[src] / denominator : 0.0F; + } + return audio; +} + +} // namespace + +struct SoproVocoderGraph { + SoproVocoderGraph( + ggml_backend_t backend_in, + engine::core::BackendType backend_type, + size_t graph_context_bytes, + const SoproVocoderConfig & config_in, + std::shared_ptr weights_in, + int64_t frames_in) + : backend(backend_in), + weights(std::move(weights_in)), + frames(frames_in), + head_dim(config_in.n_fft + 2), + config(&config_in) { + if (backend == nullptr || weights == nullptr) { + throw std::runtime_error("Sopro vocoder graph requires a backend and weights"); + } + if (frames <= 0) { + throw std::runtime_error("Sopro vocoder graph requires a positive frame count"); + } + ggml_init_params params{graph_context_bytes, nullptr, true}; + ctx.reset(ggml_init(params)); + if (ctx == nullptr) { + throw std::runtime_error("failed to initialize the Sopro vocoder graph context"); + } + engine::core::ModuleBuildContext build_ctx{ctx.get(), "sopro_tts.vocoder", backend_type}; + const auto shape = engine::core::TensorShape::from_dims({1, config_in.n_mels, frames}); + input = engine::core::make_tensor(build_ctx, GGML_TYPE_F32, shape).tensor; + ggml_set_input(input); + auto head = build_vocoder_head( + build_ctx, engine::core::wrap_tensor(input, shape, GGML_TYPE_F32), *weights, config_in); + head = engine::core::ensure_backend_addressable_layout(build_ctx, head); + output = head.tensor; + ggml_set_output(output); + graph = ggml_new_graph_custom(ctx.get(), 65536, false); + ggml_build_forward_expand(graph, output); + gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (gallocr == nullptr || !ggml_gallocr_reserve(gallocr, graph) || + !ggml_gallocr_alloc_graph(gallocr, graph)) { + throw std::runtime_error("failed to allocate the Sopro vocoder graph"); + } + } + + ~SoproVocoderGraph() { + if (gallocr != nullptr) { + ggml_gallocr_free(gallocr); + gallocr = nullptr; + } + } + + bool matches(const SoproVocoderWeights & other, int64_t other_frames) const noexcept { + return weights.get() == &other && frames == other_frames; + } + + std::vector run(const std::vector & mel, size_t threads) { + ggml_backend_tensor_set(input, mel.data(), 0, mel.size() * sizeof(float)); + const ggml_status status = engine::core::compute_backend_graph(backend, graph); + ggml_backend_synchronize(backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Sopro vocoder graph compute failed"); + } + std::vector head(static_cast(frames * head_dim), 0.0F); + ggml_backend_tensor_get(output, head.data(), 0, head.size() * sizeof(float)); + return istft_from_head(head, frames, *config, weights->istft_window, threads); + } + + ggml_backend_t backend = nullptr; + std::shared_ptr weights; + int64_t frames = 0; + int64_t head_dim = 0; + const SoproVocoderConfig * config = nullptr; + std::unique_ptr ctx; + ggml_tensor * input = nullptr; + ggml_tensor * output = nullptr; + ggml_cgraph * graph = nullptr; + ggml_gallocr_t gallocr = nullptr; +}; + +SoproVocoderRuntime::SoproVocoderRuntime( + const SoproTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) + : config_(assets.config.vocoder), + execution_context_(execution_context), + graph_context_bytes_(graph_context_bytes), + weights_(load_vocoder_weights( + execution_context.backend(), + execution_context.backend_type(), + *assets.vocoder_weights, + assets.config.vocoder, + weight_context_bytes, + matmul_storage_type, + conv_storage_type)) {} + +SoproVocoderRuntime::~SoproVocoderRuntime() = default; + +std::vector SoproVocoderRuntime::decode( + const std::vector & mel, + int64_t frames) const { + if (frames <= 0 || static_cast(mel.size()) != frames * config_.n_mels) { + throw std::runtime_error("Sopro vocoder requires a [n_mels, frames] input"); + } + if (graph_ == nullptr || !graph_->matches(*weights_, frames)) { + // Free the previous arena first; otherwise both are resident while the + // replacement is allocated, and every segment rebuilds this graph. + graph_.reset(); + graph_ = std::make_unique( + execution_context_.backend(), + execution_context_.backend_type(), + graph_context_bytes_, + config_, + weights_, + frames); + } + return graph_->run(mel, static_cast(execution_context_.config().threads)); +} + +std::vector SoproVocoderRuntime::log_mel(const std::vector & audio) const { + if (audio.empty()) { + throw std::runtime_error("Sopro vocoder mel extraction requires a non-empty waveform"); + } + const int64_t freq_bins = config_.n_fft / 2 + 1; + engine::audio::STFTConfig stft; + stft.n_fft = config_.n_fft; + stft.hop_length = config_.hop_length; + stft.win_length = config_.n_fft; + stft.center = true; + stft.pad_mode = engine::audio::STFTPadMode::Reflect; + const auto magnitude = engine::audio::STFT{}.compute_magnitude( + audio, weights_->analysis_window, 1, static_cast(audio.size()), stft, + static_cast(execution_context_.config().threads)); + if (magnitude.shape.size() != 3 || magnitude.shape[1] != freq_bins) { + throw std::runtime_error("Sopro vocoder STFT produced an unexpected layout"); + } + const int64_t frames = magnitude.shape[2]; + std::vector mel(static_cast(config_.n_mels * frames), 0.0F); + // MelScale: mel[m, t] = sum_f magnitude[f, t] * fb[f, m], then log-clamped. + for (int64_t f = 0; f < freq_bins; ++f) { + const float * fb_row = weights_->mel_filterbank.data() + static_cast(f * config_.n_mels); + const float * spec_row = magnitude.values.data() + static_cast(f * frames); + for (int64_t m = 0; m < config_.n_mels; ++m) { + const float weight = fb_row[m]; + if (weight == 0.0F) { + continue; + } + float * out_row = mel.data() + static_cast(m * frames); + for (int64_t t = 0; t < frames; ++t) { + out_row[t] += weight * spec_row[t]; + } + } + } + for (auto & value : mel) { + value = std::log(std::max(value, 1.0e-7F)); + } + return mel; +} + +int64_t SoproVocoderRuntime::mel_frames(int64_t samples) const noexcept { + return samples / config_.hop_length + 1; // centred STFT +} + +int64_t SoproVocoderRuntime::hop_length() const noexcept { return config_.hop_length; } +int64_t SoproVocoderRuntime::n_mels() const noexcept { return config_.n_mels; } +int SoproVocoderRuntime::sample_rate() const noexcept { return static_cast(config_.sample_rate); } + +} // namespace engine::community_models::sopro_tts diff --git a/tests/sopro_tts/sopro_probe.cpp b/tests/sopro_tts/sopro_probe.cpp new file mode 100644 index 000000000..1dcd6fc38 --- /dev/null +++ b/tests/sopro_tts/sopro_probe.cpp @@ -0,0 +1,383 @@ +// Stage probe for the sopro_tts port. +// +// The offline pipeline has five stages and a broken one is hard to localise +// from the final waveform. This binary exercises them in isolation: +// +// mel analysis mel -> vocoder -> waveform. Vocos is trained to invert +// its own analysis mel, so a clean round trip proves the mel front +// end and the whole vocoder graph at once. +// semantic reference waveform -> FSQ token ids (histogram + first ids). +// speaker reference waveform -> id/style/style-ctrl embedding statistics. +// +// Usage: sopro_probe [out-dir] + +#include "engine/community_models/sopro_tts/acoustic.h" +#include "engine/community_models/sopro_tts/assets.h" +#include "engine/community_models/sopro_tts/reference.h" +#include "engine/community_models/sopro_tts/semantic_encoder.h" +#include "engine/community_models/sopro_tts/speaker_encoder.h" +#include "engine/community_models/sopro_tts/vocoder.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/audio/conversion.h" +#include "engine/framework/audio/resampling.h" +#include "engine/framework/audio/wav_reader.h" +#include "engine/framework/audio/wav_writer.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/execution_context.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sopro = engine::community_models::sopro_tts; + +namespace { + +struct Stats { + float min = 0.0F; + float max = 0.0F; + double mean = 0.0; + double rms = 0.0; + size_t nonfinite = 0; +}; + +Stats describe(const std::vector & values) { + Stats out; + if (values.empty()) { + return out; + } + out.min = out.max = values.front(); + double sum = 0.0; + double square_sum = 0.0; + for (const float value : values) { + if (!std::isfinite(value)) { + ++out.nonfinite; + continue; + } + out.min = std::min(out.min, value); + out.max = std::max(out.max, value); + sum += value; + square_sum += static_cast(value) * value; + } + const auto count = static_cast(values.size()); + out.mean = sum / count; + out.rms = std::sqrt(square_sum / count); + return out; +} + +void print_stats(const char * label, const std::vector & values) { + const auto stats = describe(values); + std::printf( + " %-22s n=%-8zu min=%+.4f max=%+.4f mean=%+.5f rms=%.5f nonfinite=%zu\n", + label, values.size(), stats.min, stats.max, stats.mean, stats.rms, stats.nonfinite); +} + +// Segmental SNR between two aligned signals, in dB. +double snr_db(const std::vector & reference, const std::vector & test) { + const size_t n = std::min(reference.size(), test.size()); + double signal = 0.0; + double noise = 0.0; + for (size_t i = 0; i < n; ++i) { + const double r = reference[i]; + const double d = r - test[i]; + signal += r * r; + noise += d * d; + } + if (noise <= 0.0) { + return 999.0; + } + return 10.0 * std::log10(std::max(signal, 1e-20) / noise); +} + +} // namespace + +int main(int argc, char ** argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s [out-dir]\n", argv[0]); + return 2; + } + const std::filesystem::path model_path = argv[1]; + const std::filesystem::path reference_path = argv[2]; + const std::filesystem::path out_dir = argc > 3 ? std::filesystem::path(argv[3]) + : std::filesystem::path("."); + std::filesystem::create_directories(out_dir); + + auto assets = sopro::load_sopro_tts_assets(model_path); + const auto & config = assets->config; + const auto sample_rate = static_cast(config.sample_rate); + + engine::core::BackendConfig backend_config; + backend_config.type = engine::core::BackendType::Cpu; + backend_config.threads = 8; + engine::core::ExecutionContext execution(backend_config); + + constexpr size_t kWeightBytes = 512ull * 1024ull * 1024ull; + constexpr size_t kGraphBytes = 1024ull * 1024ull * 1024ull; + const auto storage = engine::assets::TensorStorageType::F32; + + // ---- input ---- + const auto wav = engine::audio::read_wav_f32(reference_path); + std::vector mono = wav.channels == 1 + ? wav.samples + : engine::audio::mixdown_interleaved_to_mono_average(wav.samples, wav.channels); + if (wav.sample_rate != sample_rate) { + mono = engine::audio::resample_mono_torchaudio_sinc_hann(mono, wav.sample_rate, sample_rate); + } + for (auto & value : mono) { + value = std::min(1.0F, std::max(-1.0F, value)); + } + std::printf("reference: %s\n", reference_path.string().c_str()); + std::printf(" source_rate=%d channels=%d samples=%zu -> %.3f s at %d Hz\n", + wav.sample_rate, wav.channels, mono.size(), + static_cast(mono.size()) / sample_rate, sample_rate); + + // ---- crop / level normalisation, as prepare_reference does ---- + std::mt19937_64 rng(1234); + auto cropped = sopro::audio_ops::crop_on_pause( + mono, config.generation.ref_seconds, sample_rate, rng); + const auto level = sopro::audio_ops::speech_level_db(mono, sample_rate); + std::printf(" speech_level=%.2f dB active=%.3f s\n", level.level_db, level.active_seconds); + std::printf(" crop_on_pause(%.1f s): %zu -> %zu samples (%.3f s)\n", + config.generation.ref_seconds, mono.size(), cropped.size(), + static_cast(cropped.size()) / sample_rate); + const auto crop_level = sopro::audio_ops::speech_level_db(cropped, sample_rate); + float crop_peak = 0.0F; + for (const float value : cropped) { + crop_peak = std::max(crop_peak, std::fabs(value)); + } + auto normalisation = sopro::audio_ops::normalize_reference(cropped, sample_rate); + const auto normalised = std::move(normalisation.wav); + std::printf(" normalize_reference: level %.2f -> %.2f dB (gain %+.2f dB, crop peak %.3f)\n", + crop_level.level_db, normalisation.level_db, + normalisation.level_db - crop_level.level_db, crop_peak); + + // ---- stage: vocoder round trip ---- + { + std::printf("\n[mel] analysis mel -> vocoder round trip\n"); + sopro::SoproVocoderRuntime vocoder( + *assets, execution, kWeightBytes, kGraphBytes, storage, storage); + const auto mel = vocoder.log_mel(normalised); + const int64_t n_mels = vocoder.n_mels(); + const int64_t frames = static_cast(mel.size()) / n_mels; + std::printf(" frames=%lld (expected %lld)\n", + static_cast(frames), + static_cast(vocoder.mel_frames(static_cast(normalised.size())))); + print_stats("log_mel", mel); + // Dump for an independent numpy check of the front end. + { + std::FILE * fh = std::fopen((out_dir / "probe_input_logmel.f32").string().c_str(), "wb"); + if (fh != nullptr) { + std::fwrite(mel.data(), sizeof(float), mel.size(), fh); + std::fclose(fh); + } + std::FILE * wh = std::fopen((out_dir / "probe_input_audio.f32").string().c_str(), "wb"); + if (wh != nullptr) { + std::fwrite(normalised.data(), sizeof(float), normalised.size(), wh); + std::fclose(wh); + } + } + + const auto audio = vocoder.decode(mel, frames); + print_stats("round_trip_audio", audio); + engine::audio::write_pcm16_wav(out_dir / "probe_vocoder_roundtrip.wav", sample_rate, 1, audio); + { + std::FILE * fh = std::fopen((out_dir / "probe_roundtrip_audio.f32").string().c_str(), "wb"); + if (fh != nullptr) { + std::fwrite(audio.data(), sizeof(float), audio.size(), fh); + std::fclose(fh); + } + } + + // Waveform SNR is phase-sensitive and Vocos predicts phase, so the + // meaningful check is whether re-analysing the output reproduces the + // mel the vocoder was asked to render. + std::printf(" waveform SNR vs input: %.2f dB (phase-sensitive, informative only)\n", + snr_db(normalised, audio)); + const auto remel = vocoder.log_mel(audio); + const int64_t reframes = static_cast(remel.size()) / n_mels; + const int64_t common = std::min(frames, reframes); + double abs_sum = 0.0; + double abs_max = 0.0; + size_t counted = 0; + for (int64_t c = 0; c < n_mels; ++c) { + for (int64_t t = 0; t < common; ++t) { + const double a = mel[static_cast(c * frames + t)]; + const double b = remel[static_cast(c * reframes + t)]; + // Ignore bins pinned at the log floor; they carry no signal. + if (a <= -16.0) { + continue; + } + const double d = std::fabs(a - b); + abs_sum += d; + abs_max = std::max(abs_max, d); + ++counted; + } + } + const double mae = counted > 0 ? abs_sum / static_cast(counted) : 0.0; + std::printf(" mel round-trip MAE=%.4f max=%.4f over %zu bins (log units)\n", + mae, abs_max, counted); + std::printf(" -> %s\n", + mae < 0.35 ? "vocoder + mel front end look CORRECT" + : "vocoder or mel front end is WRONG"); + } + + // ---- stage: semantic encoder ---- + { + std::printf("\n[semantic] FSQ tokeniser\n"); + sopro::SoproSemanticEncoderRuntime encoder( + *assets, execution, kWeightBytes, kGraphBytes, storage, storage); + const auto tokens = encoder.encode(normalised); + const int64_t expected = (static_cast(normalised.size()) + + config.semantic_encoder.token_samples_24k - 1) / + config.semantic_encoder.token_samples_24k; + std::printf(" tokens=%zu (expected %lld)\n", tokens.size(), + static_cast(expected)); + std::map histogram; + for (const int32_t token : tokens) { + ++histogram[token]; + } + int32_t lo = tokens.empty() ? 0 : *std::min_element(tokens.begin(), tokens.end()); + int32_t hi = tokens.empty() ? 0 : *std::max_element(tokens.begin(), tokens.end()); + std::printf(" distinct=%zu range=[%d, %d] of [0, %lld)\n", + histogram.size(), lo, hi, + static_cast(config.model.semantic_vocab_size)); + std::printf(" first ids:"); + for (size_t i = 0; i < std::min(16, tokens.size()); ++i) { + std::printf(" %d", tokens[i]); + } + std::printf("\n"); + // A collapsed tokeniser (one id repeated) means the encoder is broken. + int most = 0; + for (const auto & [id, count] : histogram) { + (void) id; + most = std::max(most, count); + } + const double share = tokens.empty() ? 0.0 : static_cast(most) / tokens.size(); + std::printf(" most common id share: %.1f%% %s\n", share * 100.0, + share > 0.5 ? "(COLLAPSED - encoder likely wrong)" : "(looks healthy)"); + } + + // ---- stage: speaker encoder ---- + { + std::printf("\n[speaker] identity / style embeddings\n"); + sopro::SoproSpeakerEncoderRuntime encoder( + *assets, execution, kWeightBytes, kGraphBytes, storage, storage); + const auto wav16 = engine::audio::resample_mono_torchaudio_sinc_hann( + normalised, sample_rate, static_cast(encoder.sample_rate())); + const auto embeddings = encoder.encode(wav16); + print_stats("id_emb", embeddings.id_emb); + print_stats("style_emb", embeddings.style_emb); + print_stats("style_ctrl", embeddings.style_ctrl); + double norm = 0.0; + for (const float value : embeddings.id_emb) { + norm += static_cast(value) * value; + } + std::printf(" |id_emb| = %.6f (must be 1.0)\n", std::sqrt(norm)); + } + + // ---- stage: acoustic head self-reconstruction ---- + // Re-render the second half of the reference from its own semantic tokens, + // conditioned on the first half as the prompt. The acoustic head is lossy + // (it only sees FSQ ids) but a correct one tracks the real mel closely; + // a broken one produces something uncorrelated with it. + { + std::printf("\n[acoustic] self-reconstruction from the reference's own tokens\n"); + sopro::SoproVocoderRuntime vocoder( + *assets, execution, kWeightBytes, kGraphBytes, storage, storage); + sopro::SoproSpeakerEncoderRuntime speaker( + *assets, execution, kWeightBytes, kGraphBytes, storage, storage); + sopro::SoproSemanticEncoderRuntime semantic( + *assets, execution, kWeightBytes, kGraphBytes, storage, storage); + sopro::SoproReferenceBuilder builder(*assets, speaker, semantic, vocoder); + std::mt19937_64 build_rng(1234); + const auto voice = builder.build(mono, config.generation.ref_seconds, build_rng); + + const int64_t n_mels = config.model.acoustic_mel_n_mels; + const int64_t hop_ratio = config.hop_ratio(); + const auto total_tokens = static_cast(voice.semantic_tokens.size()); + const int64_t split = total_tokens / 2; + const int64_t prompt_frames = std::min(voice.mel_frames, split * hop_ratio); + const int64_t gen_tokens = total_tokens - split; + const int64_t total_frames = prompt_frames + gen_tokens * hop_ratio; + std::printf(" reference: %lld tokens, %lld mel frames\n", + static_cast(total_tokens), static_cast(voice.mel_frames)); + std::printf(" prompt: %lld frames, regenerating %lld tokens -> %lld frames\n", + static_cast(prompt_frames), static_cast(gen_tokens), + static_cast(total_frames)); + + sopro::SoproAcousticRequest request; + request.semantic_tokens = voice.semantic_tokens; + request.cond_vec = voice.cond_vec; + request.prompt_mel.assign(static_cast(n_mels * prompt_frames), 0.0F); + for (int64_t c = 0; c < n_mels; ++c) { + for (int64_t t = 0; t < prompt_frames; ++t) { + request.prompt_mel[static_cast(c * prompt_frames + t)] = + voice.mel[static_cast(c * voice.mel_frames + t)]; + } + } + request.prompt_frames = prompt_frames; + request.total_frames = total_frames; + request.steps = 32; // many steps: isolate the field from the schedule + request.seed = 1234; + + sopro::SoproAcousticRuntime acoustic( + *assets, execution, kWeightBytes, kGraphBytes, storage, storage); + const auto solved = acoustic.solve(request); + print_stats("solved_mel(normalised)", solved); + + // Compare only the regenerated span against the true reference mel. + const int64_t compare_end = std::min(total_frames, voice.mel_frames); + double abs_sum = 0.0; + double ref_sq = 0.0; + double err_sq = 0.0; + size_t counted = 0; + for (int64_t c = 0; c < n_mels; ++c) { + for (int64_t t = prompt_frames; t < compare_end; ++t) { + const double truth = voice.mel[static_cast(c * voice.mel_frames + t)]; + const double got = solved[static_cast(c * total_frames + t)]; + abs_sum += std::fabs(truth - got); + ref_sq += truth * truth; + err_sq += (truth - got) * (truth - got); + ++counted; + } + } + if (counted > 0) { + const double mae = abs_sum / static_cast(counted); + const double nmse = err_sq / std::max(ref_sq, 1e-12); + std::printf(" regenerated span vs true mel: MAE=%.4f NMSE=%.4f (%zu bins)\n", + mae, nmse, counted); + std::printf(" -> %s\n", + nmse < 0.6 ? "acoustic head tracks the reference (looks CORRECT)" + : "acoustic head output is uncorrelated with the reference (WRONG)"); + } + std::FILE * fh = std::fopen((out_dir / "probe_acoustic_solved.f32").string().c_str(), "wb"); + if (fh != nullptr) { + std::fwrite(solved.data(), sizeof(float), solved.size(), fh); + std::fclose(fh); + } + std::FILE * th = std::fopen((out_dir / "probe_ref_tokens.i32").string().c_str(), "wb"); + if (th != nullptr) { + std::fwrite(voice.semantic_tokens.data(), sizeof(int32_t), voice.semantic_tokens.size(), th); + std::fclose(th); + } + std::FILE * ch = std::fopen((out_dir / "probe_cond_vec.f32").string().c_str(), "wb"); + if (ch != nullptr) { + std::fwrite(voice.cond_vec.data(), sizeof(float), voice.cond_vec.size(), ch); + std::fclose(ch); + } + std::FILE * mh = std::fopen((out_dir / "probe_ref_mel.f32").string().c_str(), "wb"); + if (mh != nullptr) { + std::fwrite(voice.mel.data(), sizeof(float), voice.mel.size(), mh); + std::fclose(mh); + } + } + + std::printf("\nwrote %s\n", (out_dir / "probe_vocoder_roundtrip.wav").string().c_str()); + return 0; +} diff --git a/tests/sopro_tts/test_sopro_tts_audio_ops.cpp b/tests/sopro_tts/test_sopro_tts_audio_ops.cpp new file mode 100644 index 000000000..fb480169e --- /dev/null +++ b/tests/sopro_tts/test_sopro_tts_audio_ops.cpp @@ -0,0 +1,182 @@ +// Host-side checks for the sopro reference level chain and the ISTFT head's +// band limit. Both are pure functions over plain buffers, so none of this needs +// the checkpoint; the weight-bound stages are covered by sopro_probe instead. +#include "engine/community_models/sopro_tts/assets.h" +#include "engine/community_models/sopro_tts/reference.h" +#include "engine/community_models/sopro_tts/vocoder.h" +#include "test_assert.h" + +#include +#include +#include +#include +#include + +namespace { + +namespace test = engine::test; +namespace sopro = engine::community_models::sopro_tts; +namespace audio_ops = sopro::audio_ops; + +constexpr int kSampleRate = 24000; + +// A constant-amplitude buffer makes speech_level_db exact: every 25 ms frame +// has the same RMS, so the 0.2-quantile gate keeps all of them and the median +// is the amplitude itself. Peak and speech level therefore agree, which keeps +// the expected values below plain arithmetic. +std::vector flat(float amplitude, float seconds = 1.0F) { + const auto samples = static_cast(static_cast(kSampleRate) * seconds); + return std::vector(samples, amplitude); +} + +float peak_of(const std::vector & wav) { + float peak = 0.0F; + for (const float value : wav) { + peak = std::max(peak, std::fabs(value)); + } + return peak; +} + +void test_speech_level_db_reads_a_flat_buffer() { + const auto level = audio_ops::speech_level_db(flat(0.1F), kSampleRate); + test::require_close(level.level_db, -20.0F, 1.0e-3F, "flat 0.1 level"); + // 98 frames at a 10 ms hop; all of them survive the activity gate. + test::require_close(level.active_seconds, 0.98F, 1.0e-3F, "flat 0.1 active seconds"); + + // Shorter than one 25 ms window: whole-buffer RMS, and no active span. + const auto tiny = audio_ops::speech_level_db(std::vector(100, 0.1F), kSampleRate); + test::require_close(tiny.level_db, -20.0F, 1.0e-3F, "short buffer level"); + test::require_close(tiny.active_seconds, 0.0F, 1.0e-6F, "short buffer active seconds"); + + // Silence floors at 1e-6 rather than diverging. + const auto silent = audio_ops::speech_level_db(std::vector(100, 0.0F), kSampleRate); + test::require_close(silent.level_db, -120.0F, 1.0e-3F, "silence level"); +} + +void test_quiet_reference_is_boosted_to_the_prompt_level() { + const auto input = flat(0.01F); // -40 dB, 20.2 dB below the prompt level + const auto out = audio_ops::normalize_reference(input, kSampleRate); + + test::require_close(out.level_db, audio_ops::kPromptLevelDb, 1.0e-3F, "boosted level"); + const float expected_gain = std::pow(10.0F, 20.2F / 20.0F); + test::require_close(out.wav.front(), 0.01F * expected_gain, 1.0e-6F, "boosted sample"); + // Well clear of the 0.95 ceiling, so the peak guard must not have bitten. + test::require(peak_of(out.wav) < 0.95F, "boosted peak stays below the ceiling"); + test::require_eq(out.wav.size(), input.size(), "boosted length"); +} + +void test_hot_reference_is_left_alone() { + // -6.02 dB, well above the prompt level. The pre-2.1 rule attenuated this + // by 13.78 dB; boost-only must pass it through untouched. + const auto input = flat(0.5F); + const auto out = audio_ops::normalize_reference(input, kSampleRate); + + test::require_close(out.level_db, -6.0206F, 1.0e-3F, "hot level is unchanged"); + for (size_t i = 0; i < input.size(); i += 997) { + test::require_eq(out.wav[i], input[i], "hot sample is unchanged"); + } +} + +void test_peak_guard_caps_the_boost() { + // An impulse train with a high crest factor: every 25 ms window holds + // exactly six 0.9 spikes, so the frame RMS is a uniform 0.09 (-20.92 dB) + // and the activity gate keeps all of it. A lone spike would not do — the + // gate would keep only the frames containing it and read the level off + // those. The buffer wants +1.12 dB but the peak leaves only +0.47 dB. + std::vector input(static_cast(kSampleRate), 0.0F); + for (size_t i = 0; i < input.size(); i += 100) { + input[i] = 0.9F; + } + const auto out = audio_ops::normalize_reference(input, kSampleRate); + + const float capped_gain = 20.0F * std::log10(0.95F / 0.9F); + test::require_close(out.level_db, -20.9151F + capped_gain, 1.0e-2F, "peak-guarded level"); + test::require_close(peak_of(out.wav), 0.95F, 1.0e-4F, "peak lands on the ceiling"); + test::require(out.level_db < audio_ops::kPromptLevelDb, "peak guard undershoots the target"); +} + +void test_boost_is_limited_to_thirty_db() { + // -60 dB with 59.5 dB of peak headroom, so the 30 dB gain limit is what + // binds rather than the ceiling. + const auto out = audio_ops::normalize_reference(flat(0.001F), kSampleRate); + test::require_close(out.level_db, -30.0F, 1.0e-2F, "gain-limited level"); +} + +void test_output_gain_tracks_the_reference_level() { + test::require_close( + audio_ops::output_gain(), std::pow(10.0F, -3.2F / 20.0F), 1.0e-6F, "default output gain"); + // A hotter reference has to be pulled down further to reach -23 dB. + test::require( + audio_ops::output_gain(-11.24F) < audio_ops::output_gain(), + "a hot reference gets a smaller output gain"); + test::require_close( + audio_ops::output_gain(-11.24F), std::pow(10.0F, -11.76F / 20.0F), 1.0e-6F, + "hot reference output gain"); +} + +void test_match_gain_falls_back_to_the_reference_level() { + // Under kMinActiveSeconds of measurable speech, so match_gain cannot level + // off the audio itself and defers to the reference it was cloned from. + const std::vector too_short(100, 0.1F); + test::require_close( + audio_ops::match_gain(too_short, kSampleRate, audio_ops::kOutputLevelDb, -11.24F), + audio_ops::output_gain(-11.24F), 1.0e-6F, "fallback uses the reference level"); + test::require( + audio_ops::match_gain(too_short, kSampleRate, audio_ops::kOutputLevelDb, -11.24F) != + audio_ops::match_gain(too_short, kSampleRate), + "fallback varies with the reference level"); + + // With enough speech to measure, the reference level is irrelevant: the + // gain comes from the audio actually produced. + const auto measurable = flat(0.1F); // -20 dB, 0.98 s active + const float expected = std::pow(10.0F, -3.0F / 20.0F); + test::require_close( + audio_ops::match_gain(measurable, kSampleRate), expected, 1.0e-4F, "measured gain"); + test::require_close( + audio_ops::match_gain(measurable, kSampleRate, audio_ops::kOutputLevelDb, -11.24F), + expected, 1.0e-4F, "measured gain ignores the reference level"); +} + +void test_band_limit_bin() { + sopro::SoproVocoderConfig config; // 24 kHz, n_fft 1024, 10900 Hz + // ceil(10900 * 1024 / 24000) = 466 of 513 bins, i.e. a 10921.9 Hz cut. + test::require_eq(sopro::band_limit_bin(config), int64_t{466}, "default cut"); + + config.band_limit_hz = 0.0F; + test::require_eq(sopro::band_limit_bin(config), int64_t{513}, "zero keeps every bin"); + config.band_limit_hz = -1.0F; + test::require_eq(sopro::band_limit_bin(config), int64_t{513}, "negative keeps every bin"); + + // Nyquist itself is bin 512, and the cut is inclusive, so a 12 kHz limit + // still drops that last bin — which is the one the unlimited head used to + // synthesise with a bogus imaginary part. + config.band_limit_hz = 12000.0F; + test::require_eq(sopro::band_limit_bin(config), int64_t{512}, "Nyquist cut"); + config.band_limit_hz = 48000.0F; + test::require_eq(sopro::band_limit_bin(config), int64_t{513}, "above Nyquist clamps"); + + // The cut follows the transform size, not a hardcoded bin index. + config.band_limit_hz = 10900.0F; + config.n_fft = 2048; + test::require_eq(sopro::band_limit_bin(config), int64_t{931}, "n_fft 2048 cut"); +} + +} // namespace + +int main() { + try { + test_speech_level_db_reads_a_flat_buffer(); + test_quiet_reference_is_boosted_to_the_prompt_level(); + test_hot_reference_is_left_alone(); + test_peak_guard_caps_the_boost(); + test_boost_is_limited_to_thirty_db(); + test_output_gain_tracks_the_reference_level(); + test_match_gain_falls_back_to_the_reference_level(); + test_band_limit_bin(); + } catch (const std::exception & error) { + std::cerr << "FAIL: " << error.what() << "\n"; + return 1; + } + std::cout << "PASS: sopro_tts audio ops checks\n"; + return 0; +} diff --git a/tools/audiocpp_cli/audiocpp_cli_longform_tts_clone_cases.json b/tools/audiocpp_cli/audiocpp_cli_longform_tts_clone_cases.json index 4ad10c62b..30db64f4f 100644 --- a/tools/audiocpp_cli/audiocpp_cli_longform_tts_clone_cases.json +++ b/tools/audiocpp_cli/audiocpp_cli_longform_tts_clone_cases.json @@ -330,6 +330,51 @@ } ] }, + { + "id": "sopro_tts_voice_clone_longform", + "coverage": "Sopro V2 Turbo reference voice clone with shared long-form text for chunking and RTF measurement", + "family": "sopro_tts", + "model": "models/sopro-v2-turbo", + "task": "tts", + "mode": "offline", + "outputs": [ + "audio" + ], + "requests": [ + { + "id": "clone_longform", + "text": "At dawn the harbor station opens its tall windows and the first clerk begins a careful report for the day. She notes the weather above the river, the slow cargo boats beyond the bridge, and the market voices arriving from the eastern road. A brass clock marks each quarter hour while porters stack wooden crates, bakers carry warm bread across the square, and a violinist practices the same bright phrase under the stone archway. By midmorning the keeper of the lighthouse sends a message about shifting currents, the museum guide unlocks a cabinet of maps, and a teacher leads a quiet line of students toward the ferry. In the afternoon a painter describes the silver color of the water, a mechanic jokes with the tram driver, and the station master reads an announcement that asks every traveler to keep close watch over letters, tickets, and parcels. After sunset the same clerk continues the report because new visitors keep arriving from the inland road. She explains that a florist carries pale roses past the fountain, two carpenters compare measurements beside the warehouse door, and the watchman checks each lock before the tide reaches its highest mark. A child laughs when the tram bell rings, a cook lowers a basket of fruit to the cellar, and three sailors unfold a chart that shows old channels, sandbars, and safe turning points for the morning crossing. Near midnight the lamps still glow on wet stone, the last cart rattles toward the market gate, and the report ends by saying that the harbor remains orderly, the wind has softened, the ferries are secure, and the town can rest until the next sunrise returns over the water. On the following morning the clerk resumes the record with even greater care because a week of inspections is about to begin. She writes that a ferry captain checks the mooring ropes one by one, a bookseller arranges travel guides beside the station cafe, and a pair of gardeners lift wet soil into bright clay pots near the west entrance. The bakery sends out trays of seed bread, the telegraph operator copies three official notices, and a tailor unfolds navy cloth across a polished wooden counter while customers wait in a line that bends toward the fountain. Before noon a surveyor compares bridge numbers against an old ledger, two cousins argue cheerfully about the best route to the fish market, and a choir director rehearses a patient scale that echoes against the warehouse wall. The lighthouse keeper reports that the northern channel is calmer than expected, the harbor pilot recommends a slower turn near the sandbar, and the customs officer stamps a packet of forms before waving a cart through the side gate. Later the schoolteacher returns with another group of students, asking them to observe the colors of rope, paint, stone, and water so they can write more exact descriptions in the classroom. A photographer kneels beside a rain barrel to capture the reflection of the clock tower, a mechanic tightens a brass hinge on the tram door, and an elderly traveler asks the clerk whether the evening ferry still stops at the orchard village beyond the marsh. As dusk arrives, lamps are trimmed again, shutters are tested against the wind, and the station kitchen sends bowls of soup to workers who remain on the late shift. The report continues with notes about a carpenter measuring floorboards in the east hall, a florist tying silver ribbon around the last stems of the day, and a violin case resting open on a bench beside the ticket window while its owner copies melody marks into a notebook. Long after the market gate closes, the clerk still writes that the harbor road stays busy, the river glints beneath scattered lamps, and the town maintains its patient rhythm of signals, footsteps, voices, bells, and distant engines. On the third day the clerk decides the record should be more precise, so she marks each event by the quarter hour and notes which sounds carry farthest through the station concourse. At first light she hears broom bristles on the stone steps, kettle lids in the cafe kitchen, and the slow scrape of crates being nudged across a loading cart beside the river wall. A messenger in a green coat delivers two canvas pouches, the ticket agent counts rolled coins into a brass tray, and a mother reads directions aloud while her son traces the painted ferry schedule with one curious finger. Midmorning brings a burst of sunlight across the waiting hall, making every brass handle shine while the museum guide escorts visitors toward the gallery of maps and navigational instruments. A porter pauses to describe the oldest compass in the display, a student sketches the harbor outline in graphite, and an apprentice clockmaker compares the station bell to a pocket watch that once belonged to his grandfather. By noon the fish market sends salt and seaweed scents through the open doors, tram wheels hiss at the curb, and the baker from the square exchanges a laugh with the florist who is carrying fresh lilies to the hotel veranda. The clerk writes that a cooper rolls three narrow barrels toward the cellar ramp, a translator copies weather bulletins for inland travelers, and a painter in a blue scarf studies the changing color of the tide as if each small wave might explain a different part of the sky. In the late afternoon the station master reviews freight tags, the customs officer checks a parcel of glassware, and a choir of children crosses the square singing a phrase so soft that the watchman removes his cap to listen. Evening settles slowly; lamps brighten in sequence, a cook inventories apples and onions in the pantry, and two sailors spread a faded chart on a crate so they can debate whether the shoals have shifted since the previous autumn. Before sleep the clerk closes the day with a final note that every vessel is accounted for, every platform has been swept, every lock has been tested twice, and the harbor seems ready to welcome another tide, another market, and another patient stream of voices at sunrise.", + "language": "en", + "voice_ref": "assets/resources/a.wav", + "seed": 1234, + "num_inference_steps": 8, + "text_chunk_size": 200 + } + ] + }, + { + "id": "sopro_tts_voice_clone_longform_streaming", + "coverage": "Sopro V2 Turbo streaming clone: one pull event per text segment over the same long-form text", + "family": "sopro_tts", + "model": "models/sopro-v2-turbo", + "task": "tts", + "mode": "streaming", + "outputs": [ + "audio", + "named_audio" + ], + "requests": [ + { + "id": "clone_longform_streaming", + "text": "At dawn the harbor station opens its tall windows and the first clerk begins a careful report for the day. She notes the weather above the river, the slow cargo boats beyond the bridge, and the market voices arriving from the eastern road. A brass clock marks each quarter hour while porters stack wooden crates, bakers carry warm bread across the square, and a violinist practices the same bright phrase under the stone archway. By midmorning the keeper of the lighthouse sends a message about shifting currents, the museum guide unlocks a cabinet of maps, and a teacher leads a quiet line of students toward the ferry. In the afternoon a painter describes the silver color of the water, a mechanic jokes with the tram driver, and the station master reads an announcement that asks every traveler to keep close watch over letters, tickets, and parcels. After sunset the same clerk continues the report because new visitors keep arriving from the inland road. She explains that a florist carries pale roses past the fountain, two carpenters compare measurements beside the warehouse door, and the watchman checks each lock before the tide reaches its highest mark. A child laughs when the tram bell rings, a cook lowers a basket of fruit to the cellar, and three sailors unfold a chart that shows old channels, sandbars, and safe turning points for the morning crossing. Near midnight the lamps still glow on wet stone, the last cart rattles toward the market gate, and the report ends by saying that the harbor remains orderly, the wind has softened, the ferries are secure, and the town can rest until the next sunrise returns over the water. On the following morning the clerk resumes the record with even greater care because a week of inspections is about to begin. She writes that a ferry captain checks the mooring ropes one by one, a bookseller arranges travel guides beside the station cafe, and a pair of gardeners lift wet soil into bright clay pots near the west entrance. The bakery sends out trays of seed bread, the telegraph operator copies three official notices, and a tailor unfolds navy cloth across a polished wooden counter while customers wait in a line that bends toward the fountain. Before noon a surveyor compares bridge numbers against an old ledger, two cousins argue cheerfully about the best route to the fish market, and a choir director rehearses a patient scale that echoes against the warehouse wall. The lighthouse keeper reports that the northern channel is calmer than expected, the harbor pilot recommends a slower turn near the sandbar, and the customs officer stamps a packet of forms before waving a cart through the side gate. Later the schoolteacher returns with another group of students, asking them to observe the colors of rope, paint, stone, and water so they can write more exact descriptions in the classroom. A photographer kneels beside a rain barrel to capture the reflection of the clock tower, a mechanic tightens a brass hinge on the tram door, and an elderly traveler asks the clerk whether the evening ferry still stops at the orchard village beyond the marsh. As dusk arrives, lamps are trimmed again, shutters are tested against the wind, and the station kitchen sends bowls of soup to workers who remain on the late shift. The report continues with notes about a carpenter measuring floorboards in the east hall, a florist tying silver ribbon around the last stems of the day, and a violin case resting open on a bench beside the ticket window while its owner copies melody marks into a notebook. Long after the market gate closes, the clerk still writes that the harbor road stays busy, the river glints beneath scattered lamps, and the town maintains its patient rhythm of signals, footsteps, voices, bells, and distant engines. On the third day the clerk decides the record should be more precise, so she marks each event by the quarter hour and notes which sounds carry farthest through the station concourse. At first light she hears broom bristles on the stone steps, kettle lids in the cafe kitchen, and the slow scrape of crates being nudged across a loading cart beside the river wall. A messenger in a green coat delivers two canvas pouches, the ticket agent counts rolled coins into a brass tray, and a mother reads directions aloud while her son traces the painted ferry schedule with one curious finger. Midmorning brings a burst of sunlight across the waiting hall, making every brass handle shine while the museum guide escorts visitors toward the gallery of maps and navigational instruments. A porter pauses to describe the oldest compass in the display, a student sketches the harbor outline in graphite, and an apprentice clockmaker compares the station bell to a pocket watch that once belonged to his grandfather. By noon the fish market sends salt and seaweed scents through the open doors, tram wheels hiss at the curb, and the baker from the square exchanges a laugh with the florist who is carrying fresh lilies to the hotel veranda. The clerk writes that a cooper rolls three narrow barrels toward the cellar ramp, a translator copies weather bulletins for inland travelers, and a painter in a blue scarf studies the changing color of the tide as if each small wave might explain a different part of the sky. In the late afternoon the station master reviews freight tags, the customs officer checks a parcel of glassware, and a choir of children crosses the square singing a phrase so soft that the watchman removes his cap to listen. Evening settles slowly; lamps brighten in sequence, a cook inventories apples and onions in the pantry, and two sailors spread a faded chart on a crate so they can debate whether the shoals have shifted since the previous autumn. Before sleep the clerk closes the day with a final note that every vessel is accounted for, every platform has been swept, every lock has been tested twice, and the harbor seems ready to welcome another tide, another market, and another patient stream of voices at sunrise.", + "language": "en", + "voice_ref": "assets/resources/a.wav", + "seed": 1234, + "num_inference_steps": 8, + "text_chunk_size": 200 + } + ] + }, { "id": "inflect_v2_tts_longform", "coverage": "Inflect v2 fixed-voice TTS with repeated requests, punctuation-aware long-form chunking, graph reuse, and RTF measurement", diff --git a/webui/configs/model_params.json b/webui/configs/model_params.json index 7b3d71bbc..98f2fd2ad 100644 --- a/webui/configs/model_params.json +++ b/webui/configs/model_params.json @@ -360,6 +360,17 @@ {"name": "temperature", "type": "slider", "label": "temperature", "default": 0.7, "minimum": 0.0, "maximum": 2.0, "step": 0.05} ], + "sopro_tts": [ + {"name": "language", "type": "choice", "label": "language", "label_en": "Language tag", "default": "", "choices": ["", "en", "pt", "fr", "de"], "info": "Optional <|lang_xx|> tag; helps pronunciation on ambiguous text."}, + {"name": "temperature", "type": "slider", "label": "temperature", "label_en": "Temperature", "default": 0.8, "minimum": 0.0, "maximum": 2.0, "step": 0.05}, + {"name": "top_p", "type": "slider", "label": "top_p", "label_en": "Top-p", "default": 0.9, "minimum": 0.0, "maximum": 1.0, "step": 0.01}, + {"name": "top_k", "type": "number", "label": "top_k", "label_en": "Top-k", "default": 25, "minimum": 0, "step": 1, "precision": 0, "info": "0 disables top-k truncation."}, + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps", "label_en": "Acoustic steps", "default": 2, "minimum": 1, "maximum": 32, "step": 1, "precision": 0, "info": "Rectified-flow Euler steps for the acoustic head."}, + {"name": "max_seconds", "type": "number", "label": "max_seconds", "label_en": "Max seconds per segment", "default": 30.0, "minimum": 1.0, "maximum": 60.0, "step": 0.5, "precision": 1}, + {"name": "min_seconds", "type": "number", "label": "min_seconds", "label_en": "Min seconds per segment", "default": 0.4, "minimum": 0.0, "maximum": 10.0, "step": 0.1, "precision": 1, "info": "Must not exceed max_seconds."}, + {"name": "ref_seconds", "type": "number", "label": "ref_seconds", "label_en": "Reference seconds", "default": 10.0, "minimum": 1.0, "maximum": 30.0, "step": 0.5, "precision": 1, "info": "Reference window used for cloning."}, + {"name": "text_chunk_size", "type": "number", "label": "text_chunk_size", "label_en": "Segment size", "default": 300, "minimum": 20, "maximum": 2000, "step": 10, "precision": 0, "info": "Max codepoints per synthesis segment."} + ], "supertonic": [ {"name": "voice", "type": "choice", "label": "voice（预置音色：M 男声 / F 女声）", "label_en": "voice (M = male, F = female presets)", "default": "M1", "choices": ["M1", "M2", "M3", "M4", "M5", "F1", "F2", "F3", "F4", "F5"]}, {"name": "speaking_rate", "type": "slider", "label": "speaking_rate（语速倍率）", "label_en": "speaking_rate", "default": 1.05, "minimum": 0.5, "maximum": 2.0, "step": 0.05}, diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index 36e8f4041..4929f484e 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -15,6 +15,7 @@ { "id": "qwen3-tts-1.7b", "display_name": "Qwen3-TTS 1.7B Base (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-Base", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_base", "min_vram_gb": 8 }, { "id": "qwen3-tts-1.7b-custom", "display_name": "Qwen3-TTS 1.7B CustomVoice (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-CustomVoice", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_custom_voice", "min_vram_gb": 8 }, { "id": "miotts", "display_name": "MioTTS 1.7B (tts; needs MioCodec)", "family": "miotts", "path": "models/MioTTS-1.7B", "task": "tts", "mode": "offline", "download_id": "miotts_1_7b", "min_vram_gb": 8 }, + { "id": "sopro-tts", "display_name": "Sopro V2 Turbo (tts + clone)", "family": "sopro_tts", "path": "models/sopro-v2-turbo", "task": "tts", "mode": "offline", "download_id": "sopro_v2_turbo_safetensors", "min_vram_gb": 2, "request_options": ["language", "temperature", "top_p", "top_k", "num_inference_steps", "max_seconds", "min_seconds", "ref_seconds", "text_chunk_size", "seed"] }, { "id": "soprano-tts", "display_name": "Soprano TTS (tts)", "family": "soprano_tts", "path": "models/Soprano-1.1-80M-GGUF", "task": "tts", "mode": "offline", "download_id": "soprano_1_1_80m_q8_0", "min_vram_gb": 1 }, { "id": "voxcpm2", "display_name": "VoxCPM2 (tts)", "family": "voxcpm2", "path": "models/VoxCPM2", "task": "tts", "mode": "offline", "download_id": "voxcpm2", "session_options": { "voxcpm2.weight_type": "q8_0" }, "min_vram_gb": 6 }, { "id": "voxcpm1", "display_name": "VoxCPM1 0.5B (tts + clone)", "family": "voxcpm1", "path": "models/VoxCPM1-GGUF", "task": "tts", "mode": "offline", "download_id": "voxcpm1_0.5b_q8_0", "min_vram_gb": 4 }, diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index 39e4b1ef9..11f5def14 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,20 +31,20 @@