diff --git a/CMakeLists.txt b/CMakeLists.txt index 01199cade..4389a9aed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -630,6 +630,19 @@ audiocpp_add_model(personaplex engine::models::personaplex::make_personaplex_loader ) +audiocpp_add_model(sanotts + SOURCES + src/community_models/sanotts/assets.cpp + src/community_models/sanotts/frontend.cpp + src/community_models/sanotts/piper_runtime.cpp + src/community_models/sanotts/runtime.cpp + src/community_models/sanotts/session.cpp + INCLUDES + engine/community_models/sanotts/session.h + LOADERS + engine::models::sanotts::make_sanotts_loader +) + audiocpp_add_model(inflect_v2 SOURCES src/community_models/inflect_v2/assets.cpp @@ -2545,6 +2558,13 @@ if (ENGINE_BUILD_TESTS) COMMAND echo_tts_host_units ) + add_engine_unittest(sanotts_frontend_test tests/unittests/test_sanotts_frontend.cpp) + target_include_directories(sanotts_frontend_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests) + add_test( + NAME sanotts_frontend_test + COMMAND sanotts_frontend_test + ) + add_engine_unittest(inflect_v2_frontend_test tests/unittests/test_inflect_v2_frontend.cpp) target_include_directories(inflect_v2_frontend_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests) add_test( diff --git a/README.md b/README.md index 7c95c6024..a9bbd328f 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ Community model ports live under `community_models` to make the ownership bounda | **moss_voicegen** | Voice Design | en, zh | GGUF | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](docs/community_models/moss_voicegen.md) speech in a voice designed from a written instruction | | **outetts** | TTS, Clone | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | GGUF | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | GGUF F32/16/Q8, Stream | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](docs/community_models/parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | +| **sanotts** | TTS | en, vi, id | GGUF FP32 | Ashish [@voidash](https://github.com/voidash) | [sanoTTS voice family](docs/community_models/sanotts.md) seven voices from 294k to 2.27M parameters, native offline synthesis | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | GGUF Q8, Stream | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](docs/community_models/sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **soprano_tts** | TTS | en | GGUF Q8, Stream | [@WalkingCat](https://github.com/WalkingCat) | [Soprano-1.1-80M](https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF) ultra-lightweight TTS with Qwen3 LM + Vocos decoder | | **vietneu_tts** | TTS, Clone | vi, en | GGUF | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](docs/community_models/vietneu_tts.md) TTS and voice cloning support | diff --git a/docs/community_models/models.md b/docs/community_models/models.md index 994556b52..d88164887 100644 --- a/docs/community_models/models.md +++ b/docs/community_models/models.md @@ -31,6 +31,7 @@ Practical expectations: | **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Mirek [@mirek190](https://github.com/mirek190) | [Llama-OuteTTS-1.0-1B](outetts.md) TTS and voice cloning support | | **voxcpm1** | TTS, voice cloning | zh, en, ja, ko | Community | [VoxCPM1](voxcpm1.md) tokenizer-free 0.5B TTS with 16 kHz output, streaming, and continuation-mode voice cloning | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | +| **sanotts** | TTS | en, vi, id | Community | [sanoTTS voice family](sanotts.md) seven voices from 294k to 2.27M parameters, FP32 offline synthesis | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **vietneu_tts** | TTS, voice cloning | vi, en | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](vietneu_tts.md) TTS and voice cloning support | | **moss_voicegen** | Voice design | en, zh | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](moss_voicegen.md) voice design from a written instruction, on the MOSS delay architecture | diff --git a/docs/community_models/sanotts.md b/docs/community_models/sanotts.md new file mode 100644 index 000000000..ff88f8071 --- /dev/null +++ b/docs/community_models/sanotts.md @@ -0,0 +1,129 @@ +# sanoTTS voice family + +`sanotts` provides native GGML inference for the +[sanoTTS](https://github.com/Ampixa/sanoTTS) voice family — very small +text-to-speech models, the smallest of which also runs on microcontrollers. +All packages download from Hugging Face +([ampixa/sanoTTS](https://huggingface.co/ampixa/sanoTTS) `gguf/`) as +standalone FP32 GGUFs with embedded model specs. Offline FP32 inference only. + +Two graphs share one family: + +- **nano** — duration student → contextual acoustic student → mel-100 → + noise-fed ConvNeXt-1D decoder → [log-magnitude | phase] head → inverse + STFT. 24 kHz. A seed picks one of many valid renderings. +- **piperlite** — duration student → contextual acoustic student → 192-channel + latent → 3-stage ConvTranspose1d decoder with dilated residual banks → + tanh waveform. 22.05 kHz. Fully deterministic (no seed). + +| Package | Voice | Graph | Params | Language | Notes | +|---|---|---|---:|---|---| +| `sanotts_heart_orig` | heart | nano | 2,272,145 | en | best quality of the nano pair | +| `sanotts_heart_nano_orig` | heart-nano | nano | 294,279 | en | microcontroller-class | +| `sanotts_amy_orig` | amy | piperlite | 1,454,284 | en | Piper-distilled | +| `sanotts_hfc_orig` | hfc | piperlite | 1,834,380 | en | largest piperlite voice | +| `sanotts_kristin_orig` | kristin | piperlite | 1,396,151 | en | carries a learned post filter | +| `sanotts_vi_orig` | vi | piperlite | 1,565,484 | vi | Vietnamese | +| `sanotts_id_orig` | id | piperlite | 1,562,124 | id | Indonesian | + +## Install + +Install eSpeak-ng and its voice data first. On Debian or Ubuntu: + +```bash +sudo apt install espeak-ng libespeak-ng1 +``` + +On macOS: + +```bash +brew install espeak-ng +``` + +Then install any package, e.g.: + +```bash +python3 tools/model_manager_v2.py install sanotts_heart_orig --models-root models +python3 tools/model_manager_v2.py install sanotts_amy_orig --models-root models +``` + +## Run + +```bash +audiocpp_cli --task tts --family sanotts \ + --model models/sanoTTS-heart-GGUF --backend cpu \ + --text "Hello from sano T T S, a very small neural text to speech model." \ + --out sanotts.wav +``` + +Swap `--model` for any installed package directory +(`models/sanoTTS-amy-GGUF`, `models/sanoTTS-vi-GGUF`, ...). The Vietnamese +and Indonesian voices accept `--language vi` / `--language id`; a session +rejects text tagged with a language the voice was not trained on. + +eSpeak-ng is loaded dynamically at runtime, never linked. If it is not on the +default library path: + +```bash +audiocpp_cli --task tts --family sanotts \ + --model models/sanoTTS-heart-GGUF --backend cpu \ + --session-option sanotts.espeak_library_path=/path/to/libespeak-ng.so \ + --session-option sanotts.espeak_data_path=/path/to/espeak-ng-data \ + --text "A configured eSpeak installation." --out sanotts.wav +``` + +## Options + +- `speaking_rate` (request, 0.5..2.0, default 1.0) — duration multiplier on + the voice's tuned length scale; larger is slower. +- `seed` (request, default 0) — nano voices only: the decoder is noise-fed, + so a given seed picks one of many valid renderings. `0` derives the seed + from each text chunk as `sha256(text)[:8]`, which is what the reference + implementations do; an explicit seed advances by one per long-form chunk. + Piperlite voices are deterministic and ignore the seed. +- `text_chunk_size` (request, default 280) — maximum codepoints per long-form + chunk; chunks split on sentence punctuation first, and a chunk that + phonemizes past the voice's token limit is bisected at whitespace. + +## Determinism and parity + +The runtimes reproduce the reference implementations' exact semantics: + +- Front ends: the phonemizer punctuation-preservation pipeline through the + same eSpeak-ng library. The nano voices add the misaki E2M rewrite with + tie characters; the piperlite voices use Piper's NFD-decompose-to- + codepoints convention, per-voice `phoneme_id_map`, `[BOS, PAD, (id, PAD)…, + EOS]` framing, and the schwa fallback for ids outside a component's + trained vocabulary. +- nano: ATen-compatible MT19937 noise (24-bit uniform, Box–Muller in blocks + of 16), torch.istft window normalisation and centre trim, and the + reference's DC blocker `H(z) = (1 - z^-1)/(1 - 0.9973 z^-1)`. +- Shared: torch.linspace / expand_features float behaviour, LayerNorm eps + 1e-6 (nano), ties-to-even duration rounding. + +Measured against the project's numpy references (same text, same +eSpeak-ng build), every voice: **correlation ≥ 0.99999996 with identical +sample counts**; max sample delta ~1.7e-05 is the WAV's own int16 +quantisation. The numpy references are themselves gated ≥ 0.987 against the +float PyTorch models. + +## Performance + +CPU-only, 12-thread x86 (default 4 backend threads), FP32, the shared 6 kB +long-form text: + +| Voice | Audio | Wall | vs real time | Peak RSS | +|---|---:|---:|---:|---:| +| heart-nano | 373 s | 1.3 s | ~283× | 220 MB | +| amy | 394 s | 18.5 s | ~21× | 497 MB | + +The nano decoder runs at frame rate with a host iSTFT; the piperlite decoder +runs convolutions at audio rate, which is why it is heavier. Graphs are +cached per token count (duration and token stages) and per frame count +(decoder); `--log` prints cache hits and per-stage timings. + +## Licensing + +The sanoTTS runtimes and weights are MIT-licensed. eSpeak-ng is GPL-3.0 and +is therefore opened with `dlopen` at runtime and never linked, matching how +`inflect_v2` treats it. diff --git a/docs/tts.md b/docs/tts.md index ac3b720e3..a85f7c4a6 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -29,6 +29,7 @@ | GLM-TTS | `glm_tts` | `tts`, `clon` | [GLM-TTS](#glm-tts) | | Inflect Micro v2 | `inflect_v2` | `tts` | [Inflect v2](#inflect-v2) | | OuteTTS | `outetts` | `tts`, `clon` | [OuteTTS](#outetts) | +| sanoTTS voice family | `sanotts` | `tts` | [sanoTTS](#sanotts) | | Supertonic | `supertonic` | `tts` | [Supertonic](#supertonic) | | VieNeu-TTS | `vietneu_tts` | `tts`, `clon` | [VieNeu-TTS](community_models/vietneu_tts.md) | | VibeVoice | `vibevoice` | `tts` | [VibeVoice](#vibevoice) | @@ -768,6 +769,27 @@ See the [Inflect v2 community model guide](community_models/inflect_v2.md) for eSpeak-ng paths, long-form behavior, source/conversion instructions, and limitations. +## sanoTTS + +sanoTTS is a family of very small offline TTS voices (English, Vietnamese, +Indonesian; 294k to 2.27M parameters) with native GGML runtimes; the +smallest voice also runs on microcontrollers. The GGUF packages are +standalone and download from Hugging Face. sanoTTS requires an external +eSpeak-ng installation: + +```bash +python3 tools/model_manager_v2.py install sanotts_heart_nano_orig --models-root models + +audiocpp_cli --task tts --family sanotts \ + --model models/sanoTTS-heart-nano-GGUF --backend cpu \ + --text "Hello from sano T T S, a very small neural text to speech model." \ + --request-option speaking_rate=1.0 \ + --out sanotts.wav +``` + +See the [sanoTTS community model guide](community_models/sanotts.md) for +eSpeak-ng paths, seed semantics, parity evidence, and performance numbers. + ## Supertonic Supertonic 3 is a preset-voice multilingual TTS model. It does not use external speaker references in the current integration. diff --git a/include/engine/community_models/sanotts/assets.h b/include/engine/community_models/sanotts/assets.h new file mode 100644 index 000000000..144bf9939 --- /dev/null +++ b/include/engine/community_models/sanotts/assets.h @@ -0,0 +1,91 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { + +struct SanoTtsConfig { + int64_t vocab_size = 62; + int64_t sample_rate = 24000; + int64_t hop_length = 256; + int64_t n_fft = 1024; + int64_t mels = 100; + int64_t dim = 0; + int64_t blocks = 0; + int64_t pw_hidden = 0; + int64_t noise_channels = 4; + int64_t dw_kernel = 7; + int64_t embed_kernel = 7; + + int64_t duration_hidden = 0; + int64_t duration_depth = 0; + int64_t duration_kernel = 5; + int64_t duration_max_tokens = 207; + int64_t duration_max_frames = 80; + + int64_t acoustic_hidden = 0; + int64_t acoustic_token_depth = 0; + int64_t acoustic_depth = 0; + int64_t acoustic_kernel = 5; + + std::string voice; +}; + +enum class SanoTtsGraph { + Nano, // mel-100 -> ConvNeXt-1D -> iSTFT, noise-fed (heart, heart-nano) + Piperlite, // 192-ch latent -> 3-stage ConvTranspose1d, deterministic (amy, ...) +}; + +struct SanoTtsPiperConfig { + std::string voice; + std::string language; // short code the session validates against: en, vi, id + std::string espeak_voice; + int64_t sample_rate = 22050; + double duration_length_scale = 1.0; + + int64_t duration_vocab = 0; + int64_t duration_hidden = 0; + int64_t duration_depth = 0; + int64_t duration_kernel = 5; + int64_t duration_max_tokens = 0; + int64_t duration_max_frames = 0; + + int64_t acoustic_vocab = 0; + int64_t acoustic_hidden = 0; + int64_t acoustic_depth = 0; + int64_t acoustic_token_depth = 0; + int64_t acoustic_kernel = 5; + int64_t acoustic_out_channels = 0; + + std::array channels = {0, 0, 0, 0}; + std::array, 3> stage_branches; + int64_t post_filter_channels = 0; + int64_t post_filter_layers = 0; + int64_t post_filter_kernel = 9; + double post_filter_scale = 0.0; + + /** Piper phoneme_id_map: one UTF-8 codepoint -> id. */ + std::unordered_map phoneme_id_map; +}; + +struct SanoTtsAssets { + assets::ResourceBundle resources; + SanoTtsGraph graph = SanoTtsGraph::Nano; + SanoTtsConfig config; // valid when graph == Nano + SanoTtsPiperConfig piper; // valid when graph == Piperlite + std::shared_ptr weights; +}; + +std::shared_ptr load_sanotts_assets( + const std::filesystem::path & model_path); + +} // namespace engine::models::sanotts diff --git a/include/engine/community_models/sanotts/frontend.h b/include/engine/community_models/sanotts/frontend.h new file mode 100644 index 000000000..9ce91e51d --- /dev/null +++ b/include/engine/community_models/sanotts/frontend.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { + +/** Thrown by encode() when a chunk phonemizes past the duration model's + * token limit; the session responds by bisecting the chunk. */ +struct SanoTtsTooLongError : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +struct SanoTtsEncoded { + std::vector token_ids; + std::string dropped; // symbols outside the vocabulary, for tracing +}; + +/** + * Text -> the 62-symbol phoneme ids the sanoTTS front end was trained on. + * + * eSpeak-ng produces the IPA; misaki's E2M table then rewrites it into the + * character-level inventory this model uses. Both steps are reproduced from + * the project's own JavaScript and Python front ends so the three agree + * symbol for symbol. + * + * eSpeak-ng is opened at runtime and never linked, matching how inflect_v2 + * treats it: it is GPL-3.0 and must not be embedded in this project. + */ +class SanoTtsFrontend { +public: + SanoTtsFrontend( + std::filesystem::path espeak_library_path, + std::filesystem::path espeak_data_path, + int64_t max_tokens); + ~SanoTtsFrontend(); + + [[nodiscard]] SanoTtsEncoded encode(const std::string & text) const; + + /** Long-form splitting on sentence punctuation, then a codepoint budget. */ + [[nodiscard]] static std::vector split_text( + const std::string & text, + int64_t max_codepoints); + + /** Pause inserted between chunks, longer after a sentence end. */ + [[nodiscard]] static double boundary_pause_seconds(const std::string & chunk); + +private: + struct Impl; + std::unique_ptr impl_; + int64_t max_tokens_; +}; + +/** + * Text -> Piper phoneme ids for the piperlite voices. + * + * Reproduces piper's phonemize_espeak / phonemes_to_ids convention through + * the same eSpeak-ng library: phonemizer-style punctuation preservation, + * NFD decomposition to single codepoints, then the voice's phoneme_id_map + * with [BOS, PAD, (id, PAD)..., EOS] framing. Deterministic per voice. + */ +class SanoTtsPiperFrontend { +public: + SanoTtsPiperFrontend( + std::filesystem::path espeak_library_path, + std::filesystem::path espeak_data_path, + std::string espeak_voice, + std::unordered_map phoneme_id_map, + int64_t max_tokens); + ~SanoTtsPiperFrontend(); + + [[nodiscard]] SanoTtsEncoded encode(const std::string & text) const; + +private: + struct Impl; + std::unique_ptr impl_; + std::unordered_map id_map_; + int64_t max_tokens_; +}; + +} // namespace engine::models::sanotts diff --git a/include/engine/community_models/sanotts/piper_runtime.h b/include/engine/community_models/sanotts/piper_runtime.h new file mode 100644 index 000000000..77d08a01e --- /dev/null +++ b/include/engine/community_models/sanotts/piper_runtime.h @@ -0,0 +1,34 @@ +#pragma once + +#include "engine/community_models/sanotts/assets.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include + +namespace engine::models::sanotts { + +struct SanoTtsPiperGenerationOptions { + /** Multiplier on the voice's tuned duration_length_scale; larger is + * slower. The decoder is deterministic -- there is no seed. */ + float speaking_rate = 1.0F; +}; + +class SanoTtsPiperRuntime { +public: + SanoTtsPiperRuntime( + std::shared_ptr assets, + core::BackendConfig backend_config); + ~SanoTtsPiperRuntime(); + + runtime::AudioBuffer synthesize( + const std::vector & token_ids, + const SanoTtsPiperGenerationOptions & options); + +private: + struct State; + std::unique_ptr state_; +}; + +} // namespace engine::models::sanotts diff --git a/include/engine/community_models/sanotts/runtime.h b/include/engine/community_models/sanotts/runtime.h new file mode 100644 index 000000000..750cb8759 --- /dev/null +++ b/include/engine/community_models/sanotts/runtime.h @@ -0,0 +1,41 @@ +#pragma once + +#include "engine/community_models/sanotts/assets.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include +#include + +namespace engine::models::sanotts { + +struct SanoTtsGenerationOptions { + /** Duration multiplier applied before rounding; larger is slower. */ + float speaking_rate = 1.0F; + /** Decoder noise seed; the session resolves the derive-from-text default + * before calling the runtime, so this is always the final seed. */ + uint64_t seed = 0; +}; + +class SanoTtsNativeRuntime { +public: + SanoTtsNativeRuntime( + std::shared_ptr assets, + core::BackendConfig backend_config); + ~SanoTtsNativeRuntime(); + + runtime::AudioBuffer synthesize( + const std::vector & token_ids, + const SanoTtsGenerationOptions & options); + +private: + struct State; + std::unique_ptr state_; +}; + +/** int.from_bytes(sha256(text).digest()[:8], "big") -- the seed the reference + * implementations derive when the caller does not pass one. */ +uint64_t sanotts_text_seed(const std::string & text); + +} // namespace engine::models::sanotts diff --git a/include/engine/community_models/sanotts/session.h b/include/engine/community_models/sanotts/session.h new file mode 100644 index 000000000..464737ad6 --- /dev/null +++ b/include/engine/community_models/sanotts/session.h @@ -0,0 +1,43 @@ +#pragma once + +#include "engine/community_models/sanotts/assets.h" +#include "engine/community_models/sanotts/frontend.h" +#include "engine/community_models/sanotts/piper_runtime.h" +#include "engine/community_models/sanotts/runtime.h" +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/session_base.h" + +#include + +namespace engine::models::sanotts { + +std::shared_ptr make_sanotts_loader(); + +class SanoTtsSession final + : public runtime::RuntimeSessionBase + , public runtime::IOfflineVoiceTaskSession { +public: + SanoTtsSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract); + ~SanoTtsSession() override; + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest & request) override; + runtime::TaskResult run(const runtime::TaskRequest & request) override; + +private: + runtime::TaskSpec task_; + std::shared_ptr assets_; + std::shared_ptr contract_; + std::unique_ptr frontend_; // nano graph + std::unique_ptr runtime_; // nano graph + std::unique_ptr piper_frontend_; // piperlite graph + std::unique_ptr piper_runtime_; // piperlite graph +}; + +} // namespace engine::models::sanotts diff --git a/model_specs/sanotts.json b/model_specs/sanotts.json new file mode 100644 index 000000000..8d98fa9e1 --- /dev/null +++ b/model_specs/sanotts.json @@ -0,0 +1,212 @@ +{ + "schema_version": 1, + "family": "sanotts", + "display_name": "sanoTTS Nano", + "description": "Very small English, Vietnamese and Indonesian text-to-speech. Two graphs: the nano lineage (duration student, contextual acoustic student to mel-100, noise-fed ConvNeXt-1D decoder with an iSTFT head; voices heart 2.27M and heart-nano 294k, 24 kHz) and the deterministic piperlite lineage (duration student, acoustic student to a 192-channel latent, 3-stage ConvTranspose1d decoder with dilated residual banks; voices amy, hfc, kristin, vi, id at 1.1-1.8M parameters, 22.05 kHz). Uses an external eSpeak-ng phonemizer.", + "category": "tts", + "status": "community", + "tasks": [ + "tts" + ], + "modes": [ + "offline" + ], + "languages": [ + "en", + "vi", + "id" + ], + "runtime": { + "tags": [ + "gguf" + ] + }, + "capabilities": { + "tts": [ + "long_form" + ] + }, + "options": { + "request": [ + { + "name": "speaking_rate", + "type": "float", + "description": "Duration multiplier on the voice's tuned length scale; larger is slower. Applied before the per-token clamp.", + "required": false, + "min": 0.5, + "max": 2.0, + "default": 1.0 + }, + { + "name": "seed", + "type": "int", + "description": "Decoder noise seed. The decoder is noise-fed, so a given seed picks one of many valid renderings; 0 derives it from the text as sha256(text)[:8], which is what the reference implementations do. Piperlite voices are deterministic and ignore the seed.", + "required": false, + "min": 0, + "default": 0 + }, + { + "name": "text_chunk_mode", + "type": "enum", + "description": "Long-form text chunking mode.", + "values": [ + "word_budget" + ], + "required": false, + "default": "word_budget" + }, + { + "name": "text_chunk_size", + "type": "int", + "description": "Maximum Unicode codepoints per long-form text chunk; default 280.", + "required": false, + "min": 1, + "default": 280 + } + ], + "session": [ + { + "name": "espeak_library_path", + "type": "path", + "description": "Optional explicit path to the eSpeak-ng shared library.", + "required": false + }, + { + "name": "espeak_data_path", + "type": "path", + "description": "Optional explicit path to the directory containing espeak-ng-data.", + "required": false + } + ], + "load": [] + }, + "package_defaults": { + "download": { + "kind": "huggingface_snapshot", + "repo": "ampixa/sanoTTS", + "revision": "main", + "gated": false + } + }, + "packages": [ + { + "id": "sanotts_heart_nano_orig", + "display_name": "sanoTTS heart-nano 294k FP32 GGUF", + "default": true, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-heart-nano-GGUF", + "files": [ + "gguf/heart-nano-f32.gguf", + "gguf/config.json" + ], + "strip_prefix": "gguf" + }, + { + "id": "sanotts_heart_orig", + "display_name": "sanoTTS heart 2.27M FP32 GGUF", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-heart-GGUF", + "files": [ + "gguf/heart/heart-f32.gguf", + "gguf/heart/config.json" + ], + "strip_prefix": "gguf/heart" + }, + { + "id": "sanotts_amy_orig", + "display_name": "sanoTTS amy 1.46M FP32 GGUF (English, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-amy-GGUF", + "files": [ + "gguf/amy/amy-f32.gguf", + "gguf/amy/config.json" + ], + "strip_prefix": "gguf/amy" + }, + { + "id": "sanotts_hfc_orig", + "display_name": "sanoTTS hfc 1.83M FP32 GGUF (English, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-hfc-GGUF", + "files": [ + "gguf/hfc/hfc-f32.gguf", + "gguf/hfc/config.json" + ], + "strip_prefix": "gguf/hfc" + }, + { + "id": "sanotts_kristin_orig", + "display_name": "sanoTTS kristin 1.40M FP32 GGUF (English, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-kristin-GGUF", + "files": [ + "gguf/kristin/kristin-f32.gguf", + "gguf/kristin/config.json" + ], + "strip_prefix": "gguf/kristin" + }, + { + "id": "sanotts_vi_orig", + "display_name": "sanoTTS vi 1.57M FP32 GGUF (Vietnamese, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-vi-GGUF", + "files": [ + "gguf/vi/vi-f32.gguf", + "gguf/vi/config.json" + ], + "strip_prefix": "gguf/vi" + }, + { + "id": "sanotts_id_orig", + "display_name": "sanoTTS id 1.56M FP32 GGUF (Indonesian, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-id-GGUF", + "files": [ + "gguf/id/id-f32.gguf", + "gguf/id/config.json" + ], + "strip_prefix": "gguf/id" + } + ], + "dependencies": [], + "ui": { + "recommended_package": "sanotts_heart_nano_orig", + "tags": [ + "TTS", + "GGUF" + ], + "docs": [ + "docs/tts.md", + "docs/community_models/sanotts.md", + "docs/gguf.md" + ] + }, + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "config": "model:config.json" + }, + "tensors": { + "weights": "weights:" + } + } + ] +} diff --git a/src/community_models/sanotts/assets.cpp b/src/community_models/sanotts/assets.cpp new file mode 100644 index 000000000..a20abb77f --- /dev/null +++ b/src/community_models/sanotts/assets.cpp @@ -0,0 +1,343 @@ +#include "engine/community_models/sanotts/assets.h" + +#include "engine/framework/io/json.h" +#include "engine/framework/io/config.h" +#include "engine/framework/model_spec/package.h" + +#include +#include +#include + +namespace engine::models::sanotts { +namespace { + +namespace json = engine::io::json; + +constexpr const char * kFamily = "sanotts"; + +SanoTtsConfig parse_nano_config(const engine::io::json::Value & root) { + SanoTtsConfig out; + out.voice = root.require("voice").as_string(); + out.vocab_size = root.require("vocab_size").as_i64(); + out.sample_rate = root.require("sample_rate").as_i64(); + out.hop_length = root.require("hop_length").as_i64(); + out.n_fft = root.require("n_fft").as_i64(); + out.mels = root.require("mels").as_i64(); + out.dim = root.require("dim").as_i64(); + out.blocks = root.require("blocks").as_i64(); + out.pw_hidden = root.require("pw_hidden").as_i64(); + out.noise_channels = root.require("noise_channels").as_i64(); + out.dw_kernel = root.require("dw_kernel").as_i64(); + out.embed_kernel = root.require("embed_kernel").as_i64(); + + const auto & duration = root.require("duration"); + out.duration_hidden = duration.require("hidden").as_i64(); + out.duration_depth = duration.require("depth").as_i64(); + out.duration_kernel = duration.require("kernel").as_i64(); + out.duration_max_tokens = duration.require("max_tokens").as_i64(); + out.duration_max_frames = duration.require("max_duration").as_i64(); + + const auto & acoustic = root.require("acoustic"); + out.acoustic_hidden = acoustic.require("hidden").as_i64(); + out.acoustic_token_depth = acoustic.require("token_depth").as_i64(); + out.acoustic_depth = acoustic.require("depth").as_i64(); + out.acoustic_kernel = acoustic.require("kernel").as_i64(); + + for (const auto & [label, value] : std::initializer_list>{ + {"sanoTTS dim", out.dim}, + {"sanoTTS blocks", out.blocks}, + {"sanoTTS pw_hidden", out.pw_hidden}, + {"sanoTTS duration hidden", out.duration_hidden}, + {"sanoTTS acoustic hidden", out.acoustic_hidden}, + {"sanoTTS sample_rate", out.sample_rate}, + }) { + engine::io::require_positive(value, label); + } + return out; +} + +/** + * Fail on a missing or wrongly-shaped tensor at load, not mid-graph. + * + * The decoder is noise-fed and ends in an iSTFT, so a weight that is present + * but wrong in shape tends to produce plausible-sounding audio rather than an + * obvious failure. Checking the whole inventory up front is what keeps a + * packaging mistake loud. + */ +void validate_nano_tensors(const SanoTtsAssets & assets) { + const auto & c = assets.config; + const auto & weights = *assets.weights; + + std::vector>> expected; + const auto conv = [&](const std::string & name, int64_t out_ch, int64_t in_ch, int64_t k) { + expected.emplace_back(name + ".weight", std::vector{out_ch, in_ch, k}); + expected.emplace_back(name + ".bias", std::vector{out_ch}); + }; + const auto linear = [&](const std::string & name, int64_t out_ch, int64_t in_ch) { + expected.emplace_back(name + ".weight", std::vector{out_ch, in_ch}); + expected.emplace_back(name + ".bias", std::vector{out_ch}); + }; + + expected.emplace_back("duration.embedding.weight", + std::vector{c.vocab_size, c.duration_hidden}); + conv("duration.input_proj", c.duration_hidden, c.duration_hidden + 3, 1); + for (int64_t b = 0; b < c.duration_depth; ++b) { + const std::string prefix = "duration.blocks." + std::to_string(b); + conv(prefix + ".net.0", c.duration_hidden, c.duration_hidden, c.duration_kernel); + conv(prefix + ".net.2", c.duration_hidden, c.duration_hidden, c.duration_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("duration.output", 1, c.duration_hidden, 1); + + expected.emplace_back("acoustic.embedding.weight", + std::vector{c.vocab_size, c.acoustic_hidden}); + conv("acoustic.token_input_proj", c.acoustic_hidden, c.acoustic_hidden + 2, 1); + for (int64_t b = 0; b < c.acoustic_token_depth; ++b) { + const std::string prefix = "acoustic.token_blocks." + std::to_string(b); + conv(prefix + ".net.0", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + conv(prefix + ".net.2", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("acoustic.frame_input_proj", c.acoustic_hidden, c.acoustic_hidden + 3, 1); + for (int64_t b = 0; b < c.acoustic_depth; ++b) { + const std::string prefix = "acoustic.frame_blocks." + std::to_string(b); + conv(prefix + ".net.0", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + conv(prefix + ".net.2", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("acoustic.output", c.mels, c.acoustic_hidden, 1); + + conv("decoder.embed", c.dim, c.mels, c.embed_kernel); + conv("decoder.noise_adapter", c.dim, c.noise_channels, c.embed_kernel); + expected.emplace_back("decoder.norm.weight", std::vector{c.dim}); + expected.emplace_back("decoder.norm.bias", std::vector{c.dim}); + for (int64_t b = 0; b < c.blocks; ++b) { + const std::string prefix = "decoder.blocks." + std::to_string(b); + conv(prefix + ".dwconv", c.dim, 1, c.dw_kernel); // groups == dim + expected.emplace_back(prefix + ".norm.weight", std::vector{c.dim}); + expected.emplace_back(prefix + ".norm.bias", std::vector{c.dim}); + linear(prefix + ".pwconv1", c.pw_hidden, c.dim); + linear(prefix + ".pwconv2", c.dim, c.pw_hidden); + expected.emplace_back(prefix + ".gamma", std::vector{c.dim}); + } + expected.emplace_back("decoder.final_norm.weight", std::vector{c.dim}); + expected.emplace_back("decoder.final_norm.bias", std::vector{c.dim}); + linear("decoder.head", c.n_fft + 2, c.dim); + + for (const auto & [name, shape] : expected) { + if (!weights.has_tensor(name)) { + throw std::runtime_error("sanoTTS missing tensor: " + name); + } + assets::require_tensor_shape(weights, name, shape); + } +} + + +SanoTtsPiperConfig parse_piper_config(const engine::io::json::Value & root) { + SanoTtsPiperConfig out; + out.voice = root.require("voice").as_string(); + out.language = root.require("language").as_string(); + out.espeak_voice = root.require("espeak_voice").as_string(); + out.sample_rate = root.require("sample_rate").as_i64(); + out.duration_length_scale = + static_cast(root.require("duration_length_scale").as_f32()); + + const auto & duration = root.require("duration"); + out.duration_vocab = duration.require("vocab_size").as_i64(); + out.duration_hidden = duration.require("hidden").as_i64(); + out.duration_depth = duration.require("depth").as_i64(); + out.duration_kernel = duration.require("kernel").as_i64(); + out.duration_max_tokens = duration.require("max_tokens").as_i64(); + out.duration_max_frames = duration.require("max_duration").as_i64(); + + const auto & acoustic = root.require("acoustic"); + out.acoustic_vocab = acoustic.require("vocab_size").as_i64(); + out.acoustic_hidden = acoustic.require("hidden").as_i64(); + out.acoustic_depth = acoustic.require("depth").as_i64(); + out.acoustic_token_depth = acoustic.require("token_depth").as_i64(); + out.acoustic_kernel = acoustic.require("kernel").as_i64(); + out.acoustic_out_channels = acoustic.require("out_channels").as_i64(); + + const auto & decoder = root.require("decoder"); + const auto channels = decoder.require("channels").as_array(); + if (channels.size() != 4) { + throw std::runtime_error("sanoTTS piperlite decoder.channels must have 4 entries"); + } + for (size_t stage = 0; stage < 4; ++stage) { + out.channels[stage] = channels[stage].as_i64(); + } + for (size_t stage = 0; stage < 3; ++stage) { + const auto & branches = + decoder.require("stage" + std::to_string(stage) + "_branches").as_array(); + for (const auto & branch : branches) { + const int64_t index = branch.as_i64(); + if (index < 0 || index > 2) { + throw std::runtime_error("sanoTTS piperlite branch index out of range"); + } + out.stage_branches[stage].push_back(index); + } + if (out.stage_branches[stage].empty()) { + throw std::runtime_error("sanoTTS piperlite stage has no branches"); + } + } + out.post_filter_channels = decoder.require("post_filter_channels").as_i64(); + out.post_filter_layers = decoder.require("post_filter_layers").as_i64(); + out.post_filter_kernel = decoder.require("post_filter_kernel").as_i64(); + out.post_filter_scale = + static_cast(decoder.require("post_filter_scale").as_f32()); + + for (const auto & [symbol, id] : root.require("phoneme_id_map").as_object()) { + out.phoneme_id_map.emplace(symbol, static_cast(id.as_i64())); + } + if (out.phoneme_id_map.empty()) { + throw std::runtime_error("sanoTTS piperlite config has an empty phoneme_id_map"); + } + + for (const auto & [label, value] : std::initializer_list>{ + {"sanoTTS piperlite duration vocab", out.duration_vocab}, + {"sanoTTS piperlite duration hidden", out.duration_hidden}, + {"sanoTTS piperlite duration max_tokens", out.duration_max_tokens}, + {"sanoTTS piperlite acoustic vocab", out.acoustic_vocab}, + {"sanoTTS piperlite acoustic hidden", out.acoustic_hidden}, + {"sanoTTS piperlite latent channels", out.acoustic_out_channels}, + {"sanoTTS piperlite sample_rate", out.sample_rate}, + }) { + engine::io::require_positive(value, label); + } + if (out.duration_length_scale <= 0.0) { + throw std::runtime_error("sanoTTS piperlite duration_length_scale must be positive"); + } + return out; +} + +// PiperResidualBank geometry, fixed by the training code. +constexpr int64_t kBankKernels[3] = {3, 5, 7}; + +/** + * The piperlite inventory. Kernel sizes the reference reads from the tensors + * themselves (decoder pre/post convs) are validated as rank/channels only, + * by looking the actual kernel up from the source. + */ +void validate_piper_tensors(const SanoTtsAssets & assets) { + const auto & c = assets.piper; + const auto & weights = *assets.weights; + + std::vector>> expected; + const auto conv = [&](const std::string & name, int64_t out_ch, int64_t in_ch, int64_t k) { + expected.emplace_back(name + ".weight", std::vector{out_ch, in_ch, k}); + expected.emplace_back(name + ".bias", std::vector{out_ch}); + }; + const auto conv_any_kernel = [&](const std::string & name, int64_t out_ch, int64_t in_ch) { + const auto metadata = weights.require_metadata(name + ".weight"); + if (metadata.shape.size() != 3 || metadata.shape[0] != out_ch || + metadata.shape[1] != in_ch || metadata.shape[2] < 1 || + metadata.shape[2] % 2 == 0) { + throw std::runtime_error("sanoTTS unexpected shape for tensor: " + name + ".weight"); + } + expected.emplace_back(name + ".bias", std::vector{out_ch}); + }; + + expected.emplace_back("duration.embedding.weight", + std::vector{c.duration_vocab, c.duration_hidden}); + conv("duration.input_proj", c.duration_hidden, c.duration_hidden + 3, 1); + for (int64_t b = 0; b < c.duration_depth; ++b) { + const std::string prefix = "duration.blocks." + std::to_string(b); + conv(prefix + ".net.0", c.duration_hidden, c.duration_hidden, c.duration_kernel); + conv(prefix + ".net.2", c.duration_hidden, c.duration_hidden, c.duration_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("duration.output", 1, c.duration_hidden, 1); + + expected.emplace_back("acoustic.embedding.weight", + std::vector{c.acoustic_vocab, c.acoustic_hidden}); + conv("acoustic.token_input_proj", c.acoustic_hidden, c.acoustic_hidden + 2, 1); + for (int64_t b = 0; b < c.acoustic_token_depth; ++b) { + const std::string prefix = "acoustic.token_blocks." + std::to_string(b); + conv(prefix + ".net.0", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + conv(prefix + ".net.2", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("acoustic.frame_input_proj", c.acoustic_hidden, c.acoustic_hidden + 3, 1); + for (int64_t b = 0; b < c.acoustic_depth; ++b) { + const std::string prefix = "acoustic.frame_blocks." + std::to_string(b); + conv(prefix + ".net.0", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + conv(prefix + ".net.2", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("acoustic.output", c.acoustic_out_channels, c.acoustic_hidden, 1); + + conv_any_kernel("decoder.pre", c.channels[0], c.acoustic_out_channels); + const int64_t up_kernels[3] = {16, 16, 8}; + for (size_t stage = 0; stage < 3; ++stage) { + const int64_t in_ch = c.channels[stage]; + const int64_t out_ch = c.channels[stage + 1]; + // ConvTranspose1d stores [in, out, K] + expected.emplace_back( + "decoder.up" + std::to_string(stage) + ".weight", + std::vector{in_ch, out_ch, up_kernels[stage]}); + expected.emplace_back( + "decoder.up" + std::to_string(stage) + ".bias", + std::vector{out_ch}); + for (const int64_t branch : c.stage_branches[stage]) { + const std::string prefix = "decoder.res" + std::to_string(stage) + ".0.blocks." + + std::to_string(branch); + conv(prefix + ".conv1", out_ch, out_ch, kBankKernels[branch]); + conv(prefix + ".conv2", out_ch, out_ch, kBankKernels[branch]); + } + } + conv_any_kernel("decoder.post", 1, c.channels[3]); + if (c.post_filter_channels > 0) { + conv_any_kernel("decoder.post_filter.in_conv", c.post_filter_channels, 1); + for (int64_t layer = 0; layer < c.post_filter_layers; ++layer) { + const std::string prefix = "decoder.post_filter.units." + std::to_string(layer); + // Unit conv kernels are a property of the checkpoint, not of + // post_filter_kernel (which sizes only the in/out convs). + conv_any_kernel(prefix + ".conv1", c.post_filter_channels, c.post_filter_channels); + conv_any_kernel(prefix + ".conv2", c.post_filter_channels, c.post_filter_channels); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv_any_kernel("decoder.post_filter.out_conv", 1, c.post_filter_channels); + } + + for (const auto & [name, shape] : expected) { + if (!weights.has_tensor(name)) { + throw std::runtime_error("sanoTTS missing tensor: " + name); + } + assets::require_tensor_shape(weights, name, shape); + } +} + +} // namespace + +std::shared_ptr load_sanotts_assets( + const std::filesystem::path & model_path) { + auto resources = engine::model_spec::load_resource_bundle_for_family(model_path, kFamily); + SanoTtsAssets out; + const auto root = resources.parse_json("config"); + const auto architecture = root.require("architecture").as_string(); + if (architecture != kFamily) { + throw std::runtime_error( + "sanoTTS config.json architecture is '" + architecture + "', expected 'sanotts'"); + } + const auto * graph = root.find("graph"); + const auto graph_name = graph == nullptr ? std::string("nano") : graph->as_string(); + if (graph_name == "nano") { + out.graph = SanoTtsGraph::Nano; + out.config = parse_nano_config(root); + } else if (graph_name == "piperlite") { + out.graph = SanoTtsGraph::Piperlite; + out.piper = parse_piper_config(root); + } else { + throw std::runtime_error("sanoTTS config.json has unknown graph '" + graph_name + "'"); + } + out.weights = resources.open_tensor_source("weights"); + out.resources = std::move(resources); + if (out.graph == SanoTtsGraph::Nano) { + validate_nano_tensors(out); + } else { + validate_piper_tensors(out); + } + return std::make_shared(std::move(out)); +} + +} // namespace engine::models::sanotts diff --git a/src/community_models/sanotts/frontend.cpp b/src/community_models/sanotts/frontend.cpp new file mode 100644 index 000000000..547eabeeb --- /dev/null +++ b/src/community_models/sanotts/frontend.cpp @@ -0,0 +1,1182 @@ +#include "engine/community_models/sanotts/frontend.h" + +#include "engine/framework/io/dynamic_library.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { +namespace { + +using InitializeFn = int (*)(int, int, const char *, int); +using SetVoiceFn = int (*)(const char *); +using TextToPhonemesFn = const char * (*)(const void **, int, int); +using TerminateFn = int (*)(); + +constexpr int kEspeakSynchronous = 2; +constexpr int kEspeakCharsUtf8 = 1; +// IPA output (0x02), tie flag (bit 7), and U+0361 COMBINING DOUBLE INVERTED +// BREVE in bits 8..23 as the tie character -- exactly the phonemes_mode +// phonemizer computes, so the E2M diphthong patterns ("a͡ɪ" -> "I") can match. +constexpr int kEspeakPhonemesIpaTie = 0x02 | (0x01 << 7) | (0x0361 << 8); +// The piperlite voices phonemize with tie=False: IPA with '_' as the phoneme +// separator in bits 8..23, exactly phonemizer's untied phonemes_mode. +constexpr int kEspeakPhonemesIpaUnderscore = 0x02 | ('_' << 8); + +constexpr std::string_view kTieDefault = "͡"; // COMBINING DOUBLE INVERTED BREVE +constexpr std::string_view kTieMisaki = "^"; +constexpr std::string_view kSyllabic = "̩"; // COMBINING VERTICAL LINE BELOW +constexpr std::string_view kNasal = "̃"; // COMBINING TILDE + +/** + * The frozen 62-symbol inventory, exactly as the trained package records it. + * + * Ids are positional and contiguous from zero; 0..2 are , , + * and never appear in text. + */ +const std::unordered_map & vocabulary() { + static const std::unordered_map table = [] { + static constexpr std::array symbols = { + " ", "!", "\"", "(", ")", ",", ".", ":", ";", + "?", "A", "I", "O", "T", "W", "Y", "b", + "d", "f", "h", "i", "j", "k", "l", "m", + "n", "p", "s", "t", "u", "v", "w", "z", + "æ", "ð", "ŋ", "ɐ", "ɑ", + "ɔ", "ə", "ɛ", "ɜ", "ɡ", + "ɪ", "ɹ", "ʃ", "ʊ", "ʌ", + "ʒ", "ʤ", "ʧ", "ˈ", "ˌ", + "θ", "ᵊ", "ᵻ", "—", "“", + "”", + }; + std::unordered_map out; + int32_t id = 3; // 0..2 are the specials + for (const char * symbol : symbols) { + out.emplace(symbol, id++); + } + if (out.size() + 3 != 62) { + throw std::runtime_error("sanoTTS compiled symbol inventory is invalid"); + } + return out; + }(); + return table; +} + +void replace_all(std::string & value, std::string_view from, std::string_view to) { + if (from.empty()) { + return; + } + size_t position = 0; + while ((position = value.find(from, position)) != std::string::npos) { + value.replace(position, from.size(), to); + position += to.size(); + } +} + +/** + * misaki EspeakFallback's E2M rewrite, british=false. + * + * Order is load-bearing and matches the Python source, which sorts by + * descending key length: "e͡ɪ" must be tried before the bare "e", + * or every diphthong collapses to the wrong symbol. + */ +std::string apply_e2m(std::string ps) { + static const std::array, 20> kE2M = {{ + {"ʔˌn̩", "ʔn"}, + {"ʔn̩", "ʔn"}, + {"a^ɪ", "I"}, {"a^ʊ", "W"}, {"d^ʒ", "ʤ"}, + {"e^ɪ", "A"}, {"t^ʃ", "ʧ"}, {"ɔ^ɪ", "Y"}, + {"ə^l", "ᵊl"}, + {"ʲo", "jo"}, {"ʲə", "jə"}, {"e", "A"}, {"ʲ", ""}, + {"ɚ", "əɹ"}, {"r", "ɹ"}, {"x", "k"}, {"ç", "k"}, + {"ɐ", "ə"}, {"ɬ", "l"}, {"̃", ""}, + }}; + // trim + const auto not_space = [](unsigned char ch) { return std::isspace(ch) == 0; }; + ps.erase(ps.begin(), std::find_if(ps.begin(), ps.end(), not_space)); + ps.erase(std::find_if(ps.rbegin(), ps.rend(), not_space).base(), ps.end()); + + for (const auto & [from, to] : kE2M) { + replace_all(ps, from, to); + } + // re.sub(r'(\S)̩', r'ᵊ\1', ps) then drop any remaining syllabic + size_t at = 0; + while ((at = ps.find(kSyllabic, at)) != std::string::npos) { + size_t start = at; + while (start > 0 && (static_cast(ps[start - 1]) & 0xC0U) == 0x80U) { + --start; // walk back over the UTF-8 continuation bytes + } + if (start == at || std::isspace(static_cast(ps[start])) != 0) { + ps.erase(at, kSyllabic.size()); + continue; + } + ps.erase(at, kSyllabic.size()); + ps.insert(start, "ᵊ"); + at = start + std::strlen("ᵊ"); + } + replace_all(ps, "o^ʊ", "O"); + replace_all(ps, "ɜːɹ", "ɜɹ"); + replace_all(ps, "ɜː", "ɜɹ"); + replace_all(ps, "ɪə", "iə"); + replace_all(ps, "ː", ""); + replace_all(ps, "o", "ɔ"); // espeak < 1.52 + replace_all(ps, "ɾ", "T"); // version != '2.0' + replace_all(ps, "ʔ", "t"); + replace_all(ps, "^", ""); + return ps; +} + + +// ---- phonemizer-fork punctuation preserve/restore ------------------------ +// +// The reference front ends run eSpeak through phonemizer with +// preserve_punctuation=True: punctuation is cut out before phonemization and +// spliced back afterwards, so marks like "," and "." survive as tokens (the +// model was trained with them). This reproduces phonemizer's Punctuation +// class for the fixed marks and separator this model uses. + +constexpr std::string_view kPunctuationMarks = "!'(),-.:;?\""; + +bool is_punctuation_mark(char ch) { + return kPunctuationMarks.find(ch) != std::string_view::npos; +} + +bool is_ascii_space(char ch) { + return std::isspace(static_cast(ch)) != 0; +} + +struct MarkIndex { + std::string mark; + char position = 'I'; // B(egin), E(nd), I(nside), A(lone) +}; + +/** Matches of phonemizer's (\s*[marks]+\s*)+ -- maximal runs of spaces and + * marks that contain at least one mark. */ +std::vector> find_mark_runs(const std::string & line) { + std::vector> runs; + size_t i = 0; + while (i < line.size()) { + if (!is_ascii_space(line[i]) && !is_punctuation_mark(line[i])) { + ++i; + continue; + } + size_t end = i; + bool has_mark = false; + while (end < line.size() && + (is_ascii_space(line[end]) || is_punctuation_mark(line[end]))) { + has_mark = has_mark || is_punctuation_mark(line[end]); + ++end; + } + if (has_mark) { + runs.emplace_back(i, end); + } + i = end; + } + return runs; +} + +/** Punctuation._preserve_line: chunks without punctuation + ordered marks. + * Empty chunks are filtered, as Punctuation.preserve() does. */ +std::pair, std::vector> preserve_punctuation( + const std::string & line) { + const auto runs = find_mark_runs(line); + if (runs.empty()) { + return {{line}, {}}; + } + if (runs.size() == 1 && runs[0].first == 0 && runs[0].second == line.size()) { + return {{}, {{line, 'A'}}}; + } + std::vector marks; + marks.reserve(runs.size()); + for (size_t index = 0; index < runs.size(); ++index) { + const auto & run = runs[index]; + char position = 'I'; + if (index == 0 && run.first == 0) { + position = 'B'; + } else if (index + 1 == runs.size() && run.second == line.size()) { + position = 'E'; + } + marks.push_back({line.substr(run.first, run.second - run.first), position}); + } + // The find-first split dance, exactly as the Python does it. + std::vector chunks; + std::string rest = line; + for (const auto & mark : marks) { + const size_t at = rest.find(mark.mark); + if (at == std::string::npos) { + chunks.push_back(rest); + rest.clear(); + continue; + } + chunks.push_back(rest.substr(0, at)); + rest.erase(0, at + mark.mark.size()); + } + chunks.push_back(rest); + chunks.erase( + std::remove_if(chunks.begin(), chunks.end(), + [](const std::string & chunk) { return chunk.empty(); }), + chunks.end()); + return {std::move(chunks), std::move(marks)}; +} + +/** Punctuation.restore for a single line, sep.word = " ", strip = False. */ +std::string restore_punctuation( + std::vector chunk_phonemes, + std::vector marks) { + std::deque text(chunk_phonemes.begin(), chunk_phonemes.end()); + std::deque pending(marks.begin(), marks.end()); + std::vector out; + size_t pos = 0; + while (!text.empty() || !pending.empty()) { + if (pending.empty()) { + for (auto & line : text) { + if (line.empty() || line.back() != ' ') { + line.push_back(' '); + } + out.push_back(std::move(line)); + } + text.clear(); + } else if (text.empty()) { + std::string joined; + for (const auto & mark : pending) { + joined += mark.mark; + } + out.push_back(std::move(joined)); + pending.clear(); + } else if (pos == 0) { // single line: every mark carries index 0 + const auto current = pending.front(); + pending.pop_front(); + if (!text.front().empty() && text.front().back() == ' ') { + text.front().pop_back(); + } + const bool mark_ends_with_sep = + !current.mark.empty() && current.mark.back() == ' '; + if (current.position == 'B') { + text.front() = current.mark + text.front(); + } else if (current.position == 'E') { + out.push_back(text.front() + current.mark + (mark_ends_with_sep ? "" : " ")); + text.pop_front(); + ++pos; + } else if (current.position == 'A') { + out.push_back(current.mark + (mark_ends_with_sep ? "" : " ")); + ++pos; + } else { // 'I' + if (text.size() == 1) { + text.front() += current.mark; + } else { + auto first = std::move(text.front()); + text.pop_front(); + text.front() = first + current.mark + text.front(); + } + } + } else { + auto & line = text.front(); + if (line.empty() || line.back() != ' ') { + line.push_back(' '); + } + out.push_back(std::move(line)); + text.pop_front(); + ++pos; + } + } + // phonemizer would return these as separate lines and the reference + // takes the first; a single input line produces one in practice. + std::string result; + for (const auto & line : out) { + result += line; + } + return result; +} + +/** phonemizer EspeakBackend._postprocess_line with tie enabled, + * with_stress=True, strip=False, word separator " ", phone separator "". */ +std::string postprocess_espeak_line(std::string line) { + const auto not_space = [](unsigned char ch) { return std::isspace(ch) == 0; }; + line.erase(line.begin(), std::find_if(line.begin(), line.end(), not_space)); + line.erase(std::find_if(line.rbegin(), line.rend(), not_space).base(), line.end()); + std::replace(line.begin(), line.end(), '\n', ' '); + replace_all(line, " ", " "); + // espeak-ng#694: stray '_' separators at word ends + std::string squeezed; + squeezed.reserve(line.size()); + for (const char ch : line) { + if (ch == '_' && !squeezed.empty() && squeezed.back() == '_') { + continue; + } + squeezed.push_back(ch); + } + line = std::move(squeezed); + replace_all(line, "_ ", " "); + // language_switch="remove-flags": strip espeak's (lang) switch flags + if (line.find('(') != std::string::npos) { + std::string unflagged; + size_t at = 0; + while (at < line.size()) { + if (line[at] == '(') { + const size_t close = line.find(')', at + 1); + if (close != std::string::npos) { + at = close + 1; + continue; + } + } + unflagged.push_back(line[at++]); + } + line = std::move(unflagged); + } + if (line.empty()) { + return line; + } + // per word: strip, drop in-word '_' (phone separator is empty), append " " + std::string out; + size_t start = 0; + while (start <= line.size()) { + size_t end = line.find(' ', start); + if (end == std::string::npos) { + end = line.size(); + } + std::string word = line.substr(start, end - start); + word.erase(std::remove(word.begin(), word.end(), '_'), word.end()); + out += word; + out.push_back(' '); + if (end == line.size()) { + break; + } + start = end + 1; + } + return out; +} + +/** The reference front end's own line post-processing: rewrite eSpeak's tie + * to '^' per word so the E2M diphthong patterns can match. */ +std::string rewrite_ties_per_word(const std::string & line_in) { + std::string line = line_in; + const auto not_space = [](unsigned char ch) { return std::isspace(ch) == 0; }; + line.erase(line.begin(), std::find_if(line.begin(), line.end(), not_space)); + line.erase(std::find_if(line.rbegin(), line.rend(), not_space).base(), line.end()); + std::replace(line.begin(), line.end(), '\n', ' '); + replace_all(line, " ", " "); + if (line.empty()) { + return line; + } + std::string out; + size_t start = 0; + while (start <= line.size()) { + size_t end = line.find(' ', start); + if (end == std::string::npos) { + end = line.size(); + } + std::string word = line.substr(start, end - start); + replace_all(word, kTieDefault, kTieMisaki); + out += word; + out.push_back(' '); + if (end == line.size()) { + break; + } + start = end + 1; + } + return out; +} + + +// ---- NFD decomposition for the Piper codepoint mapping ------------------- +// +// piper normalizes the phonemized string with NFD before mapping each +// codepoint through phoneme_id_map. eSpeak-ng's IPA output for the shipped +// languages is already NFD-normal (probed over en/vi/id corpora), so this +// table only has to cover precomposed Latin letters that could slip through; +// characters outside it pass through unchanged, and an unmapped codepoint is +// skipped exactly as piper skips it. The framework's NFKD normalizer is NOT +// usable here: compatibility decomposition rewrites IPA modifier letters. + +// Canonical (NFD) decompositions for Latin-1/Extended and Vietnamese +// precomposed letters -- everything eSpeak-ng can plausibly emit in IPA +// mode. Generated from Python unicodedata (Unicode 15); canonical +// decompositions are stable across Unicode versions by policy. +struct NfdEntry { uint32_t composed; uint32_t parts[3]; }; +constexpr NfdEntry kNfdEntries[] = { + {0x00C0, {0x0041, 0x0300, 0x0000}}, + {0x00C1, {0x0041, 0x0301, 0x0000}}, + {0x00C2, {0x0041, 0x0302, 0x0000}}, + {0x00C3, {0x0041, 0x0303, 0x0000}}, + {0x00C4, {0x0041, 0x0308, 0x0000}}, + {0x00C5, {0x0041, 0x030A, 0x0000}}, + {0x00C7, {0x0043, 0x0327, 0x0000}}, + {0x00C8, {0x0045, 0x0300, 0x0000}}, + {0x00C9, {0x0045, 0x0301, 0x0000}}, + {0x00CA, {0x0045, 0x0302, 0x0000}}, + {0x00CB, {0x0045, 0x0308, 0x0000}}, + {0x00CC, {0x0049, 0x0300, 0x0000}}, + {0x00CD, {0x0049, 0x0301, 0x0000}}, + {0x00CE, {0x0049, 0x0302, 0x0000}}, + {0x00CF, {0x0049, 0x0308, 0x0000}}, + {0x00D1, {0x004E, 0x0303, 0x0000}}, + {0x00D2, {0x004F, 0x0300, 0x0000}}, + {0x00D3, {0x004F, 0x0301, 0x0000}}, + {0x00D4, {0x004F, 0x0302, 0x0000}}, + {0x00D5, {0x004F, 0x0303, 0x0000}}, + {0x00D6, {0x004F, 0x0308, 0x0000}}, + {0x00D9, {0x0055, 0x0300, 0x0000}}, + {0x00DA, {0x0055, 0x0301, 0x0000}}, + {0x00DB, {0x0055, 0x0302, 0x0000}}, + {0x00DC, {0x0055, 0x0308, 0x0000}}, + {0x00DD, {0x0059, 0x0301, 0x0000}}, + {0x00E0, {0x0061, 0x0300, 0x0000}}, + {0x00E1, {0x0061, 0x0301, 0x0000}}, + {0x00E2, {0x0061, 0x0302, 0x0000}}, + {0x00E3, {0x0061, 0x0303, 0x0000}}, + {0x00E4, {0x0061, 0x0308, 0x0000}}, + {0x00E5, {0x0061, 0x030A, 0x0000}}, + {0x00E7, {0x0063, 0x0327, 0x0000}}, + {0x00E8, {0x0065, 0x0300, 0x0000}}, + {0x00E9, {0x0065, 0x0301, 0x0000}}, + {0x00EA, {0x0065, 0x0302, 0x0000}}, + {0x00EB, {0x0065, 0x0308, 0x0000}}, + {0x00EC, {0x0069, 0x0300, 0x0000}}, + {0x00ED, {0x0069, 0x0301, 0x0000}}, + {0x00EE, {0x0069, 0x0302, 0x0000}}, + {0x00EF, {0x0069, 0x0308, 0x0000}}, + {0x00F1, {0x006E, 0x0303, 0x0000}}, + {0x00F2, {0x006F, 0x0300, 0x0000}}, + {0x00F3, {0x006F, 0x0301, 0x0000}}, + {0x00F4, {0x006F, 0x0302, 0x0000}}, + {0x00F5, {0x006F, 0x0303, 0x0000}}, + {0x00F6, {0x006F, 0x0308, 0x0000}}, + {0x00F9, {0x0075, 0x0300, 0x0000}}, + {0x00FA, {0x0075, 0x0301, 0x0000}}, + {0x00FB, {0x0075, 0x0302, 0x0000}}, + {0x00FC, {0x0075, 0x0308, 0x0000}}, + {0x00FD, {0x0079, 0x0301, 0x0000}}, + {0x00FF, {0x0079, 0x0308, 0x0000}}, + {0x0100, {0x0041, 0x0304, 0x0000}}, + {0x0101, {0x0061, 0x0304, 0x0000}}, + {0x0102, {0x0041, 0x0306, 0x0000}}, + {0x0103, {0x0061, 0x0306, 0x0000}}, + {0x0104, {0x0041, 0x0328, 0x0000}}, + {0x0105, {0x0061, 0x0328, 0x0000}}, + {0x0106, {0x0043, 0x0301, 0x0000}}, + {0x0107, {0x0063, 0x0301, 0x0000}}, + {0x0108, {0x0043, 0x0302, 0x0000}}, + {0x0109, {0x0063, 0x0302, 0x0000}}, + {0x010A, {0x0043, 0x0307, 0x0000}}, + {0x010B, {0x0063, 0x0307, 0x0000}}, + {0x010C, {0x0043, 0x030C, 0x0000}}, + {0x010D, {0x0063, 0x030C, 0x0000}}, + {0x010E, {0x0044, 0x030C, 0x0000}}, + {0x010F, {0x0064, 0x030C, 0x0000}}, + {0x0112, {0x0045, 0x0304, 0x0000}}, + {0x0113, {0x0065, 0x0304, 0x0000}}, + {0x0114, {0x0045, 0x0306, 0x0000}}, + {0x0115, {0x0065, 0x0306, 0x0000}}, + {0x0116, {0x0045, 0x0307, 0x0000}}, + {0x0117, {0x0065, 0x0307, 0x0000}}, + {0x0118, {0x0045, 0x0328, 0x0000}}, + {0x0119, {0x0065, 0x0328, 0x0000}}, + {0x011A, {0x0045, 0x030C, 0x0000}}, + {0x011B, {0x0065, 0x030C, 0x0000}}, + {0x011C, {0x0047, 0x0302, 0x0000}}, + {0x011D, {0x0067, 0x0302, 0x0000}}, + {0x011E, {0x0047, 0x0306, 0x0000}}, + {0x011F, {0x0067, 0x0306, 0x0000}}, + {0x0120, {0x0047, 0x0307, 0x0000}}, + {0x0121, {0x0067, 0x0307, 0x0000}}, + {0x0122, {0x0047, 0x0327, 0x0000}}, + {0x0123, {0x0067, 0x0327, 0x0000}}, + {0x0124, {0x0048, 0x0302, 0x0000}}, + {0x0125, {0x0068, 0x0302, 0x0000}}, + {0x0128, {0x0049, 0x0303, 0x0000}}, + {0x0129, {0x0069, 0x0303, 0x0000}}, + {0x012A, {0x0049, 0x0304, 0x0000}}, + {0x012B, {0x0069, 0x0304, 0x0000}}, + {0x012C, {0x0049, 0x0306, 0x0000}}, + {0x012D, {0x0069, 0x0306, 0x0000}}, + {0x012E, {0x0049, 0x0328, 0x0000}}, + {0x012F, {0x0069, 0x0328, 0x0000}}, + {0x0130, {0x0049, 0x0307, 0x0000}}, + {0x0134, {0x004A, 0x0302, 0x0000}}, + {0x0135, {0x006A, 0x0302, 0x0000}}, + {0x0136, {0x004B, 0x0327, 0x0000}}, + {0x0137, {0x006B, 0x0327, 0x0000}}, + {0x0139, {0x004C, 0x0301, 0x0000}}, + {0x013A, {0x006C, 0x0301, 0x0000}}, + {0x013B, {0x004C, 0x0327, 0x0000}}, + {0x013C, {0x006C, 0x0327, 0x0000}}, + {0x013D, {0x004C, 0x030C, 0x0000}}, + {0x013E, {0x006C, 0x030C, 0x0000}}, + {0x0143, {0x004E, 0x0301, 0x0000}}, + {0x0144, {0x006E, 0x0301, 0x0000}}, + {0x0145, {0x004E, 0x0327, 0x0000}}, + {0x0146, {0x006E, 0x0327, 0x0000}}, + {0x0147, {0x004E, 0x030C, 0x0000}}, + {0x0148, {0x006E, 0x030C, 0x0000}}, + {0x014C, {0x004F, 0x0304, 0x0000}}, + {0x014D, {0x006F, 0x0304, 0x0000}}, + {0x014E, {0x004F, 0x0306, 0x0000}}, + {0x014F, {0x006F, 0x0306, 0x0000}}, + {0x0150, {0x004F, 0x030B, 0x0000}}, + {0x0151, {0x006F, 0x030B, 0x0000}}, + {0x0154, {0x0052, 0x0301, 0x0000}}, + {0x0155, {0x0072, 0x0301, 0x0000}}, + {0x0156, {0x0052, 0x0327, 0x0000}}, + {0x0157, {0x0072, 0x0327, 0x0000}}, + {0x0158, {0x0052, 0x030C, 0x0000}}, + {0x0159, {0x0072, 0x030C, 0x0000}}, + {0x015A, {0x0053, 0x0301, 0x0000}}, + {0x015B, {0x0073, 0x0301, 0x0000}}, + {0x015C, {0x0053, 0x0302, 0x0000}}, + {0x015D, {0x0073, 0x0302, 0x0000}}, + {0x015E, {0x0053, 0x0327, 0x0000}}, + {0x015F, {0x0073, 0x0327, 0x0000}}, + {0x0160, {0x0053, 0x030C, 0x0000}}, + {0x0161, {0x0073, 0x030C, 0x0000}}, + {0x0162, {0x0054, 0x0327, 0x0000}}, + {0x0163, {0x0074, 0x0327, 0x0000}}, + {0x0164, {0x0054, 0x030C, 0x0000}}, + {0x0165, {0x0074, 0x030C, 0x0000}}, + {0x0168, {0x0055, 0x0303, 0x0000}}, + {0x0169, {0x0075, 0x0303, 0x0000}}, + {0x016A, {0x0055, 0x0304, 0x0000}}, + {0x016B, {0x0075, 0x0304, 0x0000}}, + {0x016C, {0x0055, 0x0306, 0x0000}}, + {0x016D, {0x0075, 0x0306, 0x0000}}, + {0x016E, {0x0055, 0x030A, 0x0000}}, + {0x016F, {0x0075, 0x030A, 0x0000}}, + {0x0170, {0x0055, 0x030B, 0x0000}}, + {0x0171, {0x0075, 0x030B, 0x0000}}, + {0x0172, {0x0055, 0x0328, 0x0000}}, + {0x0173, {0x0075, 0x0328, 0x0000}}, + {0x0174, {0x0057, 0x0302, 0x0000}}, + {0x0175, {0x0077, 0x0302, 0x0000}}, + {0x0176, {0x0059, 0x0302, 0x0000}}, + {0x0177, {0x0079, 0x0302, 0x0000}}, + {0x0178, {0x0059, 0x0308, 0x0000}}, + {0x0179, {0x005A, 0x0301, 0x0000}}, + {0x017A, {0x007A, 0x0301, 0x0000}}, + {0x017B, {0x005A, 0x0307, 0x0000}}, + {0x017C, {0x007A, 0x0307, 0x0000}}, + {0x017D, {0x005A, 0x030C, 0x0000}}, + {0x017E, {0x007A, 0x030C, 0x0000}}, + {0x1E00, {0x0041, 0x0325, 0x0000}}, + {0x1E01, {0x0061, 0x0325, 0x0000}}, + {0x1E02, {0x0042, 0x0307, 0x0000}}, + {0x1E03, {0x0062, 0x0307, 0x0000}}, + {0x1E04, {0x0042, 0x0323, 0x0000}}, + {0x1E05, {0x0062, 0x0323, 0x0000}}, + {0x1E06, {0x0042, 0x0331, 0x0000}}, + {0x1E07, {0x0062, 0x0331, 0x0000}}, + {0x1E08, {0x0043, 0x0327, 0x0301}}, + {0x1E09, {0x0063, 0x0327, 0x0301}}, + {0x1E0A, {0x0044, 0x0307, 0x0000}}, + {0x1E0B, {0x0064, 0x0307, 0x0000}}, + {0x1E0C, {0x0044, 0x0323, 0x0000}}, + {0x1E0D, {0x0064, 0x0323, 0x0000}}, + {0x1E0E, {0x0044, 0x0331, 0x0000}}, + {0x1E0F, {0x0064, 0x0331, 0x0000}}, + {0x1E10, {0x0044, 0x0327, 0x0000}}, + {0x1E11, {0x0064, 0x0327, 0x0000}}, + {0x1E12, {0x0044, 0x032D, 0x0000}}, + {0x1E13, {0x0064, 0x032D, 0x0000}}, + {0x1E14, {0x0045, 0x0304, 0x0300}}, + {0x1E15, {0x0065, 0x0304, 0x0300}}, + {0x1E16, {0x0045, 0x0304, 0x0301}}, + {0x1E17, {0x0065, 0x0304, 0x0301}}, + {0x1E18, {0x0045, 0x032D, 0x0000}}, + {0x1E19, {0x0065, 0x032D, 0x0000}}, + {0x1E1A, {0x0045, 0x0330, 0x0000}}, + {0x1E1B, {0x0065, 0x0330, 0x0000}}, + {0x1E1C, {0x0045, 0x0327, 0x0306}}, + {0x1E1D, {0x0065, 0x0327, 0x0306}}, + {0x1E1E, {0x0046, 0x0307, 0x0000}}, + {0x1E1F, {0x0066, 0x0307, 0x0000}}, + {0x1E20, {0x0047, 0x0304, 0x0000}}, + {0x1E21, {0x0067, 0x0304, 0x0000}}, + {0x1E22, {0x0048, 0x0307, 0x0000}}, + {0x1E23, {0x0068, 0x0307, 0x0000}}, + {0x1E24, {0x0048, 0x0323, 0x0000}}, + {0x1E25, {0x0068, 0x0323, 0x0000}}, + {0x1E26, {0x0048, 0x0308, 0x0000}}, + {0x1E27, {0x0068, 0x0308, 0x0000}}, + {0x1E28, {0x0048, 0x0327, 0x0000}}, + {0x1E29, {0x0068, 0x0327, 0x0000}}, + {0x1E2A, {0x0048, 0x032E, 0x0000}}, + {0x1E2B, {0x0068, 0x032E, 0x0000}}, + {0x1E2C, {0x0049, 0x0330, 0x0000}}, + {0x1E2D, {0x0069, 0x0330, 0x0000}}, + {0x1E2E, {0x0049, 0x0308, 0x0301}}, + {0x1E2F, {0x0069, 0x0308, 0x0301}}, + {0x1E30, {0x004B, 0x0301, 0x0000}}, + {0x1E31, {0x006B, 0x0301, 0x0000}}, + {0x1E32, {0x004B, 0x0323, 0x0000}}, + {0x1E33, {0x006B, 0x0323, 0x0000}}, + {0x1E34, {0x004B, 0x0331, 0x0000}}, + {0x1E35, {0x006B, 0x0331, 0x0000}}, + {0x1E36, {0x004C, 0x0323, 0x0000}}, + {0x1E37, {0x006C, 0x0323, 0x0000}}, + {0x1E38, {0x004C, 0x0323, 0x0304}}, + {0x1E39, {0x006C, 0x0323, 0x0304}}, + {0x1E3A, {0x004C, 0x0331, 0x0000}}, + {0x1E3B, {0x006C, 0x0331, 0x0000}}, + {0x1E3C, {0x004C, 0x032D, 0x0000}}, + {0x1E3D, {0x006C, 0x032D, 0x0000}}, + {0x1E3E, {0x004D, 0x0301, 0x0000}}, + {0x1E3F, {0x006D, 0x0301, 0x0000}}, + {0x1E40, {0x004D, 0x0307, 0x0000}}, + {0x1E41, {0x006D, 0x0307, 0x0000}}, + {0x1E42, {0x004D, 0x0323, 0x0000}}, + {0x1E43, {0x006D, 0x0323, 0x0000}}, + {0x1E44, {0x004E, 0x0307, 0x0000}}, + {0x1E45, {0x006E, 0x0307, 0x0000}}, + {0x1E46, {0x004E, 0x0323, 0x0000}}, + {0x1E47, {0x006E, 0x0323, 0x0000}}, + {0x1E48, {0x004E, 0x0331, 0x0000}}, + {0x1E49, {0x006E, 0x0331, 0x0000}}, + {0x1E4A, {0x004E, 0x032D, 0x0000}}, + {0x1E4B, {0x006E, 0x032D, 0x0000}}, + {0x1E4C, {0x004F, 0x0303, 0x0301}}, + {0x1E4D, {0x006F, 0x0303, 0x0301}}, + {0x1E4E, {0x004F, 0x0303, 0x0308}}, + {0x1E4F, {0x006F, 0x0303, 0x0308}}, + {0x1E50, {0x004F, 0x0304, 0x0300}}, + {0x1E51, {0x006F, 0x0304, 0x0300}}, + {0x1E52, {0x004F, 0x0304, 0x0301}}, + {0x1E53, {0x006F, 0x0304, 0x0301}}, + {0x1E54, {0x0050, 0x0301, 0x0000}}, + {0x1E55, {0x0070, 0x0301, 0x0000}}, + {0x1E56, {0x0050, 0x0307, 0x0000}}, + {0x1E57, {0x0070, 0x0307, 0x0000}}, + {0x1E58, {0x0052, 0x0307, 0x0000}}, + {0x1E59, {0x0072, 0x0307, 0x0000}}, + {0x1E5A, {0x0052, 0x0323, 0x0000}}, + {0x1E5B, {0x0072, 0x0323, 0x0000}}, + {0x1E5C, {0x0052, 0x0323, 0x0304}}, + {0x1E5D, {0x0072, 0x0323, 0x0304}}, + {0x1E5E, {0x0052, 0x0331, 0x0000}}, + {0x1E5F, {0x0072, 0x0331, 0x0000}}, + {0x1E60, {0x0053, 0x0307, 0x0000}}, + {0x1E61, {0x0073, 0x0307, 0x0000}}, + {0x1E62, {0x0053, 0x0323, 0x0000}}, + {0x1E63, {0x0073, 0x0323, 0x0000}}, + {0x1E64, {0x0053, 0x0301, 0x0307}}, + {0x1E65, {0x0073, 0x0301, 0x0307}}, + {0x1E66, {0x0053, 0x030C, 0x0307}}, + {0x1E67, {0x0073, 0x030C, 0x0307}}, + {0x1E68, {0x0053, 0x0323, 0x0307}}, + {0x1E69, {0x0073, 0x0323, 0x0307}}, + {0x1E6A, {0x0054, 0x0307, 0x0000}}, + {0x1E6B, {0x0074, 0x0307, 0x0000}}, + {0x1E6C, {0x0054, 0x0323, 0x0000}}, + {0x1E6D, {0x0074, 0x0323, 0x0000}}, + {0x1E6E, {0x0054, 0x0331, 0x0000}}, + {0x1E6F, {0x0074, 0x0331, 0x0000}}, + {0x1E70, {0x0054, 0x032D, 0x0000}}, + {0x1E71, {0x0074, 0x032D, 0x0000}}, + {0x1E72, {0x0055, 0x0324, 0x0000}}, + {0x1E73, {0x0075, 0x0324, 0x0000}}, + {0x1E74, {0x0055, 0x0330, 0x0000}}, + {0x1E75, {0x0075, 0x0330, 0x0000}}, + {0x1E76, {0x0055, 0x032D, 0x0000}}, + {0x1E77, {0x0075, 0x032D, 0x0000}}, + {0x1E78, {0x0055, 0x0303, 0x0301}}, + {0x1E79, {0x0075, 0x0303, 0x0301}}, + {0x1E7A, {0x0055, 0x0304, 0x0308}}, + {0x1E7B, {0x0075, 0x0304, 0x0308}}, + {0x1E7C, {0x0056, 0x0303, 0x0000}}, + {0x1E7D, {0x0076, 0x0303, 0x0000}}, + {0x1E7E, {0x0056, 0x0323, 0x0000}}, + {0x1E7F, {0x0076, 0x0323, 0x0000}}, + {0x1E80, {0x0057, 0x0300, 0x0000}}, + {0x1E81, {0x0077, 0x0300, 0x0000}}, + {0x1E82, {0x0057, 0x0301, 0x0000}}, + {0x1E83, {0x0077, 0x0301, 0x0000}}, + {0x1E84, {0x0057, 0x0308, 0x0000}}, + {0x1E85, {0x0077, 0x0308, 0x0000}}, + {0x1E86, {0x0057, 0x0307, 0x0000}}, + {0x1E87, {0x0077, 0x0307, 0x0000}}, + {0x1E88, {0x0057, 0x0323, 0x0000}}, + {0x1E89, {0x0077, 0x0323, 0x0000}}, + {0x1E8A, {0x0058, 0x0307, 0x0000}}, + {0x1E8B, {0x0078, 0x0307, 0x0000}}, + {0x1E8C, {0x0058, 0x0308, 0x0000}}, + {0x1E8D, {0x0078, 0x0308, 0x0000}}, + {0x1E8E, {0x0059, 0x0307, 0x0000}}, + {0x1E8F, {0x0079, 0x0307, 0x0000}}, + {0x1E90, {0x005A, 0x0302, 0x0000}}, + {0x1E91, {0x007A, 0x0302, 0x0000}}, + {0x1E92, {0x005A, 0x0323, 0x0000}}, + {0x1E93, {0x007A, 0x0323, 0x0000}}, + {0x1E94, {0x005A, 0x0331, 0x0000}}, + {0x1E95, {0x007A, 0x0331, 0x0000}}, + {0x1E96, {0x0068, 0x0331, 0x0000}}, + {0x1E97, {0x0074, 0x0308, 0x0000}}, + {0x1E98, {0x0077, 0x030A, 0x0000}}, + {0x1E99, {0x0079, 0x030A, 0x0000}}, + {0x1E9B, {0x017F, 0x0307, 0x0000}}, + {0x1EA0, {0x0041, 0x0323, 0x0000}}, + {0x1EA1, {0x0061, 0x0323, 0x0000}}, + {0x1EA2, {0x0041, 0x0309, 0x0000}}, + {0x1EA3, {0x0061, 0x0309, 0x0000}}, + {0x1EA4, {0x0041, 0x0302, 0x0301}}, + {0x1EA5, {0x0061, 0x0302, 0x0301}}, + {0x1EA6, {0x0041, 0x0302, 0x0300}}, + {0x1EA7, {0x0061, 0x0302, 0x0300}}, + {0x1EA8, {0x0041, 0x0302, 0x0309}}, + {0x1EA9, {0x0061, 0x0302, 0x0309}}, + {0x1EAA, {0x0041, 0x0302, 0x0303}}, + {0x1EAB, {0x0061, 0x0302, 0x0303}}, + {0x1EAC, {0x0041, 0x0323, 0x0302}}, + {0x1EAD, {0x0061, 0x0323, 0x0302}}, + {0x1EAE, {0x0041, 0x0306, 0x0301}}, + {0x1EAF, {0x0061, 0x0306, 0x0301}}, + {0x1EB0, {0x0041, 0x0306, 0x0300}}, + {0x1EB1, {0x0061, 0x0306, 0x0300}}, + {0x1EB2, {0x0041, 0x0306, 0x0309}}, + {0x1EB3, {0x0061, 0x0306, 0x0309}}, + {0x1EB4, {0x0041, 0x0306, 0x0303}}, + {0x1EB5, {0x0061, 0x0306, 0x0303}}, + {0x1EB6, {0x0041, 0x0323, 0x0306}}, + {0x1EB7, {0x0061, 0x0323, 0x0306}}, + {0x1EB8, {0x0045, 0x0323, 0x0000}}, + {0x1EB9, {0x0065, 0x0323, 0x0000}}, + {0x1EBA, {0x0045, 0x0309, 0x0000}}, + {0x1EBB, {0x0065, 0x0309, 0x0000}}, + {0x1EBC, {0x0045, 0x0303, 0x0000}}, + {0x1EBD, {0x0065, 0x0303, 0x0000}}, + {0x1EBE, {0x0045, 0x0302, 0x0301}}, + {0x1EBF, {0x0065, 0x0302, 0x0301}}, + {0x1EC0, {0x0045, 0x0302, 0x0300}}, + {0x1EC1, {0x0065, 0x0302, 0x0300}}, + {0x1EC2, {0x0045, 0x0302, 0x0309}}, + {0x1EC3, {0x0065, 0x0302, 0x0309}}, + {0x1EC4, {0x0045, 0x0302, 0x0303}}, + {0x1EC5, {0x0065, 0x0302, 0x0303}}, + {0x1EC6, {0x0045, 0x0323, 0x0302}}, + {0x1EC7, {0x0065, 0x0323, 0x0302}}, + {0x1EC8, {0x0049, 0x0309, 0x0000}}, + {0x1EC9, {0x0069, 0x0309, 0x0000}}, + {0x1ECA, {0x0049, 0x0323, 0x0000}}, + {0x1ECB, {0x0069, 0x0323, 0x0000}}, + {0x1ECC, {0x004F, 0x0323, 0x0000}}, + {0x1ECD, {0x006F, 0x0323, 0x0000}}, + {0x1ECE, {0x004F, 0x0309, 0x0000}}, + {0x1ECF, {0x006F, 0x0309, 0x0000}}, + {0x1ED0, {0x004F, 0x0302, 0x0301}}, + {0x1ED1, {0x006F, 0x0302, 0x0301}}, + {0x1ED2, {0x004F, 0x0302, 0x0300}}, + {0x1ED3, {0x006F, 0x0302, 0x0300}}, + {0x1ED4, {0x004F, 0x0302, 0x0309}}, + {0x1ED5, {0x006F, 0x0302, 0x0309}}, + {0x1ED6, {0x004F, 0x0302, 0x0303}}, + {0x1ED7, {0x006F, 0x0302, 0x0303}}, + {0x1ED8, {0x004F, 0x0323, 0x0302}}, + {0x1ED9, {0x006F, 0x0323, 0x0302}}, + {0x1EDA, {0x004F, 0x031B, 0x0301}}, + {0x1EDB, {0x006F, 0x031B, 0x0301}}, + {0x1EDC, {0x004F, 0x031B, 0x0300}}, + {0x1EDD, {0x006F, 0x031B, 0x0300}}, + {0x1EDE, {0x004F, 0x031B, 0x0309}}, + {0x1EDF, {0x006F, 0x031B, 0x0309}}, + {0x1EE0, {0x004F, 0x031B, 0x0303}}, + {0x1EE1, {0x006F, 0x031B, 0x0303}}, + {0x1EE2, {0x004F, 0x031B, 0x0323}}, + {0x1EE3, {0x006F, 0x031B, 0x0323}}, + {0x1EE4, {0x0055, 0x0323, 0x0000}}, + {0x1EE5, {0x0075, 0x0323, 0x0000}}, + {0x1EE6, {0x0055, 0x0309, 0x0000}}, + {0x1EE7, {0x0075, 0x0309, 0x0000}}, + {0x1EE8, {0x0055, 0x031B, 0x0301}}, + {0x1EE9, {0x0075, 0x031B, 0x0301}}, + {0x1EEA, {0x0055, 0x031B, 0x0300}}, + {0x1EEB, {0x0075, 0x031B, 0x0300}}, + {0x1EEC, {0x0055, 0x031B, 0x0309}}, + {0x1EED, {0x0075, 0x031B, 0x0309}}, + {0x1EEE, {0x0055, 0x031B, 0x0303}}, + {0x1EEF, {0x0075, 0x031B, 0x0303}}, + {0x1EF0, {0x0055, 0x031B, 0x0323}}, + {0x1EF1, {0x0075, 0x031B, 0x0323}}, + {0x1EF2, {0x0059, 0x0300, 0x0000}}, + {0x1EF3, {0x0079, 0x0300, 0x0000}}, + {0x1EF4, {0x0059, 0x0323, 0x0000}}, + {0x1EF5, {0x0079, 0x0323, 0x0000}}, + {0x1EF6, {0x0059, 0x0309, 0x0000}}, + {0x1EF7, {0x0079, 0x0309, 0x0000}}, + {0x1EF8, {0x0059, 0x0303, 0x0000}}, + {0x1EF9, {0x0079, 0x0303, 0x0000}}, +}; + + +void append_codepoint_utf8(uint32_t codepoint, std::string & out) { + if (codepoint < 0x80U) { + out.push_back(static_cast(codepoint)); + } else if (codepoint < 0x800U) { + out.push_back(static_cast(0xC0U | (codepoint >> 6))); + out.push_back(static_cast(0x80U | (codepoint & 0x3FU))); + } else if (codepoint < 0x10000U) { + out.push_back(static_cast(0xE0U | (codepoint >> 12))); + out.push_back(static_cast(0x80U | ((codepoint >> 6) & 0x3FU))); + out.push_back(static_cast(0x80U | (codepoint & 0x3FU))); + } else { + out.push_back(static_cast(0xF0U | (codepoint >> 18))); + out.push_back(static_cast(0x80U | ((codepoint >> 12) & 0x3FU))); + out.push_back(static_cast(0x80U | ((codepoint >> 6) & 0x3FU))); + out.push_back(static_cast(0x80U | (codepoint & 0x3FU))); + } +} + +/** Iterates whole UTF-8 codepoints; invalid lead bytes pass through as one. */ +size_t utf8_sequence_length(const std::string & text, size_t at) { + const auto lead = static_cast(text[at]); + size_t len = 1; + if ((lead & 0xF8U) == 0xF0U) { len = 4; } + else if ((lead & 0xF0U) == 0xE0U) { len = 3; } + else if ((lead & 0xE0U) == 0xC0U) { len = 2; } + return std::min(len, text.size() - at); +} + +uint32_t decode_codepoint_utf8(const std::string & text, size_t at, size_t len) { + const auto lead = static_cast(text[at]); + if (len == 1) { + return lead; + } + uint32_t value = lead & (0x7FU >> len); + for (size_t i = 1; i < len; ++i) { + value = (value << 6) | (static_cast(text[at + i]) & 0x3FU); + } + return value; +} + +std::string nfd_decompose(const std::string & text) { + std::string out; + out.reserve(text.size()); + for (size_t i = 0; i < text.size();) { + const size_t len = utf8_sequence_length(text, i); + const uint32_t codepoint = decode_codepoint_utf8(text, i, len); + const auto * entry = std::lower_bound( + std::begin(kNfdEntries), + std::end(kNfdEntries), + codepoint, + [](const NfdEntry & candidate, uint32_t value) { + return candidate.composed < value; + }); + if (entry != std::end(kNfdEntries) && entry->composed == codepoint) { + for (const uint32_t part : entry->parts) { + if (part != 0) { + append_codepoint_utf8(part, out); + } + } + } else { + out.append(text, i, len); + } + i += len; + } + return out; +} + +struct EspeakApi { + io::DynamicLibraryHandle library = nullptr; + InitializeFn initialize = nullptr; + SetVoiceFn set_voice = nullptr; + TextToPhonemesFn text_to_phonemes = nullptr; + TerminateFn terminate = nullptr; + mutable std::mutex call_mutex; + + EspeakApi(const std::filesystem::path & requested_library, + const std::filesystem::path & requested_data, + const std::string & voice) { + if (!requested_library.empty() && + !std::filesystem::is_regular_file(requested_library)) { + throw std::runtime_error( + "sanoTTS eSpeak-ng library does not exist: " + requested_library.string()); + } + if (!requested_data.empty() && + (!std::filesystem::is_directory(requested_data) || + !std::filesystem::is_regular_file(requested_data / "phontab"))) { + throw std::runtime_error( + "sanoTTS eSpeak-ng data path is invalid; expected the espeak-ng-data " + "directory containing phontab: " + requested_data.string()); + } + if (!requested_library.empty()) { + library = io::open_dynamic_library(requested_library.string()); + } else { + library = io::open_dynamic_library({ +#ifdef _WIN32 + "espeak-ng.dll", "libespeak-ng.dll", +#elif defined(__APPLE__) + "libespeak-ng.dylib", "libespeak-ng.1.dylib", +#else + "libespeak-ng.so.1", "libespeak-ng.so", +#endif + }); + } + if (library == nullptr) { + throw std::runtime_error( + "sanoTTS could not load eSpeak-ng. Install it (apt install espeak-ng, " + "brew install espeak-ng) or pass " + "--session-option sanotts.espeak_library_path=/path/to/libespeak-ng.so"); + } + initialize = reinterpret_cast( + io::dynamic_library_symbol(library, "espeak_Initialize")); + set_voice = reinterpret_cast( + io::dynamic_library_symbol(library, "espeak_SetVoiceByName")); + text_to_phonemes = reinterpret_cast( + io::dynamic_library_symbol(library, "espeak_TextToPhonemes")); + terminate = reinterpret_cast( + io::dynamic_library_symbol(library, "espeak_Terminate")); + if (initialize == nullptr || set_voice == nullptr || text_to_phonemes == nullptr) { + throw std::runtime_error("sanoTTS eSpeak-ng is missing required symbols"); + } + // espeak appends "/espeak-ng-data" to the path it is given, so the + // PARENT of the data directory is what it wants. Handing it the data + // directory itself makes it fall back to its compiled-in default. + const std::string data = + requested_data.empty() ? std::string() : requested_data.parent_path().string(); + if (initialize(kEspeakSynchronous, 0, data.empty() ? nullptr : data.c_str(), 0) <= 0) { + throw std::runtime_error( + "sanoTTS eSpeak-ng failed to initialize; pass " + "--session-option sanotts.espeak_data_path=/path/to/espeak-ng-data"); + } + // Some packages name a bare language code ("en"). phonemizer, which + // the reference front end drives, rejects bare codes on every + // espeak-ng >= 1.49 and falls back to the regional variant, even + // though espeak_SetVoiceByName itself would accept "en" (and select + // a different accent). Prefer the regional variants first so both + // stacks phonemize identically; a code with no variant (vi, id) + // falls through to itself. + std::vector candidates; + if (voice.find('-') == std::string::npos) { + candidates.push_back(voice + "-us"); + candidates.push_back(voice + "-gb"); + } + candidates.push_back(voice); + bool selected = false; + for (const auto & candidate : candidates) { + if (set_voice(candidate.c_str()) == 0) { + selected = true; + break; + } + } + if (!selected) { + throw std::runtime_error("sanoTTS eSpeak-ng has no voice matching '" + voice + "'"); + } + } + + ~EspeakApi() { + if (terminate != nullptr) { + terminate(); + } + if (library != nullptr) { + io::close_dynamic_library(library); + } + } + + [[nodiscard]] std::string phonemize(const std::string & text, int phonemes_mode) const { + const std::lock_guard guard(call_mutex); + std::string out; + const char * cursor = text.c_str(); + const void * position = cursor; + // espeak consumes one clause per call and advances the pointer; it + // returns null when the input is spent. + while (position != nullptr) { + const char * clause = + text_to_phonemes(&position, kEspeakCharsUtf8, phonemes_mode); + if (clause == nullptr) { + break; + } + if (!out.empty()) { + out.push_back(' '); + } + out.append(clause); + } + return out; + } +}; + +} // namespace + +struct SanoTtsFrontend::Impl { + EspeakApi espeak; + Impl(const std::filesystem::path & library, const std::filesystem::path & data) + : espeak(library, data, "en-us") {} +}; + +SanoTtsFrontend::SanoTtsFrontend( + std::filesystem::path espeak_library_path, + std::filesystem::path espeak_data_path, + int64_t max_tokens) + : impl_(std::make_unique(espeak_library_path, espeak_data_path)), + max_tokens_(max_tokens > 2 ? max_tokens : 207) {} + +SanoTtsFrontend::~SanoTtsFrontend() = default; + +SanoTtsEncoded SanoTtsFrontend::encode(const std::string & text) const { + auto [chunks, marks] = preserve_punctuation(text); + std::vector chunk_phonemes; + chunk_phonemes.reserve(chunks.size()); + for (const auto & chunk : chunks) { + chunk_phonemes.push_back(postprocess_espeak_line( + impl_->espeak.phonemize(chunk, kEspeakPhonemesIpaTie))); + } + const std::string restored = + restore_punctuation(std::move(chunk_phonemes), std::move(marks)); + // Ties become '^' BEFORE E2M runs: every diphthong pattern in the table + // ("o^ʊ" -> "O", "t^ʃ" -> "ʧ", ...) matches on the rewritten form. + std::string ipa = apply_e2m(rewrite_ties_per_word(restored)); + + const auto & vocab = vocabulary(); + SanoTtsEncoded out; + out.token_ids.push_back(1); // + // Iterate whole UTF-8 codepoints: every vocabulary symbol is one + // codepoint, so anything else can only be dropped -- which is what the + // reference front ends do with unknown symbols. + for (size_t i = 0; i < ipa.size();) { + size_t len = 1; + const auto lead = static_cast(ipa[i]); + if ((lead & 0xF8U) == 0xF0U) { len = 4; } + else if ((lead & 0xF0U) == 0xE0U) { len = 3; } + else if ((lead & 0xE0U) == 0xC0U) { len = 2; } + len = std::min(len, ipa.size() - i); + const std::string symbol = ipa.substr(i, len); + i += len; + const auto found = vocab.find(symbol); + if (found != vocab.end()) { + out.token_ids.push_back(found->second); + } else { + out.dropped.append(symbol); + } + } + if (out.token_ids.size() == 1) { + throw std::runtime_error( + "sanoTTS phonemization produced no symbols in the packaged vocabulary"); + } + out.token_ids.push_back(2); // + if (static_cast(out.token_ids.size()) > max_tokens_) { + throw SanoTtsTooLongError( + "sanoTTS phoneme sequence has " + std::to_string(out.token_ids.size()) + + " tokens including BOS/EOS; the duration model was trained for at most " + + std::to_string(max_tokens_) + "."); + } + return out; +} + +std::vector SanoTtsFrontend::split_text( + const std::string & text, + int64_t max_codepoints) { + const size_t budget = max_codepoints > 0 ? static_cast(max_codepoints) : 280U; + std::vector chunks; + std::string current; + size_t codepoints = 0; + for (size_t i = 0; i < text.size();) { + size_t len = 1; + const auto lead = static_cast(text[i]); + if ((lead & 0xF8U) == 0xF0U) { len = 4; } + else if ((lead & 0xF0U) == 0xE0U) { len = 3; } + else if ((lead & 0xE0U) == 0xC0U) { len = 2; } + len = std::min(len, text.size() - i); + current.append(text, i, len); + i += len; + ++codepoints; + const bool sentence_end = len == 1 && (text[i - 1] == '.' || text[i - 1] == '!' || + text[i - 1] == '?'); + if ((sentence_end && codepoints >= budget / 4) || codepoints >= budget) { + chunks.push_back(current); + current.clear(); + codepoints = 0; + } + } + if (!current.empty()) { + chunks.push_back(current); + } + if (chunks.empty()) { + chunks.push_back(text); + } + return chunks; +} + +double SanoTtsFrontend::boundary_pause_seconds(const std::string & chunk) { + for (auto it = chunk.rbegin(); it != chunk.rend(); ++it) { + if (std::isspace(static_cast(*it)) != 0) { + continue; + } + return (*it == '.' || *it == '!' || *it == '?') ? 0.20 : 0.08; + } + return 0.08; +} + +// ---- piperlite front end ------------------------------------------------- + +struct SanoTtsPiperFrontend::Impl { + EspeakApi espeak; + Impl(const std::filesystem::path & library, + const std::filesystem::path & data, + const std::string & voice) + : espeak(library, data, voice) {} +}; + +SanoTtsPiperFrontend::SanoTtsPiperFrontend( + std::filesystem::path espeak_library_path, + std::filesystem::path espeak_data_path, + std::string espeak_voice, + std::unordered_map phoneme_id_map, + int64_t max_tokens) + : impl_(std::make_unique(espeak_library_path, espeak_data_path, espeak_voice)), + id_map_(std::move(phoneme_id_map)), + max_tokens_(max_tokens > 3 ? max_tokens : 3) {} + +SanoTtsPiperFrontend::~SanoTtsPiperFrontend() = default; + +SanoTtsEncoded SanoTtsPiperFrontend::encode(const std::string & text) const { + auto [chunks, marks] = preserve_punctuation(text); + std::vector chunk_phonemes; + chunk_phonemes.reserve(chunks.size()); + for (const auto & chunk : chunks) { + chunk_phonemes.push_back(postprocess_espeak_line( + impl_->espeak.phonemize(chunk, kEspeakPhonemesIpaUnderscore))); + } + std::string restored = + restore_punctuation(std::move(chunk_phonemes), std::move(marks)); + // phonemizer leaves a trailing word separator that piper's own bridge + // does not emit at true end of input; the reference rstrips before NFD. + while (!restored.empty() && + std::isspace(static_cast(restored.back())) != 0) { + restored.pop_back(); + } + const std::string decomposed = nfd_decompose(restored); + + // piper.phoneme_ids.phonemes_to_ids: bos, pad, then (id, pad) per + // codepoint, then eos. The exporter validated the framing symbols + // ('_' -> 0, '^' -> 1, '$' -> 2) against the voice's own map. + SanoTtsEncoded out; + out.token_ids.push_back(1); // + out.token_ids.push_back(0); // + for (size_t i = 0; i < decomposed.size();) { + const size_t len = utf8_sequence_length(decomposed, i); + const std::string symbol = decomposed.substr(i, len); + i += len; + const auto found = id_map_.find(symbol); + if (found != id_map_.end()) { + out.token_ids.push_back(found->second); + out.token_ids.push_back(0); + } else { + // skipped with a note, exactly as piper skips unmapped phonemes + out.dropped.append(symbol); + } + } + if (out.token_ids.size() <= 2) { + throw std::runtime_error( + "sanoTTS phonemization produced no symbols in the voice's phoneme_id_map"); + } + out.token_ids.push_back(2); // + if (static_cast(out.token_ids.size()) > max_tokens_) { + throw SanoTtsTooLongError( + "sanoTTS phoneme sequence has " + std::to_string(out.token_ids.size()) + + " ids including framing; the duration model was trained for at most " + + std::to_string(max_tokens_) + "."); + } + return out; +} + +} // namespace engine::models::sanotts diff --git a/src/community_models/sanotts/graph_common.h b/src/community_models/sanotts/graph_common.h new file mode 100644 index 000000000..4dc0e24d1 --- /dev/null +++ b/src/community_models/sanotts/graph_common.h @@ -0,0 +1,635 @@ +#pragma once + +// Internal helpers shared by the two sanoTTS runtimes (the nano lineage in +// runtime.cpp and the piperlite lineage in piper_runtime.cpp). Both lineages +// share the same convolutional front-end structure -- residual conv blocks +// over channel-major [1, C, T] values -- and the same graph plumbing. + +#include "engine/community_models/sanotts/assets.h" +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/runtime/cache_slots.h" + +#include "ggml-alloc.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts::graph { + +struct GgmlContextDeleter { + void operator()(ggml_context * context) const noexcept { + if (context != nullptr) { + ggml_free(context); + } + } +}; + +inline core::TensorValue contiguous( + core::ModuleBuildContext & ctx, + const core::TensorValue & value) { + if (core::has_backend_addressable_layout(value.tensor)) { + return value; + } + return core::wrap_tensor( + ggml_cont(ctx.ggml, value.tensor), + value.shape, + value.type); +} + +inline core::TensorValue add( + core::ModuleBuildContext & ctx, + const core::TensorValue & lhs, + const core::TensorValue & rhs) { + return modules::AddModule().build(ctx, lhs, rhs); +} + +struct SanoTtsBackendWeights { + std::shared_ptr store; + std::unordered_map tensors; +}; + +inline const core::TensorValue & weight( + const SanoTtsBackendWeights & weights, + const std::string & name) { + const auto found = weights.tensors.find(name); + if (found == weights.tensors.end()) { + throw std::runtime_error("sanoTTS missing tensor: " + name); + } + return found->second; +} + +inline std::shared_ptr load_weights( + const std::shared_ptr & assets, + ggml_backend_t backend, + core::BackendType backend_type, + size_t expected_tensors, + size_t weight_arena_bytes) { + auto out = std::make_shared(); + out->store = std::make_shared( + backend, + backend_type, + "sanotts.weights", + weight_arena_bytes); + const auto metadata = assets->weights->tensors(); + if (metadata.size() != expected_tensors) { + throw std::runtime_error( + "sanoTTS expects exactly " + std::to_string(expected_tensors) + + " tensors for this config, found " + std::to_string(metadata.size())); + } + out->tensors.reserve(metadata.size()); + for (const auto & tensor : metadata) { + if (assets::ggml_type_for_tensor_dtype(tensor.dtype) != GGML_TYPE_F32) { + throw std::runtime_error( + "sanoTTS supports FP32 weights only: " + tensor.name); + } + out->tensors.emplace( + tensor.name, + out->store->load_tensor( + *assets->weights, + tensor.name, + assets::TensorStorageType::F32, + tensor.shape)); + } + out->store->upload(); + assets->weights->release_storage(); + return out; +} + +/** Conv1d over channel-major [1, C, T] with explicit padding and dilation. + * A kernel-1 conv lowers to a matmul. */ +inline core::TensorValue conv1d( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t out_channels, + int64_t kernel, + int padding, + int dilation = 1) { + const int64_t in_channels = input.shape.dims[1]; + const int64_t input_frames = input.shape.dims[2]; + const int64_t output_frames = + input_frames + 2 * padding - static_cast(dilation) * (kernel - 1); + const auto source = contiguous(ctx, input); + auto * input_2d = ggml_reshape_2d( + ctx.ggml, + source.tensor, + input_frames, + in_channels); + auto * kernel_tensor = weight(weights, prefix + ".weight").tensor; + ggml_tensor * output = nullptr; + if (kernel == 1 && padding == 0) { + auto * kernel_2d = ggml_reshape_2d( + ctx.ggml, + kernel_tensor, + in_channels, + out_channels); + auto * input_channels_first = ggml_cont( + ctx.ggml, + ggml_permute(ctx.ggml, input_2d, 1, 0, 2, 3)); + auto * output_channels_first = + ggml_mul_mat(ctx.ggml, kernel_2d, input_channels_first); + output = ggml_reshape_2d( + ctx.ggml, + ggml_cont( + ctx.ggml, + ggml_permute(ctx.ggml, output_channels_first, 1, 0, 2, 3)), + output_frames, + out_channels); + } else { + auto * kernel_3d = ggml_reshape_3d( + ctx.ggml, + kernel_tensor, + kernel, + in_channels, + out_channels); + auto * input_3d = ggml_reshape_3d( + ctx.ggml, + input_2d, + input_frames, + in_channels, + 1); + auto * output_3d = ggml_conv_1d( + ctx.ggml, + kernel_3d, + input_3d, + 1, + padding, + dilation); + output = ggml_reshape_2d( + ctx.ggml, + output_3d, + output_frames, + out_channels); + } + auto * bias = ggml_reshape_2d( + ctx.ggml, + weight(weights, prefix + ".bias").tensor, + 1, + out_channels); + output = ggml_add(ctx.ggml, output, bias); + return core::wrap_tensor( + ggml_reshape_3d(ctx.ggml, output, output_frames, out_channels, 1), + core::TensorShape::from_dims({1, out_channels, output_frames}), + GGML_TYPE_F32); +} + +/** x + scale * conv2(silu(conv1(x))) -- the front ends' ResidualConvBlock. + * `scale` is a learned one-element tensor, broadcast by ggml_mul. */ +inline core::TensorValue residual_block( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t hidden, + int64_t kernel) { + const int padding = static_cast(kernel / 2); + auto h = conv1d(ctx, weights, input, prefix + ".net.0", hidden, kernel, padding); + h = modules::SiluModule().build(ctx, h); + h = conv1d(ctx, weights, h, prefix + ".net.2", hidden, kernel, padding); + const auto h_source = contiguous(ctx, h); + const auto scaled_h = core::wrap_tensor( + ggml_mul(ctx.ggml, h_source.tensor, weight(weights, prefix + ".scale").tensor), + h.shape, + GGML_TYPE_F32); + return add(ctx, input, scaled_h); +} + +inline core::TensorValue embed_tokens( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & tokens, + const std::string & name, + int64_t vocab, + int64_t hidden) { + auto embedded = modules::EmbeddingModule({vocab, hidden}).build( + ctx, + tokens, + weight(weights, name)); + return modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, embedded); +} + +struct GraphResources { + ~GraphResources() { + core::free_backend_graph_plan(backend, plan); + core::release_backend_graph_resources(backend, graph); + if (allocator != nullptr) { + ggml_gallocr_free(allocator); + } + if (io_buffer != nullptr) { + ggml_backend_buffer_free(io_buffer); + } + } + + std::unique_ptr io_context; + std::unique_ptr graph_context; + ggml_backend_buffer_t io_buffer = nullptr; + ggml_gallocr_t allocator = nullptr; + ggml_backend_t backend = nullptr; + ggml_backend_graph_plan_t plan = nullptr; + ggml_cgraph * graph = nullptr; +}; + +inline void allocate_graph(GraphResources & resources) { + resources.io_buffer = + ggml_backend_alloc_ctx_tensors(resources.io_context.get(), resources.backend); + if (resources.io_buffer == nullptr) { + throw std::runtime_error("sanoTTS failed to allocate graph input buffer"); + } + resources.allocator = + ggml_gallocr_new(ggml_backend_get_default_buffer_type(resources.backend)); + if (resources.allocator == nullptr || + !ggml_gallocr_reserve(resources.allocator, resources.graph) || + !ggml_gallocr_alloc_graph(resources.allocator, resources.graph)) { + throw std::runtime_error("sanoTTS failed to allocate backend graph"); + } + core::validate_backend_graph_supported( + resources.backend, + resources.graph, + "sanoTTS"); + resources.plan = + core::create_backend_graph_plan_if_host(resources.backend, resources.graph); +} + +inline void compute_graph(GraphResources & resources, const char * label) { + const auto status = core::compute_backend_graph( + resources.backend, + resources.graph, + resources.plan, + label); + ggml_backend_synchronize(resources.backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error(std::string(label) + " graph compute failed"); + } +} + +/** torch.linspace(0, 1, n) with exact CPU-kernel float semantics: step in + * fp32, the first half filled as step*i, the second as fma(-step, n-1-i, 1). */ +inline void linspace01(float * dst, int64_t n) { + if (n <= 0) { + return; + } + if (n == 1) { + dst[0] = 0.0F; + return; + } + const auto step = 1.0F / static_cast(n - 1); + const int64_t half = n / 2; + for (int64_t i = 0; i < half; ++i) { + dst[i] = step * static_cast(i); + } + for (int64_t i = half; i < n; ++i) { + dst[i] = std::fma(-step, static_cast(n - 1 - i), 1.0F); + } +} + + +// ---- shared front-end graphs --------------------------------------------- +// +// Both lineages run the same two token-level stages -- embed + positional +// features + input projection + residual blocks (+ optional 1x1 output conv) +// -- under the same tensor names; only the dimensions differ per package. + +struct FrontStageSpec { + const char * embedding = nullptr; // "duration.embedding.weight" + const char * input_proj = nullptr; // "duration.input_proj" + const char * block_prefix = nullptr; // "duration.blocks." + const char * output_conv = nullptr; // "duration.output", or nullptr + const char * label = nullptr; // graph label for contexts/tracing + int64_t feat_rows = 3; + int64_t vocab = 0; + int64_t hidden = 0; + int64_t depth = 0; + int64_t kernel = 0; + int64_t out_channels = 1; // used when output_conv is set +}; + +inline FrontStageSpec duration_stage_spec(int64_t vocab, int64_t hidden, int64_t depth, int64_t kernel) { + return {"duration.embedding.weight", "duration.input_proj", "duration.blocks.", + "duration.output", "sanotts.duration", 3, vocab, hidden, depth, kernel, 1}; +} + +inline FrontStageSpec token_stage_spec(int64_t vocab, int64_t hidden, int64_t depth, int64_t kernel) { + return {"acoustic.embedding.weight", "acoustic.token_input_proj", "acoustic.token_blocks.", + nullptr, "sanotts.token", 2, vocab, hidden, depth, kernel, 1}; +} + +struct FrontGraph : GraphResources { + int64_t token_count = 0; + ggml_tensor * tokens = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * output = nullptr; // log-durations or token context +}; + +inline std::unique_ptr build_front_graph( + const SanoTtsBackendWeights & weights, + const FrontStageSpec & spec, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t token_count, + size_t io_arena_bytes, + size_t graph_arena_bytes) { + auto out = std::make_unique(); + out->backend = backend; + out->token_count = token_count; + out->io_context.reset(ggml_init({io_arena_bytes, nullptr, true})); + out->graph_context.reset(ggml_init({graph_arena_bytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create graph contexts"); + } + core::ModuleBuildContext io_ctx{out->io_context.get(), spec.label, backend_type}; + core::ModuleBuildContext ctx{out->graph_context.get(), spec.label, backend_type}; + auto tokens = core::make_tensor( + io_ctx, + GGML_TYPE_I32, + core::TensorShape::from_dims({1, token_count})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, spec.feat_rows, token_count})); + ggml_set_input(tokens.tensor); + ggml_set_input(feats.tensor); + + auto hidden = embed_tokens(ctx, weights, tokens, spec.embedding, spec.vocab, spec.hidden); + hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); + hidden = conv1d(ctx, weights, hidden, spec.input_proj, spec.hidden, 1, 0); + for (int64_t block = 0; block < spec.depth; ++block) { + hidden = residual_block( + ctx, + weights, + hidden, + spec.block_prefix + std::to_string(block), + spec.hidden, + spec.kernel); + } + if (spec.output_conv != nullptr) { + hidden = conv1d(ctx, weights, hidden, spec.output_conv, spec.out_channels, 1, 0); + } + hidden = contiguous(ctx, hidden); + out->tokens = tokens.tensor; + out->feats = feats.tensor; + out->output = hidden.tensor; + ggml_set_output(out->output); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->output); + allocate_graph(*out); + return out; +} + +/** The frame-level acoustic stage both decoder graphs open with: expanded + * token context + [frame_pos, token_pos, duration_pos] -> residual blocks + * -> 1x1 output conv (mel-100 for nano, the 192-ch latent for piperlite). */ +inline core::TensorValue acoustic_frame_stage( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & context, + const core::TensorValue & feats, + int64_t hidden, + int64_t depth, + int64_t kernel, + int64_t out_channels) { + auto value = modules::ConcatModule({1}).build(ctx, context, feats); + value = conv1d(ctx, weights, value, "acoustic.frame_input_proj", hidden, 1, 0); + for (int64_t block = 0; block < depth; ++block) { + value = residual_block( + ctx, + weights, + value, + "acoustic.frame_blocks." + std::to_string(block), + hidden, + kernel); + } + return conv1d(ctx, weights, value, "acoustic.output", out_channels, 1, 0); +} + +// ---- shared runtime state ------------------------------------------------ + +struct BackendOwner { + ggml_backend_t value = nullptr; + BackendOwner() = default; + BackendOwner(const BackendOwner &) = delete; + BackendOwner & operator=(const BackendOwner &) = delete; + ~BackendOwner() { + if (value != nullptr) { + ggml_backend_free(value); + } + } +}; + +/** Backend + uploaded weights, with the tiny duration model mirrored to the + * host on CUDA builds: durations round to integers and gate the whole frame + * layout, so small TF32 differences must not move them (the inflect_v2 + * rationale). */ +struct BackendState { + std::shared_ptr assets; + int threads = 1; + core::BackendType backend_type = core::BackendType::Cpu; + BackendOwner backend; + std::shared_ptr weights; + BackendOwner duration_backend; + std::shared_ptr duration_weights; + + BackendState( + std::shared_ptr assets_in, + core::BackendConfig backend_config, + size_t expected_tensors, + size_t weight_arena_bytes) + : assets(std::move(assets_in)), + threads(std::max(1, backend_config.threads)) { + if (assets == nullptr) { + throw std::runtime_error("sanoTTS runtime requires assets"); + } + backend_config.threads = threads; + backend.value = core::init_backend(backend_config); + backend_type = core::backend_type(backend.value); + core::set_backend_threads(backend.value, threads); + weights = load_weights( + assets, backend.value, backend_type, expected_tensors, weight_arena_bytes); + if (backend_type == core::BackendType::Cuda) { + core::BackendConfig duration_config{core::BackendType::Cpu, 0, threads}; + duration_backend.value = core::init_backend(duration_config); + core::set_backend_threads(duration_backend.value, threads); + duration_weights = load_weights( + assets, + duration_backend.value, + core::BackendType::Cpu, + expected_tensors, + weight_arena_bytes); + } + } + + ggml_backend_t duration_backend_value() const { + return duration_backend.value != nullptr ? duration_backend.value : backend.value; + } + core::BackendType duration_backend_type() const { + return duration_backend.value != nullptr ? core::BackendType::Cpu : backend_type; + } + const SanoTtsBackendWeights & duration_weights_ref() const { + return duration_weights != nullptr ? *duration_weights : *weights; + } +}; + +template +Graph & cached_graph( + runtime::CacheSlots> & slots, + int64_t key, + const char * trace_name, + Build && build) { + if (auto * found = slots.find(key)) { + engine::debug::trace_log_scalar(trace_name, true); + return **found; + } + engine::debug::trace_log_scalar(trace_name, false); + slots.put(key, build()); + auto * created = slots.find(key); + if (created == nullptr) { + throw std::runtime_error("sanoTTS graph cache insert failed"); + } + return **created; +} + +// ---- shared host-side pieces --------------------------------------------- +// +// Hints are computed in double and cast to fp32, matching the numpy +// references both runtimes are gated against. + +/** [positions, length_hint, valid=1] rows for the duration stage. */ +inline std::vector duration_features(int64_t token_count, int64_t max_tokens) { + std::vector feats(static_cast(3 * token_count)); + linspace01(feats.data(), token_count); + const auto length_hint = static_cast( + std::log1p(static_cast(token_count)) / + std::log1p(static_cast(max_tokens))); + std::fill_n(feats.begin() + token_count, token_count, length_hint); + std::fill_n(feats.begin() + 2 * token_count, token_count, 1.0F); + return feats; +} + +/** [token_pos, duration_hint] rows for the token stage. */ +inline std::vector token_features( + int64_t token_count, + const std::vector & durations) { + std::vector feats(static_cast(2 * token_count)); + linspace01(feats.data(), token_count); + double max_duration = 1.0; + for (const int64_t duration : durations) { + max_duration = std::max(max_duration, static_cast(duration)); + } + const double log_max_duration = std::log1p(max_duration); + for (int64_t token = 0; token < token_count; ++token) { + feats[static_cast(token_count + token)] = static_cast( + std::log1p(static_cast(durations[static_cast(token)])) / + log_max_duration); + } + return feats; +} + +/** exp -> clamp_min(1) -> *scale -> round (ties to even) -> clamp, in + * double, exactly as the references compute it. Returns per-token frame + * counts and validates the total against the trained limits. */ +inline std::vector round_durations( + const std::vector & log_duration, + double scale, + int64_t max_frames_per_token, + int64_t max_total_frames, + int64_t & total_frames) { + std::vector durations(log_duration.size()); + total_frames = 0; + for (size_t token = 0; token < log_duration.size(); ++token) { + double value = std::exp(static_cast(log_duration[token])); + if (!std::isfinite(value)) { + throw std::runtime_error("sanoTTS duration predictor produced a non-finite duration"); + } + if (value < 1.0) { + value = 1.0; + } + value = std::rint(value * scale); + if (value < 1.0) { + value = 1.0; + } + if (value > static_cast(max_frames_per_token)) { + value = static_cast(max_frames_per_token); + } + durations[token] = static_cast(value); + total_frames += durations[token]; + } + if (total_frames < 1 || total_frames > max_total_frames) { + throw std::runtime_error( + "sanoTTS expanded to " + std::to_string(total_frames) + + " frames, outside the supported range"); + } + return durations; +} + +/** Repeat each token's context vector across its own frames. */ +inline std::vector expand_context( + const std::vector & token_context, + const std::vector & durations, + int64_t hidden, + int64_t token_count, + int64_t frames) { + std::vector expanded(static_cast(hidden * frames)); + for (int64_t channel = 0; channel < hidden; ++channel) { + const float * row = token_context.data() + channel * token_count; + float * out_row = expanded.data() + channel * frames; + int64_t at = 0; + for (int64_t token = 0; token < token_count; ++token) { + const float value = row[token]; + for (int64_t j = 0; j < durations[static_cast(token)]; ++j) { + out_row[at++] = value; + } + } + } + return expanded; +} + +/** [frame_pos, token_pos, duration_pos] rows -- expand_features' documented + * float semantics: doubles cast to fp32, a duration of 1 contributing 0. */ +inline std::vector frame_features( + int64_t token_count, + const std::vector & durations, + int64_t frames) { + std::vector feats(static_cast(3 * frames)); + linspace01(feats.data(), frames); + const int64_t denominator = token_count > 1 ? token_count - 1 : 1; + float * token_pos = feats.data() + frames; + float * duration_pos = feats.data() + 2 * frames; + int64_t at = 0; + for (int64_t token = 0; token < token_count; ++token) { + const int64_t count = durations[static_cast(token)]; + const auto position = static_cast( + static_cast(token) / static_cast(denominator)); + for (int64_t j = 0; j < count; ++j) { + token_pos[at] = position; + duration_pos[at] = count == 1 + ? 0.0F + : static_cast( + static_cast(j) / static_cast(count - 1)); + ++at; + } + } + return feats; +} + +inline void write_i32_input(ggml_tensor * tensor, const core::TensorShape & shape, const std::vector & values) { + core::write_tensor_i32(core::wrap_tensor(tensor, shape, GGML_TYPE_I32), values); +} + +inline void write_f32_input(ggml_tensor * tensor, const core::TensorShape & shape, const std::vector & values) { + core::write_tensor_f32(core::wrap_tensor(tensor, shape, GGML_TYPE_F32), values); +} + +} // namespace engine::models::sanotts::graph diff --git a/src/community_models/sanotts/piper_runtime.cpp b/src/community_models/sanotts/piper_runtime.cpp new file mode 100644 index 000000000..4b80ef346 --- /dev/null +++ b/src/community_models/sanotts/piper_runtime.cpp @@ -0,0 +1,552 @@ +#include "engine/community_models/sanotts/piper_runtime.h" + +#include "graph_common.h" + +#include "engine/framework/modules/conv_modules.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { +namespace { + +namespace core = engine::core; +namespace modules = engine::modules; +using namespace engine::models::sanotts::graph; // shared graph helpers + +constexpr size_t kIoArenaBytes = 8ULL * 1024ULL * 1024ULL; +constexpr size_t kGraphArenaBytes = 128ULL * 1024ULL * 1024ULL; +constexpr size_t kWeightArenaBytes = 32ULL * 1024ULL * 1024ULL; +constexpr float kLeakyReluSlope = 0.1F; + +// PiperResidualBank geometry, fixed by the training code: branch b uses +// kernel kBankKernels[b] with dilation kBankDilations1[b] on its first conv +// and kBankDilations2[b] on its second. +constexpr int64_t kBankKernels[3] = {3, 5, 7}; +constexpr int kBankDilations1[3] = {1, 2, 3}; +constexpr int kBankDilations2[3] = {2, 6, 12}; + +// The three ConvTranspose1d stages: stride and PyTorch padding. Kernel sizes +// are read from the weights themselves. +constexpr int kUpStrides[3] = {8, 8, 4}; +constexpr int64_t kUpPaddings[3] = {4, 4, 2}; + +/** Ids outside a component's trained vocab remap to schwa -- the same + * fallback the reference runtimes use for the shared-frontend/table + * vocab-size mismatch (e.g. duration vocab 127 vs table ids to 156). */ +constexpr int32_t kSchwaFallbackId = 59; + +size_t expected_tensor_count(const SanoTtsPiperConfig & config) { + const auto duration = 5 + 5 * config.duration_depth; + const auto acoustic = + 7 + 5 * (config.acoustic_token_depth + config.acoustic_depth); + int64_t decoder = 4; // pre + post + for (const auto & branches : config.stage_branches) { + decoder += 2 + 4 * static_cast(branches.size()); + } + if (config.post_filter_channels > 0) { + decoder += 4 + 5 * config.post_filter_layers; + } + return static_cast(duration + acoustic + decoder); +} + +std::vector clamp_ids_to_vocab( + const std::vector & ids, + int64_t vocab_size) { + const int32_t fallback = + kSchwaFallbackId < vocab_size ? kSchwaFallbackId : 0; + std::vector out(ids.size()); + for (size_t i = 0; i < ids.size(); ++i) { + const int32_t id = ids[i]; + out[i] = (id < 0 || id >= vocab_size) ? fallback : id; + } + return out; +} + +int64_t kernel_of(const SanoTtsBackendWeights & weights, const std::string & prefix) { + return weight(weights, prefix + ".weight").shape.dims[2]; +} + +core::TensorValue leaky_relu( + core::ModuleBuildContext & ctx, + const core::TensorValue & input, + float slope) { + const auto source = contiguous(ctx, input); + return core::wrap_tensor( + ggml_leaky_relu(ctx.ggml, source.tensor, slope, false), + input.shape, + GGML_TYPE_F32); +} + +core::TensorValue scaled( + core::ModuleBuildContext & ctx, + const core::TensorValue & value, + float scale) { + const auto source = contiguous(ctx, value); + return core::wrap_tensor( + ggml_scale(ctx.ggml, source.tensor, scale), + value.shape, + GGML_TYPE_F32); +} + +/** 'same'-padded conv: pad = dilation * (kernel / 2). */ +core::TensorValue conv1d_same( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t out_channels, + int64_t kernel, + int dilation = 1) { + return conv1d( + ctx, + weights, + input, + prefix, + out_channels, + kernel, + dilation * static_cast(kernel / 2), + dilation); +} + +/** ConvTranspose1d producing stride * input_frames samples, PyTorch padding + * semantics -- the same construction inflect_v2 uses. */ +core::TensorValue conv_transpose1d( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t out_channels, + int64_t kernel, + int stride, + int64_t padding) { + const int64_t input_frames = input.shape.dims[2]; + const int64_t output_frames = input_frames * stride; + if (ctx.backend_type == core::BackendType::Cpu && padding > 0) { + const auto source = contiguous(ctx, input); + auto * input_2d = ggml_reshape_2d( + ctx.ggml, + source.tensor, + input_frames, + input.shape.dims[1]); + auto * output = ggml_conv_transpose_1d( + ctx.ggml, + weight(weights, prefix + ".weight").tensor, + input_2d, + stride, + 0, + 1); + const int64_t full_frames = (input_frames - 1) * stride + kernel; + output = ggml_reshape_2d(ctx.ggml, output, full_frames, out_channels); + output = ggml_view_2d( + ctx.ggml, + output, + output_frames, + out_channels, + ggml_row_size(output->type, full_frames), + ggml_row_size(output->type, padding)); + output = ggml_cont(ctx.ggml, output); + auto * bias = ggml_reshape_2d( + ctx.ggml, + weight(weights, prefix + ".bias").tensor, + 1, + out_channels); + output = ggml_add(ctx.ggml, output, bias); + return core::wrap_tensor( + ggml_reshape_3d(ctx.ggml, output, output_frames, out_channels, 1), + core::TensorShape::from_dims({1, out_channels, output_frames}), + GGML_TYPE_F32); + } + auto output = modules::ConvTranspose1dModule({ + input.shape.dims[1], + out_channels, + kernel, + stride, + 0, + 1, + true, + }).build( + ctx, + input, + { + weight(weights, prefix + ".weight"), + weight(weights, prefix + ".bias"), + }); + return modules::SliceModule({2, padding, output_frames}).build(ctx, output); +} + +/** PiperResidualBank: mean over active branches of + * y2 = conv2(lrelu(y1)) + y1, y1 = conv1(lrelu(x)) + x. */ +core::TensorValue residual_bank( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + const std::vector & branches) { + core::TensorValue sum; + for (const int64_t branch : branches) { + const std::string branch_prefix = + prefix + ".blocks." + std::to_string(branch); + const int64_t kernel = kBankKernels[branch]; + auto value = leaky_relu(ctx, input, kLeakyReluSlope); + value = conv1d_same( + ctx, + weights, + value, + branch_prefix + ".conv1", + input.shape.dims[1], + kernel, + kBankDilations1[branch]); + const auto y1 = add(ctx, value, input); + value = leaky_relu(ctx, y1, kLeakyReluSlope); + value = conv1d_same( + ctx, + weights, + value, + branch_prefix + ".conv2", + input.shape.dims[1], + kernel, + kBankDilations2[branch]); + const auto y2 = add(ctx, value, y1); + sum = sum.valid() ? add(ctx, sum, y2) : y2; + } + return scaled(ctx, sum, 1.0F / static_cast(branches.size())); +} + +core::TensorValue post_filter( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const SanoTtsPiperConfig & config, + const core::TensorValue & audio) { + auto value = conv1d_same( + ctx, + weights, + audio, + "decoder.post_filter.in_conv", + config.post_filter_channels, + kernel_of(weights, "decoder.post_filter.in_conv")); + for (int64_t layer = 0; layer < config.post_filter_layers; ++layer) { + const std::string prefix = + "decoder.post_filter.units." + std::to_string(layer); + auto branch = leaky_relu(ctx, value, kLeakyReluSlope); + branch = conv1d_same( + ctx, + weights, + branch, + prefix + ".conv1", + config.post_filter_channels, + kernel_of(weights, prefix + ".conv1"), + static_cast(1 + layer)); + branch = leaky_relu(ctx, branch, kLeakyReluSlope); + branch = conv1d_same( + ctx, + weights, + branch, + prefix + ".conv2", + config.post_filter_channels, + kernel_of(weights, prefix + ".conv2")); + const auto branch_source = contiguous(ctx, branch); + branch = core::wrap_tensor( + ggml_mul( + ctx.ggml, + branch_source.tensor, + weight(weights, prefix + ".scale").tensor), + branch.shape, + GGML_TYPE_F32); + value = add(ctx, value, branch); + } + auto correction = conv1d_same( + ctx, + weights, + value, + "decoder.post_filter.out_conv", + 1, + kernel_of(weights, "decoder.post_filter.out_conv")); + correction = scaled(ctx, correction, static_cast(config.post_filter_scale)); + auto mixed = add(ctx, audio, correction); + const auto mixed_source = contiguous(ctx, mixed); + return core::wrap_tensor( + ggml_tanh(ctx.ggml, mixed_source.tensor), + mixed.shape, + GGML_TYPE_F32); +} + +struct DecoderGraph : GraphResources { + int64_t frames = 0; + ggml_tensor * context = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * waveform = nullptr; +}; + +/** Frame-stage acoustic blocks -> latent -> 3-stage ConvTranspose decoder + * with dilated residual banks -> tanh waveform (plus kristin's post + * filter when the config carries one). */ +std::unique_ptr build_decoder_graph( + const SanoTtsBackendWeights & weights, + const SanoTtsPiperConfig & config, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t frames) { + auto out = std::make_unique(); + out->backend = backend; + out->frames = frames; + out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); + out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create decoder graph contexts"); + } + core::ModuleBuildContext io_ctx{ + out->io_context.get(), + "sanotts.piper.decoder.io", + backend_type, + }; + core::ModuleBuildContext ctx{ + out->graph_context.get(), + "sanotts.piper.decoder", + backend_type, + }; + auto context = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, 3, frames})); + ggml_set_input(context.tensor); + ggml_set_input(feats.tensor); + + auto latent = acoustic_frame_stage( + ctx, + weights, + context, + feats, + config.acoustic_hidden, + config.acoustic_depth, + config.acoustic_kernel, + config.acoustic_out_channels); + + auto value = conv1d_same( + ctx, + weights, + latent, + "decoder.pre", + config.channels[0], + kernel_of(weights, "decoder.pre")); + for (size_t stage = 0; stage < 3; ++stage) { + const std::string up_name = "decoder.up" + std::to_string(stage); + value = leaky_relu(ctx, value, kLeakyReluSlope); + value = conv_transpose1d( + ctx, + weights, + value, + up_name, + config.channels[stage + 1], + kernel_of(weights, up_name), + kUpStrides[stage], + kUpPaddings[stage]); + value = residual_bank( + ctx, + weights, + value, + "decoder.res" + std::to_string(stage) + ".0", + config.stage_branches[stage]); + } + value = leaky_relu(ctx, value, 0.01F); + auto audio = conv1d_same( + ctx, + weights, + value, + "decoder.post", + 1, + kernel_of(weights, "decoder.post")); + { + const auto audio_source = contiguous(ctx, audio); + audio = core::wrap_tensor( + ggml_tanh(ctx.ggml, audio_source.tensor), + audio.shape, + GGML_TYPE_F32); + } + if (config.post_filter_channels > 0) { + audio = post_filter(ctx, weights, config, audio); + } + audio = contiguous(ctx, audio); + out->context = context.tensor; + out->feats = feats.tensor; + out->waveform = audio.tensor; + ggml_set_output(out->waveform); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->waveform); + allocate_graph(*out); + return out; +} + +} // namespace + +struct SanoTtsPiperRuntime::State : BackendState { + State(const std::shared_ptr & assets_in, + core::BackendConfig backend_config) + : BackendState( + assets_in, + backend_config, + assets_in == nullptr ? 0 : expected_tensor_count(assets_in->piper), + kWeightArenaBytes) {} + + runtime::CacheSlots> duration_graphs{4}; + runtime::CacheSlots> token_graphs{4}; + runtime::CacheSlots> decoder_graphs{2}; +}; + +SanoTtsPiperRuntime::SanoTtsPiperRuntime( + std::shared_ptr assets, + core::BackendConfig backend_config) + : state_(std::make_unique(std::move(assets), backend_config)) {} + +SanoTtsPiperRuntime::~SanoTtsPiperRuntime() = default; + +runtime::AudioBuffer SanoTtsPiperRuntime::synthesize( + const std::vector & token_ids, + const SanoTtsPiperGenerationOptions & options) { + const auto & config = state_->assets->piper; + const auto token_count = static_cast(token_ids.size()); + if (token_count <= 0) { + throw std::runtime_error("sanoTTS requires at least one phoneme token"); + } + const auto total_start = std::chrono::steady_clock::now(); + + // -- durations --------------------------------------------------------- + const auto duration_start = std::chrono::steady_clock::now(); + auto & duration = cached_graph( + state_->duration_graphs, + token_count, + "sanotts.duration_graph.cache_hit", + [&] { + return build_front_graph( + state_->duration_weights_ref(), + duration_stage_spec( + config.duration_vocab, + config.duration_hidden, + config.duration_depth, + config.duration_kernel), + state_->duration_backend_value(), + state_->duration_backend_type(), + token_count, + kIoArenaBytes, + kGraphArenaBytes); + }); + write_i32_input( + duration.tokens, + core::TensorShape::from_dims({1, token_count}), + clamp_ids_to_vocab(token_ids, config.duration_vocab)); + write_f32_input( + duration.feats, + core::TensorShape::from_dims({1, 3, token_count}), + duration_features(token_count, config.duration_max_tokens)); + compute_graph(duration, "sanoTTS piper duration"); + const auto log_duration = core::read_tensor_f32(duration.output); + if (static_cast(log_duration.size()) != token_count) { + throw std::runtime_error("sanoTTS duration graph returned invalid output"); + } + // The user's speaking_rate multiplies the voice's tuned length scale. + int64_t frames = 0; + const auto durations = round_durations( + log_duration, + config.duration_length_scale * static_cast(options.speaking_rate), + config.duration_max_frames, + config.duration_max_tokens * config.duration_max_frames, + frames); + engine::debug::timing_log_scalar( + "sanotts.duration_ms", + engine::debug::elapsed_ms(duration_start)); + + // -- token-stage acoustic context -------------------------------------- + const auto acoustic_start = std::chrono::steady_clock::now(); + auto & token_graph = cached_graph( + state_->token_graphs, + token_count, + "sanotts.token_graph.cache_hit", + [&] { + return build_front_graph( + *state_->weights, + token_stage_spec( + config.acoustic_vocab, + config.acoustic_hidden, + config.acoustic_token_depth, + config.acoustic_kernel), + state_->backend.value, + state_->backend_type, + token_count, + kIoArenaBytes, + kGraphArenaBytes); + }); + write_i32_input( + token_graph.tokens, + core::TensorShape::from_dims({1, token_count}), + clamp_ids_to_vocab(token_ids, config.acoustic_vocab)); + write_f32_input( + token_graph.feats, + core::TensorShape::from_dims({1, 2, token_count}), + token_features(token_count, durations)); + compute_graph(token_graph, "sanoTTS piper token context"); + const auto token_context = core::read_tensor_f32(token_graph.output); + if (static_cast(token_context.size()) != + config.acoustic_hidden * token_count) { + throw std::runtime_error("sanoTTS token graph returned invalid output"); + } + engine::debug::timing_log_scalar( + "sanotts.acoustic_ms", + engine::debug::elapsed_ms(acoustic_start)); + + // -- frame stage + decoder --------------------------------------------- + const auto decoder_start = std::chrono::steady_clock::now(); + auto & decoder = cached_graph( + state_->decoder_graphs, + frames, + "sanotts.decoder_graph.cache_hit", + [&] { + return build_decoder_graph( + *state_->weights, + config, + state_->backend.value, + state_->backend_type, + frames); + }); + write_f32_input( + decoder.context, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames}), + expand_context(token_context, durations, config.acoustic_hidden, token_count, frames)); + write_f32_input( + decoder.feats, + core::TensorShape::from_dims({1, 3, frames}), + frame_features(token_count, durations, frames)); + compute_graph(decoder, "sanoTTS piper decoder"); + runtime::AudioBuffer out; + out.sample_rate = static_cast(config.sample_rate); + out.channels = 1; + out.samples = core::read_tensor_f32(decoder.waveform); + const auto expected_samples = static_cast(frames) * 256U; + if (out.samples.size() != expected_samples) { + throw std::runtime_error("sanoTTS decoder returned an unexpected sample count"); + } + engine::debug::timing_log_scalar( + "sanotts.decoder_ms", + engine::debug::elapsed_ms(decoder_start)); + engine::debug::trace_log_scalar("sanotts.token_count", token_count); + engine::debug::trace_log_scalar("sanotts.frames", frames); + engine::debug::trace_log_scalar( + "sanotts.output_samples", + static_cast(out.samples.size())); + engine::debug::timing_log_scalar( + "session.wall_ms", + engine::debug::elapsed_ms(total_start)); + return out; +} + +} // namespace engine::models::sanotts diff --git a/src/community_models/sanotts/runtime.cpp b/src/community_models/sanotts/runtime.cpp new file mode 100644 index 000000000..64efa1ba0 --- /dev/null +++ b/src/community_models/sanotts/runtime.cpp @@ -0,0 +1,658 @@ +#include "engine/community_models/sanotts/runtime.h" + +#include "graph_common.h" + +#include "engine/framework/audio/istft_graph.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { +namespace { + +namespace core = engine::core; +namespace modules = engine::modules; +using namespace engine::models::sanotts::graph; // shared graph helpers + +constexpr size_t kIoArenaBytes = 8ULL * 1024ULL * 1024ULL; +constexpr size_t kGraphArenaBytes = 128ULL * 1024ULL * 1024ULL; +constexpr size_t kWeightArenaBytes = 32ULL * 1024ULL * 1024ULL; + +// The decoder's norms are nn.LayerNorm(eps=1e-6), NOT torch's 1e-5 default. +// The difference compounds through the ConvNeXt blocks and is then amplified +// by the exp() in the magnitude head; the reference implementations document +// losing 0.06 of correlation and a third of the output amplitude to exactly +// this constant. +constexpr float kLayerNormEps = 1.0e-6F; +constexpr float kDcBlockPole = 0.9973F; +constexpr double kPi = 3.14159265358979323846; + +/** Tensor count implied by the config -- 103 for heart-nano, 117 for heart. + * Kept in lockstep with the inventory validate_nano_tensors() builds. */ +size_t expected_tensor_count(const SanoTtsConfig & config) { + const auto duration = 5 + 5 * config.duration_depth; + const auto acoustic = + 7 + 5 * (config.acoustic_token_depth + config.acoustic_depth); + const auto decoder = 10 + 9 * config.blocks; + return static_cast(duration + acoustic + decoder); +} + +core::TensorValue channel_last_layer_norm( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t channels) { + return modules::LayerNormModule({channels, kLayerNormEps, true, true}).build( + ctx, + input, + { + weight(weights, prefix + ".weight"), + weight(weights, prefix + ".bias"), + }); +} + +// ---- ATen-compatible noise ---------------------------------------------- +// +// The decoder is noise-fed, so a rendering is only reproducible if the noise +// stream is. This is PyTorch's CPU path exactly: MT19937 seeded from the low +// 32 bits, a 24-bit float uniform, and Box-Muller in blocks of 16 +// (aten/src/ATen/native/DistributionTemplates.h). The integer half is +// bit-exact against torch.rand; the Gaussian half goes through logf/cosf/ +// sinf, which differ by a few ulp across libm builds -- the reference +// implementations document the same bound (at most ~2e-6 per draw). + +constexpr int kMtN = 624; +constexpr int kMtM = 397; + +struct AtenMt19937 { + uint32_t state[kMtN]; + int left = 1; + int next = 0; + + explicit AtenMt19937(uint64_t seed) { + // ATen mt19937_engine::init_with_uint32 -- only the low 32 bits used. + state[0] = static_cast(seed & 0xFFFFFFFFULL); + for (int i = 1; i < kMtN; ++i) { + state[i] = 1812433253U * (state[i - 1] ^ (state[i - 1] >> 30)) + + static_cast(i); + } + } + + static uint32_t twist(uint32_t u, uint32_t v) { + const uint32_t mixed = (u & 0x80000000U) | (v & 0x7FFFFFFFU); + return (mixed >> 1) ^ ((v & 1U) != 0U ? 0x9908B0DFU : 0U); + } + + void next_state() { + left = kMtN; + next = 0; + int i = 0; + for (; i < kMtN - kMtM; ++i) { + state[i] = state[i + kMtM] ^ twist(state[i], state[i + 1]); + } + for (; i < kMtN - 1; ++i) { + state[i] = state[i + kMtM - kMtN] ^ twist(state[i], state[i + 1]); + } + state[kMtN - 1] = state[kMtM - 1] ^ twist(state[kMtN - 1], state[0]); + } + + uint32_t random() { + if (--left <= 0) { + next_state(); + } + uint32_t y = state[next++]; + y ^= (y >> 11); + y ^= (y << 7) & 0x9D2C5680U; + y ^= (y << 15) & 0xEFC60000U; + y ^= (y >> 18); + return y; + } + + /** at::uniform_real_distribution: (raw & (2^24 - 1)) * 2^-24. */ + float uniform() { + return static_cast(random() & 0xFFFFFFU) * (1.0F / 16777216.0F); + } +}; + +/** ATen normal_fill_16, mean 0 std 1, float32 throughout. In place. */ +void normal_fill_16(float * d) { + for (int j = 0; j < 8; ++j) { + const float u1 = 1.0F - d[j]; + const float u2 = d[j + 8]; + const float radius = std::sqrt(-2.0F * std::log(u1)); + const float theta = static_cast(2.0 * kPi) * u2; + d[j] = radius * std::cos(theta); + d[j + 8] = radius * std::sin(theta); + } +} + +std::vector seeded_noise(uint64_t seed, int64_t channels, int64_t frames) { + const int64_t size = channels * frames; + if (size < 16) { + // torch dispatches sizes under 16 to a scalar path this does not + // implement; the decoder always asks for channels*frames above that. + throw std::runtime_error("sanoTTS seeded noise needs at least 16 values"); + } + AtenMt19937 gen(seed); + std::vector out(static_cast(size)); + for (auto & value : out) { + value = gen.uniform(); + } + int64_t i = 0; + for (; i + 16 <= size; i += 16) { + normal_fill_16(out.data() + i); + } + if (size % 16 != 0) { + // Torch draws a FRESH block of 16 (continuing the same stream) and + // overwrites the last 16 values with it; the loop's remainder is + // discarded, so the tail is not simply its leftover. + float tail[16]; + for (float & value : tail) { + value = gen.uniform(); + } + normal_fill_16(tail); + std::memcpy(out.data() + size - 16, tail, sizeof(tail)); + } + return out; +} + +// ---- decoder graph ------------------------------------------------------- + +struct DecoderGraph : GraphResources { + int64_t frames = 0; + ggml_tensor * context = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * noise = nullptr; + ggml_tensor * spectrum = nullptr; +}; + +/** Frame-stage acoustic blocks -> mel-100 -> ConvNeXt decoder -> the + * [log-magnitude | phase] spectrum rows the host iSTFT consumes. */ +std::unique_ptr build_decoder_graph( + const SanoTtsBackendWeights & weights, + const SanoTtsConfig & config, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t frames) { + auto out = std::make_unique(); + out->backend = backend; + out->frames = frames; + out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); + out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create decoder graph contexts"); + } + core::ModuleBuildContext io_ctx{ + out->io_context.get(), + "sanotts.decoder.io", + backend_type, + }; + core::ModuleBuildContext ctx{ + out->graph_context.get(), + "sanotts.decoder", + backend_type, + }; + auto context = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, 3, frames})); + auto noise = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, config.noise_channels, frames})); + ggml_set_input(context.tensor); + ggml_set_input(feats.tensor); + ggml_set_input(noise.tensor); + + auto mel = acoustic_frame_stage( + ctx, + weights, + context, + feats, + config.acoustic_hidden, + config.acoustic_depth, + config.acoustic_kernel, + config.mels); + + // ConvNeXt decoder. Noise-fed: the noise adapter's output is added to the + // mel embedding before the first norm. + const int embed_padding = static_cast(config.embed_kernel / 2); + auto value = conv1d( + ctx, + weights, + mel, + "decoder.embed", + config.dim, + config.embed_kernel, + embed_padding); + value = add( + ctx, + value, + conv1d( + ctx, + weights, + noise, + "decoder.noise_adapter", + config.dim, + config.embed_kernel, + embed_padding)); + + // Channel-last from here: LayerNorm and the pointwise projections act on + // the channel axis, exactly as the PyTorch modules do. + auto value_cl = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, value); + value_cl = channel_last_layer_norm(ctx, weights, value_cl, "decoder.norm", config.dim); + for (int64_t block = 0; block < config.blocks; ++block) { + const std::string prefix = "decoder.blocks." + std::to_string(block); + const auto residual = value_cl; + auto branch_cm = contiguous( + ctx, + modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, value_cl)); + auto branch = core::wrap_tensor( + ggml_conv_1d_dw( + ctx.ggml, + contiguous(ctx, weight(weights, prefix + ".dwconv.weight")).tensor, + branch_cm.tensor, + 1, + static_cast(config.dw_kernel / 2), + 1), + core::TensorShape::from_dims({1, config.dim, frames}), + GGML_TYPE_F32); + auto * dw_bias = ggml_reshape_3d( + ctx.ggml, + weight(weights, prefix + ".dwconv.bias").tensor, + 1, + config.dim, + 1); + branch = core::wrap_tensor( + ggml_add(ctx.ggml, branch.tensor, dw_bias), + branch.shape, + GGML_TYPE_F32); + auto branch_cl = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, branch); + branch_cl = channel_last_layer_norm(ctx, weights, branch_cl, prefix + ".norm", config.dim); + branch_cl = modules::LinearModule({config.dim, config.pw_hidden, true}).build( + ctx, + branch_cl, + { + weight(weights, prefix + ".pwconv1.weight"), + weight(weights, prefix + ".pwconv1.bias"), + }); + // Exact erf GELU -- what nn.GELU() computes by default, NOT the tanh + // approximation. + branch_cl = modules::GeluModule({modules::GeluApproximation::ExactErf}) + .build(ctx, branch_cl); + branch_cl = modules::LinearModule({config.pw_hidden, config.dim, true}).build( + ctx, + branch_cl, + { + weight(weights, prefix + ".pwconv2.weight"), + weight(weights, prefix + ".pwconv2.bias"), + }); + const auto branch_source = contiguous(ctx, branch_cl); + branch_cl = core::wrap_tensor( + ggml_mul( + ctx.ggml, + branch_source.tensor, + weight(weights, prefix + ".gamma").tensor), + branch_cl.shape, + GGML_TYPE_F32); + value_cl = add(ctx, residual, branch_cl); + } + value_cl = channel_last_layer_norm(ctx, weights, value_cl, "decoder.final_norm", config.dim); + auto spectrum = modules::LinearModule({config.dim, config.n_fft + 2, true}).build( + ctx, + value_cl, + { + weight(weights, "decoder.head.weight"), + weight(weights, "decoder.head.bias"), + }); + spectrum = contiguous(ctx, spectrum); + out->context = context.tensor; + out->feats = feats.tensor; + out->noise = noise.tensor; + out->spectrum = spectrum.tensor; + ggml_set_output(out->spectrum); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->spectrum); + allocate_graph(*out); + return out; +} + +// ---- SHA-256 for the derive-from-text seed ------------------------------- + +constexpr uint32_t kShaK[64] = { + 0x428A2F98U, 0x71374491U, 0xB5C0FBCFU, 0xE9B5DBA5U, 0x3956C25BU, 0x59F111F1U, + 0x923F82A4U, 0xAB1C5ED5U, 0xD807AA98U, 0x12835B01U, 0x243185BEU, 0x550C7DC3U, + 0x72BE5D74U, 0x80DEB1FEU, 0x9BDC06A7U, 0xC19BF174U, 0xE49B69C1U, 0xEFBE4786U, + 0x0FC19DC6U, 0x240CA1CCU, 0x2DE92C6FU, 0x4A7484AAU, 0x5CB0A9DCU, 0x76F988DAU, + 0x983E5152U, 0xA831C66DU, 0xB00327C8U, 0xBF597FC7U, 0xC6E00BF3U, 0xD5A79147U, + 0x06CA6351U, 0x14292967U, 0x27B70A85U, 0x2E1B2138U, 0x4D2C6DFCU, 0x53380D13U, + 0x650A7354U, 0x766A0ABBU, 0x81C2C92EU, 0x92722C85U, 0xA2BFE8A1U, 0xA81A664BU, + 0xC24B8B70U, 0xC76C51A3U, 0xD192E819U, 0xD6990624U, 0xF40E3585U, 0x106AA070U, + 0x19A4C116U, 0x1E376C08U, 0x2748774CU, 0x34B0BCB5U, 0x391C0CB3U, 0x4ED8AA4AU, + 0x5B9CCA4FU, 0x682E6FF3U, 0x748F82EEU, 0x78A5636FU, 0x84C87814U, 0x8CC70208U, + 0x90BEFFFAU, 0xA4506CEBU, 0xBEF9A3F7U, 0xC67178F2U, +}; + +uint32_t rotr32(uint32_t v, int n) { + return (v >> n) | (v << (32 - n)); +} + +void sha256_block(uint32_t * h, const unsigned char * block) { + uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(block[4 * i]) << 24) | + (static_cast(block[4 * i + 1]) << 16) | + (static_cast(block[4 * i + 2]) << 8) | + static_cast(block[4 * i + 3]); + } + for (int i = 16; i < 64; ++i) { + const uint32_t s0 = rotr32(w[i - 15], 7) ^ rotr32(w[i - 15], 18) ^ (w[i - 15] >> 3); + const uint32_t s1 = rotr32(w[i - 2], 17) ^ rotr32(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0]; + uint32_t b = h[1]; + uint32_t c = h[2]; + uint32_t d = h[3]; + uint32_t e = h[4]; + uint32_t f = h[5]; + uint32_t g = h[6]; + uint32_t hh = h[7]; + for (int i = 0; i < 64; ++i) { + const uint32_t s1 = rotr32(e, 6) ^ rotr32(e, 11) ^ rotr32(e, 25); + const uint32_t ch = (e & f) ^ ((~e) & g); + const uint32_t s0 = rotr32(a, 2) ^ rotr32(a, 13) ^ rotr32(a, 22); + const uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + const uint32_t t1 = hh + s1 + ch + kShaK[i] + w[i]; + const uint32_t t2 = s0 + maj; + hh = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + h[5] += f; + h[6] += g; + h[7] += hh; +} + +// ---- host post-processing ------------------------------------------------ + +std::vector periodic_hann_window(int64_t n_fft) { + std::vector window(static_cast(n_fft)); + for (int64_t i = 0; i < n_fft; ++i) { + window[static_cast(i)] = static_cast( + 0.5 - 0.5 * std::cos(2.0 * kPi * static_cast(i) / + static_cast(n_fft))); + } + return window; +} + +/** H(z) = (1 - z^-1) / (1 - R z^-1), 2 MAC/sample, zero initial state -- + * the same DC blocker the reference runtimes apply after the iSTFT. */ +void dc_block_in_place(std::vector & samples) { + float x1 = 0.0F; + float y1 = 0.0F; + for (float & sample : samples) { + const float x = sample; + const float y = x - x1 + kDcBlockPole * y1; + x1 = x; + y1 = y; + sample = y; + } +} + +} // namespace + +uint64_t sanotts_text_seed(const std::string & text) { + uint32_t h[8] = {0x6A09E667U, 0xBB67AE85U, 0x3C6EF372U, 0xA54FF53AU, + 0x510E527FU, 0x9B05688CU, 0x1F83D9ABU, 0x5BE0CD19U}; + const auto * data = reinterpret_cast(text.data()); + const size_t len = text.size(); + const size_t full = len / 64; + const size_t rem = len % 64; + for (size_t i = 0; i < full; ++i) { + sha256_block(h, data + i * 64); + } + unsigned char tail[128] = {0}; + std::memcpy(tail, data + full * 64, rem); + tail[rem] = 0x80; + const size_t tail_len = rem < 56 ? 64 : 128; + const uint64_t bits = static_cast(len) * 8U; + for (int i = 0; i < 8; ++i) { + tail[tail_len - 1 - static_cast(i)] = + static_cast((bits >> (8 * i)) & 0xFFU); + } + sha256_block(h, tail); + if (tail_len == 128) { + sha256_block(h, tail + 64); + } + // == int.from_bytes(sha256(text).digest()[:8], "big"); ATen seeding then + // keeps only the low 32 bits, exactly as the reference runtimes do. + return (static_cast(h[0]) << 32) | static_cast(h[1]); +} + +struct SanoTtsNativeRuntime::State : BackendState { + State(const std::shared_ptr & assets_in, + core::BackendConfig backend_config) + : BackendState( + assets_in, + backend_config, + assets_in == nullptr ? 0 : expected_tensor_count(assets_in->config), + kWeightArenaBytes) {} + + runtime::CacheSlots> duration_graphs{4}; + runtime::CacheSlots> token_graphs{4}; + runtime::CacheSlots> decoder_graphs{2}; +}; + +SanoTtsNativeRuntime::SanoTtsNativeRuntime( + std::shared_ptr assets, + core::BackendConfig backend_config) + : state_(std::make_unique(std::move(assets), backend_config)) {} + +SanoTtsNativeRuntime::~SanoTtsNativeRuntime() = default; + +runtime::AudioBuffer SanoTtsNativeRuntime::synthesize( + const std::vector & token_ids, + const SanoTtsGenerationOptions & options) { + const auto & config = state_->assets->config; + const auto token_count = static_cast(token_ids.size()); + if (token_count <= 0) { + throw std::runtime_error("sanoTTS requires at least one phoneme token"); + } + const auto total_start = std::chrono::steady_clock::now(); + + // -- durations --------------------------------------------------------- + const auto duration_start = std::chrono::steady_clock::now(); + auto & duration = cached_graph( + state_->duration_graphs, + token_count, + "sanotts.duration_graph.cache_hit", + [&] { + return build_front_graph( + state_->duration_weights_ref(), + duration_stage_spec( + config.vocab_size, + config.duration_hidden, + config.duration_depth, + config.duration_kernel), + state_->duration_backend_value(), + state_->duration_backend_type(), + token_count, + kIoArenaBytes, + kGraphArenaBytes); + }); + write_i32_input( + duration.tokens, core::TensorShape::from_dims({1, token_count}), token_ids); + write_f32_input( + duration.feats, + core::TensorShape::from_dims({1, 3, token_count}), + duration_features(token_count, config.duration_max_tokens)); + compute_graph(duration, "sanoTTS duration"); + const auto log_duration = core::read_tensor_f32(duration.output); + if (static_cast(log_duration.size()) != token_count) { + throw std::runtime_error("sanoTTS duration graph returned invalid output"); + } + int64_t frames = 0; + const auto durations = round_durations( + log_duration, + static_cast(options.speaking_rate), + config.duration_max_frames, + config.duration_max_tokens * config.duration_max_frames, + frames); + if (frames < 2) { + throw std::runtime_error("sanoTTS duration predictor produced too few frames"); + } + engine::debug::timing_log_scalar( + "sanotts.duration_ms", + engine::debug::elapsed_ms(duration_start)); + + // -- token-stage acoustic context -------------------------------------- + const auto acoustic_start = std::chrono::steady_clock::now(); + auto & token_graph = cached_graph( + state_->token_graphs, + token_count, + "sanotts.token_graph.cache_hit", + [&] { + return build_front_graph( + *state_->weights, + token_stage_spec( + config.vocab_size, + config.acoustic_hidden, + config.acoustic_token_depth, + config.acoustic_kernel), + state_->backend.value, + state_->backend_type, + token_count, + kIoArenaBytes, + kGraphArenaBytes); + }); + write_i32_input( + token_graph.tokens, core::TensorShape::from_dims({1, token_count}), token_ids); + write_f32_input( + token_graph.feats, + core::TensorShape::from_dims({1, 2, token_count}), + token_features(token_count, durations)); + compute_graph(token_graph, "sanoTTS token context"); + const auto token_context = core::read_tensor_f32(token_graph.output); + if (static_cast(token_context.size()) != + config.acoustic_hidden * token_count) { + throw std::runtime_error("sanoTTS token graph returned invalid output"); + } + const auto noise = seeded_noise(options.seed, config.noise_channels, frames); + engine::debug::timing_log_scalar( + "sanotts.acoustic_ms", + engine::debug::elapsed_ms(acoustic_start)); + + // -- frame stage + decoder --------------------------------------------- + const auto decoder_start = std::chrono::steady_clock::now(); + auto & decoder = cached_graph( + state_->decoder_graphs, + frames, + "sanotts.decoder_graph.cache_hit", + [&] { + return build_decoder_graph( + *state_->weights, + config, + state_->backend.value, + state_->backend_type, + frames); + }); + write_f32_input( + decoder.context, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames}), + expand_context(token_context, durations, config.acoustic_hidden, token_count, frames)); + write_f32_input( + decoder.feats, + core::TensorShape::from_dims({1, 3, frames}), + frame_features(token_count, durations, frames)); + write_f32_input( + decoder.noise, + core::TensorShape::from_dims({1, config.noise_channels, frames}), + noise); + compute_graph(decoder, "sanoTTS decoder"); + auto spectrum = core::read_tensor_f32(decoder.spectrum); + const int64_t out_dim = config.n_fft + 2; + if (static_cast(spectrum.size()) != frames * out_dim) { + throw std::runtime_error("sanoTTS decoder graph returned invalid output"); + } + engine::debug::timing_log_scalar( + "sanotts.decoder_ms", + engine::debug::elapsed_ms(decoder_start)); + + // -- iSTFT + DC block --------------------------------------------------- + const auto istft_start = std::chrono::steady_clock::now(); + // Bin 0 and Nyquist stay zeroed: the mag*(cos,sin) parametrisation + // phase-collapses at bin 0, which is where the frame-DC artefact came + // from. -inf log-magnitude exps to exactly 0. + const int64_t bins = config.n_fft / 2 + 1; + for (int64_t frame = 0; frame < frames; ++frame) { + float * row = spectrum.data() + frame * out_dim; + row[0] = -std::numeric_limits::infinity(); + row[bins - 1] = -std::numeric_limits::infinity(); + } + engine::audio::HostLogMagnitudePhaseISTFTConfig istft_config; + istft_config.frames = frames; + istft_config.n_fft = config.n_fft; + istft_config.hop_length = config.hop_length; + istft_config.out_dim = out_dim; + istft_config.threads = static_cast(state_->threads); + engine::audio::HostLogMagnitudePhaseISTFT istft(istft_config); + auto istft_result = istft.compute(spectrum, periodic_hann_window(config.n_fft)); + // The framework trims (n_fft - hop)/2 per side; torch.istft(center=True) + // trims n_fft/2. Drop the extra hop/2 per side so lengths and content + // match the reference exactly: (frames - 1) * hop samples. + const auto edge = static_cast(config.hop_length / 2); + const size_t expected = static_cast(frames) * static_cast(config.hop_length); + if (istft_result.audio.size() != expected || istft_result.audio.size() < 2 * edge) { + throw std::runtime_error("sanoTTS iSTFT returned an unexpected sample count"); + } + std::vector samples( + istft_result.audio.begin() + static_cast(edge), + istft_result.audio.end() - static_cast(edge)); + dc_block_in_place(samples); + engine::debug::timing_log_scalar( + "sanotts.istft_ms", + engine::debug::elapsed_ms(istft_start)); + + runtime::AudioBuffer out; + out.sample_rate = static_cast(config.sample_rate); + out.channels = 1; + out.samples = std::move(samples); + engine::debug::trace_log_scalar("sanotts.token_count", token_count); + engine::debug::trace_log_scalar("sanotts.frames", frames); + engine::debug::trace_log_scalar( + "sanotts.output_samples", + static_cast(out.samples.size())); + engine::debug::timing_log_scalar( + "session.wall_ms", + engine::debug::elapsed_ms(total_start)); + return out; +} + +} // namespace engine::models::sanotts diff --git a/src/community_models/sanotts/session.cpp b/src/community_models/sanotts/session.cpp new file mode 100644 index 000000000..68e50d23f --- /dev/null +++ b/src/community_models/sanotts/session.cpp @@ -0,0 +1,282 @@ +#include "engine/community_models/sanotts/session.h" + +#include "engine/framework/debug/profiler.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { +namespace { + +constexpr const char * kFamily = "sanotts"; + +std::shared_ptr require_assets( + std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("sanoTTS session requires assets"); + } + return assets; +} + +std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("sanoTTS session requires a model contract"); + } + return contract; +} + +std::filesystem::path session_path( + const runtime::SessionOptions & options, + const char * key) { + const auto found = options.options.find(key); + return found == options.options.end() + ? std::filesystem::path{} + : std::filesystem::path(found->second); +} + +void validate_session_options( + const runtime::SessionOptions & options, + const engine::model_spec::ModelContract & contract) { + const std::string family_prefix = std::string(kFamily) + "."; + for (const auto & [key, _] : options.options) { + if (key.rfind(family_prefix, 0) == 0 && + contract.session_option_keys.find(key) == + contract.session_option_keys.end()) { + throw std::runtime_error("unknown sanoTTS session option: " + key); + } + } +} + +int64_t chunk_size_from_request(const runtime::TaskRequest & request) { + const auto value = runtime::parse_i64_option( + request.options, + {"text_chunk_size", "chunk_size"}); + const int64_t chunk_size = value.value_or(280); + if (chunk_size <= 0) { + throw std::runtime_error("sanoTTS text_chunk_size must be positive"); + } + return chunk_size; +} + +void validate_chunk_mode(const runtime::TaskRequest & request) { + if (const auto value = runtime::find_option( + request.options, + {"text_chunk_mode", "chunk_mode"})) { + if (*value != "word_budget" && *value != "default") { + throw std::runtime_error("sanoTTS text_chunk_mode must be word_budget"); + } + } +} + +struct RequestOptions { + float speaking_rate = 1.0F; + uint64_t seed = 0; + bool seed_from_text = true; +}; + +RequestOptions parse_request_options(const runtime::TaskRequest & request) { + RequestOptions out; + if (const auto value = runtime::parse_finite_float_option( + request.options, + {"speaking_rate"})) { + out.speaking_rate = *value; + } + if (out.speaking_rate < 0.5F || out.speaking_rate > 2.0F) { + throw std::runtime_error("sanoTTS speaking_rate must be between 0.5 and 2.0"); + } + if (const auto value = runtime::parse_i64_option(request.options, {"seed"})) { + if (*value < 0) { + throw std::runtime_error("sanoTTS seed must not be negative"); + } + out.seed = static_cast(*value); + out.seed_from_text = *value == 0; + } + return out; +} + +void append_pause(runtime::AudioBuffer & output, double seconds) { + if (output.sample_rate <= 0 || seconds <= 0.0) { + return; + } + const auto count = static_cast( + std::llround(seconds * static_cast(output.sample_rate))); + output.samples.insert(output.samples.end(), count, 0.0F); +} + +} // namespace + +SanoTtsSession::SanoTtsSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(options), + task_(task), + assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))) { + if (task_.task != runtime::VoiceTaskKind::Tts || + task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("sanoTTS only supports offline TTS"); + } + validate_session_options(options, *contract_); + if (assets_->graph == SanoTtsGraph::Nano) { + frontend_ = std::make_unique( + session_path(options, "sanotts.espeak_library_path"), + session_path(options, "sanotts.espeak_data_path"), + assets_->config.duration_max_tokens); + runtime_ = std::make_unique(assets_, options.backend); + } else { + piper_frontend_ = std::make_unique( + session_path(options, "sanotts.espeak_library_path"), + session_path(options, "sanotts.espeak_data_path"), + assets_->piper.espeak_voice, + assets_->piper.phoneme_id_map, + assets_->piper.duration_max_tokens); + piper_runtime_ = std::make_unique(assets_, options.backend); + } +} + +SanoTtsSession::~SanoTtsSession() = default; + +std::string SanoTtsSession::family() const { return kFamily; } +runtime::VoiceTaskKind SanoTtsSession::task_kind() const { return task_.task; } +runtime::RunMode SanoTtsSession::run_mode() const { return task_.mode; } + +void SanoTtsSession::prepare(const runtime::SessionPreparationRequest & request) { + (void)request; + mark_prepared(); +} + +runtime::TaskResult SanoTtsSession::run(const runtime::TaskRequest & request) { + require_prepared("sanoTTS run"); + if (!request.text_input.has_value() || request.text_input->text.empty()) { + throw std::runtime_error("sanoTTS requires --text input"); + } + if (request.audio_input.has_value()) { + throw std::runtime_error("sanoTTS does not accept audio input"); + } + const std::string voice_language = + assets_->graph == SanoTtsGraph::Nano ? "en" : assets_->piper.language; + if (!request.text_input->language.empty() && + request.text_input->language != voice_language && + !(voice_language == "en" && + (request.text_input->language == "en-us" || + request.text_input->language == "English"))) { + throw std::runtime_error( + "this sanoTTS voice supports language '" + voice_language + "' only"); + } + validate_chunk_mode(request); + const int64_t chunk_size = chunk_size_from_request(request); + auto chunks = SanoTtsFrontend::split_text(request.text_input->text, chunk_size); + if (chunks.empty()) { + throw std::runtime_error("sanoTTS text must not be empty"); + } + + const auto request_options = parse_request_options(request); + runtime::AudioBuffer merged; + uint64_t rendered_chunks = 0; + // Renders one chunk, bisecting at whitespace when it phonemizes past the + // duration model's token limit -- the codepoint budget cannot see phoneme + // counts, so a dense 280-codepoint chunk can exceed 207 tokens. + const std::function render_chunk = + [&](const std::string & chunk, int depth) { + SanoTtsEncoded encoded; + try { + encoded = frontend_ != nullptr + ? frontend_->encode(chunk) + : piper_frontend_->encode(chunk); + } catch (const SanoTtsTooLongError &) { + const size_t middle = chunk.size() / 2; + size_t split = std::string::npos; + for (size_t offset = 0; offset < chunk.size(); ++offset) { + const size_t after = middle + offset; + if (after < chunk.size() && chunk[after] == ' ') { + split = after; + break; + } + if (offset <= middle && chunk[middle - offset] == ' ') { + split = middle - offset; + break; + } + } + if (depth >= 8 || split == std::string::npos) { + throw; + } + const std::string left = chunk.substr(0, split); + const std::string right = chunk.substr(split + 1); + render_chunk(left, depth + 1); + append_pause(merged, SanoTtsFrontend::boundary_pause_seconds(left)); + render_chunk(right, depth + 1); + return; + } + runtime::AudioBuffer audio; + if (runtime_ != nullptr) { + SanoTtsGenerationOptions chunk_options; + chunk_options.speaking_rate = request_options.speaking_rate; + // The default seed is derived from the chunk's own text -- + // the reference implementations' sha256(text)[:8] convention + // -- so a given sentence renders identically wherever it + // appears. An explicit seed advances per chunk instead, so + // long-form noise is not reused across chunks. + chunk_options.seed = request_options.seed_from_text + ? sanotts_text_seed(chunk) + : request_options.seed + rendered_chunks; + audio = runtime_->synthesize(encoded.token_ids, chunk_options); + } else { + // The piperlite decoder is deterministic; the seed option is + // documented as ignored for these voices. + SanoTtsPiperGenerationOptions chunk_options; + chunk_options.speaking_rate = request_options.speaking_rate; + audio = piper_runtime_->synthesize(encoded.token_ids, chunk_options); + } + ++rendered_chunks; + runtime::append_audio_buffer(merged, audio); + }; + for (size_t index = 0; index < chunks.size(); ++index) { + if (index != 0) { + append_pause( + merged, + SanoTtsFrontend::boundary_pause_seconds(chunks[index - 1])); + } + render_chunk(chunks[index], 0); + } + for (float & sample : merged.samples) { + sample = std::clamp(sample, -1.0F, 1.0F); + } + engine::debug::trace_log_scalar("sanotts.text_chunk_size", chunk_size); + engine::debug::trace_log_scalar( + "sanotts.text_chunk_count", + static_cast(chunks.size())); + runtime::TaskResult result; + result.audio_output = std::move(merged); + return result; +} + +std::shared_ptr make_sanotts_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_sanotts_assets; + config.create_session = []( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, + options, + std::move(assets), + std::move(contract)); + }; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::models::sanotts diff --git a/tests/unittests/test_sanotts_frontend.cpp b/tests/unittests/test_sanotts_frontend.cpp new file mode 100644 index 000000000..23e79dd34 --- /dev/null +++ b/tests/unittests/test_sanotts_frontend.cpp @@ -0,0 +1,56 @@ +#include "engine/community_models/sanotts/frontend.h" +#include "engine/community_models/sanotts/runtime.h" + +#include "test_assert.h" + +#include +#include +#include + +int main() try { + using engine::models::sanotts::SanoTtsFrontend; + using engine::models::sanotts::sanotts_text_seed; + + // sha256 known-answer vectors: the seed is the first 8 digest bytes, + // big-endian, exactly int.from_bytes(sha256(text).digest()[:8], "big"). + engine::test::require( + sanotts_text_seed("abc") == 0xBA7816BF8F01CFEAULL, + "sha256 seed of 'abc'"); + engine::test::require( + sanotts_text_seed("") == 0xE3B0C44298FC1C14ULL, + "sha256 seed of the empty string"); + // 64 bytes forces the two-block tail path (rem >= 56). + engine::test::require( + sanotts_text_seed(std::string(64, 'a')) == 0xFFE054FE7AE0CB6DULL, + "sha256 seed across the two-block tail"); + + const auto chunks = SanoTtsFrontend::split_text( + "First sentence ends here. Second sentence is also short. " + "And a third one rounds it out.", + 40); + engine::test::require(chunks.size() >= 2, "long text must split"); + for (const auto & chunk : chunks) { + engine::test::require(!chunk.empty(), "no empty chunks"); + } + + const auto single = SanoTtsFrontend::split_text("Tiny.", 280); + engine::test::require( + single.size() == 1 && single[0] == "Tiny.", + "short text stays one chunk"); + + engine::test::require( + SanoTtsFrontend::boundary_pause_seconds("Ends with a period.") == 0.20, + "sentence end pause"); + engine::test::require( + SanoTtsFrontend::boundary_pause_seconds("ends with a comma,") == 0.08, + "clause pause"); + engine::test::require( + SanoTtsFrontend::boundary_pause_seconds("trailing space. ") == 0.20, + "pause looks through trailing whitespace"); + + std::cout << "sanotts frontend tests passed\n"; + return 0; +} catch (const std::exception & error) { + std::cerr << "sanotts frontend test failed: " << error.what() << "\n"; + return 1; +}