From 6a1afe21ab0886d4753f18614fd4e966b206eda0 Mon Sep 17 00:00:00 2001 From: ashish Date: Fri, 4 Sep 2026 14:55:35 +0545 Subject: [PATCH 1/7] model_specs: add the sanoTTS community family Two GGUF packages from ampixa/sanoTTS on Hugging Face: heart-nano (294,279 parameters, int8, 357 KB) as the default, and heart (2,272,145, f32, 9.1 MB). Both are converted losslessly from the blobs the project already ships. Rebuilding those blobs from the GGUF reproduces them byte for byte, and the golden gate on the rebuilt weights gives the same correlation against the float PyTorch references as the originals -- 0.989703 and 1.000000 against a 0.98 threshold -- including when the GGUF is fetched from Hugging Face rather than built locally. Session options mirror inflect_v2's, since sanoTTS needs the same external eSpeak-ng phonemizer and must not embed it. --- model_specs/sanotts.json | 132 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 model_specs/sanotts.json diff --git a/model_specs/sanotts.json b/model_specs/sanotts.json new file mode 100644 index 000000000..5522beea1 --- /dev/null +++ b/model_specs/sanotts.json @@ -0,0 +1,132 @@ +{ + "schema_version": 1, + "family": "sanotts", + "display_name": "sanoTTS Nano", + "description": "Very small English text-to-speech: a duration student, a contextual acoustic student producing mel-100, and a ConvNeXt-1D decoder with an iSTFT head. 294k parameters at 24 kHz, with a 2.27M variant. Uses an external eSpeak-ng phonemizer.", + "category": "tts", + "status": "community", + "tasks": [ + "tts" + ], + "modes": [ + "offline" + ], + "languages": [ + "en" + ], + "runtime": { + "tags": [ + "gguf" + ] + }, + "capabilities": { + "tts": [ + "long_form" + ] + }, + "options": { + "request": [ + { + "name": "speaking_rate", + "type": "float", + "description": "Duration multiplier; larger is slower. Applied before the per-token clamp.", + "required": false, + "min": 0.5, + "max": 2.0, + "default": 1.0 + }, + { + "name": "seed", + "type": "int", + "description": "Decoder noise seed. The decoder is noise-fed, so a given seed picks one of many valid renderings; 0 derives it from the text as sha256(text)[:8], which is what the reference implementations do.", + "required": false, + "min": 0, + "default": 0 + }, + { + "name": "text_chunk_mode", + "type": "enum", + "description": "Long-form text chunking mode.", + "values": [ + "word_budget" + ], + "required": false, + "default": "word_budget" + }, + { + "name": "text_chunk_size", + "type": "int", + "description": "Maximum Unicode codepoints per long-form text chunk; default 280.", + "required": false, + "min": 1, + "default": 280 + } + ], + "session": [ + { + "name": "espeak_library_path", + "type": "path", + "description": "Optional explicit path to the eSpeak-ng shared library.", + "required": false + }, + { + "name": "espeak_data_path", + "type": "path", + "description": "Optional explicit path to the directory containing espeak-ng-data.", + "required": false + } + ], + "load": [] + }, + "package_defaults": { + "download": { + "kind": "huggingface_snapshot", + "repo": "ampixa/sanoTTS", + "revision": "main", + "gated": false + } + }, + "packages": [ + { + "id": "sanotts_heart_nano_orig", + "display_name": "sanoTTS heart-nano 294k GGUF", + "default": true, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-heart-nano-GGUF", + "files": [ + "gguf/heart-nano.gguf" + ], + "strip_prefix": "gguf" + }, + { + "id": "sanotts_heart_orig", + "display_name": "sanoTTS heart 2.27M GGUF", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-heart-GGUF", + "files": [ + "gguf/heart.gguf" + ], + "strip_prefix": "gguf" + } + ], + "dependencies": [], + "ui": {}, + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "tensors": { + "weights": { + "source": "weights:", + "prefix": "weights" + } + } + } + ] +} From eb2dd72592d6666de84e87e3d4c262a6f49b5f71 Mon Sep 17 00:00:00 2001 From: ashish Date: Fri, 4 Sep 2026 15:43:43 +0545 Subject: [PATCH 2/7] sanotts: model spec, GGUF assets and the eSpeak-ng front end Groundwork for a ggml-native sanoTTS community model. model_specs/sanotts.json heart-nano (294,279 params, 24 kHz) from ampixa/sanoTTS on Hugging Face. Session options mirror inflect_v2's, since sanoTTS needs the same external eSpeak-ng phonemizer. assets.{h,cpp} Reads config.json and the GGUF tensors. config.json carries (tensor, offset) regions emitted by the packaging tool, so nothing here translates region names and the two cannot drift. Every shape constant is compared against the build's own, because a lineage mismatch would read weights at the wrong offsets and synthesize noise rather than fail. frontend.{h,cpp} Text -> the 62-symbol phoneme ids the model was trained on: eSpeak-ng IPA, then misaki's E2M rewrite, then the character-level vocabulary. Ported from the project's own JavaScript and Python front ends so all three agree symbol for symbol. eSpeak-ng is opened at runtime and never linked -- it is GPL-3.0 and must not be embedded here, the same treatment inflect_v2 gives it. The packaging is verified upstream: rebuilding both weight blobs from the GGUF reproduces the originals byte for byte, and the golden gate on the rebuilt weights matches the float PyTorch reference at 0.989703 against a 0.98 threshold -- including when the GGUF is fetched from Hugging Face. The inference graph is next, built on the framework's module library rather than a vendored runtime, so sanoTTS gets the shared backends like every other model here. --- .../engine/community_models/sanotts/assets.h | 61 ++++ .../community_models/sanotts/frontend.h | 51 +++ model_specs/sanotts.json | 20 +- src/community_models/sanotts/assets.cpp | 159 +++++++++ src/community_models/sanotts/frontend.cpp | 329 ++++++++++++++++++ 5 files changed, 606 insertions(+), 14 deletions(-) create mode 100644 include/engine/community_models/sanotts/assets.h create mode 100644 include/engine/community_models/sanotts/frontend.h create mode 100644 src/community_models/sanotts/assets.cpp create mode 100644 src/community_models/sanotts/frontend.cpp diff --git a/include/engine/community_models/sanotts/assets.h b/include/engine/community_models/sanotts/assets.h new file mode 100644 index 000000000..5eb11169f --- /dev/null +++ b/include/engine/community_models/sanotts/assets.h @@ -0,0 +1,61 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" + +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { + +/** + * Shape constants for one sanoTTS lineage. + * + * These arrive in the GGUF key/value header as `sanotts.*` and are checked + * against the constants the vendored runtime was compiled with. A lineage + * mismatch -- loading the 2.27M `heart` weights into a binary built for the + * 294k `heart-nano` -- would otherwise read the blobs at the wrong offsets and + * produce confident noise rather than an error. + */ +struct SanoTtsConfig { + int64_t vocab = 0; + int64_t sample_rate = 24000; + int64_t hop = 256; + int64_t n_fft = 1024; + int64_t mels = 0; + int64_t dim = 0; + int64_t blocks = 0; + int64_t noise_channels = 0; + int64_t dur_hidden = 0; + int64_t dur_depth = 0; + int64_t ac_hidden = 0; + int64_t ac_depth = 0; + int64_t max_tokens = 0; + int64_t weight_format = 0; // 0 = int8 rows with per-row scale, 1 = f32 rows + std::string voice; +}; + +/** + * The two flat weight blobs the runtime consumes, rebuilt from GGUF tensors. + * + * The runtime addresses weights by byte offset because it was written for + * microcontrollers, where parsing a container at load time is not affordable. + * Rather than give it a second addressing scheme, the loader reassembles the + * exact byte layout it expects. That reassembly is verified upstream: the + * packaging tool rebuilds both blobs from the GGUF and requires byte equality + * with the originals. + */ +struct SanoTtsAssets { + assets::ResourceBundle resources; + SanoTtsConfig config; + std::vector front_blob; + std::vector decoder_blob; +}; + +std::shared_ptr load_sanotts_assets( + const std::filesystem::path & model_path); + +} // namespace engine::models::sanotts diff --git a/include/engine/community_models/sanotts/frontend.h b/include/engine/community_models/sanotts/frontend.h new file mode 100644 index 000000000..693e0bfa3 --- /dev/null +++ b/include/engine/community_models/sanotts/frontend.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { + +struct SanoTtsEncoded { + std::vector token_ids; + std::string dropped; // symbols outside the vocabulary, for tracing +}; + +/** + * Text -> the 62-symbol phoneme ids the sanoTTS front end was trained on. + * + * eSpeak-ng produces the IPA; misaki's E2M table then rewrites it into the + * character-level inventory this model uses. Both steps are reproduced from + * the project's own JavaScript and Python front ends so the three agree + * symbol for symbol. + * + * eSpeak-ng is opened at runtime and never linked, matching how inflect_v2 + * treats it: it is GPL-3.0 and must not be embedded in this project. + */ +class SanoTtsFrontend { +public: + SanoTtsFrontend( + std::filesystem::path espeak_library_path, + std::filesystem::path espeak_data_path, + int64_t max_tokens); + ~SanoTtsFrontend(); + + [[nodiscard]] SanoTtsEncoded encode(const std::string & text) const; + + /** Long-form splitting on sentence punctuation, then a codepoint budget. */ + [[nodiscard]] static std::vector split_text( + const std::string & text, + int64_t max_codepoints); + + /** Pause inserted between chunks, longer after a sentence end. */ + [[nodiscard]] static double boundary_pause_seconds(const std::string & chunk); + +private: + struct Impl; + std::unique_ptr impl_; + int64_t max_tokens_; +}; + +} // namespace engine::models::sanotts diff --git a/model_specs/sanotts.json b/model_specs/sanotts.json index 5522beea1..b272cfaac 100644 --- a/model_specs/sanotts.json +++ b/model_specs/sanotts.json @@ -2,7 +2,7 @@ "schema_version": 1, "family": "sanotts", "display_name": "sanoTTS Nano", - "description": "Very small English text-to-speech: a duration student, a contextual acoustic student producing mel-100, and a ConvNeXt-1D decoder with an iSTFT head. 294k parameters at 24 kHz, with a 2.27M variant. Uses an external eSpeak-ng phonemizer.", + "description": "Very small English text-to-speech: a duration student, a contextual acoustic student producing mel-100, and a ConvNeXt-1D decoder with an iSTFT head. 294,279 parameters at 24 kHz. Uses an external eSpeak-ng phonemizer.", "category": "tts", "status": "community", "tasks": [ @@ -95,19 +95,8 @@ "precision": "orig", "target_directory": "sanoTTS-heart-nano-GGUF", "files": [ - "gguf/heart-nano.gguf" - ], - "strip_prefix": "gguf" - }, - { - "id": "sanotts_heart_orig", - "display_name": "sanoTTS heart 2.27M GGUF", - "default": false, - "format": "gguf", - "precision": "orig", - "target_directory": "sanoTTS-heart-GGUF", - "files": [ - "gguf/heart.gguf" + "gguf/heart-nano.gguf", + "gguf/config.json" ], "strip_prefix": "gguf" } @@ -121,6 +110,9 @@ "model": ".", "weights": "$gguf" }, + "files": { + "config": "model:config.json" + }, "tensors": { "weights": { "source": "weights:", diff --git a/src/community_models/sanotts/assets.cpp b/src/community_models/sanotts/assets.cpp new file mode 100644 index 000000000..8c6207310 --- /dev/null +++ b/src/community_models/sanotts/assets.cpp @@ -0,0 +1,159 @@ +#include "engine/community_models/sanotts/assets.h" + +#include "engine/framework/io/json.h" +#include "engine/framework/io/validation.h" +#include "engine/framework/model_spec/resource_bundle_loader.h" + +#include +#include +#include +#include + +extern "C" { +#include "nano_q8_meta.h" +} + +namespace engine::models::sanotts { +namespace { + +namespace json = engine::io::json; + +constexpr const char * kFamily = "sanotts"; + +/** + * The lineage the vendored runtime was compiled against. + * + * snt_nano.c takes its widths, block counts and kernel sizes from + * nano_q8_meta.h at compile time, which is what lets it run on a + * microcontroller with no parsing and no allocation. The consequence here is + * that one binary serves one lineage. Loading another lineage's weights would + * read the blobs at the wrong offsets and synthesize noise rather than fail, + * so every constant is compared against the package's config.json before any + * of it is used. + */ +struct CompiledLineage { + static constexpr int64_t vocab = NANO_VOCAB; + static constexpr int64_t mels = NANO_MELS; + static constexpr int64_t dim = NANO_DIM; + static constexpr int64_t blocks = NANO_BLOCKS; + static constexpr int64_t noise_ch = NANO_NOISE_CH; + static constexpr int64_t dur_hidden = NANO_DUR_HIDDEN; + static constexpr int64_t dur_depth = NANO_DUR_DEPTH; + static constexpr int64_t ac_hidden = NANO_AC_HIDDEN; + static constexpr int64_t ac_depth = NANO_AC_DEPTH; + static constexpr int64_t hop = NANO_HOP; + static constexpr int64_t n_fft = NANO_N_FFT; + static constexpr int64_t front_bytes = NANO_FRONT_BYTES; + static constexpr int64_t decoder_bytes = NANO_DEC_BYTES; + static constexpr int64_t weight_format = NANO_WEIGHT_FORMAT; +}; + +int64_t require_shape( + const json::Value & shapes, + const std::string & key, + int64_t compiled) { + const int64_t value = shapes.require(key).as_i64(); + if (value != compiled) { + throw std::runtime_error( + "sanoTTS lineage mismatch: config.json " + key + "=" + + std::to_string(value) + " but this build was compiled for " + + std::to_string(compiled) + + ". This package is for a different sanoTTS lineage."); + } + return value; +} + +SanoTtsConfig parse_config(const assets::ResourceBundle & resources) { + const auto root = resources.parse_json("config"); + const auto architecture = root.require("architecture").as_string(); + if (architecture != kFamily) { + throw std::runtime_error( + "sanoTTS config.json architecture is '" + architecture + "', expected 'sanotts'"); + } + const json::Value & shapes = root.require("shapes"); + + SanoTtsConfig out; + out.voice = root.require("voice").as_string(); + out.vocab = require_shape(shapes, "vocab", CompiledLineage::vocab); + out.mels = require_shape(shapes, "mels", CompiledLineage::mels); + out.dim = require_shape(shapes, "dim", CompiledLineage::dim); + out.blocks = require_shape(shapes, "blocks", CompiledLineage::blocks); + out.noise_channels = require_shape(shapes, "noise_ch", CompiledLineage::noise_ch); + out.dur_hidden = require_shape(shapes, "dur_hidden", CompiledLineage::dur_hidden); + out.dur_depth = require_shape(shapes, "dur_depth", CompiledLineage::dur_depth); + out.ac_hidden = require_shape(shapes, "ac_hidden", CompiledLineage::ac_hidden); + out.ac_depth = require_shape(shapes, "ac_depth", CompiledLineage::ac_depth); + out.hop = require_shape(shapes, "hop", CompiledLineage::hop); + out.n_fft = require_shape(shapes, "n_fft", CompiledLineage::n_fft); + out.max_tokens = shapes.require("dur_max_tokens").as_i64(); + out.weight_format = shapes.require("weight_format").as_i64(); + if (out.weight_format != CompiledLineage::weight_format) { + throw std::runtime_error( + "sanoTTS weight format mismatch: package is " + + std::string(out.weight_format == 1 ? "f32" : "int8") + + " but this build expects " + + std::string(CompiledLineage::weight_format == 1 ? "f32" : "int8")); + } + out.sample_rate = root.require("sample_rate").as_i64(); + engine::io::require_positive(out.sample_rate, "sanoTTS sample_rate"); + return out; +} + +/** + * Rebuild one flat blob by writing each named tensor at the byte offset the + * runtime expects. + * + * config.json carries the (tensor, offset) pairs, so nothing here has to + * translate region names -- the packaging tool emits the mapping from the + * same code that verifies the reassembly, and it proves the result equals the + * original blob byte for byte before publishing. + * + * Regions are 16-byte aligned, so a few hundred bytes between them are never + * written; the blob is zero-initialised and the originals carry zeros there, + * which is exactly what makes the round-trip byte-identical. + */ +void rebuild_blob( + std::vector & blob, + const assets::TensorSource & weights, + const json::Value & root, + const char * key) { + const auto & regions = root.require(key).as_array(); + if (regions.empty()) { + throw std::runtime_error(std::string("sanoTTS config.json ") + key + " is empty"); + } + for (const auto & region : regions) { + const auto & tensor = region.require("tensor").as_string(); + const int64_t offset = region.require("offset").as_i64(); + const auto data = weights.require_tensor_data(tensor); + if (offset < 0 || + static_cast(offset) + data.bytes.size() > blob.size()) { + throw std::runtime_error( + "sanoTTS tensor '" + tensor + "' does not fit its blob at offset " + + std::to_string(offset)); + } + std::memcpy(blob.data() + offset, data.bytes.data(), data.bytes.size()); + } +} + +} // namespace + +std::shared_ptr load_sanotts_assets( + const std::filesystem::path & model_path) { + auto resources = engine::model_spec::load_resource_bundle_for_family(model_path, kFamily); + + SanoTtsAssets out; + out.config = parse_config(resources); + const auto weights = resources.open_tensor_source("weights"); + const auto root = resources.parse_json("config"); + + out.front_blob.assign(static_cast(CompiledLineage::front_bytes), 0U); + out.decoder_blob.assign(static_cast(CompiledLineage::decoder_bytes), 0U); + + rebuild_blob(out.front_blob, *weights, root, "front_regions"); + rebuild_blob(out.decoder_blob, *weights, root, "decoder_regions"); + + out.resources = std::move(resources); + return std::make_shared(std::move(out)); +} + +} // namespace engine::models::sanotts diff --git a/src/community_models/sanotts/frontend.cpp b/src/community_models/sanotts/frontend.cpp new file mode 100644 index 000000000..d8fa4483d --- /dev/null +++ b/src/community_models/sanotts/frontend.cpp @@ -0,0 +1,329 @@ +#include "engine/community_models/sanotts/frontend.h" + +#include "engine/framework/io/dynamic_library.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { +namespace { + +using InitializeFn = int (*)(int, int, const char *, int); +using SetVoiceFn = int (*)(const char *); +using TextToPhonemesFn = const char * (*)(const void **, int, int); +using TerminateFn = int (*)(); + +constexpr int kEspeakSynchronous = 2; +constexpr int kEspeakCharsUtf8 = 1; +// IPA, plus 0x10 for tie characters: the E2M table below matches on ties +// ("a͡ɪ" -> "I"), so phonemes must be emitted with them. +constexpr int kEspeakPhonemesIpaTie = 2 | 0x10; + +constexpr std::string_view kTieDefault = "͡"; // COMBINING DOUBLE INVERTED BREVE +constexpr std::string_view kTieMisaki = "^"; +constexpr std::string_view kSyllabic = "̩"; // COMBINING VERTICAL LINE BELOW +constexpr std::string_view kNasal = "̃"; // COMBINING TILDE + +/** + * The frozen 62-symbol inventory, exactly as the trained package records it. + * + * Ids are positional and contiguous from zero; 0..2 are , , + * and never appear in text. + */ +const std::unordered_map & vocabulary() { + static const std::unordered_map table = [] { + static constexpr std::array symbols = { + " ", "!", "\"", "(", ")", ",", ".", ":", ";", + "?", "A", "I", "O", "T", "W", "Y", "b", + "d", "f", "h", "i", "j", "k", "l", "m", + "n", "p", "s", "t", "u", "v", "w", "z", + "æ", "ð", "ŋ", "ɐ", "ɑ", + "ɔ", "ə", "ɛ", "ɜ", "ɡ", + "ɪ", "ɹ", "ʃ", "ʊ", "ʌ", + "ʒ", "ʤ", "ʧ", "ˈ", "ˌ", + "θ", "ᵊ", "ᵻ", "—", "“", + "”", + }; + std::unordered_map out; + int32_t id = 3; // 0..2 are the specials + for (const char * symbol : symbols) { + out.emplace(symbol, id++); + } + if (out.size() + 3 != 62) { + throw std::runtime_error("sanoTTS compiled symbol inventory is invalid"); + } + return out; + }(); + return table; +} + +void replace_all(std::string & value, std::string_view from, std::string_view to) { + if (from.empty()) { + return; + } + size_t position = 0; + while ((position = value.find(from, position)) != std::string::npos) { + value.replace(position, from.size(), to); + position += to.size(); + } +} + +/** + * misaki EspeakFallback's E2M rewrite, british=false. + * + * Order is load-bearing and matches the Python source, which sorts by + * descending key length: "e͡ɪ" must be tried before the bare "e", + * or every diphthong collapses to the wrong symbol. + */ +std::string apply_e2m(std::string ps) { + static const std::array, 20> kE2M = {{ + {"ʔˌn̩", "ʔn"}, + {"ʔn̩", "ʔn"}, + {"a^ɪ", "I"}, {"a^ʊ", "W"}, {"d^ʒ", "ʤ"}, + {"e^ɪ", "A"}, {"t^ʃ", "ʧ"}, {"ɔ^ɪ", "Y"}, + {"ə^l", "ᵊl"}, + {"ʲo", "jo"}, {"ʲə", "jə"}, {"e", "A"}, {"ʲ", ""}, + {"ɚ", "əɹ"}, {"r", "ɹ"}, {"x", "k"}, {"ç", "k"}, + {"ɐ", "ə"}, {"ɬ", "l"}, {"̃", ""}, + }}; + // trim + const auto not_space = [](unsigned char ch) { return std::isspace(ch) == 0; }; + ps.erase(ps.begin(), std::find_if(ps.begin(), ps.end(), not_space)); + ps.erase(std::find_if(ps.rbegin(), ps.rend(), not_space).base(), ps.end()); + + for (const auto & [from, to] : kE2M) { + replace_all(ps, from, to); + } + // re.sub(r'(\S)̩', r'ᵊ\1', ps) then drop any remaining syllabic + size_t at = 0; + while ((at = ps.find(kSyllabic, at)) != std::string::npos) { + size_t start = at; + while (start > 0 && (static_cast(ps[start - 1]) & 0xC0U) == 0x80U) { + --start; // walk back over the UTF-8 continuation bytes + } + if (start == at || std::isspace(static_cast(ps[start])) != 0) { + ps.erase(at, kSyllabic.size()); + continue; + } + ps.erase(at, kSyllabic.size()); + ps.insert(start, "ᵊ"); + at = start + std::strlen("ᵊ"); + } + replace_all(ps, "o^ʊ", "O"); + replace_all(ps, "ɜːɹ", "ɜɹ"); + replace_all(ps, "ɜː", "ɜɹ"); + replace_all(ps, "ɪə", "iə"); + replace_all(ps, "ː", ""); + replace_all(ps, "o", "ɔ"); // espeak < 1.52 + replace_all(ps, "ɾ", "T"); // version != '2.0' + replace_all(ps, "ʔ", "t"); + replace_all(ps, "^", ""); + return ps; +} + +struct EspeakApi { + io::DynamicLibraryHandle library = nullptr; + InitializeFn initialize = nullptr; + SetVoiceFn set_voice = nullptr; + TextToPhonemesFn text_to_phonemes = nullptr; + TerminateFn terminate = nullptr; + mutable std::mutex call_mutex; + + EspeakApi(const std::filesystem::path & requested_library, + const std::filesystem::path & requested_data) { + if (!requested_library.empty() && + !std::filesystem::is_regular_file(requested_library)) { + throw std::runtime_error( + "sanoTTS eSpeak-ng library does not exist: " + requested_library.string()); + } + if (!requested_data.empty() && + (!std::filesystem::is_directory(requested_data) || + !std::filesystem::is_regular_file(requested_data / "phontab"))) { + throw std::runtime_error( + "sanoTTS eSpeak-ng data path is invalid; expected the espeak-ng-data " + "directory containing phontab: " + requested_data.string()); + } + if (!requested_library.empty()) { + library = io::open_dynamic_library(requested_library.string()); + } else { + library = io::open_dynamic_library({ +#ifdef _WIN32 + "espeak-ng.dll", "libespeak-ng.dll", +#elif defined(__APPLE__) + "libespeak-ng.dylib", "libespeak-ng.1.dylib", +#else + "libespeak-ng.so.1", "libespeak-ng.so", +#endif + }); + } + if (library == nullptr) { + throw std::runtime_error( + "sanoTTS could not load eSpeak-ng. Install it (apt install espeak-ng, " + "brew install espeak-ng) or pass " + "--session-option sanotts.espeak_library_path=/path/to/libespeak-ng.so"); + } + initialize = reinterpret_cast( + io::dynamic_library_symbol(library, "espeak_Initialize")); + set_voice = reinterpret_cast( + io::dynamic_library_symbol(library, "espeak_SetVoiceByName")); + text_to_phonemes = reinterpret_cast( + io::dynamic_library_symbol(library, "espeak_TextToPhonemes")); + terminate = reinterpret_cast( + io::dynamic_library_symbol(library, "espeak_Terminate")); + if (initialize == nullptr || set_voice == nullptr || text_to_phonemes == nullptr) { + throw std::runtime_error("sanoTTS eSpeak-ng is missing required symbols"); + } + // espeak appends "/espeak-ng-data" to the path it is given, so the + // PARENT of the data directory is what it wants. Handing it the data + // directory itself makes it fall back to its compiled-in default. + const std::string data = + requested_data.empty() ? std::string() : requested_data.parent_path().string(); + if (initialize(kEspeakSynchronous, 0, data.empty() ? nullptr : data.c_str(), 0) <= 0) { + throw std::runtime_error( + "sanoTTS eSpeak-ng failed to initialize; pass " + "--session-option sanotts.espeak_data_path=/path/to/espeak-ng-data"); + } + if (set_voice("en-us") != 0) { + throw std::runtime_error("sanoTTS eSpeak-ng has no en-us voice"); + } + } + + ~EspeakApi() { + if (terminate != nullptr) { + terminate(); + } + if (library != nullptr) { + io::close_dynamic_library(library); + } + } + + [[nodiscard]] std::string phonemize(const std::string & text) const { + const std::lock_guard guard(call_mutex); + std::string out; + const char * cursor = text.c_str(); + const void * position = cursor; + // espeak consumes one clause per call and advances the pointer; it + // returns null when the input is spent. + while (position != nullptr) { + const char * clause = + text_to_phonemes(&position, kEspeakCharsUtf8, kEspeakPhonemesIpaTie); + if (clause == nullptr) { + break; + } + if (!out.empty()) { + out.push_back(' '); + } + out.append(clause); + } + return out; + } +}; + +} // namespace + +struct SanoTtsFrontend::Impl { + EspeakApi espeak; + Impl(const std::filesystem::path & library, const std::filesystem::path & data) + : espeak(library, data) {} +}; + +SanoTtsFrontend::SanoTtsFrontend( + std::filesystem::path espeak_library_path, + std::filesystem::path espeak_data_path, + int64_t max_tokens) + : impl_(std::make_unique(espeak_library_path, espeak_data_path)), + max_tokens_(max_tokens > 2 ? max_tokens : 207) {} + +SanoTtsFrontend::~SanoTtsFrontend() = default; + +SanoTtsEncoded SanoTtsFrontend::encode(const std::string & text) const { + std::string ipa = apply_e2m(impl_->espeak.phonemize(text)); + replace_all(ipa, kTieDefault, kTieMisaki); + + const auto & vocab = vocabulary(); + SanoTtsEncoded out; + out.token_ids.push_back(1); // + // Iterate whole UTF-8 codepoints: every vocabulary symbol is one + // codepoint, so anything else can only be dropped -- which is what the + // reference front ends do with unknown symbols. + for (size_t i = 0; i < ipa.size();) { + size_t len = 1; + const auto lead = static_cast(ipa[i]); + if ((lead & 0xF8U) == 0xF0U) { len = 4; } + else if ((lead & 0xF0U) == 0xE0U) { len = 3; } + else if ((lead & 0xE0U) == 0xC0U) { len = 2; } + len = std::min(len, ipa.size() - i); + const std::string symbol = ipa.substr(i, len); + i += len; + const auto found = vocab.find(symbol); + if (found != vocab.end()) { + out.token_ids.push_back(found->second); + } else { + out.dropped.append(symbol); + } + } + if (out.token_ids.size() == 1) { + throw std::runtime_error( + "sanoTTS phonemization produced no symbols in the packaged vocabulary"); + } + out.token_ids.push_back(2); // + if (static_cast(out.token_ids.size()) > max_tokens_) { + throw std::runtime_error( + "sanoTTS phoneme sequence has " + std::to_string(out.token_ids.size()) + + " tokens including BOS/EOS; the duration model was trained for at most " + + std::to_string(max_tokens_) + ". Lower text_chunk_size."); + } + return out; +} + +std::vector SanoTtsFrontend::split_text( + const std::string & text, + int64_t max_codepoints) { + const size_t budget = max_codepoints > 0 ? static_cast(max_codepoints) : 280U; + std::vector chunks; + std::string current; + size_t codepoints = 0; + for (size_t i = 0; i < text.size();) { + size_t len = 1; + const auto lead = static_cast(text[i]); + if ((lead & 0xF8U) == 0xF0U) { len = 4; } + else if ((lead & 0xF0U) == 0xE0U) { len = 3; } + else if ((lead & 0xE0U) == 0xC0U) { len = 2; } + len = std::min(len, text.size() - i); + current.append(text, i, len); + i += len; + ++codepoints; + const bool sentence_end = len == 1 && (text[i - 1] == '.' || text[i - 1] == '!' || + text[i - 1] == '?'); + if ((sentence_end && codepoints >= budget / 4) || codepoints >= budget) { + chunks.push_back(current); + current.clear(); + codepoints = 0; + } + } + if (!current.empty()) { + chunks.push_back(current); + } + if (chunks.empty()) { + chunks.push_back(text); + } + return chunks; +} + +double SanoTtsFrontend::boundary_pause_seconds(const std::string & chunk) { + for (auto it = chunk.rbegin(); it != chunk.rend(); ++it) { + if (std::isspace(static_cast(*it)) != 0) { + continue; + } + return (*it == '.' || *it == '!' || *it == '?') ? 0.20 : 0.08; + } + return 0.08; +} + +} // namespace engine::models::sanotts From 54078ba7015975548c57d2c655cf510ad90b9961 Mon Sep 17 00:00:00 2001 From: ashish Date: Fri, 4 Sep 2026 16:37:13 +0545 Subject: [PATCH 3/7] sanotts: ggml-native runtime, session and docs Three cached graphs (duration, token stage, frame stage + ConvNeXt decoder) with the reference implementations' exact semantics: ATen-compatible MT19937 noise, torch.linspace/expand_features float behavior, LayerNorm eps 1e-6, erf GELU, torch.istft trim, and the 0.9973-pole DC blocker. The frontend gains phonemizer-compatible punctuation preservation and the correct eSpeak-ng tie mode so token streams are byte-identical to the Python front end. Verified against the project's numpy reference (same text, seed, and eSpeak-ng build): correlation 0.999999985, identical sample count; the reference is itself gated 0.987-1.000 against float PyTorch. 38.7 s of audio renders in 0.22 s wall on CPU (peak RSS 76 MB). Claude-Session: https://claude.ai/code/session_01P1iL37FdfJkGxdGrpjH1we --- CMakeLists.txt | 19 + README.md | 1 + docs/community_models/models.md | 1 + docs/community_models/sanotts.md | 103 ++ docs/tts.md | 21 + .../engine/community_models/sanotts/assets.h | 52 +- .../engine/community_models/sanotts/runtime.h | 41 + .../engine/community_models/sanotts/session.h | 40 + model_specs/sanotts.json | 22 +- src/community_models/sanotts/assets.cpp | 219 ++- src/community_models/sanotts/frontend.cpp | 272 +++- src/community_models/sanotts/runtime.cpp | 1259 +++++++++++++++++ src/community_models/sanotts/session.cpp | 221 +++ tests/unittests/test_sanotts_frontend.cpp | 56 + 14 files changed, 2171 insertions(+), 156 deletions(-) create mode 100644 docs/community_models/sanotts.md create mode 100644 include/engine/community_models/sanotts/runtime.h create mode 100644 include/engine/community_models/sanotts/session.h create mode 100644 src/community_models/sanotts/runtime.cpp create mode 100644 src/community_models/sanotts/session.cpp create mode 100644 tests/unittests/test_sanotts_frontend.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 01199cade..0d25e40d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -630,6 +630,18 @@ audiocpp_add_model(personaplex engine::models::personaplex::make_personaplex_loader ) +audiocpp_add_model(sanotts + SOURCES + src/community_models/sanotts/assets.cpp + src/community_models/sanotts/frontend.cpp + src/community_models/sanotts/runtime.cpp + src/community_models/sanotts/session.cpp + INCLUDES + engine/community_models/sanotts/session.h + LOADERS + engine::models::sanotts::make_sanotts_loader +) + audiocpp_add_model(inflect_v2 SOURCES src/community_models/inflect_v2/assets.cpp @@ -2545,6 +2557,13 @@ if (ENGINE_BUILD_TESTS) COMMAND echo_tts_host_units ) + add_engine_unittest(sanotts_frontend_test tests/unittests/test_sanotts_frontend.cpp) + target_include_directories(sanotts_frontend_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests) + add_test( + NAME sanotts_frontend_test + COMMAND sanotts_frontend_test + ) + add_engine_unittest(inflect_v2_frontend_test tests/unittests/test_inflect_v2_frontend.cpp) target_include_directories(inflect_v2_frontend_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests) add_test( diff --git a/README.md b/README.md index 7c95c6024..c0ba10798 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ Community model ports live under `community_models` to make the ownership bounda | **moss_voicegen** | Voice Design | en, zh | GGUF | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](docs/community_models/moss_voicegen.md) speech in a voice designed from a written instruction | | **outetts** | TTS, Clone | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | GGUF | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | GGUF F32/16/Q8, Stream | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](docs/community_models/parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | +| **sanotts** | TTS | en | GGUF FP32 | Ashish [@voidash](https://github.com/voidash) | [sanoTTS heart-nano](docs/community_models/sanotts.md) 294k-parameter native offline synthesis | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | GGUF Q8, Stream | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](docs/community_models/sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **soprano_tts** | TTS | en | GGUF Q8, Stream | [@WalkingCat](https://github.com/WalkingCat) | [Soprano-1.1-80M](https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF) ultra-lightweight TTS with Qwen3 LM + Vocos decoder | | **vietneu_tts** | TTS, Clone | vi, en | GGUF | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](docs/community_models/vietneu_tts.md) TTS and voice cloning support | diff --git a/docs/community_models/models.md b/docs/community_models/models.md index 994556b52..1ef662670 100644 --- a/docs/community_models/models.md +++ b/docs/community_models/models.md @@ -31,6 +31,7 @@ Practical expectations: | **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Mirek [@mirek190](https://github.com/mirek190) | [Llama-OuteTTS-1.0-1B](outetts.md) TTS and voice cloning support | | **voxcpm1** | TTS, voice cloning | zh, en, ja, ko | Community | [VoxCPM1](voxcpm1.md) tokenizer-free 0.5B TTS with 16 kHz output, streaming, and continuation-mode voice cloning | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | +| **sanotts** | TTS | en | Community | [sanoTTS heart-nano](sanotts.md) 294k-parameter FP32 offline synthesis | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **vietneu_tts** | TTS, voice cloning | vi, en | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](vietneu_tts.md) TTS and voice cloning support | | **moss_voicegen** | Voice design | en, zh | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](moss_voicegen.md) voice design from a written instruction, on the MOSS delay architecture | diff --git a/docs/community_models/sanotts.md b/docs/community_models/sanotts.md new file mode 100644 index 000000000..1823f4451 --- /dev/null +++ b/docs/community_models/sanotts.md @@ -0,0 +1,103 @@ +# sanoTTS heart-nano + +`sanotts` provides native GGML inference for +[sanoTTS](https://github.com/Ampixa/sanoTTS) **heart-nano**, a +294,279-parameter English text-to-speech model that also runs on +microcontrollers. The graph is a duration student, a contextual acoustic +student producing a mel-100 spectrogram, and a noise-fed ConvNeXt-1D decoder +whose [log-magnitude | phase] head feeds an inverse STFT. Output is 24 kHz +mono. Offline FP32 inference only. + +The GGUF is published on Hugging Face at +[ampixa/sanoTTS](https://huggingface.co/ampixa/sanoTTS) under `gguf/`: the +int8 on-device rows are dequantised to FP32 tensors in PyTorch shapes, with +the audio.cpp exact-shape metadata and an embedded model spec, so the package +is standalone. + +## Install + +Install eSpeak-ng and its English voice data first. On Debian or Ubuntu: + +```bash +sudo apt install espeak-ng libespeak-ng1 +``` + +On macOS: + +```bash +brew install espeak-ng +``` + +Then install the GGUF package: + +```bash +python tools/model_manager_v2.py install sanotts_heart_nano_orig --models-root models +``` + +## Run + +```bash +audiocpp_cli --task tts --family sanotts \ + --model models/sanoTTS-heart-nano-GGUF --backend cpu \ + --text "Hello from sano T T S, a very small neural text to speech model." \ + --out sanotts.wav +``` + +eSpeak-ng is loaded dynamically at runtime, never linked. If it is not on the +default library path: + +```bash +audiocpp_cli --task tts --family sanotts \ + --model models/sanoTTS-heart-nano-GGUF --backend cpu \ + --session-option sanotts.espeak_library_path=/path/to/libespeak-ng.so \ + --session-option sanotts.espeak_data_path=/path/to/espeak-ng-data \ + --text "A configured eSpeak installation." --out sanotts.wav +``` + +## Options + +- `speaking_rate` (request, 0.5..2.0, default 1.0) — duration multiplier + applied before rounding; larger is slower. +- `seed` (request, default 0) — decoder noise seed. The decoder is noise-fed, + so a given seed picks one of many valid renderings. `0` derives the seed + from each text chunk as `sha256(text)[:8]`, which is what the reference + implementations do; an explicit seed advances by one per long-form chunk. +- `text_chunk_size` (request, default 280) — maximum codepoints per long-form + chunk; chunks are split on sentence punctuation first. + +## Determinism and parity + +The runtime reproduces the reference implementations' exact semantics: + +- ATen-compatible MT19937 noise (24-bit uniform, Box–Muller in blocks of 16), + so a seed renders the same waveform as the PyTorch and MCU runtimes up to a + few ulp of libm difference. +- The phonemizer punctuation-preservation pipeline and misaki E2M rewrite, + byte-identical token streams against the Python front end across a + punctuation corpus. +- torch.istft window normalisation and centre trim, and the reference's + DC-blocking filter `H(z) = (1 - z^-1)/(1 - 0.9973 z^-1)`. + +Measured against the project's numpy reference (same text, same seed, same +eSpeak-ng build): correlation **0.999999985**, max sample delta 1.7e-05 +(the WAV's own int16 quantisation), identical sample count. The numpy +reference is itself gated at 0.987–1.000 against the float PyTorch model. + +## Performance + +CPU-only, 12-thread x86 (default 4 backend threads), FP32: + +- 38.7 s of audio synthesized in 0.22 s wall including model load + (about 175x faster than real time); peak RSS 76 MB. +- Per stage on a 5.7 s utterance (`--log`): duration 0.4 ms, acoustic + 0.5 ms, decoder 15.1 ms, host iSTFT 5.3 ms. + +Graphs are cached per token count (duration and token stages) and per frame +count (decoder), so repeated lengths skip graph construction; `--log` prints +the cache hits and stage timings. + +## Licensing + +The sanoTTS runtime and weights are MIT-licensed. eSpeak-ng is GPL-3.0 and is +therefore opened with `dlopen` at runtime and never linked, matching how +`inflect_v2` treats it. diff --git a/docs/tts.md b/docs/tts.md index ac3b720e3..3b7cfbc67 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -29,6 +29,7 @@ | GLM-TTS | `glm_tts` | `tts`, `clon` | [GLM-TTS](#glm-tts) | | Inflect Micro v2 | `inflect_v2` | `tts` | [Inflect v2](#inflect-v2) | | OuteTTS | `outetts` | `tts`, `clon` | [OuteTTS](#outetts) | +| sanoTTS heart-nano | `sanotts` | `tts` | [sanoTTS](#sanotts) | | Supertonic | `supertonic` | `tts` | [Supertonic](#supertonic) | | VieNeu-TTS | `vietneu_tts` | `tts`, `clon` | [VieNeu-TTS](community_models/vietneu_tts.md) | | VibeVoice | `vibevoice` | `tts` | [VibeVoice](#vibevoice) | @@ -768,6 +769,26 @@ See the [Inflect v2 community model guide](community_models/inflect_v2.md) for eSpeak-ng paths, long-form behavior, source/conversion instructions, and limitations. +## sanoTTS + +sanoTTS heart-nano is a 294,279-parameter English offline TTS model with a +native GGML runtime, small enough that the same weights also run on +microcontrollers. The GGUF package is standalone and downloads from Hugging +Face. sanoTTS requires an external eSpeak-ng installation: + +```bash +python3 tools/model_manager_v2.py install sanotts_heart_nano_orig --models-root models + +audiocpp_cli --task tts --family sanotts \ + --model models/sanoTTS-heart-nano-GGUF --backend cpu \ + --text "Hello from sano T T S, a very small neural text to speech model." \ + --request-option speaking_rate=1.0 \ + --out sanotts.wav +``` + +See the [sanoTTS community model guide](community_models/sanotts.md) for +eSpeak-ng paths, seed semantics, parity evidence, and performance numbers. + ## Supertonic Supertonic 3 is a preset-voice multilingual TTS model. It does not use external speaker references in the current integration. diff --git a/include/engine/community_models/sanotts/assets.h b/include/engine/community_models/sanotts/assets.h index 5eb11169f..635959d3e 100644 --- a/include/engine/community_models/sanotts/assets.h +++ b/include/engine/community_models/sanotts/assets.h @@ -7,52 +7,40 @@ #include #include #include -#include namespace engine::models::sanotts { -/** - * Shape constants for one sanoTTS lineage. - * - * These arrive in the GGUF key/value header as `sanotts.*` and are checked - * against the constants the vendored runtime was compiled with. A lineage - * mismatch -- loading the 2.27M `heart` weights into a binary built for the - * 294k `heart-nano` -- would otherwise read the blobs at the wrong offsets and - * produce confident noise rather than an error. - */ struct SanoTtsConfig { - int64_t vocab = 0; + int64_t vocab_size = 62; int64_t sample_rate = 24000; - int64_t hop = 256; + int64_t hop_length = 256; int64_t n_fft = 1024; - int64_t mels = 0; + int64_t mels = 100; int64_t dim = 0; int64_t blocks = 0; - int64_t noise_channels = 0; - int64_t dur_hidden = 0; - int64_t dur_depth = 0; - int64_t ac_hidden = 0; - int64_t ac_depth = 0; - int64_t max_tokens = 0; - int64_t weight_format = 0; // 0 = int8 rows with per-row scale, 1 = f32 rows + int64_t pw_hidden = 0; + int64_t noise_channels = 4; + int64_t dw_kernel = 7; + int64_t embed_kernel = 7; + + int64_t duration_hidden = 0; + int64_t duration_depth = 0; + int64_t duration_kernel = 5; + int64_t duration_max_tokens = 207; + int64_t duration_max_frames = 80; + + int64_t acoustic_hidden = 0; + int64_t acoustic_token_depth = 0; + int64_t acoustic_depth = 0; + int64_t acoustic_kernel = 5; + std::string voice; }; -/** - * The two flat weight blobs the runtime consumes, rebuilt from GGUF tensors. - * - * The runtime addresses weights by byte offset because it was written for - * microcontrollers, where parsing a container at load time is not affordable. - * Rather than give it a second addressing scheme, the loader reassembles the - * exact byte layout it expects. That reassembly is verified upstream: the - * packaging tool rebuilds both blobs from the GGUF and requires byte equality - * with the originals. - */ struct SanoTtsAssets { assets::ResourceBundle resources; SanoTtsConfig config; - std::vector front_blob; - std::vector decoder_blob; + std::shared_ptr weights; }; std::shared_ptr load_sanotts_assets( diff --git a/include/engine/community_models/sanotts/runtime.h b/include/engine/community_models/sanotts/runtime.h new file mode 100644 index 000000000..750cb8759 --- /dev/null +++ b/include/engine/community_models/sanotts/runtime.h @@ -0,0 +1,41 @@ +#pragma once + +#include "engine/community_models/sanotts/assets.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include +#include + +namespace engine::models::sanotts { + +struct SanoTtsGenerationOptions { + /** Duration multiplier applied before rounding; larger is slower. */ + float speaking_rate = 1.0F; + /** Decoder noise seed; the session resolves the derive-from-text default + * before calling the runtime, so this is always the final seed. */ + uint64_t seed = 0; +}; + +class SanoTtsNativeRuntime { +public: + SanoTtsNativeRuntime( + std::shared_ptr assets, + core::BackendConfig backend_config); + ~SanoTtsNativeRuntime(); + + runtime::AudioBuffer synthesize( + const std::vector & token_ids, + const SanoTtsGenerationOptions & options); + +private: + struct State; + std::unique_ptr state_; +}; + +/** int.from_bytes(sha256(text).digest()[:8], "big") -- the seed the reference + * implementations derive when the caller does not pass one. */ +uint64_t sanotts_text_seed(const std::string & text); + +} // namespace engine::models::sanotts diff --git a/include/engine/community_models/sanotts/session.h b/include/engine/community_models/sanotts/session.h new file mode 100644 index 000000000..ba259d1fc --- /dev/null +++ b/include/engine/community_models/sanotts/session.h @@ -0,0 +1,40 @@ +#pragma once + +#include "engine/community_models/sanotts/assets.h" +#include "engine/community_models/sanotts/frontend.h" +#include "engine/community_models/sanotts/runtime.h" +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/session_base.h" + +#include + +namespace engine::models::sanotts { + +std::shared_ptr make_sanotts_loader(); + +class SanoTtsSession final + : public runtime::RuntimeSessionBase + , public runtime::IOfflineVoiceTaskSession { +public: + SanoTtsSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract); + ~SanoTtsSession() override; + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest & request) override; + runtime::TaskResult run(const runtime::TaskRequest & request) override; + +private: + runtime::TaskSpec task_; + std::shared_ptr assets_; + std::shared_ptr contract_; + std::unique_ptr frontend_; + std::unique_ptr runtime_; +}; + +} // namespace engine::models::sanotts diff --git a/model_specs/sanotts.json b/model_specs/sanotts.json index b272cfaac..893419519 100644 --- a/model_specs/sanotts.json +++ b/model_specs/sanotts.json @@ -89,20 +89,31 @@ "packages": [ { "id": "sanotts_heart_nano_orig", - "display_name": "sanoTTS heart-nano 294k GGUF", + "display_name": "sanoTTS heart-nano 294k FP32 GGUF", "default": true, "format": "gguf", "precision": "orig", "target_directory": "sanoTTS-heart-nano-GGUF", "files": [ - "gguf/heart-nano.gguf", + "gguf/heart-nano-f32.gguf", "gguf/config.json" ], "strip_prefix": "gguf" } ], "dependencies": [], - "ui": {}, + "ui": { + "recommended_package": "sanotts_heart_nano_orig", + "tags": [ + "TTS", + "GGUF" + ], + "docs": [ + "docs/tts.md", + "docs/community_models/sanotts.md", + "docs/gguf.md" + ] + }, "sources": [ { "format": "gguf", @@ -114,10 +125,7 @@ "config": "model:config.json" }, "tensors": { - "weights": { - "source": "weights:", - "prefix": "weights" - } + "weights": "weights:" } } ] diff --git a/src/community_models/sanotts/assets.cpp b/src/community_models/sanotts/assets.cpp index 8c6207310..13c8b4c8b 100644 --- a/src/community_models/sanotts/assets.cpp +++ b/src/community_models/sanotts/assets.cpp @@ -1,17 +1,12 @@ #include "engine/community_models/sanotts/assets.h" #include "engine/framework/io/json.h" -#include "engine/framework/io/validation.h" -#include "engine/framework/model_spec/resource_bundle_loader.h" +#include "engine/framework/io/config.h" +#include "engine/framework/model_spec/package.h" -#include #include #include -#include - -extern "C" { -#include "nano_q8_meta.h" -} +#include namespace engine::models::sanotts { namespace { @@ -20,49 +15,6 @@ namespace json = engine::io::json; constexpr const char * kFamily = "sanotts"; -/** - * The lineage the vendored runtime was compiled against. - * - * snt_nano.c takes its widths, block counts and kernel sizes from - * nano_q8_meta.h at compile time, which is what lets it run on a - * microcontroller with no parsing and no allocation. The consequence here is - * that one binary serves one lineage. Loading another lineage's weights would - * read the blobs at the wrong offsets and synthesize noise rather than fail, - * so every constant is compared against the package's config.json before any - * of it is used. - */ -struct CompiledLineage { - static constexpr int64_t vocab = NANO_VOCAB; - static constexpr int64_t mels = NANO_MELS; - static constexpr int64_t dim = NANO_DIM; - static constexpr int64_t blocks = NANO_BLOCKS; - static constexpr int64_t noise_ch = NANO_NOISE_CH; - static constexpr int64_t dur_hidden = NANO_DUR_HIDDEN; - static constexpr int64_t dur_depth = NANO_DUR_DEPTH; - static constexpr int64_t ac_hidden = NANO_AC_HIDDEN; - static constexpr int64_t ac_depth = NANO_AC_DEPTH; - static constexpr int64_t hop = NANO_HOP; - static constexpr int64_t n_fft = NANO_N_FFT; - static constexpr int64_t front_bytes = NANO_FRONT_BYTES; - static constexpr int64_t decoder_bytes = NANO_DEC_BYTES; - static constexpr int64_t weight_format = NANO_WEIGHT_FORMAT; -}; - -int64_t require_shape( - const json::Value & shapes, - const std::string & key, - int64_t compiled) { - const int64_t value = shapes.require(key).as_i64(); - if (value != compiled) { - throw std::runtime_error( - "sanoTTS lineage mismatch: config.json " + key + "=" + - std::to_string(value) + " but this build was compiled for " + - std::to_string(compiled) + - ". This package is for a different sanoTTS lineage."); - } - return value; -} - SanoTtsConfig parse_config(const assets::ResourceBundle & resources) { const auto root = resources.parse_json("config"); const auto architecture = root.require("architecture").as_string(); @@ -70,68 +22,119 @@ SanoTtsConfig parse_config(const assets::ResourceBundle & resources) { throw std::runtime_error( "sanoTTS config.json architecture is '" + architecture + "', expected 'sanotts'"); } - const json::Value & shapes = root.require("shapes"); - SanoTtsConfig out; out.voice = root.require("voice").as_string(); - out.vocab = require_shape(shapes, "vocab", CompiledLineage::vocab); - out.mels = require_shape(shapes, "mels", CompiledLineage::mels); - out.dim = require_shape(shapes, "dim", CompiledLineage::dim); - out.blocks = require_shape(shapes, "blocks", CompiledLineage::blocks); - out.noise_channels = require_shape(shapes, "noise_ch", CompiledLineage::noise_ch); - out.dur_hidden = require_shape(shapes, "dur_hidden", CompiledLineage::dur_hidden); - out.dur_depth = require_shape(shapes, "dur_depth", CompiledLineage::dur_depth); - out.ac_hidden = require_shape(shapes, "ac_hidden", CompiledLineage::ac_hidden); - out.ac_depth = require_shape(shapes, "ac_depth", CompiledLineage::ac_depth); - out.hop = require_shape(shapes, "hop", CompiledLineage::hop); - out.n_fft = require_shape(shapes, "n_fft", CompiledLineage::n_fft); - out.max_tokens = shapes.require("dur_max_tokens").as_i64(); - out.weight_format = shapes.require("weight_format").as_i64(); - if (out.weight_format != CompiledLineage::weight_format) { - throw std::runtime_error( - "sanoTTS weight format mismatch: package is " + - std::string(out.weight_format == 1 ? "f32" : "int8") + - " but this build expects " + - std::string(CompiledLineage::weight_format == 1 ? "f32" : "int8")); - } + out.vocab_size = root.require("vocab_size").as_i64(); out.sample_rate = root.require("sample_rate").as_i64(); - engine::io::require_positive(out.sample_rate, "sanoTTS sample_rate"); + out.hop_length = root.require("hop_length").as_i64(); + out.n_fft = root.require("n_fft").as_i64(); + out.mels = root.require("mels").as_i64(); + out.dim = root.require("dim").as_i64(); + out.blocks = root.require("blocks").as_i64(); + out.pw_hidden = root.require("pw_hidden").as_i64(); + out.noise_channels = root.require("noise_channels").as_i64(); + out.dw_kernel = root.require("dw_kernel").as_i64(); + out.embed_kernel = root.require("embed_kernel").as_i64(); + + const auto & duration = root.require("duration"); + out.duration_hidden = duration.require("hidden").as_i64(); + out.duration_depth = duration.require("depth").as_i64(); + out.duration_kernel = duration.require("kernel").as_i64(); + out.duration_max_tokens = duration.require("max_tokens").as_i64(); + out.duration_max_frames = duration.require("max_duration").as_i64(); + + const auto & acoustic = root.require("acoustic"); + out.acoustic_hidden = acoustic.require("hidden").as_i64(); + out.acoustic_token_depth = acoustic.require("token_depth").as_i64(); + out.acoustic_depth = acoustic.require("depth").as_i64(); + out.acoustic_kernel = acoustic.require("kernel").as_i64(); + + for (const auto & [label, value] : std::initializer_list>{ + {"sanoTTS dim", out.dim}, + {"sanoTTS blocks", out.blocks}, + {"sanoTTS pw_hidden", out.pw_hidden}, + {"sanoTTS duration hidden", out.duration_hidden}, + {"sanoTTS acoustic hidden", out.acoustic_hidden}, + {"sanoTTS sample_rate", out.sample_rate}, + }) { + engine::io::require_positive(value, label); + } return out; } /** - * Rebuild one flat blob by writing each named tensor at the byte offset the - * runtime expects. - * - * config.json carries the (tensor, offset) pairs, so nothing here has to - * translate region names -- the packaging tool emits the mapping from the - * same code that verifies the reassembly, and it proves the result equals the - * original blob byte for byte before publishing. + * Fail on a missing or wrongly-shaped tensor at load, not mid-graph. * - * Regions are 16-byte aligned, so a few hundred bytes between them are never - * written; the blob is zero-initialised and the originals carry zeros there, - * which is exactly what makes the round-trip byte-identical. + * The decoder is noise-fed and ends in an iSTFT, so a weight that is present + * but wrong in shape tends to produce plausible-sounding audio rather than an + * obvious failure. Checking the whole inventory up front is what keeps a + * packaging mistake loud. */ -void rebuild_blob( - std::vector & blob, - const assets::TensorSource & weights, - const json::Value & root, - const char * key) { - const auto & regions = root.require(key).as_array(); - if (regions.empty()) { - throw std::runtime_error(std::string("sanoTTS config.json ") + key + " is empty"); +void validate_tensors(const SanoTtsAssets & assets) { + const auto & c = assets.config; + const auto & weights = *assets.weights; + + std::vector>> expected; + const auto conv = [&](const std::string & name, int64_t out_ch, int64_t in_ch, int64_t k) { + expected.emplace_back(name + ".weight", std::vector{out_ch, in_ch, k}); + expected.emplace_back(name + ".bias", std::vector{out_ch}); + }; + const auto linear = [&](const std::string & name, int64_t out_ch, int64_t in_ch) { + expected.emplace_back(name + ".weight", std::vector{out_ch, in_ch}); + expected.emplace_back(name + ".bias", std::vector{out_ch}); + }; + + expected.emplace_back("duration.embedding.weight", + std::vector{c.vocab_size, c.duration_hidden}); + conv("duration.input_proj", c.duration_hidden, c.duration_hidden + 3, 1); + for (int64_t b = 0; b < c.duration_depth; ++b) { + const std::string prefix = "duration.blocks." + std::to_string(b); + conv(prefix + ".net.0", c.duration_hidden, c.duration_hidden, c.duration_kernel); + conv(prefix + ".net.2", c.duration_hidden, c.duration_hidden, c.duration_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("duration.output", 1, c.duration_hidden, 1); + + expected.emplace_back("acoustic.embedding.weight", + std::vector{c.vocab_size, c.acoustic_hidden}); + conv("acoustic.token_input_proj", c.acoustic_hidden, c.acoustic_hidden + 2, 1); + for (int64_t b = 0; b < c.acoustic_token_depth; ++b) { + const std::string prefix = "acoustic.token_blocks." + std::to_string(b); + conv(prefix + ".net.0", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + conv(prefix + ".net.2", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); } - for (const auto & region : regions) { - const auto & tensor = region.require("tensor").as_string(); - const int64_t offset = region.require("offset").as_i64(); - const auto data = weights.require_tensor_data(tensor); - if (offset < 0 || - static_cast(offset) + data.bytes.size() > blob.size()) { - throw std::runtime_error( - "sanoTTS tensor '" + tensor + "' does not fit its blob at offset " + - std::to_string(offset)); + conv("acoustic.frame_input_proj", c.acoustic_hidden, c.acoustic_hidden + 3, 1); + for (int64_t b = 0; b < c.acoustic_depth; ++b) { + const std::string prefix = "acoustic.frame_blocks." + std::to_string(b); + conv(prefix + ".net.0", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + conv(prefix + ".net.2", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("acoustic.output", c.mels, c.acoustic_hidden, 1); + + conv("decoder.embed", c.dim, c.mels, c.embed_kernel); + conv("decoder.noise_adapter", c.dim, c.noise_channels, c.embed_kernel); + expected.emplace_back("decoder.norm.weight", std::vector{c.dim}); + expected.emplace_back("decoder.norm.bias", std::vector{c.dim}); + for (int64_t b = 0; b < c.blocks; ++b) { + const std::string prefix = "decoder.blocks." + std::to_string(b); + conv(prefix + ".dwconv", c.dim, 1, c.dw_kernel); // groups == dim + expected.emplace_back(prefix + ".norm.weight", std::vector{c.dim}); + expected.emplace_back(prefix + ".norm.bias", std::vector{c.dim}); + linear(prefix + ".pwconv1", c.pw_hidden, c.dim); + linear(prefix + ".pwconv2", c.dim, c.pw_hidden); + expected.emplace_back(prefix + ".gamma", std::vector{c.dim}); + } + expected.emplace_back("decoder.final_norm.weight", std::vector{c.dim}); + expected.emplace_back("decoder.final_norm.bias", std::vector{c.dim}); + linear("decoder.head", c.n_fft + 2, c.dim); + + for (const auto & [name, shape] : expected) { + if (!weights.has_tensor(name)) { + throw std::runtime_error("sanoTTS missing tensor: " + name); } - std::memcpy(blob.data() + offset, data.bytes.data(), data.bytes.size()); + assets::require_tensor_shape(weights, name, shape); } } @@ -140,19 +143,11 @@ void rebuild_blob( std::shared_ptr load_sanotts_assets( const std::filesystem::path & model_path) { auto resources = engine::model_spec::load_resource_bundle_for_family(model_path, kFamily); - SanoTtsAssets out; out.config = parse_config(resources); - const auto weights = resources.open_tensor_source("weights"); - const auto root = resources.parse_json("config"); - - out.front_blob.assign(static_cast(CompiledLineage::front_bytes), 0U); - out.decoder_blob.assign(static_cast(CompiledLineage::decoder_bytes), 0U); - - rebuild_blob(out.front_blob, *weights, root, "front_regions"); - rebuild_blob(out.decoder_blob, *weights, root, "decoder_regions"); - + out.weights = resources.open_tensor_source("weights"); out.resources = std::move(resources); + validate_tensors(out); return std::make_shared(std::move(out)); } diff --git a/src/community_models/sanotts/frontend.cpp b/src/community_models/sanotts/frontend.cpp index d8fa4483d..340467f36 100644 --- a/src/community_models/sanotts/frontend.cpp +++ b/src/community_models/sanotts/frontend.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -21,9 +22,10 @@ using TerminateFn = int (*)(); constexpr int kEspeakSynchronous = 2; constexpr int kEspeakCharsUtf8 = 1; -// IPA, plus 0x10 for tie characters: the E2M table below matches on ties -// ("a͡ɪ" -> "I"), so phonemes must be emitted with them. -constexpr int kEspeakPhonemesIpaTie = 2 | 0x10; +// IPA output (0x02), tie flag (bit 7), and U+0361 COMBINING DOUBLE INVERTED +// BREVE in bits 8..23 as the tie character -- exactly the phonemes_mode +// phonemizer computes, so the E2M diphthong patterns ("a͡ɪ" -> "I") can match. +constexpr int kEspeakPhonemesIpaTie = 0x02 | (0x01 << 7) | (0x0361 << 8); constexpr std::string_view kTieDefault = "͡"; // COMBINING DOUBLE INVERTED BREVE constexpr std::string_view kTieMisaki = "^"; @@ -127,6 +129,257 @@ std::string apply_e2m(std::string ps) { return ps; } + +// ---- phonemizer-fork punctuation preserve/restore ------------------------ +// +// The reference front ends run eSpeak through phonemizer with +// preserve_punctuation=True: punctuation is cut out before phonemization and +// spliced back afterwards, so marks like "," and "." survive as tokens (the +// model was trained with them). This reproduces phonemizer's Punctuation +// class for the fixed marks and separator this model uses. + +constexpr std::string_view kPunctuationMarks = "!'(),-.:;?\""; + +bool is_punctuation_mark(char ch) { + return kPunctuationMarks.find(ch) != std::string_view::npos; +} + +bool is_ascii_space(char ch) { + return std::isspace(static_cast(ch)) != 0; +} + +struct MarkIndex { + std::string mark; + char position = 'I'; // B(egin), E(nd), I(nside), A(lone) +}; + +/** Matches of phonemizer's (\s*[marks]+\s*)+ -- maximal runs of spaces and + * marks that contain at least one mark. */ +std::vector> find_mark_runs(const std::string & line) { + std::vector> runs; + size_t i = 0; + while (i < line.size()) { + if (!is_ascii_space(line[i]) && !is_punctuation_mark(line[i])) { + ++i; + continue; + } + size_t end = i; + bool has_mark = false; + while (end < line.size() && + (is_ascii_space(line[end]) || is_punctuation_mark(line[end]))) { + has_mark = has_mark || is_punctuation_mark(line[end]); + ++end; + } + if (has_mark) { + runs.emplace_back(i, end); + } + i = end; + } + return runs; +} + +/** Punctuation._preserve_line: chunks without punctuation + ordered marks. + * Empty chunks are filtered, as Punctuation.preserve() does. */ +std::pair, std::vector> preserve_punctuation( + const std::string & line) { + const auto runs = find_mark_runs(line); + if (runs.empty()) { + return {{line}, {}}; + } + if (runs.size() == 1 && runs[0].first == 0 && runs[0].second == line.size()) { + return {{}, {{line, 'A'}}}; + } + std::vector marks; + marks.reserve(runs.size()); + for (size_t index = 0; index < runs.size(); ++index) { + const auto & run = runs[index]; + char position = 'I'; + if (index == 0 && run.first == 0) { + position = 'B'; + } else if (index + 1 == runs.size() && run.second == line.size()) { + position = 'E'; + } + marks.push_back({line.substr(run.first, run.second - run.first), position}); + } + // The find-first split dance, exactly as the Python does it. + std::vector chunks; + std::string rest = line; + for (const auto & mark : marks) { + const size_t at = rest.find(mark.mark); + if (at == std::string::npos) { + chunks.push_back(rest); + rest.clear(); + continue; + } + chunks.push_back(rest.substr(0, at)); + rest.erase(0, at + mark.mark.size()); + } + chunks.push_back(rest); + chunks.erase( + std::remove_if(chunks.begin(), chunks.end(), + [](const std::string & chunk) { return chunk.empty(); }), + chunks.end()); + return {std::move(chunks), std::move(marks)}; +} + +/** Punctuation.restore for a single line, sep.word = " ", strip = False. */ +std::string restore_punctuation( + std::vector chunk_phonemes, + std::vector marks) { + std::deque text(chunk_phonemes.begin(), chunk_phonemes.end()); + std::deque pending(marks.begin(), marks.end()); + std::vector out; + size_t pos = 0; + while (!text.empty() || !pending.empty()) { + if (pending.empty()) { + for (auto & line : text) { + if (line.empty() || line.back() != ' ') { + line.push_back(' '); + } + out.push_back(std::move(line)); + } + text.clear(); + } else if (text.empty()) { + std::string joined; + for (const auto & mark : pending) { + joined += mark.mark; + } + out.push_back(std::move(joined)); + pending.clear(); + } else if (pos == 0) { // single line: every mark carries index 0 + const auto current = pending.front(); + pending.pop_front(); + if (!text.front().empty() && text.front().back() == ' ') { + text.front().pop_back(); + } + const bool mark_ends_with_sep = + !current.mark.empty() && current.mark.back() == ' '; + if (current.position == 'B') { + text.front() = current.mark + text.front(); + } else if (current.position == 'E') { + out.push_back(text.front() + current.mark + (mark_ends_with_sep ? "" : " ")); + text.pop_front(); + ++pos; + } else if (current.position == 'A') { + out.push_back(current.mark + (mark_ends_with_sep ? "" : " ")); + ++pos; + } else { // 'I' + if (text.size() == 1) { + text.front() += current.mark; + } else { + auto first = std::move(text.front()); + text.pop_front(); + text.front() = first + current.mark + text.front(); + } + } + } else { + auto & line = text.front(); + if (line.empty() || line.back() != ' ') { + line.push_back(' '); + } + out.push_back(std::move(line)); + text.pop_front(); + ++pos; + } + } + // phonemizer would return these as separate lines and the reference + // takes the first; a single input line produces one in practice. + std::string result; + for (const auto & line : out) { + result += line; + } + return result; +} + +/** phonemizer EspeakBackend._postprocess_line with tie enabled, + * with_stress=True, strip=False, word separator " ", phone separator "". */ +std::string postprocess_espeak_line(std::string line) { + const auto not_space = [](unsigned char ch) { return std::isspace(ch) == 0; }; + line.erase(line.begin(), std::find_if(line.begin(), line.end(), not_space)); + line.erase(std::find_if(line.rbegin(), line.rend(), not_space).base(), line.end()); + std::replace(line.begin(), line.end(), '\n', ' '); + replace_all(line, " ", " "); + // espeak-ng#694: stray '_' separators at word ends + std::string squeezed; + squeezed.reserve(line.size()); + for (const char ch : line) { + if (ch == '_' && !squeezed.empty() && squeezed.back() == '_') { + continue; + } + squeezed.push_back(ch); + } + line = std::move(squeezed); + replace_all(line, "_ ", " "); + // language_switch="remove-flags": strip espeak's (lang) switch flags + if (line.find('(') != std::string::npos) { + std::string unflagged; + size_t at = 0; + while (at < line.size()) { + if (line[at] == '(') { + const size_t close = line.find(')', at + 1); + if (close != std::string::npos) { + at = close + 1; + continue; + } + } + unflagged.push_back(line[at++]); + } + line = std::move(unflagged); + } + if (line.empty()) { + return line; + } + // per word: strip, drop in-word '_' (phone separator is empty), append " " + std::string out; + size_t start = 0; + while (start <= line.size()) { + size_t end = line.find(' ', start); + if (end == std::string::npos) { + end = line.size(); + } + std::string word = line.substr(start, end - start); + word.erase(std::remove(word.begin(), word.end(), '_'), word.end()); + out += word; + out.push_back(' '); + if (end == line.size()) { + break; + } + start = end + 1; + } + return out; +} + +/** The reference front end's own line post-processing: rewrite eSpeak's tie + * to '^' per word so the E2M diphthong patterns can match. */ +std::string rewrite_ties_per_word(const std::string & line_in) { + std::string line = line_in; + const auto not_space = [](unsigned char ch) { return std::isspace(ch) == 0; }; + line.erase(line.begin(), std::find_if(line.begin(), line.end(), not_space)); + line.erase(std::find_if(line.rbegin(), line.rend(), not_space).base(), line.end()); + std::replace(line.begin(), line.end(), '\n', ' '); + replace_all(line, " ", " "); + if (line.empty()) { + return line; + } + std::string out; + size_t start = 0; + while (start <= line.size()) { + size_t end = line.find(' ', start); + if (end == std::string::npos) { + end = line.size(); + } + std::string word = line.substr(start, end - start); + replace_all(word, kTieDefault, kTieMisaki); + out += word; + out.push_back(' '); + if (end == line.size()) { + break; + } + start = end + 1; + } + return out; +} + struct EspeakApi { io::DynamicLibraryHandle library = nullptr; InitializeFn initialize = nullptr; @@ -243,8 +496,17 @@ SanoTtsFrontend::SanoTtsFrontend( SanoTtsFrontend::~SanoTtsFrontend() = default; SanoTtsEncoded SanoTtsFrontend::encode(const std::string & text) const { - std::string ipa = apply_e2m(impl_->espeak.phonemize(text)); - replace_all(ipa, kTieDefault, kTieMisaki); + auto [chunks, marks] = preserve_punctuation(text); + std::vector chunk_phonemes; + chunk_phonemes.reserve(chunks.size()); + for (const auto & chunk : chunks) { + chunk_phonemes.push_back(postprocess_espeak_line(impl_->espeak.phonemize(chunk))); + } + const std::string restored = + restore_punctuation(std::move(chunk_phonemes), std::move(marks)); + // Ties become '^' BEFORE E2M runs: every diphthong pattern in the table + // ("o^ʊ" -> "O", "t^ʃ" -> "ʧ", ...) matches on the rewritten form. + std::string ipa = apply_e2m(rewrite_ties_per_word(restored)); const auto & vocab = vocabulary(); SanoTtsEncoded out; diff --git a/src/community_models/sanotts/runtime.cpp b/src/community_models/sanotts/runtime.cpp new file mode 100644 index 000000000..92e7f721e --- /dev/null +++ b/src/community_models/sanotts/runtime.cpp @@ -0,0 +1,1259 @@ +#include "engine/community_models/sanotts/runtime.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/audio/istft_graph.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/runtime/cache_slots.h" + +#include "ggml-alloc.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { +namespace { + +namespace core = engine::core; +namespace modules = engine::modules; + +constexpr size_t kIoArenaBytes = 8ULL * 1024ULL * 1024ULL; +constexpr size_t kGraphArenaBytes = 128ULL * 1024ULL * 1024ULL; +constexpr size_t kWeightArenaBytes = 8ULL * 1024ULL * 1024ULL; +constexpr size_t kExpectedTensors = 103; + +// The decoder's norms are nn.LayerNorm(eps=1e-6), NOT torch's 1e-5 default. +// The difference compounds through the four ConvNeXt blocks and is then +// amplified by the exp() in the magnitude head; the reference implementations +// document losing 0.06 of correlation and a third of the output amplitude to +// exactly this constant. +constexpr float kLayerNormEps = 1.0e-6F; +constexpr float kDcBlockPole = 0.9973F; +constexpr double kPi = 3.14159265358979323846; + +struct GgmlContextDeleter { + void operator()(ggml_context * context) const noexcept { + if (context != nullptr) { + ggml_free(context); + } + } +}; + +core::TensorValue contiguous( + core::ModuleBuildContext & ctx, + const core::TensorValue & value) { + if (core::has_backend_addressable_layout(value.tensor)) { + return value; + } + return core::wrap_tensor( + ggml_cont(ctx.ggml, value.tensor), + value.shape, + value.type); +} + +core::TensorValue add( + core::ModuleBuildContext & ctx, + const core::TensorValue & lhs, + const core::TensorValue & rhs) { + return modules::AddModule().build(ctx, lhs, rhs); +} + +struct SanoTtsBackendWeights { + std::shared_ptr store; + std::unordered_map tensors; +}; + +const core::TensorValue & weight( + const SanoTtsBackendWeights & weights, + const std::string & name) { + const auto found = weights.tensors.find(name); + if (found == weights.tensors.end()) { + throw std::runtime_error("sanoTTS missing tensor: " + name); + } + return found->second; +} + +std::shared_ptr load_weights( + const std::shared_ptr & assets, + ggml_backend_t backend, + core::BackendType backend_type) { + auto out = std::make_shared(); + out->store = std::make_shared( + backend, + backend_type, + "sanotts.weights", + kWeightArenaBytes); + const auto metadata = assets->weights->tensors(); + if (metadata.size() != kExpectedTensors) { + throw std::runtime_error( + "sanoTTS expects exactly " + std::to_string(kExpectedTensors) + + " tensors, found " + std::to_string(metadata.size())); + } + out->tensors.reserve(metadata.size()); + for (const auto & tensor : metadata) { + if (assets::ggml_type_for_tensor_dtype(tensor.dtype) != GGML_TYPE_F32) { + throw std::runtime_error( + "sanoTTS supports FP32 weights only: " + tensor.name); + } + out->tensors.emplace( + tensor.name, + out->store->load_tensor( + *assets->weights, + tensor.name, + assets::TensorStorageType::F32, + tensor.shape)); + } + out->store->upload(); + assets->weights->release_storage(); + return out; +} + +// ---- graph builders ------------------------------------------------------ +// +// Values are carried channel-major as [1, C, T] (ggml ne0 = T) through the +// convolutional front ends, and channel-last as [1, T, C] through the +// ConvNeXt decoder blocks, matching the PyTorch modules they reproduce. + +core::TensorValue conv1d( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t out_channels, + int64_t kernel, + int padding) { + const int64_t in_channels = input.shape.dims[1]; + const int64_t input_frames = input.shape.dims[2]; + const int64_t output_frames = input_frames + 2 * padding - (kernel - 1); + const auto source = contiguous(ctx, input); + auto * input_2d = ggml_reshape_2d( + ctx.ggml, + source.tensor, + input_frames, + in_channels); + auto * kernel_tensor = weight(weights, prefix + ".weight").tensor; + ggml_tensor * output = nullptr; + if (kernel == 1 && padding == 0) { + auto * kernel_2d = ggml_reshape_2d( + ctx.ggml, + kernel_tensor, + in_channels, + out_channels); + auto * input_channels_first = ggml_cont( + ctx.ggml, + ggml_permute(ctx.ggml, input_2d, 1, 0, 2, 3)); + auto * output_channels_first = + ggml_mul_mat(ctx.ggml, kernel_2d, input_channels_first); + output = ggml_reshape_2d( + ctx.ggml, + ggml_cont( + ctx.ggml, + ggml_permute(ctx.ggml, output_channels_first, 1, 0, 2, 3)), + output_frames, + out_channels); + } else { + auto * kernel_3d = ggml_reshape_3d( + ctx.ggml, + kernel_tensor, + kernel, + in_channels, + out_channels); + auto * input_3d = ggml_reshape_3d( + ctx.ggml, + input_2d, + input_frames, + in_channels, + 1); + auto * output_3d = ggml_conv_1d( + ctx.ggml, + kernel_3d, + input_3d, + 1, + padding, + 1); + output = ggml_reshape_2d( + ctx.ggml, + output_3d, + output_frames, + out_channels); + } + auto * bias = ggml_reshape_2d( + ctx.ggml, + weight(weights, prefix + ".bias").tensor, + 1, + out_channels); + output = ggml_add(ctx.ggml, output, bias); + return core::wrap_tensor( + ggml_reshape_3d(ctx.ggml, output, output_frames, out_channels, 1), + core::TensorShape::from_dims({1, out_channels, output_frames}), + GGML_TYPE_F32); +} + +/** x + scale * conv2(silu(conv1(x))) -- the front ends' ResidualConvBlock. + * `scale` is a learned one-element tensor, broadcast by ggml_mul. */ +core::TensorValue residual_block( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t hidden, + int64_t kernel) { + const int padding = static_cast(kernel / 2); + auto h = conv1d(ctx, weights, input, prefix + ".net.0", hidden, kernel, padding); + h = modules::SiluModule().build(ctx, h); + h = conv1d(ctx, weights, h, prefix + ".net.2", hidden, kernel, padding); + const auto h_source = contiguous(ctx, h); + const auto scaled_h = core::wrap_tensor( + ggml_mul(ctx.ggml, h_source.tensor, weight(weights, prefix + ".scale").tensor), + h.shape, + GGML_TYPE_F32); + return add(ctx, input, scaled_h); +} + +core::TensorValue embed_tokens( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & tokens, + const std::string & name, + int64_t vocab, + int64_t hidden) { + auto embedded = modules::EmbeddingModule({vocab, hidden}).build( + ctx, + tokens, + weight(weights, name)); + return modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, embedded); +} + +core::TensorValue channel_last_layer_norm( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t channels) { + return modules::LayerNormModule({channels, kLayerNormEps, true, true}).build( + ctx, + input, + { + weight(weights, prefix + ".weight"), + weight(weights, prefix + ".bias"), + }); +} + +struct GraphResources { + ~GraphResources() { + core::free_backend_graph_plan(backend, plan); + core::release_backend_graph_resources(backend, graph); + if (allocator != nullptr) { + ggml_gallocr_free(allocator); + } + if (io_buffer != nullptr) { + ggml_backend_buffer_free(io_buffer); + } + } + + std::unique_ptr io_context; + std::unique_ptr graph_context; + ggml_backend_buffer_t io_buffer = nullptr; + ggml_gallocr_t allocator = nullptr; + ggml_backend_t backend = nullptr; + ggml_backend_graph_plan_t plan = nullptr; + ggml_cgraph * graph = nullptr; +}; + +void allocate_graph(GraphResources & resources) { + resources.io_buffer = + ggml_backend_alloc_ctx_tensors(resources.io_context.get(), resources.backend); + if (resources.io_buffer == nullptr) { + throw std::runtime_error("sanoTTS failed to allocate graph input buffer"); + } + resources.allocator = + ggml_gallocr_new(ggml_backend_get_default_buffer_type(resources.backend)); + if (resources.allocator == nullptr || + !ggml_gallocr_reserve(resources.allocator, resources.graph) || + !ggml_gallocr_alloc_graph(resources.allocator, resources.graph)) { + throw std::runtime_error("sanoTTS failed to allocate backend graph"); + } + core::validate_backend_graph_supported( + resources.backend, + resources.graph, + "sanoTTS"); + resources.plan = + core::create_backend_graph_plan_if_host(resources.backend, resources.graph); +} + +void compute_graph(GraphResources & resources, const char * label) { + const auto status = core::compute_backend_graph( + resources.backend, + resources.graph, + resources.plan, + label); + ggml_backend_synchronize(resources.backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error(std::string(label) + " graph compute failed"); + } +} + +// ---- ATen-compatible noise ---------------------------------------------- +// +// The decoder is noise-fed, so a rendering is only reproducible if the noise +// stream is. This is PyTorch's CPU path exactly: MT19937 seeded from the low +// 32 bits, a 24-bit float uniform, and Box-Muller in blocks of 16 +// (aten/src/ATen/native/DistributionTemplates.h). The integer half is +// bit-exact against torch.rand; the Gaussian half goes through logf/cosf/ +// sinf, which differ by a few ulp across libm builds -- the reference +// implementations document the same bound (at most ~2e-6 per draw). + +constexpr int kMtN = 624; +constexpr int kMtM = 397; + +struct AtenMt19937 { + uint32_t state[kMtN]; + int left = 1; + int next = 0; + + explicit AtenMt19937(uint64_t seed) { + // ATen mt19937_engine::init_with_uint32 -- only the low 32 bits used. + state[0] = static_cast(seed & 0xFFFFFFFFULL); + for (int i = 1; i < kMtN; ++i) { + state[i] = 1812433253U * (state[i - 1] ^ (state[i - 1] >> 30)) + + static_cast(i); + } + } + + static uint32_t twist(uint32_t u, uint32_t v) { + const uint32_t mixed = (u & 0x80000000U) | (v & 0x7FFFFFFFU); + return (mixed >> 1) ^ ((v & 1U) != 0U ? 0x9908B0DFU : 0U); + } + + void next_state() { + left = kMtN; + next = 0; + int i = 0; + for (; i < kMtN - kMtM; ++i) { + state[i] = state[i + kMtM] ^ twist(state[i], state[i + 1]); + } + for (; i < kMtN - 1; ++i) { + state[i] = state[i + kMtM - kMtN] ^ twist(state[i], state[i + 1]); + } + state[kMtN - 1] = state[kMtM - 1] ^ twist(state[kMtN - 1], state[0]); + } + + uint32_t random() { + if (--left <= 0) { + next_state(); + } + uint32_t y = state[next++]; + y ^= (y >> 11); + y ^= (y << 7) & 0x9D2C5680U; + y ^= (y << 15) & 0xEFC60000U; + y ^= (y >> 18); + return y; + } + + /** at::uniform_real_distribution: (raw & (2^24 - 1)) * 2^-24. */ + float uniform() { + return static_cast(random() & 0xFFFFFFU) * (1.0F / 16777216.0F); + } +}; + +/** ATen normal_fill_16, mean 0 std 1, float32 throughout. In place. */ +void normal_fill_16(float * d) { + for (int j = 0; j < 8; ++j) { + const float u1 = 1.0F - d[j]; + const float u2 = d[j + 8]; + const float radius = std::sqrt(-2.0F * std::log(u1)); + const float theta = static_cast(2.0 * kPi) * u2; + d[j] = radius * std::cos(theta); + d[j + 8] = radius * std::sin(theta); + } +} + +std::vector seeded_noise(uint64_t seed, int64_t channels, int64_t frames) { + const int64_t size = channels * frames; + if (size < 16) { + // torch dispatches sizes under 16 to a scalar path this does not + // implement; the decoder always asks for channels*frames above that. + throw std::runtime_error("sanoTTS seeded noise needs at least 16 values"); + } + AtenMt19937 gen(seed); + std::vector out(static_cast(size)); + for (auto & value : out) { + value = gen.uniform(); + } + int64_t i = 0; + for (; i + 16 <= size; i += 16) { + normal_fill_16(out.data() + i); + } + if (size % 16 != 0) { + // Torch draws a FRESH block of 16 (continuing the same stream) and + // overwrites the last 16 values with it; the loop's remainder is + // discarded, so the tail is not simply its leftover. + float tail[16]; + for (float & value : tail) { + value = gen.uniform(); + } + normal_fill_16(tail); + std::memcpy(out.data() + size - 16, tail, sizeof(tail)); + } + return out; +} + +// ---- host float semantics shared with the reference front end ------------ + +/** torch.linspace(0, 1, n) with exact CPU-kernel float semantics: step in + * fp32, the first half filled as step*i, the second as fma(-step, n-1-i, 1). */ +void linspace01(float * dst, int64_t n) { + if (n <= 0) { + return; + } + if (n == 1) { + dst[0] = 0.0F; + return; + } + const auto step = 1.0F / static_cast(n - 1); + const int64_t half = n / 2; + for (int64_t i = 0; i < half; ++i) { + dst[i] = step * static_cast(i); + } + for (int64_t i = half; i < n; ++i) { + dst[i] = std::fma(-step, static_cast(n - 1 - i), 1.0F); + } +} + +// ---- graphs -------------------------------------------------------------- + +struct DurationGraph : GraphResources { + int64_t token_count = 0; + ggml_tensor * tokens = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * log_duration = nullptr; +}; + +std::unique_ptr build_duration_graph( + const SanoTtsBackendWeights & weights, + const SanoTtsConfig & config, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t token_count) { + auto out = std::make_unique(); + out->backend = backend; + out->token_count = token_count; + out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); + out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create duration graph contexts"); + } + core::ModuleBuildContext io_ctx{ + out->io_context.get(), + "sanotts.duration.io", + backend_type, + }; + core::ModuleBuildContext ctx{ + out->graph_context.get(), + "sanotts.duration", + backend_type, + }; + auto tokens = core::make_tensor( + io_ctx, + GGML_TYPE_I32, + core::TensorShape::from_dims({1, token_count})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, 3, token_count})); + ggml_set_input(tokens.tensor); + ggml_set_input(feats.tensor); + + auto hidden = embed_tokens( + ctx, + weights, + tokens, + "duration.embedding.weight", + config.vocab_size, + config.duration_hidden); + hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); + hidden = conv1d( + ctx, + weights, + hidden, + "duration.input_proj", + config.duration_hidden, + 1, + 0); + for (int64_t block = 0; block < config.duration_depth; ++block) { + hidden = residual_block( + ctx, + weights, + hidden, + "duration.blocks." + std::to_string(block), + config.duration_hidden, + config.duration_kernel); + } + auto log_duration = conv1d(ctx, weights, hidden, "duration.output", 1, 1, 0); + log_duration = contiguous(ctx, log_duration); + out->tokens = tokens.tensor; + out->feats = feats.tensor; + out->log_duration = log_duration.tensor; + ggml_set_output(out->log_duration); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->log_duration); + allocate_graph(*out); + return out; +} + +struct TokenGraph : GraphResources { + int64_t token_count = 0; + ggml_tensor * tokens = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * context = nullptr; +}; + +std::unique_ptr build_token_graph( + const SanoTtsBackendWeights & weights, + const SanoTtsConfig & config, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t token_count) { + auto out = std::make_unique(); + out->backend = backend; + out->token_count = token_count; + out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); + out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create token graph contexts"); + } + core::ModuleBuildContext io_ctx{ + out->io_context.get(), + "sanotts.token.io", + backend_type, + }; + core::ModuleBuildContext ctx{ + out->graph_context.get(), + "sanotts.token", + backend_type, + }; + auto tokens = core::make_tensor( + io_ctx, + GGML_TYPE_I32, + core::TensorShape::from_dims({1, token_count})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, 2, token_count})); + ggml_set_input(tokens.tensor); + ggml_set_input(feats.tensor); + + auto hidden = embed_tokens( + ctx, + weights, + tokens, + "acoustic.embedding.weight", + config.vocab_size, + config.acoustic_hidden); + hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); + hidden = conv1d( + ctx, + weights, + hidden, + "acoustic.token_input_proj", + config.acoustic_hidden, + 1, + 0); + for (int64_t block = 0; block < config.acoustic_token_depth; ++block) { + hidden = residual_block( + ctx, + weights, + hidden, + "acoustic.token_blocks." + std::to_string(block), + config.acoustic_hidden, + config.acoustic_kernel); + } + hidden = contiguous(ctx, hidden); + out->tokens = tokens.tensor; + out->feats = feats.tensor; + out->context = hidden.tensor; + ggml_set_output(out->context); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->context); + allocate_graph(*out); + return out; +} + +struct DecoderGraph : GraphResources { + int64_t frames = 0; + ggml_tensor * context = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * noise = nullptr; + ggml_tensor * spectrum = nullptr; +}; + +/** Frame-stage acoustic blocks -> mel-100 -> ConvNeXt decoder -> the + * [log-magnitude | phase] spectrum rows the host iSTFT consumes. */ +std::unique_ptr build_decoder_graph( + const SanoTtsBackendWeights & weights, + const SanoTtsConfig & config, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t frames) { + auto out = std::make_unique(); + out->backend = backend; + out->frames = frames; + out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); + out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create decoder graph contexts"); + } + core::ModuleBuildContext io_ctx{ + out->io_context.get(), + "sanotts.decoder.io", + backend_type, + }; + core::ModuleBuildContext ctx{ + out->graph_context.get(), + "sanotts.decoder", + backend_type, + }; + auto context = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, 3, frames})); + auto noise = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, config.noise_channels, frames})); + ggml_set_input(context.tensor); + ggml_set_input(feats.tensor); + ggml_set_input(noise.tensor); + + // Acoustic frame stage: expanded token context + positional features. + auto hidden = modules::ConcatModule({1}).build(ctx, context, feats); + hidden = conv1d( + ctx, + weights, + hidden, + "acoustic.frame_input_proj", + config.acoustic_hidden, + 1, + 0); + for (int64_t block = 0; block < config.acoustic_depth; ++block) { + hidden = residual_block( + ctx, + weights, + hidden, + "acoustic.frame_blocks." + std::to_string(block), + config.acoustic_hidden, + config.acoustic_kernel); + } + auto mel = conv1d(ctx, weights, hidden, "acoustic.output", config.mels, 1, 0); + + // ConvNeXt decoder. Noise-fed: the noise adapter's output is added to the + // mel embedding before the first norm. + const int embed_padding = static_cast(config.embed_kernel / 2); + auto value = conv1d( + ctx, + weights, + mel, + "decoder.embed", + config.dim, + config.embed_kernel, + embed_padding); + value = add( + ctx, + value, + conv1d( + ctx, + weights, + noise, + "decoder.noise_adapter", + config.dim, + config.embed_kernel, + embed_padding)); + + // Channel-last from here: LayerNorm and the pointwise projections act on + // the channel axis, exactly as the PyTorch modules do. + auto value_cl = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, value); + value_cl = channel_last_layer_norm(ctx, weights, value_cl, "decoder.norm", config.dim); + for (int64_t block = 0; block < config.blocks; ++block) { + const std::string prefix = "decoder.blocks." + std::to_string(block); + const auto residual = value_cl; + auto branch_cm = contiguous( + ctx, + modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, value_cl)); + auto branch = core::wrap_tensor( + ggml_conv_1d_dw( + ctx.ggml, + contiguous(ctx, weight(weights, prefix + ".dwconv.weight")).tensor, + branch_cm.tensor, + 1, + static_cast(config.dw_kernel / 2), + 1), + core::TensorShape::from_dims({1, config.dim, frames}), + GGML_TYPE_F32); + auto * dw_bias = ggml_reshape_3d( + ctx.ggml, + weight(weights, prefix + ".dwconv.bias").tensor, + 1, + config.dim, + 1); + branch = core::wrap_tensor( + ggml_add(ctx.ggml, branch.tensor, dw_bias), + branch.shape, + GGML_TYPE_F32); + auto branch_cl = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, branch); + branch_cl = channel_last_layer_norm(ctx, weights, branch_cl, prefix + ".norm", config.dim); + branch_cl = modules::LinearModule({config.dim, config.pw_hidden, true}).build( + ctx, + branch_cl, + { + weight(weights, prefix + ".pwconv1.weight"), + weight(weights, prefix + ".pwconv1.bias"), + }); + // Exact erf GELU -- what nn.GELU() computes by default, NOT the tanh + // approximation. + branch_cl = modules::GeluModule({modules::GeluApproximation::ExactErf}) + .build(ctx, branch_cl); + branch_cl = modules::LinearModule({config.pw_hidden, config.dim, true}).build( + ctx, + branch_cl, + { + weight(weights, prefix + ".pwconv2.weight"), + weight(weights, prefix + ".pwconv2.bias"), + }); + const auto branch_source = contiguous(ctx, branch_cl); + branch_cl = core::wrap_tensor( + ggml_mul( + ctx.ggml, + branch_source.tensor, + weight(weights, prefix + ".gamma").tensor), + branch_cl.shape, + GGML_TYPE_F32); + value_cl = add(ctx, residual, branch_cl); + } + value_cl = channel_last_layer_norm(ctx, weights, value_cl, "decoder.final_norm", config.dim); + auto spectrum = modules::LinearModule({config.dim, config.n_fft + 2, true}).build( + ctx, + value_cl, + { + weight(weights, "decoder.head.weight"), + weight(weights, "decoder.head.bias"), + }); + spectrum = contiguous(ctx, spectrum); + out->context = context.tensor; + out->feats = feats.tensor; + out->noise = noise.tensor; + out->spectrum = spectrum.tensor; + ggml_set_output(out->spectrum); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->spectrum); + allocate_graph(*out); + return out; +} + +// ---- SHA-256 for the derive-from-text seed ------------------------------- + +constexpr uint32_t kShaK[64] = { + 0x428A2F98U, 0x71374491U, 0xB5C0FBCFU, 0xE9B5DBA5U, 0x3956C25BU, 0x59F111F1U, + 0x923F82A4U, 0xAB1C5ED5U, 0xD807AA98U, 0x12835B01U, 0x243185BEU, 0x550C7DC3U, + 0x72BE5D74U, 0x80DEB1FEU, 0x9BDC06A7U, 0xC19BF174U, 0xE49B69C1U, 0xEFBE4786U, + 0x0FC19DC6U, 0x240CA1CCU, 0x2DE92C6FU, 0x4A7484AAU, 0x5CB0A9DCU, 0x76F988DAU, + 0x983E5152U, 0xA831C66DU, 0xB00327C8U, 0xBF597FC7U, 0xC6E00BF3U, 0xD5A79147U, + 0x06CA6351U, 0x14292967U, 0x27B70A85U, 0x2E1B2138U, 0x4D2C6DFCU, 0x53380D13U, + 0x650A7354U, 0x766A0ABBU, 0x81C2C92EU, 0x92722C85U, 0xA2BFE8A1U, 0xA81A664BU, + 0xC24B8B70U, 0xC76C51A3U, 0xD192E819U, 0xD6990624U, 0xF40E3585U, 0x106AA070U, + 0x19A4C116U, 0x1E376C08U, 0x2748774CU, 0x34B0BCB5U, 0x391C0CB3U, 0x4ED8AA4AU, + 0x5B9CCA4FU, 0x682E6FF3U, 0x748F82EEU, 0x78A5636FU, 0x84C87814U, 0x8CC70208U, + 0x90BEFFFAU, 0xA4506CEBU, 0xBEF9A3F7U, 0xC67178F2U, +}; + +uint32_t rotr32(uint32_t v, int n) { + return (v >> n) | (v << (32 - n)); +} + +void sha256_block(uint32_t * h, const unsigned char * block) { + uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(block[4 * i]) << 24) | + (static_cast(block[4 * i + 1]) << 16) | + (static_cast(block[4 * i + 2]) << 8) | + static_cast(block[4 * i + 3]); + } + for (int i = 16; i < 64; ++i) { + const uint32_t s0 = rotr32(w[i - 15], 7) ^ rotr32(w[i - 15], 18) ^ (w[i - 15] >> 3); + const uint32_t s1 = rotr32(w[i - 2], 17) ^ rotr32(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0]; + uint32_t b = h[1]; + uint32_t c = h[2]; + uint32_t d = h[3]; + uint32_t e = h[4]; + uint32_t f = h[5]; + uint32_t g = h[6]; + uint32_t hh = h[7]; + for (int i = 0; i < 64; ++i) { + const uint32_t s1 = rotr32(e, 6) ^ rotr32(e, 11) ^ rotr32(e, 25); + const uint32_t ch = (e & f) ^ ((~e) & g); + const uint32_t s0 = rotr32(a, 2) ^ rotr32(a, 13) ^ rotr32(a, 22); + const uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + const uint32_t t1 = hh + s1 + ch + kShaK[i] + w[i]; + const uint32_t t2 = s0 + maj; + hh = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + h[5] += f; + h[6] += g; + h[7] += hh; +} + +// ---- host post-processing ------------------------------------------------ + +std::vector periodic_hann_window(int64_t n_fft) { + std::vector window(static_cast(n_fft)); + for (int64_t i = 0; i < n_fft; ++i) { + window[static_cast(i)] = static_cast( + 0.5 - 0.5 * std::cos(2.0 * kPi * static_cast(i) / + static_cast(n_fft))); + } + return window; +} + +/** H(z) = (1 - z^-1) / (1 - R z^-1), 2 MAC/sample, zero initial state -- + * the same DC blocker the reference runtimes apply after the iSTFT. */ +void dc_block_in_place(std::vector & samples) { + float x1 = 0.0F; + float y1 = 0.0F; + for (float & sample : samples) { + const float x = sample; + const float y = x - x1 + kDcBlockPole * y1; + x1 = x; + y1 = y; + sample = y; + } +} + +} // namespace + +uint64_t sanotts_text_seed(const std::string & text) { + uint32_t h[8] = {0x6A09E667U, 0xBB67AE85U, 0x3C6EF372U, 0xA54FF53AU, + 0x510E527FU, 0x9B05688CU, 0x1F83D9ABU, 0x5BE0CD19U}; + const auto * data = reinterpret_cast(text.data()); + const size_t len = text.size(); + const size_t full = len / 64; + const size_t rem = len % 64; + for (size_t i = 0; i < full; ++i) { + sha256_block(h, data + i * 64); + } + unsigned char tail[128] = {0}; + std::memcpy(tail, data + full * 64, rem); + tail[rem] = 0x80; + const size_t tail_len = rem < 56 ? 64 : 128; + const uint64_t bits = static_cast(len) * 8U; + for (int i = 0; i < 8; ++i) { + tail[tail_len - 1 - static_cast(i)] = + static_cast((bits >> (8 * i)) & 0xFFU); + } + sha256_block(h, tail); + if (tail_len == 128) { + sha256_block(h, tail + 64); + } + // == int.from_bytes(sha256(text).digest()[:8], "big"); ATen seeding then + // keeps only the low 32 bits, exactly as the reference runtimes do. + return (static_cast(h[0]) << 32) | static_cast(h[1]); +} + +struct SanoTtsNativeRuntime::State { + struct BackendOwner { + ggml_backend_t value = nullptr; + ~BackendOwner() { + if (value != nullptr) { + ggml_backend_free(value); + } + } + }; + + State( + std::shared_ptr assets_in, + core::BackendConfig backend_config) + : assets(std::move(assets_in)), + threads(std::max(1, backend_config.threads)) { + if (assets == nullptr) { + throw std::runtime_error("sanoTTS native runtime requires assets"); + } + backend_config.threads = threads; + backend.value = core::init_backend(backend_config); + backend_type = core::backend_type(backend.value); + core::set_backend_threads(backend.value, threads); + weights = load_weights(assets, backend.value, backend_type); + if (backend_type == core::BackendType::Cuda) { + // Durations round to integers and gate the whole frame layout, so + // small TF32 differences would move frame counts. Keep the tiny + // duration model on the host; the decoder stays on CUDA. + core::BackendConfig duration_config{ + core::BackendType::Cpu, + 0, + threads, + }; + duration_backend.value = core::init_backend(duration_config); + core::set_backend_threads(duration_backend.value, threads); + duration_weights = load_weights( + assets, + duration_backend.value, + core::BackendType::Cpu); + } + } + + DurationGraph & duration_graph(int64_t token_count) { + if (auto * found = duration_graphs.find(token_count)) { + engine::debug::trace_log_scalar("sanotts.duration_graph.cache_hit", true); + return **found; + } + engine::debug::trace_log_scalar("sanotts.duration_graph.cache_hit", false); + const auto backend_value = + duration_backend.value != nullptr ? duration_backend.value : backend.value; + const auto graph_backend_type = + duration_backend.value != nullptr ? core::BackendType::Cpu : backend_type; + const auto & selected_weights = + duration_weights != nullptr ? duration_weights : weights; + duration_graphs.put( + token_count, + build_duration_graph( + *selected_weights, + assets->config, + backend_value, + graph_backend_type, + token_count)); + auto * created = duration_graphs.find(token_count); + if (created == nullptr) { + throw std::runtime_error("sanoTTS duration graph cache insert failed"); + } + return **created; + } + + TokenGraph & token_graph(int64_t token_count) { + if (auto * found = token_graphs.find(token_count)) { + engine::debug::trace_log_scalar("sanotts.token_graph.cache_hit", true); + return **found; + } + engine::debug::trace_log_scalar("sanotts.token_graph.cache_hit", false); + token_graphs.put( + token_count, + build_token_graph( + *weights, + assets->config, + backend.value, + backend_type, + token_count)); + auto * created = token_graphs.find(token_count); + if (created == nullptr) { + throw std::runtime_error("sanoTTS token graph cache insert failed"); + } + return **created; + } + + DecoderGraph & decoder_graph(int64_t frames) { + if (auto * found = decoder_graphs.find(frames)) { + engine::debug::trace_log_scalar("sanotts.decoder_graph.cache_hit", true); + return **found; + } + engine::debug::trace_log_scalar("sanotts.decoder_graph.cache_hit", false); + decoder_graphs.put( + frames, + build_decoder_graph( + *weights, + assets->config, + backend.value, + backend_type, + frames)); + auto * created = decoder_graphs.find(frames); + if (created == nullptr) { + throw std::runtime_error("sanoTTS decoder graph cache insert failed"); + } + return **created; + } + + std::shared_ptr assets; + int threads = 1; + core::BackendType backend_type = core::BackendType::Cpu; + BackendOwner backend; + std::shared_ptr weights; + BackendOwner duration_backend; + std::shared_ptr duration_weights; + runtime::CacheSlots> duration_graphs{4}; + runtime::CacheSlots> token_graphs{4}; + runtime::CacheSlots> decoder_graphs{2}; +}; + +SanoTtsNativeRuntime::SanoTtsNativeRuntime( + std::shared_ptr assets, + core::BackendConfig backend_config) + : state_(std::make_unique(std::move(assets), backend_config)) {} + +SanoTtsNativeRuntime::~SanoTtsNativeRuntime() = default; + +runtime::AudioBuffer SanoTtsNativeRuntime::synthesize( + const std::vector & token_ids, + const SanoTtsGenerationOptions & options) { + const auto & config = state_->assets->config; + const auto token_count = static_cast(token_ids.size()); + if (token_count <= 0) { + throw std::runtime_error("sanoTTS requires at least one phoneme token"); + } + const auto total_start = std::chrono::steady_clock::now(); + + // -- durations --------------------------------------------------------- + const auto duration_start = std::chrono::steady_clock::now(); + auto & duration = state_->duration_graph(token_count); + core::write_tensor_i32( + core::wrap_tensor( + duration.tokens, + core::TensorShape::from_dims({1, token_count}), + GGML_TYPE_I32), + token_ids); + { + // [positions, length_hint, valid=1] rows, the exact float semantics + // of the reference front end (mcu/src/snt_front_f32.c). + std::vector feats(static_cast(3 * token_count)); + linspace01(feats.data(), token_count); + const float length_hint = + std::log1p(static_cast(token_count)) / + static_cast(std::log1p(static_cast(config.duration_max_tokens))); + std::fill_n(feats.begin() + token_count, token_count, length_hint); + std::fill_n(feats.begin() + 2 * token_count, token_count, 1.0F); + core::write_tensor_f32( + core::wrap_tensor( + duration.feats, + core::TensorShape::from_dims({1, 3, token_count}), + GGML_TYPE_F32), + feats); + } + compute_graph(duration, "sanoTTS duration"); + const auto log_duration = core::read_tensor_f32(duration.log_duration); + if (static_cast(log_duration.size()) != token_count) { + throw std::runtime_error("sanoTTS duration graph returned invalid output"); + } + // predict_durations: exp -> clamp_min(1) -> *scale -> round -> clamp. + // rintf under the default FE_TONEAREST matches torch.round's ties-to-even. + std::vector durations(static_cast(token_count)); + int64_t frames = 0; + for (int64_t token = 0; token < token_count; ++token) { + float value = std::exp(log_duration[static_cast(token)]); + if (!std::isfinite(value)) { + throw std::runtime_error("sanoTTS duration predictor produced a non-finite duration"); + } + if (value < 1.0F) { + value = 1.0F; + } + value = std::rint(value * options.speaking_rate); + if (value < 1.0F) { + value = 1.0F; + } + if (value > static_cast(config.duration_max_frames)) { + value = static_cast(config.duration_max_frames); + } + durations[static_cast(token)] = static_cast(value); + frames += durations[static_cast(token)]; + } + const int64_t max_frames = config.duration_max_tokens * config.duration_max_frames; + if (frames < 2 || frames > max_frames) { + throw std::runtime_error( + "sanoTTS expanded to " + std::to_string(frames) + + " frames, outside the supported range"); + } + engine::debug::timing_log_scalar( + "sanotts.duration_ms", + engine::debug::elapsed_ms(duration_start)); + + // -- token-stage acoustic context -------------------------------------- + const auto acoustic_start = std::chrono::steady_clock::now(); + auto & token_graph = state_->token_graph(token_count); + core::write_tensor_i32( + core::wrap_tensor( + token_graph.tokens, + core::TensorShape::from_dims({1, token_count}), + GGML_TYPE_I32), + token_ids); + { + // [token_pos, duration_hint] rows. + std::vector feats(static_cast(2 * token_count)); + linspace01(feats.data(), token_count); + float max_duration = 1.0F; + for (int64_t token = 0; token < token_count; ++token) { + max_duration = std::max( + max_duration, + static_cast(durations[static_cast(token)])); + } + const float log_max_duration = std::log1p(max_duration); + for (int64_t token = 0; token < token_count; ++token) { + feats[static_cast(token_count + token)] = + std::log1p(static_cast(durations[static_cast(token)])) / + log_max_duration; + } + core::write_tensor_f32( + core::wrap_tensor( + token_graph.feats, + core::TensorShape::from_dims({1, 2, token_count}), + GGML_TYPE_F32), + feats); + } + compute_graph(token_graph, "sanoTTS token context"); + const auto token_context = core::read_tensor_f32(token_graph.context); + if (static_cast(token_context.size()) != + config.acoustic_hidden * token_count) { + throw std::runtime_error("sanoTTS token graph returned invalid output"); + } + + // -- expand to frames (host: pure copies plus the documented float + // semantics of expand_features: doubles cast to fp32) --------------- + std::vector expanded(static_cast(config.acoustic_hidden * frames)); + for (int64_t channel = 0; channel < config.acoustic_hidden; ++channel) { + const float * row = token_context.data() + channel * token_count; + float * out_row = expanded.data() + channel * frames; + int64_t at = 0; + for (int64_t token = 0; token < token_count; ++token) { + const float value = row[token]; + for (int64_t j = 0; j < durations[static_cast(token)]; ++j) { + out_row[at++] = value; + } + } + } + std::vector frame_feats(static_cast(3 * frames)); + linspace01(frame_feats.data(), frames); + { + const int64_t denominator = token_count > 1 ? token_count - 1 : 1; + float * token_pos = frame_feats.data() + frames; + float * duration_pos = frame_feats.data() + 2 * frames; + int64_t at = 0; + for (int64_t token = 0; token < token_count; ++token) { + const int64_t count = durations[static_cast(token)]; + const auto position = static_cast( + static_cast(token) / static_cast(denominator)); + for (int64_t j = 0; j < count; ++j) { + token_pos[at] = position; + duration_pos[at] = count == 1 + ? 0.0F + : static_cast( + static_cast(j) / static_cast(count - 1)); + ++at; + } + } + } + const auto noise = seeded_noise(options.seed, config.noise_channels, frames); + engine::debug::timing_log_scalar( + "sanotts.acoustic_ms", + engine::debug::elapsed_ms(acoustic_start)); + + // -- frame stage + decoder --------------------------------------------- + const auto decoder_start = std::chrono::steady_clock::now(); + auto & decoder = state_->decoder_graph(frames); + core::write_tensor_f32( + core::wrap_tensor( + decoder.context, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames}), + GGML_TYPE_F32), + expanded); + core::write_tensor_f32( + core::wrap_tensor( + decoder.feats, + core::TensorShape::from_dims({1, 3, frames}), + GGML_TYPE_F32), + frame_feats); + core::write_tensor_f32( + core::wrap_tensor( + decoder.noise, + core::TensorShape::from_dims({1, config.noise_channels, frames}), + GGML_TYPE_F32), + noise); + compute_graph(decoder, "sanoTTS decoder"); + auto spectrum = core::read_tensor_f32(decoder.spectrum); + const int64_t out_dim = config.n_fft + 2; + if (static_cast(spectrum.size()) != frames * out_dim) { + throw std::runtime_error("sanoTTS decoder graph returned invalid output"); + } + engine::debug::timing_log_scalar( + "sanotts.decoder_ms", + engine::debug::elapsed_ms(decoder_start)); + + // -- iSTFT + DC block --------------------------------------------------- + const auto istft_start = std::chrono::steady_clock::now(); + // Bin 0 and Nyquist stay zeroed: the mag*(cos,sin) parametrisation + // phase-collapses at bin 0, which is where the frame-DC artefact came + // from. -inf log-magnitude exps to exactly 0. + const int64_t bins = config.n_fft / 2 + 1; + for (int64_t frame = 0; frame < frames; ++frame) { + float * row = spectrum.data() + frame * out_dim; + row[0] = -std::numeric_limits::infinity(); + row[bins - 1] = -std::numeric_limits::infinity(); + } + engine::audio::HostLogMagnitudePhaseISTFTConfig istft_config; + istft_config.frames = frames; + istft_config.n_fft = config.n_fft; + istft_config.hop_length = config.hop_length; + istft_config.out_dim = out_dim; + istft_config.threads = static_cast(state_->threads); + engine::audio::HostLogMagnitudePhaseISTFT istft(istft_config); + auto istft_result = istft.compute(spectrum, periodic_hann_window(config.n_fft)); + // The framework trims (n_fft - hop)/2 per side; torch.istft(center=True) + // trims n_fft/2. Drop the extra hop/2 per side so lengths and content + // match the reference exactly: (frames - 1) * hop samples. + const auto edge = static_cast(config.hop_length / 2); + const size_t expected = static_cast(frames) * static_cast(config.hop_length); + if (istft_result.audio.size() != expected || istft_result.audio.size() < 2 * edge) { + throw std::runtime_error("sanoTTS iSTFT returned an unexpected sample count"); + } + std::vector samples( + istft_result.audio.begin() + static_cast(edge), + istft_result.audio.end() - static_cast(edge)); + dc_block_in_place(samples); + engine::debug::timing_log_scalar( + "sanotts.istft_ms", + engine::debug::elapsed_ms(istft_start)); + + runtime::AudioBuffer out; + out.sample_rate = static_cast(config.sample_rate); + out.channels = 1; + out.samples = std::move(samples); + engine::debug::trace_log_scalar("sanotts.token_count", token_count); + engine::debug::trace_log_scalar("sanotts.frames", frames); + engine::debug::trace_log_scalar( + "sanotts.output_samples", + static_cast(out.samples.size())); + engine::debug::timing_log_scalar( + "session.wall_ms", + engine::debug::elapsed_ms(total_start)); + return out; +} + +} // namespace engine::models::sanotts diff --git a/src/community_models/sanotts/session.cpp b/src/community_models/sanotts/session.cpp new file mode 100644 index 000000000..9e5c2f361 --- /dev/null +++ b/src/community_models/sanotts/session.cpp @@ -0,0 +1,221 @@ +#include "engine/community_models/sanotts/session.h" + +#include "engine/framework/debug/profiler.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { +namespace { + +constexpr const char * kFamily = "sanotts"; + +std::shared_ptr require_assets( + std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("sanoTTS session requires assets"); + } + return assets; +} + +std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("sanoTTS session requires a model contract"); + } + return contract; +} + +std::filesystem::path session_path( + const runtime::SessionOptions & options, + const char * key) { + const auto found = options.options.find(key); + return found == options.options.end() + ? std::filesystem::path{} + : std::filesystem::path(found->second); +} + +void validate_session_options( + const runtime::SessionOptions & options, + const engine::model_spec::ModelContract & contract) { + const std::string family_prefix = std::string(kFamily) + "."; + for (const auto & [key, _] : options.options) { + if (key.rfind(family_prefix, 0) == 0 && + contract.session_option_keys.find(key) == + contract.session_option_keys.end()) { + throw std::runtime_error("unknown sanoTTS session option: " + key); + } + } +} + +int64_t chunk_size_from_request(const runtime::TaskRequest & request) { + const auto value = runtime::parse_i64_option( + request.options, + {"text_chunk_size", "chunk_size"}); + const int64_t chunk_size = value.value_or(280); + if (chunk_size <= 0) { + throw std::runtime_error("sanoTTS text_chunk_size must be positive"); + } + return chunk_size; +} + +void validate_chunk_mode(const runtime::TaskRequest & request) { + if (const auto value = runtime::find_option( + request.options, + {"text_chunk_mode", "chunk_mode"})) { + if (*value != "word_budget" && *value != "default") { + throw std::runtime_error("sanoTTS text_chunk_mode must be word_budget"); + } + } +} + +struct RequestOptions { + float speaking_rate = 1.0F; + uint64_t seed = 0; + bool seed_from_text = true; +}; + +RequestOptions parse_request_options(const runtime::TaskRequest & request) { + RequestOptions out; + if (const auto value = runtime::parse_finite_float_option( + request.options, + {"speaking_rate"})) { + out.speaking_rate = *value; + } + if (out.speaking_rate < 0.5F || out.speaking_rate > 2.0F) { + throw std::runtime_error("sanoTTS speaking_rate must be between 0.5 and 2.0"); + } + if (const auto value = runtime::parse_i64_option(request.options, {"seed"})) { + if (*value < 0) { + throw std::runtime_error("sanoTTS seed must not be negative"); + } + out.seed = static_cast(*value); + out.seed_from_text = *value == 0; + } + return out; +} + +void append_pause(runtime::AudioBuffer & output, double seconds) { + if (output.sample_rate <= 0 || seconds <= 0.0) { + return; + } + const auto count = static_cast( + std::llround(seconds * static_cast(output.sample_rate))); + output.samples.insert(output.samples.end(), count, 0.0F); +} + +} // namespace + +SanoTtsSession::SanoTtsSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(options), + task_(task), + assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))) { + if (task_.task != runtime::VoiceTaskKind::Tts || + task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("sanoTTS only supports offline TTS"); + } + validate_session_options(options, *contract_); + frontend_ = std::make_unique( + session_path(options, "sanotts.espeak_library_path"), + session_path(options, "sanotts.espeak_data_path"), + assets_->config.duration_max_tokens); + runtime_ = std::make_unique(assets_, options.backend); +} + +SanoTtsSession::~SanoTtsSession() = default; + +std::string SanoTtsSession::family() const { return kFamily; } +runtime::VoiceTaskKind SanoTtsSession::task_kind() const { return task_.task; } +runtime::RunMode SanoTtsSession::run_mode() const { return task_.mode; } + +void SanoTtsSession::prepare(const runtime::SessionPreparationRequest & request) { + (void)request; + mark_prepared(); +} + +runtime::TaskResult SanoTtsSession::run(const runtime::TaskRequest & request) { + require_prepared("sanoTTS run"); + if (!request.text_input.has_value() || request.text_input->text.empty()) { + throw std::runtime_error("sanoTTS requires --text input"); + } + if (request.audio_input.has_value()) { + throw std::runtime_error("sanoTTS does not accept audio input"); + } + if (!request.text_input->language.empty() && + request.text_input->language != "en" && + request.text_input->language != "en-us" && + request.text_input->language != "English") { + throw std::runtime_error("sanoTTS supports English only"); + } + validate_chunk_mode(request); + const int64_t chunk_size = chunk_size_from_request(request); + auto chunks = SanoTtsFrontend::split_text(request.text_input->text, chunk_size); + if (chunks.empty()) { + throw std::runtime_error("sanoTTS text must not be empty"); + } + + const auto request_options = parse_request_options(request); + runtime::AudioBuffer merged; + for (size_t index = 0; index < chunks.size(); ++index) { + if (index != 0) { + append_pause( + merged, + SanoTtsFrontend::boundary_pause_seconds(chunks[index - 1])); + } + const auto encoded = frontend_->encode(chunks[index]); + SanoTtsGenerationOptions chunk_options; + chunk_options.speaking_rate = request_options.speaking_rate; + // The default seed is derived from the chunk's own text -- the + // reference implementations' sha256(text)[:8] convention -- so a + // given sentence renders identically wherever it appears. An + // explicit seed advances per chunk instead, so long-form noise is + // not reused across chunks. + chunk_options.seed = request_options.seed_from_text + ? sanotts_text_seed(chunks[index]) + : request_options.seed + static_cast(index); + auto audio = runtime_->synthesize(encoded.token_ids, chunk_options); + runtime::append_audio_buffer(merged, audio); + } + for (float & sample : merged.samples) { + sample = std::clamp(sample, -1.0F, 1.0F); + } + engine::debug::trace_log_scalar("sanotts.text_chunk_size", chunk_size); + engine::debug::trace_log_scalar( + "sanotts.text_chunk_count", + static_cast(chunks.size())); + runtime::TaskResult result; + result.audio_output = std::move(merged); + return result; +} + +std::shared_ptr make_sanotts_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_sanotts_assets; + config.create_session = []( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, + options, + std::move(assets), + std::move(contract)); + }; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::models::sanotts diff --git a/tests/unittests/test_sanotts_frontend.cpp b/tests/unittests/test_sanotts_frontend.cpp new file mode 100644 index 000000000..23e79dd34 --- /dev/null +++ b/tests/unittests/test_sanotts_frontend.cpp @@ -0,0 +1,56 @@ +#include "engine/community_models/sanotts/frontend.h" +#include "engine/community_models/sanotts/runtime.h" + +#include "test_assert.h" + +#include +#include +#include + +int main() try { + using engine::models::sanotts::SanoTtsFrontend; + using engine::models::sanotts::sanotts_text_seed; + + // sha256 known-answer vectors: the seed is the first 8 digest bytes, + // big-endian, exactly int.from_bytes(sha256(text).digest()[:8], "big"). + engine::test::require( + sanotts_text_seed("abc") == 0xBA7816BF8F01CFEAULL, + "sha256 seed of 'abc'"); + engine::test::require( + sanotts_text_seed("") == 0xE3B0C44298FC1C14ULL, + "sha256 seed of the empty string"); + // 64 bytes forces the two-block tail path (rem >= 56). + engine::test::require( + sanotts_text_seed(std::string(64, 'a')) == 0xFFE054FE7AE0CB6DULL, + "sha256 seed across the two-block tail"); + + const auto chunks = SanoTtsFrontend::split_text( + "First sentence ends here. Second sentence is also short. " + "And a third one rounds it out.", + 40); + engine::test::require(chunks.size() >= 2, "long text must split"); + for (const auto & chunk : chunks) { + engine::test::require(!chunk.empty(), "no empty chunks"); + } + + const auto single = SanoTtsFrontend::split_text("Tiny.", 280); + engine::test::require( + single.size() == 1 && single[0] == "Tiny.", + "short text stays one chunk"); + + engine::test::require( + SanoTtsFrontend::boundary_pause_seconds("Ends with a period.") == 0.20, + "sentence end pause"); + engine::test::require( + SanoTtsFrontend::boundary_pause_seconds("ends with a comma,") == 0.08, + "clause pause"); + engine::test::require( + SanoTtsFrontend::boundary_pause_seconds("trailing space. ") == 0.20, + "pause looks through trailing whitespace"); + + std::cout << "sanotts frontend tests passed\n"; + return 0; +} catch (const std::exception & error) { + std::cerr << "sanotts frontend test failed: " << error.what() << "\n"; + return 1; +} From 89a789a4a4770b5bd833cd84d1845fd0ccc21bf7 Mon Sep 17 00:00:00 2001 From: ashish Date: Fri, 4 Sep 2026 16:39:24 +0545 Subject: [PATCH 4/7] sanotts: bisect chunks that phonemize past the duration token limit The codepoint chunker cannot see phoneme counts, so a dense 280-codepoint chunk can exceed the duration model's 207-token training limit. encode() now throws a typed SanoTtsTooLongError and the session splits the chunk at the whitespace nearest its middle and recurses, so the shared long-form case (6 kB of text, 6.2 minutes of audio) renders instead of failing. Claude-Session: https://claude.ai/code/session_01P1iL37FdfJkGxdGrpjH1we --- .../community_models/sanotts/frontend.h | 7 +++ src/community_models/sanotts/frontend.cpp | 4 +- src/community_models/sanotts/session.cpp | 62 +++++++++++++++---- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/include/engine/community_models/sanotts/frontend.h b/include/engine/community_models/sanotts/frontend.h index 693e0bfa3..7deffd133 100644 --- a/include/engine/community_models/sanotts/frontend.h +++ b/include/engine/community_models/sanotts/frontend.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -8,6 +9,12 @@ namespace engine::models::sanotts { +/** Thrown by encode() when a chunk phonemizes past the duration model's + * token limit; the session responds by bisecting the chunk. */ +struct SanoTtsTooLongError : std::runtime_error { + using std::runtime_error::runtime_error; +}; + struct SanoTtsEncoded { std::vector token_ids; std::string dropped; // symbols outside the vocabulary, for tracing diff --git a/src/community_models/sanotts/frontend.cpp b/src/community_models/sanotts/frontend.cpp index 340467f36..949a0be01 100644 --- a/src/community_models/sanotts/frontend.cpp +++ b/src/community_models/sanotts/frontend.cpp @@ -536,10 +536,10 @@ SanoTtsEncoded SanoTtsFrontend::encode(const std::string & text) const { } out.token_ids.push_back(2); // if (static_cast(out.token_ids.size()) > max_tokens_) { - throw std::runtime_error( + throw SanoTtsTooLongError( "sanoTTS phoneme sequence has " + std::to_string(out.token_ids.size()) + " tokens including BOS/EOS; the duration model was trained for at most " + - std::to_string(max_tokens_) + ". Lower text_chunk_size."); + std::to_string(max_tokens_) + "."); } return out; } diff --git a/src/community_models/sanotts/session.cpp b/src/community_models/sanotts/session.cpp index 9e5c2f361..cc4b36c1d 100644 --- a/src/community_models/sanotts/session.cpp +++ b/src/community_models/sanotts/session.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -168,25 +169,60 @@ runtime::TaskResult SanoTtsSession::run(const runtime::TaskRequest & request) { const auto request_options = parse_request_options(request); runtime::AudioBuffer merged; + uint64_t rendered_chunks = 0; + // Renders one chunk, bisecting at whitespace when it phonemizes past the + // duration model's token limit -- the codepoint budget cannot see phoneme + // counts, so a dense 280-codepoint chunk can exceed 207 tokens. + const std::function render_chunk = + [&](const std::string & chunk, int depth) { + SanoTtsEncoded encoded; + try { + encoded = frontend_->encode(chunk); + } catch (const SanoTtsTooLongError &) { + const size_t middle = chunk.size() / 2; + size_t split = std::string::npos; + for (size_t offset = 0; offset < chunk.size(); ++offset) { + const size_t after = middle + offset; + if (after < chunk.size() && chunk[after] == ' ') { + split = after; + break; + } + if (offset <= middle && chunk[middle - offset] == ' ') { + split = middle - offset; + break; + } + } + if (depth >= 8 || split == std::string::npos) { + throw; + } + const std::string left = chunk.substr(0, split); + const std::string right = chunk.substr(split + 1); + render_chunk(left, depth + 1); + append_pause(merged, SanoTtsFrontend::boundary_pause_seconds(left)); + render_chunk(right, depth + 1); + return; + } + SanoTtsGenerationOptions chunk_options; + chunk_options.speaking_rate = request_options.speaking_rate; + // The default seed is derived from the chunk's own text -- the + // reference implementations' sha256(text)[:8] convention -- so a + // given sentence renders identically wherever it appears. An + // explicit seed advances per chunk instead, so long-form noise + // is not reused across chunks. + chunk_options.seed = request_options.seed_from_text + ? sanotts_text_seed(chunk) + : request_options.seed + rendered_chunks; + ++rendered_chunks; + auto audio = runtime_->synthesize(encoded.token_ids, chunk_options); + runtime::append_audio_buffer(merged, audio); + }; for (size_t index = 0; index < chunks.size(); ++index) { if (index != 0) { append_pause( merged, SanoTtsFrontend::boundary_pause_seconds(chunks[index - 1])); } - const auto encoded = frontend_->encode(chunks[index]); - SanoTtsGenerationOptions chunk_options; - chunk_options.speaking_rate = request_options.speaking_rate; - // The default seed is derived from the chunk's own text -- the - // reference implementations' sha256(text)[:8] convention -- so a - // given sentence renders identically wherever it appears. An - // explicit seed advances per chunk instead, so long-form noise is - // not reused across chunks. - chunk_options.seed = request_options.seed_from_text - ? sanotts_text_seed(chunks[index]) - : request_options.seed + static_cast(index); - auto audio = runtime_->synthesize(encoded.token_ids, chunk_options); - runtime::append_audio_buffer(merged, audio); + render_chunk(chunks[index], 0); } for (float & sample : merged.samples) { sample = std::clamp(sample, -1.0F, 1.0F); From 5a93b87732a98b33a08051b5dfa183ea45338e61 Mon Sep 17 00:00:00 2001 From: ashish Date: Fri, 4 Sep 2026 16:45:32 +0545 Subject: [PATCH 5/7] sanotts: add the heart 2.27M voice as a second package Same graph, wider and deeper; the runtime now derives the expected tensor count from the config instead of hardcoding heart-nano's 103, and the weight arena covers the 9.1 MB FP32 payload. Verified like heart-nano: correlation 0.999999985 against the numpy reference at identical sample count, installed end to end from the published Hugging Face package. Claude-Session: https://claude.ai/code/session_01P1iL37FdfJkGxdGrpjH1we --- README.md | 2 +- docs/community_models/models.md | 2 +- docs/community_models/sanotts.md | 31 ++++++++++++++---------- docs/tts.md | 2 +- model_specs/sanotts.json | 15 +++++++++++- src/community_models/sanotts/runtime.cpp | 20 +++++++++++---- 6 files changed, 50 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c0ba10798..353e6be03 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ Community model ports live under `community_models` to make the ownership bounda | **moss_voicegen** | Voice Design | en, zh | GGUF | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](docs/community_models/moss_voicegen.md) speech in a voice designed from a written instruction | | **outetts** | TTS, Clone | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | GGUF | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | GGUF F32/16/Q8, Stream | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](docs/community_models/parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | -| **sanotts** | TTS | en | GGUF FP32 | Ashish [@voidash](https://github.com/voidash) | [sanoTTS heart-nano](docs/community_models/sanotts.md) 294k-parameter native offline synthesis | +| **sanotts** | TTS | en | GGUF FP32 | Ashish [@voidash](https://github.com/voidash) | [sanoTTS heart and heart-nano](docs/community_models/sanotts.md) 2.27M and 294k-parameter native offline synthesis | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | GGUF Q8, Stream | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](docs/community_models/sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **soprano_tts** | TTS | en | GGUF Q8, Stream | [@WalkingCat](https://github.com/WalkingCat) | [Soprano-1.1-80M](https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF) ultra-lightweight TTS with Qwen3 LM + Vocos decoder | | **vietneu_tts** | TTS, Clone | vi, en | GGUF | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](docs/community_models/vietneu_tts.md) TTS and voice cloning support | diff --git a/docs/community_models/models.md b/docs/community_models/models.md index 1ef662670..bb2d5aa6b 100644 --- a/docs/community_models/models.md +++ b/docs/community_models/models.md @@ -31,7 +31,7 @@ Practical expectations: | **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Mirek [@mirek190](https://github.com/mirek190) | [Llama-OuteTTS-1.0-1B](outetts.md) TTS and voice cloning support | | **voxcpm1** | TTS, voice cloning | zh, en, ja, ko | Community | [VoxCPM1](voxcpm1.md) tokenizer-free 0.5B TTS with 16 kHz output, streaming, and continuation-mode voice cloning | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | -| **sanotts** | TTS | en | Community | [sanoTTS heart-nano](sanotts.md) 294k-parameter FP32 offline synthesis | +| **sanotts** | TTS | en | Community | [sanoTTS heart and heart-nano](sanotts.md) 2.27M and 294k-parameter FP32 offline synthesis | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **vietneu_tts** | TTS, voice cloning | vi, en | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](vietneu_tts.md) TTS and voice cloning support | | **moss_voicegen** | Voice design | en, zh | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](moss_voicegen.md) voice design from a written instruction, on the MOSS delay architecture | diff --git a/docs/community_models/sanotts.md b/docs/community_models/sanotts.md index 1823f4451..34fd381b6 100644 --- a/docs/community_models/sanotts.md +++ b/docs/community_models/sanotts.md @@ -1,12 +1,14 @@ -# sanoTTS heart-nano - -`sanotts` provides native GGML inference for -[sanoTTS](https://github.com/Ampixa/sanoTTS) **heart-nano**, a -294,279-parameter English text-to-speech model that also runs on -microcontrollers. The graph is a duration student, a contextual acoustic -student producing a mel-100 spectrogram, and a noise-fed ConvNeXt-1D decoder -whose [log-magnitude | phase] head feeds an inverse STFT. Output is 24 kHz -mono. Offline FP32 inference only. +# sanoTTS heart and heart-nano + +`sanotts` provides native GGML inference for the +[sanoTTS](https://github.com/Ampixa/sanoTTS) nano lineage: **heart** +(2,272,145 parameters, the higher-quality voice) and **heart-nano** +(294,279 parameters, small enough that the same weights also run on +microcontrollers). Both share one graph -- a duration student, a contextual +acoustic student producing a mel-100 spectrogram, and a noise-fed +ConvNeXt-1D decoder whose [log-magnitude | phase] head feeds an inverse +STFT -- differing only in width and depth, which the runtime reads from the +package config. Output is 24 kHz mono. Offline FP32 inference only. The GGUF is published on Hugging Face at [ampixa/sanoTTS](https://huggingface.co/ampixa/sanoTTS) under `gguf/`: the @@ -38,11 +40,13 @@ python tools/model_manager_v2.py install sanotts_heart_nano_orig --models-root m ```bash audiocpp_cli --task tts --family sanotts \ - --model models/sanoTTS-heart-nano-GGUF --backend cpu \ + --model models/sanoTTS-heart-GGUF --backend cpu \ --text "Hello from sano T T S, a very small neural text to speech model." \ --out sanotts.wav ``` +Use `--model models/sanoTTS-heart-nano-GGUF` for the 294k voice. + eSpeak-ng is loaded dynamically at runtime, never linked. If it is not on the default library path: @@ -79,9 +83,10 @@ The runtime reproduces the reference implementations' exact semantics: DC-blocking filter `H(z) = (1 - z^-1)/(1 - 0.9973 z^-1)`. Measured against the project's numpy reference (same text, same seed, same -eSpeak-ng build): correlation **0.999999985**, max sample delta 1.7e-05 -(the WAV's own int16 quantisation), identical sample count. The numpy -reference is itself gated at 0.987–1.000 against the float PyTorch model. +eSpeak-ng build), both voices: correlation **0.999999985**, max sample delta +1.7e-05 (the WAV's own int16 quantisation), identical sample count. The +numpy reference is itself gated at 0.987–1.000 against the float PyTorch +model. ## Performance diff --git a/docs/tts.md b/docs/tts.md index 3b7cfbc67..98a57ea0d 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -29,7 +29,7 @@ | GLM-TTS | `glm_tts` | `tts`, `clon` | [GLM-TTS](#glm-tts) | | Inflect Micro v2 | `inflect_v2` | `tts` | [Inflect v2](#inflect-v2) | | OuteTTS | `outetts` | `tts`, `clon` | [OuteTTS](#outetts) | -| sanoTTS heart-nano | `sanotts` | `tts` | [sanoTTS](#sanotts) | +| sanoTTS heart / heart-nano | `sanotts` | `tts` | [sanoTTS](#sanotts) | | Supertonic | `supertonic` | `tts` | [Supertonic](#supertonic) | | VieNeu-TTS | `vietneu_tts` | `tts`, `clon` | [VieNeu-TTS](community_models/vietneu_tts.md) | | VibeVoice | `vibevoice` | `tts` | [VibeVoice](#vibevoice) | diff --git a/model_specs/sanotts.json b/model_specs/sanotts.json index 893419519..cc01e6d09 100644 --- a/model_specs/sanotts.json +++ b/model_specs/sanotts.json @@ -2,7 +2,7 @@ "schema_version": 1, "family": "sanotts", "display_name": "sanoTTS Nano", - "description": "Very small English text-to-speech: a duration student, a contextual acoustic student producing mel-100, and a ConvNeXt-1D decoder with an iSTFT head. 294,279 parameters at 24 kHz. Uses an external eSpeak-ng phonemizer.", + "description": "Very small English text-to-speech: a duration student, a contextual acoustic student producing mel-100, and a ConvNeXt-1D decoder with an iSTFT head. Two voices: heart (2.27M parameters, higher quality) and heart-nano (294k parameters, also runs on microcontrollers). 24 kHz output. Uses an external eSpeak-ng phonemizer.", "category": "tts", "status": "community", "tasks": [ @@ -99,6 +99,19 @@ "gguf/config.json" ], "strip_prefix": "gguf" + }, + { + "id": "sanotts_heart_orig", + "display_name": "sanoTTS heart 2.27M FP32 GGUF", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-heart-GGUF", + "files": [ + "gguf/heart/heart-f32.gguf", + "gguf/heart/config.json" + ], + "strip_prefix": "gguf/heart" } ], "dependencies": [], diff --git a/src/community_models/sanotts/runtime.cpp b/src/community_models/sanotts/runtime.cpp index 92e7f721e..2bbf45611 100644 --- a/src/community_models/sanotts/runtime.cpp +++ b/src/community_models/sanotts/runtime.cpp @@ -37,8 +37,17 @@ namespace modules = engine::modules; constexpr size_t kIoArenaBytes = 8ULL * 1024ULL * 1024ULL; constexpr size_t kGraphArenaBytes = 128ULL * 1024ULL * 1024ULL; -constexpr size_t kWeightArenaBytes = 8ULL * 1024ULL * 1024ULL; -constexpr size_t kExpectedTensors = 103; +constexpr size_t kWeightArenaBytes = 32ULL * 1024ULL * 1024ULL; + +/** Tensor count implied by the config -- 103 for heart-nano, 115 for heart. + * Kept in lockstep with the inventory validate_tensors() builds. */ +size_t expected_tensor_count(const SanoTtsConfig & config) { + const auto duration = 5 + 5 * config.duration_depth; + const auto acoustic = + 7 + 5 * (config.acoustic_token_depth + config.acoustic_depth); + const auto decoder = 10 + 9 * config.blocks; + return static_cast(duration + acoustic + decoder); +} // The decoder's norms are nn.LayerNorm(eps=1e-6), NOT torch's 1e-5 default. // The difference compounds through the four ConvNeXt blocks and is then @@ -102,10 +111,11 @@ std::shared_ptr load_weights( "sanotts.weights", kWeightArenaBytes); const auto metadata = assets->weights->tensors(); - if (metadata.size() != kExpectedTensors) { + const size_t expected = expected_tensor_count(assets->config); + if (metadata.size() != expected) { throw std::runtime_error( - "sanoTTS expects exactly " + std::to_string(kExpectedTensors) + - " tensors, found " + std::to_string(metadata.size())); + "sanoTTS expects exactly " + std::to_string(expected) + + " tensors for this config, found " + std::to_string(metadata.size())); } out->tensors.reserve(metadata.size()); for (const auto & tensor : metadata) { From 246c73e038868354a465de948335c58ca74f79fc Mon Sep 17 00:00:00 2001 From: ashish Date: Fri, 4 Sep 2026 17:05:47 +0545 Subject: [PATCH 6/7] sanotts: piperlite lineage -- amy, hfc, kristin, vi and id voices Second graph in the family: duration and acoustic students into a 192-channel latent, then a 3-stage ConvTranspose1d decoder with dilated residual banks (kristin adds a learned post filter). Deterministic, 22.05 kHz. The shared front-end structure moves into graph_common.h; the session dispatches on the config's graph field. The piperlite front end reproduces Piper's convention exactly: untied eSpeak-ng phonemes through the phonemizer punctuation pipeline, NFD decomposition to codepoints, the per-voice phoneme_id_map with [BOS, PAD, (id, PAD)..., EOS] framing, the schwa fallback for ids outside a component's trained vocab, and regional-variant-first voice selection (phonemizer rejects bare language codes on espeak-ng >= 1.49, so 'en' must resolve to en-us in both stacks). All five voices verified against the project's numpy reference with the same eSpeak-ng build: correlation >= 0.99999996 at identical sample counts, installed end to end from the published Hugging Face packages. Vietnamese and Indonesian exercise their own espeak voices and language validation. Claude-Session: https://claude.ai/code/session_01P1iL37FdfJkGxdGrpjH1we --- CMakeLists.txt | 1 + README.md | 2 +- docs/community_models/models.md | 2 +- docs/community_models/sanotts.md | 121 ++- docs/tts.md | 11 +- .../engine/community_models/sanotts/assets.h | 44 +- .../community_models/sanotts/frontend.h | 28 + .../community_models/sanotts/piper_runtime.h | 34 + .../engine/community_models/sanotts/session.h | 7 +- model_specs/sanotts.json | 75 +- src/community_models/sanotts/assets.cpp | 209 +++- src/community_models/sanotts/frontend.cpp | 605 +++++++++++- src/community_models/sanotts/graph_common.h | 296 ++++++ .../sanotts/piper_runtime.cpp | 897 ++++++++++++++++++ src/community_models/sanotts/runtime.cpp | 278 +----- src/community_models/sanotts/session.cpp | 67 +- 16 files changed, 2309 insertions(+), 368 deletions(-) create mode 100644 include/engine/community_models/sanotts/piper_runtime.h create mode 100644 src/community_models/sanotts/graph_common.h create mode 100644 src/community_models/sanotts/piper_runtime.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d25e40d4..4389a9aed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -634,6 +634,7 @@ audiocpp_add_model(sanotts SOURCES src/community_models/sanotts/assets.cpp src/community_models/sanotts/frontend.cpp + src/community_models/sanotts/piper_runtime.cpp src/community_models/sanotts/runtime.cpp src/community_models/sanotts/session.cpp INCLUDES diff --git a/README.md b/README.md index 353e6be03..a9bbd328f 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ Community model ports live under `community_models` to make the ownership bounda | **moss_voicegen** | Voice Design | en, zh | GGUF | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](docs/community_models/moss_voicegen.md) speech in a voice designed from a written instruction | | **outetts** | TTS, Clone | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | GGUF | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | GGUF F32/16/Q8, Stream | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](docs/community_models/parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | -| **sanotts** | TTS | en | GGUF FP32 | Ashish [@voidash](https://github.com/voidash) | [sanoTTS heart and heart-nano](docs/community_models/sanotts.md) 2.27M and 294k-parameter native offline synthesis | +| **sanotts** | TTS | en, vi, id | GGUF FP32 | Ashish [@voidash](https://github.com/voidash) | [sanoTTS voice family](docs/community_models/sanotts.md) seven voices from 294k to 2.27M parameters, native offline synthesis | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | GGUF Q8, Stream | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](docs/community_models/sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **soprano_tts** | TTS | en | GGUF Q8, Stream | [@WalkingCat](https://github.com/WalkingCat) | [Soprano-1.1-80M](https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF) ultra-lightweight TTS with Qwen3 LM + Vocos decoder | | **vietneu_tts** | TTS, Clone | vi, en | GGUF | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](docs/community_models/vietneu_tts.md) TTS and voice cloning support | diff --git a/docs/community_models/models.md b/docs/community_models/models.md index bb2d5aa6b..d88164887 100644 --- a/docs/community_models/models.md +++ b/docs/community_models/models.md @@ -31,7 +31,7 @@ Practical expectations: | **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Mirek [@mirek190](https://github.com/mirek190) | [Llama-OuteTTS-1.0-1B](outetts.md) TTS and voice cloning support | | **voxcpm1** | TTS, voice cloning | zh, en, ja, ko | Community | [VoxCPM1](voxcpm1.md) tokenizer-free 0.5B TTS with 16 kHz output, streaming, and continuation-mode voice cloning | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | -| **sanotts** | TTS | en | Community | [sanoTTS heart and heart-nano](sanotts.md) 2.27M and 294k-parameter FP32 offline synthesis | +| **sanotts** | TTS | en, vi, id | Community | [sanoTTS voice family](sanotts.md) seven voices from 294k to 2.27M parameters, FP32 offline synthesis | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **vietneu_tts** | TTS, voice cloning | vi, en | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](vietneu_tts.md) TTS and voice cloning support | | **moss_voicegen** | Voice design | en, zh | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](moss_voicegen.md) voice design from a written instruction, on the MOSS delay architecture | diff --git a/docs/community_models/sanotts.md b/docs/community_models/sanotts.md index 34fd381b6..ff88f8071 100644 --- a/docs/community_models/sanotts.md +++ b/docs/community_models/sanotts.md @@ -1,24 +1,34 @@ -# sanoTTS heart and heart-nano +# sanoTTS voice family `sanotts` provides native GGML inference for the -[sanoTTS](https://github.com/Ampixa/sanoTTS) nano lineage: **heart** -(2,272,145 parameters, the higher-quality voice) and **heart-nano** -(294,279 parameters, small enough that the same weights also run on -microcontrollers). Both share one graph -- a duration student, a contextual -acoustic student producing a mel-100 spectrogram, and a noise-fed -ConvNeXt-1D decoder whose [log-magnitude | phase] head feeds an inverse -STFT -- differing only in width and depth, which the runtime reads from the -package config. Output is 24 kHz mono. Offline FP32 inference only. - -The GGUF is published on Hugging Face at -[ampixa/sanoTTS](https://huggingface.co/ampixa/sanoTTS) under `gguf/`: the -int8 on-device rows are dequantised to FP32 tensors in PyTorch shapes, with -the audio.cpp exact-shape metadata and an embedded model spec, so the package -is standalone. +[sanoTTS](https://github.com/Ampixa/sanoTTS) voice family — very small +text-to-speech models, the smallest of which also runs on microcontrollers. +All packages download from Hugging Face +([ampixa/sanoTTS](https://huggingface.co/ampixa/sanoTTS) `gguf/`) as +standalone FP32 GGUFs with embedded model specs. Offline FP32 inference only. + +Two graphs share one family: + +- **nano** — duration student → contextual acoustic student → mel-100 → + noise-fed ConvNeXt-1D decoder → [log-magnitude | phase] head → inverse + STFT. 24 kHz. A seed picks one of many valid renderings. +- **piperlite** — duration student → contextual acoustic student → 192-channel + latent → 3-stage ConvTranspose1d decoder with dilated residual banks → + tanh waveform. 22.05 kHz. Fully deterministic (no seed). + +| Package | Voice | Graph | Params | Language | Notes | +|---|---|---|---:|---|---| +| `sanotts_heart_orig` | heart | nano | 2,272,145 | en | best quality of the nano pair | +| `sanotts_heart_nano_orig` | heart-nano | nano | 294,279 | en | microcontroller-class | +| `sanotts_amy_orig` | amy | piperlite | 1,454,284 | en | Piper-distilled | +| `sanotts_hfc_orig` | hfc | piperlite | 1,834,380 | en | largest piperlite voice | +| `sanotts_kristin_orig` | kristin | piperlite | 1,396,151 | en | carries a learned post filter | +| `sanotts_vi_orig` | vi | piperlite | 1,565,484 | vi | Vietnamese | +| `sanotts_id_orig` | id | piperlite | 1,562,124 | id | Indonesian | ## Install -Install eSpeak-ng and its English voice data first. On Debian or Ubuntu: +Install eSpeak-ng and its voice data first. On Debian or Ubuntu: ```bash sudo apt install espeak-ng libespeak-ng1 @@ -30,10 +40,11 @@ On macOS: brew install espeak-ng ``` -Then install the GGUF package: +Then install any package, e.g.: ```bash -python tools/model_manager_v2.py install sanotts_heart_nano_orig --models-root models +python3 tools/model_manager_v2.py install sanotts_heart_orig --models-root models +python3 tools/model_manager_v2.py install sanotts_amy_orig --models-root models ``` ## Run @@ -45,14 +56,17 @@ audiocpp_cli --task tts --family sanotts \ --out sanotts.wav ``` -Use `--model models/sanoTTS-heart-nano-GGUF` for the 294k voice. +Swap `--model` for any installed package directory +(`models/sanoTTS-amy-GGUF`, `models/sanoTTS-vi-GGUF`, ...). The Vietnamese +and Indonesian voices accept `--language vi` / `--language id`; a session +rejects text tagged with a language the voice was not trained on. eSpeak-ng is loaded dynamically at runtime, never linked. If it is not on the default library path: ```bash audiocpp_cli --task tts --family sanotts \ - --model models/sanoTTS-heart-nano-GGUF --backend cpu \ + --model models/sanoTTS-heart-GGUF --backend cpu \ --session-option sanotts.espeak_library_path=/path/to/libespeak-ng.so \ --session-option sanotts.espeak_data_path=/path/to/espeak-ng-data \ --text "A configured eSpeak installation." --out sanotts.wav @@ -60,49 +74,56 @@ audiocpp_cli --task tts --family sanotts \ ## Options -- `speaking_rate` (request, 0.5..2.0, default 1.0) — duration multiplier - applied before rounding; larger is slower. -- `seed` (request, default 0) — decoder noise seed. The decoder is noise-fed, +- `speaking_rate` (request, 0.5..2.0, default 1.0) — duration multiplier on + the voice's tuned length scale; larger is slower. +- `seed` (request, default 0) — nano voices only: the decoder is noise-fed, so a given seed picks one of many valid renderings. `0` derives the seed from each text chunk as `sha256(text)[:8]`, which is what the reference implementations do; an explicit seed advances by one per long-form chunk. + Piperlite voices are deterministic and ignore the seed. - `text_chunk_size` (request, default 280) — maximum codepoints per long-form - chunk; chunks are split on sentence punctuation first. + chunk; chunks split on sentence punctuation first, and a chunk that + phonemizes past the voice's token limit is bisected at whitespace. ## Determinism and parity -The runtime reproduces the reference implementations' exact semantics: - -- ATen-compatible MT19937 noise (24-bit uniform, Box–Muller in blocks of 16), - so a seed renders the same waveform as the PyTorch and MCU runtimes up to a - few ulp of libm difference. -- The phonemizer punctuation-preservation pipeline and misaki E2M rewrite, - byte-identical token streams against the Python front end across a - punctuation corpus. -- torch.istft window normalisation and centre trim, and the reference's - DC-blocking filter `H(z) = (1 - z^-1)/(1 - 0.9973 z^-1)`. - -Measured against the project's numpy reference (same text, same seed, same -eSpeak-ng build), both voices: correlation **0.999999985**, max sample delta -1.7e-05 (the WAV's own int16 quantisation), identical sample count. The -numpy reference is itself gated at 0.987–1.000 against the float PyTorch -model. +The runtimes reproduce the reference implementations' exact semantics: + +- Front ends: the phonemizer punctuation-preservation pipeline through the + same eSpeak-ng library. The nano voices add the misaki E2M rewrite with + tie characters; the piperlite voices use Piper's NFD-decompose-to- + codepoints convention, per-voice `phoneme_id_map`, `[BOS, PAD, (id, PAD)…, + EOS]` framing, and the schwa fallback for ids outside a component's + trained vocabulary. +- nano: ATen-compatible MT19937 noise (24-bit uniform, Box–Muller in blocks + of 16), torch.istft window normalisation and centre trim, and the + reference's DC blocker `H(z) = (1 - z^-1)/(1 - 0.9973 z^-1)`. +- Shared: torch.linspace / expand_features float behaviour, LayerNorm eps + 1e-6 (nano), ties-to-even duration rounding. + +Measured against the project's numpy references (same text, same +eSpeak-ng build), every voice: **correlation ≥ 0.99999996 with identical +sample counts**; max sample delta ~1.7e-05 is the WAV's own int16 +quantisation. The numpy references are themselves gated ≥ 0.987 against the +float PyTorch models. ## Performance -CPU-only, 12-thread x86 (default 4 backend threads), FP32: +CPU-only, 12-thread x86 (default 4 backend threads), FP32, the shared 6 kB +long-form text: -- 38.7 s of audio synthesized in 0.22 s wall including model load - (about 175x faster than real time); peak RSS 76 MB. -- Per stage on a 5.7 s utterance (`--log`): duration 0.4 ms, acoustic - 0.5 ms, decoder 15.1 ms, host iSTFT 5.3 ms. +| Voice | Audio | Wall | vs real time | Peak RSS | +|---|---:|---:|---:|---:| +| heart-nano | 373 s | 1.3 s | ~283× | 220 MB | +| amy | 394 s | 18.5 s | ~21× | 497 MB | -Graphs are cached per token count (duration and token stages) and per frame -count (decoder), so repeated lengths skip graph construction; `--log` prints -the cache hits and stage timings. +The nano decoder runs at frame rate with a host iSTFT; the piperlite decoder +runs convolutions at audio rate, which is why it is heavier. Graphs are +cached per token count (duration and token stages) and per frame count +(decoder); `--log` prints cache hits and per-stage timings. ## Licensing -The sanoTTS runtime and weights are MIT-licensed. eSpeak-ng is GPL-3.0 and is -therefore opened with `dlopen` at runtime and never linked, matching how +The sanoTTS runtimes and weights are MIT-licensed. eSpeak-ng is GPL-3.0 and +is therefore opened with `dlopen` at runtime and never linked, matching how `inflect_v2` treats it. diff --git a/docs/tts.md b/docs/tts.md index 98a57ea0d..a85f7c4a6 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -29,7 +29,7 @@ | GLM-TTS | `glm_tts` | `tts`, `clon` | [GLM-TTS](#glm-tts) | | Inflect Micro v2 | `inflect_v2` | `tts` | [Inflect v2](#inflect-v2) | | OuteTTS | `outetts` | `tts`, `clon` | [OuteTTS](#outetts) | -| sanoTTS heart / heart-nano | `sanotts` | `tts` | [sanoTTS](#sanotts) | +| sanoTTS voice family | `sanotts` | `tts` | [sanoTTS](#sanotts) | | Supertonic | `supertonic` | `tts` | [Supertonic](#supertonic) | | VieNeu-TTS | `vietneu_tts` | `tts`, `clon` | [VieNeu-TTS](community_models/vietneu_tts.md) | | VibeVoice | `vibevoice` | `tts` | [VibeVoice](#vibevoice) | @@ -771,10 +771,11 @@ limitations. ## sanoTTS -sanoTTS heart-nano is a 294,279-parameter English offline TTS model with a -native GGML runtime, small enough that the same weights also run on -microcontrollers. The GGUF package is standalone and downloads from Hugging -Face. sanoTTS requires an external eSpeak-ng installation: +sanoTTS is a family of very small offline TTS voices (English, Vietnamese, +Indonesian; 294k to 2.27M parameters) with native GGML runtimes; the +smallest voice also runs on microcontrollers. The GGUF packages are +standalone and download from Hugging Face. sanoTTS requires an external +eSpeak-ng installation: ```bash python3 tools/model_manager_v2.py install sanotts_heart_nano_orig --models-root models diff --git a/include/engine/community_models/sanotts/assets.h b/include/engine/community_models/sanotts/assets.h index 635959d3e..144bf9939 100644 --- a/include/engine/community_models/sanotts/assets.h +++ b/include/engine/community_models/sanotts/assets.h @@ -3,10 +3,13 @@ #include "engine/framework/assets/resource_bundle.h" #include "engine/framework/assets/tensor_source.h" +#include #include #include #include #include +#include +#include namespace engine::models::sanotts { @@ -37,9 +40,48 @@ struct SanoTtsConfig { std::string voice; }; +enum class SanoTtsGraph { + Nano, // mel-100 -> ConvNeXt-1D -> iSTFT, noise-fed (heart, heart-nano) + Piperlite, // 192-ch latent -> 3-stage ConvTranspose1d, deterministic (amy, ...) +}; + +struct SanoTtsPiperConfig { + std::string voice; + std::string language; // short code the session validates against: en, vi, id + std::string espeak_voice; + int64_t sample_rate = 22050; + double duration_length_scale = 1.0; + + int64_t duration_vocab = 0; + int64_t duration_hidden = 0; + int64_t duration_depth = 0; + int64_t duration_kernel = 5; + int64_t duration_max_tokens = 0; + int64_t duration_max_frames = 0; + + int64_t acoustic_vocab = 0; + int64_t acoustic_hidden = 0; + int64_t acoustic_depth = 0; + int64_t acoustic_token_depth = 0; + int64_t acoustic_kernel = 5; + int64_t acoustic_out_channels = 0; + + std::array channels = {0, 0, 0, 0}; + std::array, 3> stage_branches; + int64_t post_filter_channels = 0; + int64_t post_filter_layers = 0; + int64_t post_filter_kernel = 9; + double post_filter_scale = 0.0; + + /** Piper phoneme_id_map: one UTF-8 codepoint -> id. */ + std::unordered_map phoneme_id_map; +}; + struct SanoTtsAssets { assets::ResourceBundle resources; - SanoTtsConfig config; + SanoTtsGraph graph = SanoTtsGraph::Nano; + SanoTtsConfig config; // valid when graph == Nano + SanoTtsPiperConfig piper; // valid when graph == Piperlite std::shared_ptr weights; }; diff --git a/include/engine/community_models/sanotts/frontend.h b/include/engine/community_models/sanotts/frontend.h index 7deffd133..9ce91e51d 100644 --- a/include/engine/community_models/sanotts/frontend.h +++ b/include/engine/community_models/sanotts/frontend.h @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace engine::models::sanotts { @@ -55,4 +56,31 @@ class SanoTtsFrontend { int64_t max_tokens_; }; +/** + * Text -> Piper phoneme ids for the piperlite voices. + * + * Reproduces piper's phonemize_espeak / phonemes_to_ids convention through + * the same eSpeak-ng library: phonemizer-style punctuation preservation, + * NFD decomposition to single codepoints, then the voice's phoneme_id_map + * with [BOS, PAD, (id, PAD)..., EOS] framing. Deterministic per voice. + */ +class SanoTtsPiperFrontend { +public: + SanoTtsPiperFrontend( + std::filesystem::path espeak_library_path, + std::filesystem::path espeak_data_path, + std::string espeak_voice, + std::unordered_map phoneme_id_map, + int64_t max_tokens); + ~SanoTtsPiperFrontend(); + + [[nodiscard]] SanoTtsEncoded encode(const std::string & text) const; + +private: + struct Impl; + std::unique_ptr impl_; + std::unordered_map id_map_; + int64_t max_tokens_; +}; + } // namespace engine::models::sanotts diff --git a/include/engine/community_models/sanotts/piper_runtime.h b/include/engine/community_models/sanotts/piper_runtime.h new file mode 100644 index 000000000..77d08a01e --- /dev/null +++ b/include/engine/community_models/sanotts/piper_runtime.h @@ -0,0 +1,34 @@ +#pragma once + +#include "engine/community_models/sanotts/assets.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include + +namespace engine::models::sanotts { + +struct SanoTtsPiperGenerationOptions { + /** Multiplier on the voice's tuned duration_length_scale; larger is + * slower. The decoder is deterministic -- there is no seed. */ + float speaking_rate = 1.0F; +}; + +class SanoTtsPiperRuntime { +public: + SanoTtsPiperRuntime( + std::shared_ptr assets, + core::BackendConfig backend_config); + ~SanoTtsPiperRuntime(); + + runtime::AudioBuffer synthesize( + const std::vector & token_ids, + const SanoTtsPiperGenerationOptions & options); + +private: + struct State; + std::unique_ptr state_; +}; + +} // namespace engine::models::sanotts diff --git a/include/engine/community_models/sanotts/session.h b/include/engine/community_models/sanotts/session.h index ba259d1fc..464737ad6 100644 --- a/include/engine/community_models/sanotts/session.h +++ b/include/engine/community_models/sanotts/session.h @@ -2,6 +2,7 @@ #include "engine/community_models/sanotts/assets.h" #include "engine/community_models/sanotts/frontend.h" +#include "engine/community_models/sanotts/piper_runtime.h" #include "engine/community_models/sanotts/runtime.h" #include "engine/framework/model_spec/metadata.h" #include "engine/framework/runtime/session_base.h" @@ -33,8 +34,10 @@ class SanoTtsSession final runtime::TaskSpec task_; std::shared_ptr assets_; std::shared_ptr contract_; - std::unique_ptr frontend_; - std::unique_ptr runtime_; + std::unique_ptr frontend_; // nano graph + std::unique_ptr runtime_; // nano graph + std::unique_ptr piper_frontend_; // piperlite graph + std::unique_ptr piper_runtime_; // piperlite graph }; } // namespace engine::models::sanotts diff --git a/model_specs/sanotts.json b/model_specs/sanotts.json index cc01e6d09..8d98fa9e1 100644 --- a/model_specs/sanotts.json +++ b/model_specs/sanotts.json @@ -2,7 +2,7 @@ "schema_version": 1, "family": "sanotts", "display_name": "sanoTTS Nano", - "description": "Very small English text-to-speech: a duration student, a contextual acoustic student producing mel-100, and a ConvNeXt-1D decoder with an iSTFT head. Two voices: heart (2.27M parameters, higher quality) and heart-nano (294k parameters, also runs on microcontrollers). 24 kHz output. Uses an external eSpeak-ng phonemizer.", + "description": "Very small English, Vietnamese and Indonesian text-to-speech. Two graphs: the nano lineage (duration student, contextual acoustic student to mel-100, noise-fed ConvNeXt-1D decoder with an iSTFT head; voices heart 2.27M and heart-nano 294k, 24 kHz) and the deterministic piperlite lineage (duration student, acoustic student to a 192-channel latent, 3-stage ConvTranspose1d decoder with dilated residual banks; voices amy, hfc, kristin, vi, id at 1.1-1.8M parameters, 22.05 kHz). Uses an external eSpeak-ng phonemizer.", "category": "tts", "status": "community", "tasks": [ @@ -12,7 +12,9 @@ "offline" ], "languages": [ - "en" + "en", + "vi", + "id" ], "runtime": { "tags": [ @@ -29,7 +31,7 @@ { "name": "speaking_rate", "type": "float", - "description": "Duration multiplier; larger is slower. Applied before the per-token clamp.", + "description": "Duration multiplier on the voice's tuned length scale; larger is slower. Applied before the per-token clamp.", "required": false, "min": 0.5, "max": 2.0, @@ -38,7 +40,7 @@ { "name": "seed", "type": "int", - "description": "Decoder noise seed. The decoder is noise-fed, so a given seed picks one of many valid renderings; 0 derives it from the text as sha256(text)[:8], which is what the reference implementations do.", + "description": "Decoder noise seed. The decoder is noise-fed, so a given seed picks one of many valid renderings; 0 derives it from the text as sha256(text)[:8], which is what the reference implementations do. Piperlite voices are deterministic and ignore the seed.", "required": false, "min": 0, "default": 0 @@ -112,6 +114,71 @@ "gguf/heart/config.json" ], "strip_prefix": "gguf/heart" + }, + { + "id": "sanotts_amy_orig", + "display_name": "sanoTTS amy 1.46M FP32 GGUF (English, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-amy-GGUF", + "files": [ + "gguf/amy/amy-f32.gguf", + "gguf/amy/config.json" + ], + "strip_prefix": "gguf/amy" + }, + { + "id": "sanotts_hfc_orig", + "display_name": "sanoTTS hfc 1.83M FP32 GGUF (English, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-hfc-GGUF", + "files": [ + "gguf/hfc/hfc-f32.gguf", + "gguf/hfc/config.json" + ], + "strip_prefix": "gguf/hfc" + }, + { + "id": "sanotts_kristin_orig", + "display_name": "sanoTTS kristin 1.40M FP32 GGUF (English, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-kristin-GGUF", + "files": [ + "gguf/kristin/kristin-f32.gguf", + "gguf/kristin/config.json" + ], + "strip_prefix": "gguf/kristin" + }, + { + "id": "sanotts_vi_orig", + "display_name": "sanoTTS vi 1.57M FP32 GGUF (Vietnamese, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-vi-GGUF", + "files": [ + "gguf/vi/vi-f32.gguf", + "gguf/vi/config.json" + ], + "strip_prefix": "gguf/vi" + }, + { + "id": "sanotts_id_orig", + "display_name": "sanoTTS id 1.56M FP32 GGUF (Indonesian, piperlite)", + "default": false, + "format": "gguf", + "precision": "orig", + "target_directory": "sanoTTS-id-GGUF", + "files": [ + "gguf/id/id-f32.gguf", + "gguf/id/config.json" + ], + "strip_prefix": "gguf/id" } ], "dependencies": [], diff --git a/src/community_models/sanotts/assets.cpp b/src/community_models/sanotts/assets.cpp index 13c8b4c8b..a20abb77f 100644 --- a/src/community_models/sanotts/assets.cpp +++ b/src/community_models/sanotts/assets.cpp @@ -15,13 +15,7 @@ namespace json = engine::io::json; constexpr const char * kFamily = "sanotts"; -SanoTtsConfig parse_config(const assets::ResourceBundle & resources) { - const auto root = resources.parse_json("config"); - const auto architecture = root.require("architecture").as_string(); - if (architecture != kFamily) { - throw std::runtime_error( - "sanoTTS config.json architecture is '" + architecture + "', expected 'sanotts'"); - } +SanoTtsConfig parse_nano_config(const engine::io::json::Value & root) { SanoTtsConfig out; out.voice = root.require("voice").as_string(); out.vocab_size = root.require("vocab_size").as_i64(); @@ -70,7 +64,7 @@ SanoTtsConfig parse_config(const assets::ResourceBundle & resources) { * obvious failure. Checking the whole inventory up front is what keeps a * packaging mistake loud. */ -void validate_tensors(const SanoTtsAssets & assets) { +void validate_nano_tensors(const SanoTtsAssets & assets) { const auto & c = assets.config; const auto & weights = *assets.weights; @@ -138,16 +132,211 @@ void validate_tensors(const SanoTtsAssets & assets) { } } + +SanoTtsPiperConfig parse_piper_config(const engine::io::json::Value & root) { + SanoTtsPiperConfig out; + out.voice = root.require("voice").as_string(); + out.language = root.require("language").as_string(); + out.espeak_voice = root.require("espeak_voice").as_string(); + out.sample_rate = root.require("sample_rate").as_i64(); + out.duration_length_scale = + static_cast(root.require("duration_length_scale").as_f32()); + + const auto & duration = root.require("duration"); + out.duration_vocab = duration.require("vocab_size").as_i64(); + out.duration_hidden = duration.require("hidden").as_i64(); + out.duration_depth = duration.require("depth").as_i64(); + out.duration_kernel = duration.require("kernel").as_i64(); + out.duration_max_tokens = duration.require("max_tokens").as_i64(); + out.duration_max_frames = duration.require("max_duration").as_i64(); + + const auto & acoustic = root.require("acoustic"); + out.acoustic_vocab = acoustic.require("vocab_size").as_i64(); + out.acoustic_hidden = acoustic.require("hidden").as_i64(); + out.acoustic_depth = acoustic.require("depth").as_i64(); + out.acoustic_token_depth = acoustic.require("token_depth").as_i64(); + out.acoustic_kernel = acoustic.require("kernel").as_i64(); + out.acoustic_out_channels = acoustic.require("out_channels").as_i64(); + + const auto & decoder = root.require("decoder"); + const auto channels = decoder.require("channels").as_array(); + if (channels.size() != 4) { + throw std::runtime_error("sanoTTS piperlite decoder.channels must have 4 entries"); + } + for (size_t stage = 0; stage < 4; ++stage) { + out.channels[stage] = channels[stage].as_i64(); + } + for (size_t stage = 0; stage < 3; ++stage) { + const auto & branches = + decoder.require("stage" + std::to_string(stage) + "_branches").as_array(); + for (const auto & branch : branches) { + const int64_t index = branch.as_i64(); + if (index < 0 || index > 2) { + throw std::runtime_error("sanoTTS piperlite branch index out of range"); + } + out.stage_branches[stage].push_back(index); + } + if (out.stage_branches[stage].empty()) { + throw std::runtime_error("sanoTTS piperlite stage has no branches"); + } + } + out.post_filter_channels = decoder.require("post_filter_channels").as_i64(); + out.post_filter_layers = decoder.require("post_filter_layers").as_i64(); + out.post_filter_kernel = decoder.require("post_filter_kernel").as_i64(); + out.post_filter_scale = + static_cast(decoder.require("post_filter_scale").as_f32()); + + for (const auto & [symbol, id] : root.require("phoneme_id_map").as_object()) { + out.phoneme_id_map.emplace(symbol, static_cast(id.as_i64())); + } + if (out.phoneme_id_map.empty()) { + throw std::runtime_error("sanoTTS piperlite config has an empty phoneme_id_map"); + } + + for (const auto & [label, value] : std::initializer_list>{ + {"sanoTTS piperlite duration vocab", out.duration_vocab}, + {"sanoTTS piperlite duration hidden", out.duration_hidden}, + {"sanoTTS piperlite duration max_tokens", out.duration_max_tokens}, + {"sanoTTS piperlite acoustic vocab", out.acoustic_vocab}, + {"sanoTTS piperlite acoustic hidden", out.acoustic_hidden}, + {"sanoTTS piperlite latent channels", out.acoustic_out_channels}, + {"sanoTTS piperlite sample_rate", out.sample_rate}, + }) { + engine::io::require_positive(value, label); + } + if (out.duration_length_scale <= 0.0) { + throw std::runtime_error("sanoTTS piperlite duration_length_scale must be positive"); + } + return out; +} + +// PiperResidualBank geometry, fixed by the training code. +constexpr int64_t kBankKernels[3] = {3, 5, 7}; + +/** + * The piperlite inventory. Kernel sizes the reference reads from the tensors + * themselves (decoder pre/post convs) are validated as rank/channels only, + * by looking the actual kernel up from the source. + */ +void validate_piper_tensors(const SanoTtsAssets & assets) { + const auto & c = assets.piper; + const auto & weights = *assets.weights; + + std::vector>> expected; + const auto conv = [&](const std::string & name, int64_t out_ch, int64_t in_ch, int64_t k) { + expected.emplace_back(name + ".weight", std::vector{out_ch, in_ch, k}); + expected.emplace_back(name + ".bias", std::vector{out_ch}); + }; + const auto conv_any_kernel = [&](const std::string & name, int64_t out_ch, int64_t in_ch) { + const auto metadata = weights.require_metadata(name + ".weight"); + if (metadata.shape.size() != 3 || metadata.shape[0] != out_ch || + metadata.shape[1] != in_ch || metadata.shape[2] < 1 || + metadata.shape[2] % 2 == 0) { + throw std::runtime_error("sanoTTS unexpected shape for tensor: " + name + ".weight"); + } + expected.emplace_back(name + ".bias", std::vector{out_ch}); + }; + + expected.emplace_back("duration.embedding.weight", + std::vector{c.duration_vocab, c.duration_hidden}); + conv("duration.input_proj", c.duration_hidden, c.duration_hidden + 3, 1); + for (int64_t b = 0; b < c.duration_depth; ++b) { + const std::string prefix = "duration.blocks." + std::to_string(b); + conv(prefix + ".net.0", c.duration_hidden, c.duration_hidden, c.duration_kernel); + conv(prefix + ".net.2", c.duration_hidden, c.duration_hidden, c.duration_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("duration.output", 1, c.duration_hidden, 1); + + expected.emplace_back("acoustic.embedding.weight", + std::vector{c.acoustic_vocab, c.acoustic_hidden}); + conv("acoustic.token_input_proj", c.acoustic_hidden, c.acoustic_hidden + 2, 1); + for (int64_t b = 0; b < c.acoustic_token_depth; ++b) { + const std::string prefix = "acoustic.token_blocks." + std::to_string(b); + conv(prefix + ".net.0", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + conv(prefix + ".net.2", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("acoustic.frame_input_proj", c.acoustic_hidden, c.acoustic_hidden + 3, 1); + for (int64_t b = 0; b < c.acoustic_depth; ++b) { + const std::string prefix = "acoustic.frame_blocks." + std::to_string(b); + conv(prefix + ".net.0", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + conv(prefix + ".net.2", c.acoustic_hidden, c.acoustic_hidden, c.acoustic_kernel); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv("acoustic.output", c.acoustic_out_channels, c.acoustic_hidden, 1); + + conv_any_kernel("decoder.pre", c.channels[0], c.acoustic_out_channels); + const int64_t up_kernels[3] = {16, 16, 8}; + for (size_t stage = 0; stage < 3; ++stage) { + const int64_t in_ch = c.channels[stage]; + const int64_t out_ch = c.channels[stage + 1]; + // ConvTranspose1d stores [in, out, K] + expected.emplace_back( + "decoder.up" + std::to_string(stage) + ".weight", + std::vector{in_ch, out_ch, up_kernels[stage]}); + expected.emplace_back( + "decoder.up" + std::to_string(stage) + ".bias", + std::vector{out_ch}); + for (const int64_t branch : c.stage_branches[stage]) { + const std::string prefix = "decoder.res" + std::to_string(stage) + ".0.blocks." + + std::to_string(branch); + conv(prefix + ".conv1", out_ch, out_ch, kBankKernels[branch]); + conv(prefix + ".conv2", out_ch, out_ch, kBankKernels[branch]); + } + } + conv_any_kernel("decoder.post", 1, c.channels[3]); + if (c.post_filter_channels > 0) { + conv_any_kernel("decoder.post_filter.in_conv", c.post_filter_channels, 1); + for (int64_t layer = 0; layer < c.post_filter_layers; ++layer) { + const std::string prefix = "decoder.post_filter.units." + std::to_string(layer); + // Unit conv kernels are a property of the checkpoint, not of + // post_filter_kernel (which sizes only the in/out convs). + conv_any_kernel(prefix + ".conv1", c.post_filter_channels, c.post_filter_channels); + conv_any_kernel(prefix + ".conv2", c.post_filter_channels, c.post_filter_channels); + expected.emplace_back(prefix + ".scale", std::vector{1}); + } + conv_any_kernel("decoder.post_filter.out_conv", 1, c.post_filter_channels); + } + + for (const auto & [name, shape] : expected) { + if (!weights.has_tensor(name)) { + throw std::runtime_error("sanoTTS missing tensor: " + name); + } + assets::require_tensor_shape(weights, name, shape); + } +} + } // namespace std::shared_ptr load_sanotts_assets( const std::filesystem::path & model_path) { auto resources = engine::model_spec::load_resource_bundle_for_family(model_path, kFamily); SanoTtsAssets out; - out.config = parse_config(resources); + const auto root = resources.parse_json("config"); + const auto architecture = root.require("architecture").as_string(); + if (architecture != kFamily) { + throw std::runtime_error( + "sanoTTS config.json architecture is '" + architecture + "', expected 'sanotts'"); + } + const auto * graph = root.find("graph"); + const auto graph_name = graph == nullptr ? std::string("nano") : graph->as_string(); + if (graph_name == "nano") { + out.graph = SanoTtsGraph::Nano; + out.config = parse_nano_config(root); + } else if (graph_name == "piperlite") { + out.graph = SanoTtsGraph::Piperlite; + out.piper = parse_piper_config(root); + } else { + throw std::runtime_error("sanoTTS config.json has unknown graph '" + graph_name + "'"); + } out.weights = resources.open_tensor_source("weights"); out.resources = std::move(resources); - validate_tensors(out); + if (out.graph == SanoTtsGraph::Nano) { + validate_nano_tensors(out); + } else { + validate_piper_tensors(out); + } return std::make_shared(std::move(out)); } diff --git a/src/community_models/sanotts/frontend.cpp b/src/community_models/sanotts/frontend.cpp index 949a0be01..547eabeeb 100644 --- a/src/community_models/sanotts/frontend.cpp +++ b/src/community_models/sanotts/frontend.cpp @@ -26,6 +26,9 @@ constexpr int kEspeakCharsUtf8 = 1; // BREVE in bits 8..23 as the tie character -- exactly the phonemes_mode // phonemizer computes, so the E2M diphthong patterns ("a͡ɪ" -> "I") can match. constexpr int kEspeakPhonemesIpaTie = 0x02 | (0x01 << 7) | (0x0361 << 8); +// The piperlite voices phonemize with tie=False: IPA with '_' as the phoneme +// separator in bits 8..23, exactly phonemizer's untied phonemes_mode. +constexpr int kEspeakPhonemesIpaUnderscore = 0x02 | ('_' << 8); constexpr std::string_view kTieDefault = "͡"; // COMBINING DOUBLE INVERTED BREVE constexpr std::string_view kTieMisaki = "^"; @@ -380,6 +383,499 @@ std::string rewrite_ties_per_word(const std::string & line_in) { return out; } + +// ---- NFD decomposition for the Piper codepoint mapping ------------------- +// +// piper normalizes the phonemized string with NFD before mapping each +// codepoint through phoneme_id_map. eSpeak-ng's IPA output for the shipped +// languages is already NFD-normal (probed over en/vi/id corpora), so this +// table only has to cover precomposed Latin letters that could slip through; +// characters outside it pass through unchanged, and an unmapped codepoint is +// skipped exactly as piper skips it. The framework's NFKD normalizer is NOT +// usable here: compatibility decomposition rewrites IPA modifier letters. + +// Canonical (NFD) decompositions for Latin-1/Extended and Vietnamese +// precomposed letters -- everything eSpeak-ng can plausibly emit in IPA +// mode. Generated from Python unicodedata (Unicode 15); canonical +// decompositions are stable across Unicode versions by policy. +struct NfdEntry { uint32_t composed; uint32_t parts[3]; }; +constexpr NfdEntry kNfdEntries[] = { + {0x00C0, {0x0041, 0x0300, 0x0000}}, + {0x00C1, {0x0041, 0x0301, 0x0000}}, + {0x00C2, {0x0041, 0x0302, 0x0000}}, + {0x00C3, {0x0041, 0x0303, 0x0000}}, + {0x00C4, {0x0041, 0x0308, 0x0000}}, + {0x00C5, {0x0041, 0x030A, 0x0000}}, + {0x00C7, {0x0043, 0x0327, 0x0000}}, + {0x00C8, {0x0045, 0x0300, 0x0000}}, + {0x00C9, {0x0045, 0x0301, 0x0000}}, + {0x00CA, {0x0045, 0x0302, 0x0000}}, + {0x00CB, {0x0045, 0x0308, 0x0000}}, + {0x00CC, {0x0049, 0x0300, 0x0000}}, + {0x00CD, {0x0049, 0x0301, 0x0000}}, + {0x00CE, {0x0049, 0x0302, 0x0000}}, + {0x00CF, {0x0049, 0x0308, 0x0000}}, + {0x00D1, {0x004E, 0x0303, 0x0000}}, + {0x00D2, {0x004F, 0x0300, 0x0000}}, + {0x00D3, {0x004F, 0x0301, 0x0000}}, + {0x00D4, {0x004F, 0x0302, 0x0000}}, + {0x00D5, {0x004F, 0x0303, 0x0000}}, + {0x00D6, {0x004F, 0x0308, 0x0000}}, + {0x00D9, {0x0055, 0x0300, 0x0000}}, + {0x00DA, {0x0055, 0x0301, 0x0000}}, + {0x00DB, {0x0055, 0x0302, 0x0000}}, + {0x00DC, {0x0055, 0x0308, 0x0000}}, + {0x00DD, {0x0059, 0x0301, 0x0000}}, + {0x00E0, {0x0061, 0x0300, 0x0000}}, + {0x00E1, {0x0061, 0x0301, 0x0000}}, + {0x00E2, {0x0061, 0x0302, 0x0000}}, + {0x00E3, {0x0061, 0x0303, 0x0000}}, + {0x00E4, {0x0061, 0x0308, 0x0000}}, + {0x00E5, {0x0061, 0x030A, 0x0000}}, + {0x00E7, {0x0063, 0x0327, 0x0000}}, + {0x00E8, {0x0065, 0x0300, 0x0000}}, + {0x00E9, {0x0065, 0x0301, 0x0000}}, + {0x00EA, {0x0065, 0x0302, 0x0000}}, + {0x00EB, {0x0065, 0x0308, 0x0000}}, + {0x00EC, {0x0069, 0x0300, 0x0000}}, + {0x00ED, {0x0069, 0x0301, 0x0000}}, + {0x00EE, {0x0069, 0x0302, 0x0000}}, + {0x00EF, {0x0069, 0x0308, 0x0000}}, + {0x00F1, {0x006E, 0x0303, 0x0000}}, + {0x00F2, {0x006F, 0x0300, 0x0000}}, + {0x00F3, {0x006F, 0x0301, 0x0000}}, + {0x00F4, {0x006F, 0x0302, 0x0000}}, + {0x00F5, {0x006F, 0x0303, 0x0000}}, + {0x00F6, {0x006F, 0x0308, 0x0000}}, + {0x00F9, {0x0075, 0x0300, 0x0000}}, + {0x00FA, {0x0075, 0x0301, 0x0000}}, + {0x00FB, {0x0075, 0x0302, 0x0000}}, + {0x00FC, {0x0075, 0x0308, 0x0000}}, + {0x00FD, {0x0079, 0x0301, 0x0000}}, + {0x00FF, {0x0079, 0x0308, 0x0000}}, + {0x0100, {0x0041, 0x0304, 0x0000}}, + {0x0101, {0x0061, 0x0304, 0x0000}}, + {0x0102, {0x0041, 0x0306, 0x0000}}, + {0x0103, {0x0061, 0x0306, 0x0000}}, + {0x0104, {0x0041, 0x0328, 0x0000}}, + {0x0105, {0x0061, 0x0328, 0x0000}}, + {0x0106, {0x0043, 0x0301, 0x0000}}, + {0x0107, {0x0063, 0x0301, 0x0000}}, + {0x0108, {0x0043, 0x0302, 0x0000}}, + {0x0109, {0x0063, 0x0302, 0x0000}}, + {0x010A, {0x0043, 0x0307, 0x0000}}, + {0x010B, {0x0063, 0x0307, 0x0000}}, + {0x010C, {0x0043, 0x030C, 0x0000}}, + {0x010D, {0x0063, 0x030C, 0x0000}}, + {0x010E, {0x0044, 0x030C, 0x0000}}, + {0x010F, {0x0064, 0x030C, 0x0000}}, + {0x0112, {0x0045, 0x0304, 0x0000}}, + {0x0113, {0x0065, 0x0304, 0x0000}}, + {0x0114, {0x0045, 0x0306, 0x0000}}, + {0x0115, {0x0065, 0x0306, 0x0000}}, + {0x0116, {0x0045, 0x0307, 0x0000}}, + {0x0117, {0x0065, 0x0307, 0x0000}}, + {0x0118, {0x0045, 0x0328, 0x0000}}, + {0x0119, {0x0065, 0x0328, 0x0000}}, + {0x011A, {0x0045, 0x030C, 0x0000}}, + {0x011B, {0x0065, 0x030C, 0x0000}}, + {0x011C, {0x0047, 0x0302, 0x0000}}, + {0x011D, {0x0067, 0x0302, 0x0000}}, + {0x011E, {0x0047, 0x0306, 0x0000}}, + {0x011F, {0x0067, 0x0306, 0x0000}}, + {0x0120, {0x0047, 0x0307, 0x0000}}, + {0x0121, {0x0067, 0x0307, 0x0000}}, + {0x0122, {0x0047, 0x0327, 0x0000}}, + {0x0123, {0x0067, 0x0327, 0x0000}}, + {0x0124, {0x0048, 0x0302, 0x0000}}, + {0x0125, {0x0068, 0x0302, 0x0000}}, + {0x0128, {0x0049, 0x0303, 0x0000}}, + {0x0129, {0x0069, 0x0303, 0x0000}}, + {0x012A, {0x0049, 0x0304, 0x0000}}, + {0x012B, {0x0069, 0x0304, 0x0000}}, + {0x012C, {0x0049, 0x0306, 0x0000}}, + {0x012D, {0x0069, 0x0306, 0x0000}}, + {0x012E, {0x0049, 0x0328, 0x0000}}, + {0x012F, {0x0069, 0x0328, 0x0000}}, + {0x0130, {0x0049, 0x0307, 0x0000}}, + {0x0134, {0x004A, 0x0302, 0x0000}}, + {0x0135, {0x006A, 0x0302, 0x0000}}, + {0x0136, {0x004B, 0x0327, 0x0000}}, + {0x0137, {0x006B, 0x0327, 0x0000}}, + {0x0139, {0x004C, 0x0301, 0x0000}}, + {0x013A, {0x006C, 0x0301, 0x0000}}, + {0x013B, {0x004C, 0x0327, 0x0000}}, + {0x013C, {0x006C, 0x0327, 0x0000}}, + {0x013D, {0x004C, 0x030C, 0x0000}}, + {0x013E, {0x006C, 0x030C, 0x0000}}, + {0x0143, {0x004E, 0x0301, 0x0000}}, + {0x0144, {0x006E, 0x0301, 0x0000}}, + {0x0145, {0x004E, 0x0327, 0x0000}}, + {0x0146, {0x006E, 0x0327, 0x0000}}, + {0x0147, {0x004E, 0x030C, 0x0000}}, + {0x0148, {0x006E, 0x030C, 0x0000}}, + {0x014C, {0x004F, 0x0304, 0x0000}}, + {0x014D, {0x006F, 0x0304, 0x0000}}, + {0x014E, {0x004F, 0x0306, 0x0000}}, + {0x014F, {0x006F, 0x0306, 0x0000}}, + {0x0150, {0x004F, 0x030B, 0x0000}}, + {0x0151, {0x006F, 0x030B, 0x0000}}, + {0x0154, {0x0052, 0x0301, 0x0000}}, + {0x0155, {0x0072, 0x0301, 0x0000}}, + {0x0156, {0x0052, 0x0327, 0x0000}}, + {0x0157, {0x0072, 0x0327, 0x0000}}, + {0x0158, {0x0052, 0x030C, 0x0000}}, + {0x0159, {0x0072, 0x030C, 0x0000}}, + {0x015A, {0x0053, 0x0301, 0x0000}}, + {0x015B, {0x0073, 0x0301, 0x0000}}, + {0x015C, {0x0053, 0x0302, 0x0000}}, + {0x015D, {0x0073, 0x0302, 0x0000}}, + {0x015E, {0x0053, 0x0327, 0x0000}}, + {0x015F, {0x0073, 0x0327, 0x0000}}, + {0x0160, {0x0053, 0x030C, 0x0000}}, + {0x0161, {0x0073, 0x030C, 0x0000}}, + {0x0162, {0x0054, 0x0327, 0x0000}}, + {0x0163, {0x0074, 0x0327, 0x0000}}, + {0x0164, {0x0054, 0x030C, 0x0000}}, + {0x0165, {0x0074, 0x030C, 0x0000}}, + {0x0168, {0x0055, 0x0303, 0x0000}}, + {0x0169, {0x0075, 0x0303, 0x0000}}, + {0x016A, {0x0055, 0x0304, 0x0000}}, + {0x016B, {0x0075, 0x0304, 0x0000}}, + {0x016C, {0x0055, 0x0306, 0x0000}}, + {0x016D, {0x0075, 0x0306, 0x0000}}, + {0x016E, {0x0055, 0x030A, 0x0000}}, + {0x016F, {0x0075, 0x030A, 0x0000}}, + {0x0170, {0x0055, 0x030B, 0x0000}}, + {0x0171, {0x0075, 0x030B, 0x0000}}, + {0x0172, {0x0055, 0x0328, 0x0000}}, + {0x0173, {0x0075, 0x0328, 0x0000}}, + {0x0174, {0x0057, 0x0302, 0x0000}}, + {0x0175, {0x0077, 0x0302, 0x0000}}, + {0x0176, {0x0059, 0x0302, 0x0000}}, + {0x0177, {0x0079, 0x0302, 0x0000}}, + {0x0178, {0x0059, 0x0308, 0x0000}}, + {0x0179, {0x005A, 0x0301, 0x0000}}, + {0x017A, {0x007A, 0x0301, 0x0000}}, + {0x017B, {0x005A, 0x0307, 0x0000}}, + {0x017C, {0x007A, 0x0307, 0x0000}}, + {0x017D, {0x005A, 0x030C, 0x0000}}, + {0x017E, {0x007A, 0x030C, 0x0000}}, + {0x1E00, {0x0041, 0x0325, 0x0000}}, + {0x1E01, {0x0061, 0x0325, 0x0000}}, + {0x1E02, {0x0042, 0x0307, 0x0000}}, + {0x1E03, {0x0062, 0x0307, 0x0000}}, + {0x1E04, {0x0042, 0x0323, 0x0000}}, + {0x1E05, {0x0062, 0x0323, 0x0000}}, + {0x1E06, {0x0042, 0x0331, 0x0000}}, + {0x1E07, {0x0062, 0x0331, 0x0000}}, + {0x1E08, {0x0043, 0x0327, 0x0301}}, + {0x1E09, {0x0063, 0x0327, 0x0301}}, + {0x1E0A, {0x0044, 0x0307, 0x0000}}, + {0x1E0B, {0x0064, 0x0307, 0x0000}}, + {0x1E0C, {0x0044, 0x0323, 0x0000}}, + {0x1E0D, {0x0064, 0x0323, 0x0000}}, + {0x1E0E, {0x0044, 0x0331, 0x0000}}, + {0x1E0F, {0x0064, 0x0331, 0x0000}}, + {0x1E10, {0x0044, 0x0327, 0x0000}}, + {0x1E11, {0x0064, 0x0327, 0x0000}}, + {0x1E12, {0x0044, 0x032D, 0x0000}}, + {0x1E13, {0x0064, 0x032D, 0x0000}}, + {0x1E14, {0x0045, 0x0304, 0x0300}}, + {0x1E15, {0x0065, 0x0304, 0x0300}}, + {0x1E16, {0x0045, 0x0304, 0x0301}}, + {0x1E17, {0x0065, 0x0304, 0x0301}}, + {0x1E18, {0x0045, 0x032D, 0x0000}}, + {0x1E19, {0x0065, 0x032D, 0x0000}}, + {0x1E1A, {0x0045, 0x0330, 0x0000}}, + {0x1E1B, {0x0065, 0x0330, 0x0000}}, + {0x1E1C, {0x0045, 0x0327, 0x0306}}, + {0x1E1D, {0x0065, 0x0327, 0x0306}}, + {0x1E1E, {0x0046, 0x0307, 0x0000}}, + {0x1E1F, {0x0066, 0x0307, 0x0000}}, + {0x1E20, {0x0047, 0x0304, 0x0000}}, + {0x1E21, {0x0067, 0x0304, 0x0000}}, + {0x1E22, {0x0048, 0x0307, 0x0000}}, + {0x1E23, {0x0068, 0x0307, 0x0000}}, + {0x1E24, {0x0048, 0x0323, 0x0000}}, + {0x1E25, {0x0068, 0x0323, 0x0000}}, + {0x1E26, {0x0048, 0x0308, 0x0000}}, + {0x1E27, {0x0068, 0x0308, 0x0000}}, + {0x1E28, {0x0048, 0x0327, 0x0000}}, + {0x1E29, {0x0068, 0x0327, 0x0000}}, + {0x1E2A, {0x0048, 0x032E, 0x0000}}, + {0x1E2B, {0x0068, 0x032E, 0x0000}}, + {0x1E2C, {0x0049, 0x0330, 0x0000}}, + {0x1E2D, {0x0069, 0x0330, 0x0000}}, + {0x1E2E, {0x0049, 0x0308, 0x0301}}, + {0x1E2F, {0x0069, 0x0308, 0x0301}}, + {0x1E30, {0x004B, 0x0301, 0x0000}}, + {0x1E31, {0x006B, 0x0301, 0x0000}}, + {0x1E32, {0x004B, 0x0323, 0x0000}}, + {0x1E33, {0x006B, 0x0323, 0x0000}}, + {0x1E34, {0x004B, 0x0331, 0x0000}}, + {0x1E35, {0x006B, 0x0331, 0x0000}}, + {0x1E36, {0x004C, 0x0323, 0x0000}}, + {0x1E37, {0x006C, 0x0323, 0x0000}}, + {0x1E38, {0x004C, 0x0323, 0x0304}}, + {0x1E39, {0x006C, 0x0323, 0x0304}}, + {0x1E3A, {0x004C, 0x0331, 0x0000}}, + {0x1E3B, {0x006C, 0x0331, 0x0000}}, + {0x1E3C, {0x004C, 0x032D, 0x0000}}, + {0x1E3D, {0x006C, 0x032D, 0x0000}}, + {0x1E3E, {0x004D, 0x0301, 0x0000}}, + {0x1E3F, {0x006D, 0x0301, 0x0000}}, + {0x1E40, {0x004D, 0x0307, 0x0000}}, + {0x1E41, {0x006D, 0x0307, 0x0000}}, + {0x1E42, {0x004D, 0x0323, 0x0000}}, + {0x1E43, {0x006D, 0x0323, 0x0000}}, + {0x1E44, {0x004E, 0x0307, 0x0000}}, + {0x1E45, {0x006E, 0x0307, 0x0000}}, + {0x1E46, {0x004E, 0x0323, 0x0000}}, + {0x1E47, {0x006E, 0x0323, 0x0000}}, + {0x1E48, {0x004E, 0x0331, 0x0000}}, + {0x1E49, {0x006E, 0x0331, 0x0000}}, + {0x1E4A, {0x004E, 0x032D, 0x0000}}, + {0x1E4B, {0x006E, 0x032D, 0x0000}}, + {0x1E4C, {0x004F, 0x0303, 0x0301}}, + {0x1E4D, {0x006F, 0x0303, 0x0301}}, + {0x1E4E, {0x004F, 0x0303, 0x0308}}, + {0x1E4F, {0x006F, 0x0303, 0x0308}}, + {0x1E50, {0x004F, 0x0304, 0x0300}}, + {0x1E51, {0x006F, 0x0304, 0x0300}}, + {0x1E52, {0x004F, 0x0304, 0x0301}}, + {0x1E53, {0x006F, 0x0304, 0x0301}}, + {0x1E54, {0x0050, 0x0301, 0x0000}}, + {0x1E55, {0x0070, 0x0301, 0x0000}}, + {0x1E56, {0x0050, 0x0307, 0x0000}}, + {0x1E57, {0x0070, 0x0307, 0x0000}}, + {0x1E58, {0x0052, 0x0307, 0x0000}}, + {0x1E59, {0x0072, 0x0307, 0x0000}}, + {0x1E5A, {0x0052, 0x0323, 0x0000}}, + {0x1E5B, {0x0072, 0x0323, 0x0000}}, + {0x1E5C, {0x0052, 0x0323, 0x0304}}, + {0x1E5D, {0x0072, 0x0323, 0x0304}}, + {0x1E5E, {0x0052, 0x0331, 0x0000}}, + {0x1E5F, {0x0072, 0x0331, 0x0000}}, + {0x1E60, {0x0053, 0x0307, 0x0000}}, + {0x1E61, {0x0073, 0x0307, 0x0000}}, + {0x1E62, {0x0053, 0x0323, 0x0000}}, + {0x1E63, {0x0073, 0x0323, 0x0000}}, + {0x1E64, {0x0053, 0x0301, 0x0307}}, + {0x1E65, {0x0073, 0x0301, 0x0307}}, + {0x1E66, {0x0053, 0x030C, 0x0307}}, + {0x1E67, {0x0073, 0x030C, 0x0307}}, + {0x1E68, {0x0053, 0x0323, 0x0307}}, + {0x1E69, {0x0073, 0x0323, 0x0307}}, + {0x1E6A, {0x0054, 0x0307, 0x0000}}, + {0x1E6B, {0x0074, 0x0307, 0x0000}}, + {0x1E6C, {0x0054, 0x0323, 0x0000}}, + {0x1E6D, {0x0074, 0x0323, 0x0000}}, + {0x1E6E, {0x0054, 0x0331, 0x0000}}, + {0x1E6F, {0x0074, 0x0331, 0x0000}}, + {0x1E70, {0x0054, 0x032D, 0x0000}}, + {0x1E71, {0x0074, 0x032D, 0x0000}}, + {0x1E72, {0x0055, 0x0324, 0x0000}}, + {0x1E73, {0x0075, 0x0324, 0x0000}}, + {0x1E74, {0x0055, 0x0330, 0x0000}}, + {0x1E75, {0x0075, 0x0330, 0x0000}}, + {0x1E76, {0x0055, 0x032D, 0x0000}}, + {0x1E77, {0x0075, 0x032D, 0x0000}}, + {0x1E78, {0x0055, 0x0303, 0x0301}}, + {0x1E79, {0x0075, 0x0303, 0x0301}}, + {0x1E7A, {0x0055, 0x0304, 0x0308}}, + {0x1E7B, {0x0075, 0x0304, 0x0308}}, + {0x1E7C, {0x0056, 0x0303, 0x0000}}, + {0x1E7D, {0x0076, 0x0303, 0x0000}}, + {0x1E7E, {0x0056, 0x0323, 0x0000}}, + {0x1E7F, {0x0076, 0x0323, 0x0000}}, + {0x1E80, {0x0057, 0x0300, 0x0000}}, + {0x1E81, {0x0077, 0x0300, 0x0000}}, + {0x1E82, {0x0057, 0x0301, 0x0000}}, + {0x1E83, {0x0077, 0x0301, 0x0000}}, + {0x1E84, {0x0057, 0x0308, 0x0000}}, + {0x1E85, {0x0077, 0x0308, 0x0000}}, + {0x1E86, {0x0057, 0x0307, 0x0000}}, + {0x1E87, {0x0077, 0x0307, 0x0000}}, + {0x1E88, {0x0057, 0x0323, 0x0000}}, + {0x1E89, {0x0077, 0x0323, 0x0000}}, + {0x1E8A, {0x0058, 0x0307, 0x0000}}, + {0x1E8B, {0x0078, 0x0307, 0x0000}}, + {0x1E8C, {0x0058, 0x0308, 0x0000}}, + {0x1E8D, {0x0078, 0x0308, 0x0000}}, + {0x1E8E, {0x0059, 0x0307, 0x0000}}, + {0x1E8F, {0x0079, 0x0307, 0x0000}}, + {0x1E90, {0x005A, 0x0302, 0x0000}}, + {0x1E91, {0x007A, 0x0302, 0x0000}}, + {0x1E92, {0x005A, 0x0323, 0x0000}}, + {0x1E93, {0x007A, 0x0323, 0x0000}}, + {0x1E94, {0x005A, 0x0331, 0x0000}}, + {0x1E95, {0x007A, 0x0331, 0x0000}}, + {0x1E96, {0x0068, 0x0331, 0x0000}}, + {0x1E97, {0x0074, 0x0308, 0x0000}}, + {0x1E98, {0x0077, 0x030A, 0x0000}}, + {0x1E99, {0x0079, 0x030A, 0x0000}}, + {0x1E9B, {0x017F, 0x0307, 0x0000}}, + {0x1EA0, {0x0041, 0x0323, 0x0000}}, + {0x1EA1, {0x0061, 0x0323, 0x0000}}, + {0x1EA2, {0x0041, 0x0309, 0x0000}}, + {0x1EA3, {0x0061, 0x0309, 0x0000}}, + {0x1EA4, {0x0041, 0x0302, 0x0301}}, + {0x1EA5, {0x0061, 0x0302, 0x0301}}, + {0x1EA6, {0x0041, 0x0302, 0x0300}}, + {0x1EA7, {0x0061, 0x0302, 0x0300}}, + {0x1EA8, {0x0041, 0x0302, 0x0309}}, + {0x1EA9, {0x0061, 0x0302, 0x0309}}, + {0x1EAA, {0x0041, 0x0302, 0x0303}}, + {0x1EAB, {0x0061, 0x0302, 0x0303}}, + {0x1EAC, {0x0041, 0x0323, 0x0302}}, + {0x1EAD, {0x0061, 0x0323, 0x0302}}, + {0x1EAE, {0x0041, 0x0306, 0x0301}}, + {0x1EAF, {0x0061, 0x0306, 0x0301}}, + {0x1EB0, {0x0041, 0x0306, 0x0300}}, + {0x1EB1, {0x0061, 0x0306, 0x0300}}, + {0x1EB2, {0x0041, 0x0306, 0x0309}}, + {0x1EB3, {0x0061, 0x0306, 0x0309}}, + {0x1EB4, {0x0041, 0x0306, 0x0303}}, + {0x1EB5, {0x0061, 0x0306, 0x0303}}, + {0x1EB6, {0x0041, 0x0323, 0x0306}}, + {0x1EB7, {0x0061, 0x0323, 0x0306}}, + {0x1EB8, {0x0045, 0x0323, 0x0000}}, + {0x1EB9, {0x0065, 0x0323, 0x0000}}, + {0x1EBA, {0x0045, 0x0309, 0x0000}}, + {0x1EBB, {0x0065, 0x0309, 0x0000}}, + {0x1EBC, {0x0045, 0x0303, 0x0000}}, + {0x1EBD, {0x0065, 0x0303, 0x0000}}, + {0x1EBE, {0x0045, 0x0302, 0x0301}}, + {0x1EBF, {0x0065, 0x0302, 0x0301}}, + {0x1EC0, {0x0045, 0x0302, 0x0300}}, + {0x1EC1, {0x0065, 0x0302, 0x0300}}, + {0x1EC2, {0x0045, 0x0302, 0x0309}}, + {0x1EC3, {0x0065, 0x0302, 0x0309}}, + {0x1EC4, {0x0045, 0x0302, 0x0303}}, + {0x1EC5, {0x0065, 0x0302, 0x0303}}, + {0x1EC6, {0x0045, 0x0323, 0x0302}}, + {0x1EC7, {0x0065, 0x0323, 0x0302}}, + {0x1EC8, {0x0049, 0x0309, 0x0000}}, + {0x1EC9, {0x0069, 0x0309, 0x0000}}, + {0x1ECA, {0x0049, 0x0323, 0x0000}}, + {0x1ECB, {0x0069, 0x0323, 0x0000}}, + {0x1ECC, {0x004F, 0x0323, 0x0000}}, + {0x1ECD, {0x006F, 0x0323, 0x0000}}, + {0x1ECE, {0x004F, 0x0309, 0x0000}}, + {0x1ECF, {0x006F, 0x0309, 0x0000}}, + {0x1ED0, {0x004F, 0x0302, 0x0301}}, + {0x1ED1, {0x006F, 0x0302, 0x0301}}, + {0x1ED2, {0x004F, 0x0302, 0x0300}}, + {0x1ED3, {0x006F, 0x0302, 0x0300}}, + {0x1ED4, {0x004F, 0x0302, 0x0309}}, + {0x1ED5, {0x006F, 0x0302, 0x0309}}, + {0x1ED6, {0x004F, 0x0302, 0x0303}}, + {0x1ED7, {0x006F, 0x0302, 0x0303}}, + {0x1ED8, {0x004F, 0x0323, 0x0302}}, + {0x1ED9, {0x006F, 0x0323, 0x0302}}, + {0x1EDA, {0x004F, 0x031B, 0x0301}}, + {0x1EDB, {0x006F, 0x031B, 0x0301}}, + {0x1EDC, {0x004F, 0x031B, 0x0300}}, + {0x1EDD, {0x006F, 0x031B, 0x0300}}, + {0x1EDE, {0x004F, 0x031B, 0x0309}}, + {0x1EDF, {0x006F, 0x031B, 0x0309}}, + {0x1EE0, {0x004F, 0x031B, 0x0303}}, + {0x1EE1, {0x006F, 0x031B, 0x0303}}, + {0x1EE2, {0x004F, 0x031B, 0x0323}}, + {0x1EE3, {0x006F, 0x031B, 0x0323}}, + {0x1EE4, {0x0055, 0x0323, 0x0000}}, + {0x1EE5, {0x0075, 0x0323, 0x0000}}, + {0x1EE6, {0x0055, 0x0309, 0x0000}}, + {0x1EE7, {0x0075, 0x0309, 0x0000}}, + {0x1EE8, {0x0055, 0x031B, 0x0301}}, + {0x1EE9, {0x0075, 0x031B, 0x0301}}, + {0x1EEA, {0x0055, 0x031B, 0x0300}}, + {0x1EEB, {0x0075, 0x031B, 0x0300}}, + {0x1EEC, {0x0055, 0x031B, 0x0309}}, + {0x1EED, {0x0075, 0x031B, 0x0309}}, + {0x1EEE, {0x0055, 0x031B, 0x0303}}, + {0x1EEF, {0x0075, 0x031B, 0x0303}}, + {0x1EF0, {0x0055, 0x031B, 0x0323}}, + {0x1EF1, {0x0075, 0x031B, 0x0323}}, + {0x1EF2, {0x0059, 0x0300, 0x0000}}, + {0x1EF3, {0x0079, 0x0300, 0x0000}}, + {0x1EF4, {0x0059, 0x0323, 0x0000}}, + {0x1EF5, {0x0079, 0x0323, 0x0000}}, + {0x1EF6, {0x0059, 0x0309, 0x0000}}, + {0x1EF7, {0x0079, 0x0309, 0x0000}}, + {0x1EF8, {0x0059, 0x0303, 0x0000}}, + {0x1EF9, {0x0079, 0x0303, 0x0000}}, +}; + + +void append_codepoint_utf8(uint32_t codepoint, std::string & out) { + if (codepoint < 0x80U) { + out.push_back(static_cast(codepoint)); + } else if (codepoint < 0x800U) { + out.push_back(static_cast(0xC0U | (codepoint >> 6))); + out.push_back(static_cast(0x80U | (codepoint & 0x3FU))); + } else if (codepoint < 0x10000U) { + out.push_back(static_cast(0xE0U | (codepoint >> 12))); + out.push_back(static_cast(0x80U | ((codepoint >> 6) & 0x3FU))); + out.push_back(static_cast(0x80U | (codepoint & 0x3FU))); + } else { + out.push_back(static_cast(0xF0U | (codepoint >> 18))); + out.push_back(static_cast(0x80U | ((codepoint >> 12) & 0x3FU))); + out.push_back(static_cast(0x80U | ((codepoint >> 6) & 0x3FU))); + out.push_back(static_cast(0x80U | (codepoint & 0x3FU))); + } +} + +/** Iterates whole UTF-8 codepoints; invalid lead bytes pass through as one. */ +size_t utf8_sequence_length(const std::string & text, size_t at) { + const auto lead = static_cast(text[at]); + size_t len = 1; + if ((lead & 0xF8U) == 0xF0U) { len = 4; } + else if ((lead & 0xF0U) == 0xE0U) { len = 3; } + else if ((lead & 0xE0U) == 0xC0U) { len = 2; } + return std::min(len, text.size() - at); +} + +uint32_t decode_codepoint_utf8(const std::string & text, size_t at, size_t len) { + const auto lead = static_cast(text[at]); + if (len == 1) { + return lead; + } + uint32_t value = lead & (0x7FU >> len); + for (size_t i = 1; i < len; ++i) { + value = (value << 6) | (static_cast(text[at + i]) & 0x3FU); + } + return value; +} + +std::string nfd_decompose(const std::string & text) { + std::string out; + out.reserve(text.size()); + for (size_t i = 0; i < text.size();) { + const size_t len = utf8_sequence_length(text, i); + const uint32_t codepoint = decode_codepoint_utf8(text, i, len); + const auto * entry = std::lower_bound( + std::begin(kNfdEntries), + std::end(kNfdEntries), + codepoint, + [](const NfdEntry & candidate, uint32_t value) { + return candidate.composed < value; + }); + if (entry != std::end(kNfdEntries) && entry->composed == codepoint) { + for (const uint32_t part : entry->parts) { + if (part != 0) { + append_codepoint_utf8(part, out); + } + } + } else { + out.append(text, i, len); + } + i += len; + } + return out; +} + struct EspeakApi { io::DynamicLibraryHandle library = nullptr; InitializeFn initialize = nullptr; @@ -389,7 +885,8 @@ struct EspeakApi { mutable std::mutex call_mutex; EspeakApi(const std::filesystem::path & requested_library, - const std::filesystem::path & requested_data) { + const std::filesystem::path & requested_data, + const std::string & voice) { if (!requested_library.empty() && !std::filesystem::is_regular_file(requested_library)) { throw std::runtime_error( @@ -442,8 +939,28 @@ struct EspeakApi { "sanoTTS eSpeak-ng failed to initialize; pass " "--session-option sanotts.espeak_data_path=/path/to/espeak-ng-data"); } - if (set_voice("en-us") != 0) { - throw std::runtime_error("sanoTTS eSpeak-ng has no en-us voice"); + // Some packages name a bare language code ("en"). phonemizer, which + // the reference front end drives, rejects bare codes on every + // espeak-ng >= 1.49 and falls back to the regional variant, even + // though espeak_SetVoiceByName itself would accept "en" (and select + // a different accent). Prefer the regional variants first so both + // stacks phonemize identically; a code with no variant (vi, id) + // falls through to itself. + std::vector candidates; + if (voice.find('-') == std::string::npos) { + candidates.push_back(voice + "-us"); + candidates.push_back(voice + "-gb"); + } + candidates.push_back(voice); + bool selected = false; + for (const auto & candidate : candidates) { + if (set_voice(candidate.c_str()) == 0) { + selected = true; + break; + } + } + if (!selected) { + throw std::runtime_error("sanoTTS eSpeak-ng has no voice matching '" + voice + "'"); } } @@ -456,7 +973,7 @@ struct EspeakApi { } } - [[nodiscard]] std::string phonemize(const std::string & text) const { + [[nodiscard]] std::string phonemize(const std::string & text, int phonemes_mode) const { const std::lock_guard guard(call_mutex); std::string out; const char * cursor = text.c_str(); @@ -465,7 +982,7 @@ struct EspeakApi { // returns null when the input is spent. while (position != nullptr) { const char * clause = - text_to_phonemes(&position, kEspeakCharsUtf8, kEspeakPhonemesIpaTie); + text_to_phonemes(&position, kEspeakCharsUtf8, phonemes_mode); if (clause == nullptr) { break; } @@ -483,7 +1000,7 @@ struct EspeakApi { struct SanoTtsFrontend::Impl { EspeakApi espeak; Impl(const std::filesystem::path & library, const std::filesystem::path & data) - : espeak(library, data) {} + : espeak(library, data, "en-us") {} }; SanoTtsFrontend::SanoTtsFrontend( @@ -500,7 +1017,8 @@ SanoTtsEncoded SanoTtsFrontend::encode(const std::string & text) const { std::vector chunk_phonemes; chunk_phonemes.reserve(chunks.size()); for (const auto & chunk : chunks) { - chunk_phonemes.push_back(postprocess_espeak_line(impl_->espeak.phonemize(chunk))); + chunk_phonemes.push_back(postprocess_espeak_line( + impl_->espeak.phonemize(chunk, kEspeakPhonemesIpaTie))); } const std::string restored = restore_punctuation(std::move(chunk_phonemes), std::move(marks)); @@ -588,4 +1106,77 @@ double SanoTtsFrontend::boundary_pause_seconds(const std::string & chunk) { return 0.08; } +// ---- piperlite front end ------------------------------------------------- + +struct SanoTtsPiperFrontend::Impl { + EspeakApi espeak; + Impl(const std::filesystem::path & library, + const std::filesystem::path & data, + const std::string & voice) + : espeak(library, data, voice) {} +}; + +SanoTtsPiperFrontend::SanoTtsPiperFrontend( + std::filesystem::path espeak_library_path, + std::filesystem::path espeak_data_path, + std::string espeak_voice, + std::unordered_map phoneme_id_map, + int64_t max_tokens) + : impl_(std::make_unique(espeak_library_path, espeak_data_path, espeak_voice)), + id_map_(std::move(phoneme_id_map)), + max_tokens_(max_tokens > 3 ? max_tokens : 3) {} + +SanoTtsPiperFrontend::~SanoTtsPiperFrontend() = default; + +SanoTtsEncoded SanoTtsPiperFrontend::encode(const std::string & text) const { + auto [chunks, marks] = preserve_punctuation(text); + std::vector chunk_phonemes; + chunk_phonemes.reserve(chunks.size()); + for (const auto & chunk : chunks) { + chunk_phonemes.push_back(postprocess_espeak_line( + impl_->espeak.phonemize(chunk, kEspeakPhonemesIpaUnderscore))); + } + std::string restored = + restore_punctuation(std::move(chunk_phonemes), std::move(marks)); + // phonemizer leaves a trailing word separator that piper's own bridge + // does not emit at true end of input; the reference rstrips before NFD. + while (!restored.empty() && + std::isspace(static_cast(restored.back())) != 0) { + restored.pop_back(); + } + const std::string decomposed = nfd_decompose(restored); + + // piper.phoneme_ids.phonemes_to_ids: bos, pad, then (id, pad) per + // codepoint, then eos. The exporter validated the framing symbols + // ('_' -> 0, '^' -> 1, '$' -> 2) against the voice's own map. + SanoTtsEncoded out; + out.token_ids.push_back(1); // + out.token_ids.push_back(0); // + for (size_t i = 0; i < decomposed.size();) { + const size_t len = utf8_sequence_length(decomposed, i); + const std::string symbol = decomposed.substr(i, len); + i += len; + const auto found = id_map_.find(symbol); + if (found != id_map_.end()) { + out.token_ids.push_back(found->second); + out.token_ids.push_back(0); + } else { + // skipped with a note, exactly as piper skips unmapped phonemes + out.dropped.append(symbol); + } + } + if (out.token_ids.size() <= 2) { + throw std::runtime_error( + "sanoTTS phonemization produced no symbols in the voice's phoneme_id_map"); + } + out.token_ids.push_back(2); // + if (static_cast(out.token_ids.size()) > max_tokens_) { + throw SanoTtsTooLongError( + "sanoTTS phoneme sequence has " + std::to_string(out.token_ids.size()) + + " ids including framing; the duration model was trained for at most " + + std::to_string(max_tokens_) + "."); + } + return out; +} + } // namespace engine::models::sanotts diff --git a/src/community_models/sanotts/graph_common.h b/src/community_models/sanotts/graph_common.h new file mode 100644 index 000000000..dd0b5a1c5 --- /dev/null +++ b/src/community_models/sanotts/graph_common.h @@ -0,0 +1,296 @@ +#pragma once + +// Internal helpers shared by the two sanoTTS runtimes (the nano lineage in +// runtime.cpp and the piperlite lineage in piper_runtime.cpp). Both lineages +// share the same convolutional front-end structure -- residual conv blocks +// over channel-major [1, C, T] values -- and the same graph plumbing. + +#include "engine/community_models/sanotts/assets.h" +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" + +#include "ggml-alloc.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts::graph { + +struct GgmlContextDeleter { + void operator()(ggml_context * context) const noexcept { + if (context != nullptr) { + ggml_free(context); + } + } +}; + +inline core::TensorValue contiguous( + core::ModuleBuildContext & ctx, + const core::TensorValue & value) { + if (core::has_backend_addressable_layout(value.tensor)) { + return value; + } + return core::wrap_tensor( + ggml_cont(ctx.ggml, value.tensor), + value.shape, + value.type); +} + +inline core::TensorValue add( + core::ModuleBuildContext & ctx, + const core::TensorValue & lhs, + const core::TensorValue & rhs) { + return modules::AddModule().build(ctx, lhs, rhs); +} + +struct SanoTtsBackendWeights { + std::shared_ptr store; + std::unordered_map tensors; +}; + +inline const core::TensorValue & weight( + const SanoTtsBackendWeights & weights, + const std::string & name) { + const auto found = weights.tensors.find(name); + if (found == weights.tensors.end()) { + throw std::runtime_error("sanoTTS missing tensor: " + name); + } + return found->second; +} + +inline std::shared_ptr load_weights( + const std::shared_ptr & assets, + ggml_backend_t backend, + core::BackendType backend_type, + size_t expected_tensors, + size_t weight_arena_bytes) { + auto out = std::make_shared(); + out->store = std::make_shared( + backend, + backend_type, + "sanotts.weights", + weight_arena_bytes); + const auto metadata = assets->weights->tensors(); + if (metadata.size() != expected_tensors) { + throw std::runtime_error( + "sanoTTS expects exactly " + std::to_string(expected_tensors) + + " tensors for this config, found " + std::to_string(metadata.size())); + } + out->tensors.reserve(metadata.size()); + for (const auto & tensor : metadata) { + if (assets::ggml_type_for_tensor_dtype(tensor.dtype) != GGML_TYPE_F32) { + throw std::runtime_error( + "sanoTTS supports FP32 weights only: " + tensor.name); + } + out->tensors.emplace( + tensor.name, + out->store->load_tensor( + *assets->weights, + tensor.name, + assets::TensorStorageType::F32, + tensor.shape)); + } + out->store->upload(); + assets->weights->release_storage(); + return out; +} + +/** Conv1d over channel-major [1, C, T] with explicit padding and dilation. + * A kernel-1 conv lowers to a matmul. */ +inline core::TensorValue conv1d( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t out_channels, + int64_t kernel, + int padding, + int dilation = 1) { + const int64_t in_channels = input.shape.dims[1]; + const int64_t input_frames = input.shape.dims[2]; + const int64_t output_frames = + input_frames + 2 * padding - static_cast(dilation) * (kernel - 1); + const auto source = contiguous(ctx, input); + auto * input_2d = ggml_reshape_2d( + ctx.ggml, + source.tensor, + input_frames, + in_channels); + auto * kernel_tensor = weight(weights, prefix + ".weight").tensor; + ggml_tensor * output = nullptr; + if (kernel == 1 && padding == 0) { + auto * kernel_2d = ggml_reshape_2d( + ctx.ggml, + kernel_tensor, + in_channels, + out_channels); + auto * input_channels_first = ggml_cont( + ctx.ggml, + ggml_permute(ctx.ggml, input_2d, 1, 0, 2, 3)); + auto * output_channels_first = + ggml_mul_mat(ctx.ggml, kernel_2d, input_channels_first); + output = ggml_reshape_2d( + ctx.ggml, + ggml_cont( + ctx.ggml, + ggml_permute(ctx.ggml, output_channels_first, 1, 0, 2, 3)), + output_frames, + out_channels); + } else { + auto * kernel_3d = ggml_reshape_3d( + ctx.ggml, + kernel_tensor, + kernel, + in_channels, + out_channels); + auto * input_3d = ggml_reshape_3d( + ctx.ggml, + input_2d, + input_frames, + in_channels, + 1); + auto * output_3d = ggml_conv_1d( + ctx.ggml, + kernel_3d, + input_3d, + 1, + padding, + dilation); + output = ggml_reshape_2d( + ctx.ggml, + output_3d, + output_frames, + out_channels); + } + auto * bias = ggml_reshape_2d( + ctx.ggml, + weight(weights, prefix + ".bias").tensor, + 1, + out_channels); + output = ggml_add(ctx.ggml, output, bias); + return core::wrap_tensor( + ggml_reshape_3d(ctx.ggml, output, output_frames, out_channels, 1), + core::TensorShape::from_dims({1, out_channels, output_frames}), + GGML_TYPE_F32); +} + +/** x + scale * conv2(silu(conv1(x))) -- the front ends' ResidualConvBlock. + * `scale` is a learned one-element tensor, broadcast by ggml_mul. */ +inline core::TensorValue residual_block( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t hidden, + int64_t kernel) { + const int padding = static_cast(kernel / 2); + auto h = conv1d(ctx, weights, input, prefix + ".net.0", hidden, kernel, padding); + h = modules::SiluModule().build(ctx, h); + h = conv1d(ctx, weights, h, prefix + ".net.2", hidden, kernel, padding); + const auto h_source = contiguous(ctx, h); + const auto scaled_h = core::wrap_tensor( + ggml_mul(ctx.ggml, h_source.tensor, weight(weights, prefix + ".scale").tensor), + h.shape, + GGML_TYPE_F32); + return add(ctx, input, scaled_h); +} + +inline core::TensorValue embed_tokens( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & tokens, + const std::string & name, + int64_t vocab, + int64_t hidden) { + auto embedded = modules::EmbeddingModule({vocab, hidden}).build( + ctx, + tokens, + weight(weights, name)); + return modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, embedded); +} + +struct GraphResources { + ~GraphResources() { + core::free_backend_graph_plan(backend, plan); + core::release_backend_graph_resources(backend, graph); + if (allocator != nullptr) { + ggml_gallocr_free(allocator); + } + if (io_buffer != nullptr) { + ggml_backend_buffer_free(io_buffer); + } + } + + std::unique_ptr io_context; + std::unique_ptr graph_context; + ggml_backend_buffer_t io_buffer = nullptr; + ggml_gallocr_t allocator = nullptr; + ggml_backend_t backend = nullptr; + ggml_backend_graph_plan_t plan = nullptr; + ggml_cgraph * graph = nullptr; +}; + +inline void allocate_graph(GraphResources & resources) { + resources.io_buffer = + ggml_backend_alloc_ctx_tensors(resources.io_context.get(), resources.backend); + if (resources.io_buffer == nullptr) { + throw std::runtime_error("sanoTTS failed to allocate graph input buffer"); + } + resources.allocator = + ggml_gallocr_new(ggml_backend_get_default_buffer_type(resources.backend)); + if (resources.allocator == nullptr || + !ggml_gallocr_reserve(resources.allocator, resources.graph) || + !ggml_gallocr_alloc_graph(resources.allocator, resources.graph)) { + throw std::runtime_error("sanoTTS failed to allocate backend graph"); + } + core::validate_backend_graph_supported( + resources.backend, + resources.graph, + "sanoTTS"); + resources.plan = + core::create_backend_graph_plan_if_host(resources.backend, resources.graph); +} + +inline void compute_graph(GraphResources & resources, const char * label) { + const auto status = core::compute_backend_graph( + resources.backend, + resources.graph, + resources.plan, + label); + ggml_backend_synchronize(resources.backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error(std::string(label) + " graph compute failed"); + } +} + +/** torch.linspace(0, 1, n) with exact CPU-kernel float semantics: step in + * fp32, the first half filled as step*i, the second as fma(-step, n-1-i, 1). */ +inline void linspace01(float * dst, int64_t n) { + if (n <= 0) { + return; + } + if (n == 1) { + dst[0] = 0.0F; + return; + } + const auto step = 1.0F / static_cast(n - 1); + const int64_t half = n / 2; + for (int64_t i = 0; i < half; ++i) { + dst[i] = step * static_cast(i); + } + for (int64_t i = half; i < n; ++i) { + dst[i] = std::fma(-step, static_cast(n - 1 - i), 1.0F); + } +} + +} // namespace engine::models::sanotts::graph diff --git a/src/community_models/sanotts/piper_runtime.cpp b/src/community_models/sanotts/piper_runtime.cpp new file mode 100644 index 000000000..1e23c6d50 --- /dev/null +++ b/src/community_models/sanotts/piper_runtime.cpp @@ -0,0 +1,897 @@ +#include "engine/community_models/sanotts/piper_runtime.h" + +#include "graph_common.h" + +#include "engine/framework/debug/profiler.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/runtime/cache_slots.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::sanotts { +namespace { + +namespace core = engine::core; +namespace modules = engine::modules; +using namespace engine::models::sanotts::graph; // shared graph helpers + +constexpr size_t kIoArenaBytes = 8ULL * 1024ULL * 1024ULL; +constexpr size_t kGraphArenaBytes = 128ULL * 1024ULL * 1024ULL; +constexpr size_t kWeightArenaBytes = 32ULL * 1024ULL * 1024ULL; +constexpr float kLeakyReluSlope = 0.1F; + +// PiperResidualBank geometry, fixed by the training code: branch b uses +// kernel kBankKernels[b] with dilation kBankDilations1[b] on its first conv +// and kBankDilations2[b] on its second. +constexpr int64_t kBankKernels[3] = {3, 5, 7}; +constexpr int kBankDilations1[3] = {1, 2, 3}; +constexpr int kBankDilations2[3] = {2, 6, 12}; + +// The three ConvTranspose1d stages: stride and PyTorch padding. Kernel sizes +// are read from the weights themselves. +constexpr int kUpStrides[3] = {8, 8, 4}; +constexpr int64_t kUpPaddings[3] = {4, 4, 2}; + +/** Ids outside a component's trained vocab remap to schwa -- the same + * fallback the reference runtimes use for the shared-frontend/table + * vocab-size mismatch (e.g. duration vocab 127 vs table ids to 156). */ +constexpr int32_t kSchwaFallbackId = 59; + +size_t expected_piper_tensor_count(const SanoTtsPiperConfig & config) { + const auto duration = 5 + 5 * config.duration_depth; + const auto acoustic = + 7 + 5 * (config.acoustic_token_depth + config.acoustic_depth); + int64_t decoder = 4; // pre + post + for (const auto & branches : config.stage_branches) { + decoder += 2 + 4 * static_cast(branches.size()); + } + if (config.post_filter_channels > 0) { + decoder += 4 + 5 * config.post_filter_layers; + } + return static_cast(duration + acoustic + decoder); +} + +std::vector clamp_ids_to_vocab( + const std::vector & ids, + int64_t vocab_size) { + const int32_t fallback = + kSchwaFallbackId < vocab_size ? kSchwaFallbackId : 0; + std::vector out(ids.size()); + for (size_t i = 0; i < ids.size(); ++i) { + const int32_t id = ids[i]; + out[i] = (id < 0 || id >= vocab_size) ? fallback : id; + } + return out; +} + +int64_t kernel_of(const SanoTtsBackendWeights & weights, const std::string & prefix) { + return weight(weights, prefix + ".weight").shape.dims[2]; +} + +core::TensorValue leaky_relu( + core::ModuleBuildContext & ctx, + const core::TensorValue & input, + float slope) { + const auto source = contiguous(ctx, input); + return core::wrap_tensor( + ggml_leaky_relu(ctx.ggml, source.tensor, slope, false), + input.shape, + GGML_TYPE_F32); +} + +core::TensorValue scaled( + core::ModuleBuildContext & ctx, + const core::TensorValue & value, + float scale) { + const auto source = contiguous(ctx, value); + return core::wrap_tensor( + ggml_scale(ctx.ggml, source.tensor, scale), + value.shape, + GGML_TYPE_F32); +} + +/** 'same'-padded conv: pad = dilation * (kernel / 2). */ +core::TensorValue conv1d_same( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t out_channels, + int64_t kernel, + int dilation = 1) { + return conv1d( + ctx, + weights, + input, + prefix, + out_channels, + kernel, + dilation * static_cast(kernel / 2), + dilation); +} + +/** ConvTranspose1d producing stride * input_frames samples, PyTorch padding + * semantics -- the same construction inflect_v2 uses. */ +core::TensorValue conv_transpose1d( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + int64_t out_channels, + int64_t kernel, + int stride, + int64_t padding) { + const int64_t input_frames = input.shape.dims[2]; + const int64_t output_frames = input_frames * stride; + if (ctx.backend_type == core::BackendType::Cpu && padding > 0) { + const auto source = contiguous(ctx, input); + auto * input_2d = ggml_reshape_2d( + ctx.ggml, + source.tensor, + input_frames, + input.shape.dims[1]); + auto * output = ggml_conv_transpose_1d( + ctx.ggml, + weight(weights, prefix + ".weight").tensor, + input_2d, + stride, + 0, + 1); + const int64_t full_frames = (input_frames - 1) * stride + kernel; + output = ggml_reshape_2d(ctx.ggml, output, full_frames, out_channels); + output = ggml_view_2d( + ctx.ggml, + output, + output_frames, + out_channels, + ggml_row_size(output->type, full_frames), + ggml_row_size(output->type, padding)); + output = ggml_cont(ctx.ggml, output); + auto * bias = ggml_reshape_2d( + ctx.ggml, + weight(weights, prefix + ".bias").tensor, + 1, + out_channels); + output = ggml_add(ctx.ggml, output, bias); + return core::wrap_tensor( + ggml_reshape_3d(ctx.ggml, output, output_frames, out_channels, 1), + core::TensorShape::from_dims({1, out_channels, output_frames}), + GGML_TYPE_F32); + } + auto output = modules::ConvTranspose1dModule({ + input.shape.dims[1], + out_channels, + kernel, + stride, + 0, + 1, + true, + }).build( + ctx, + input, + { + weight(weights, prefix + ".weight"), + weight(weights, prefix + ".bias"), + }); + return modules::SliceModule({2, padding, output_frames}).build(ctx, output); +} + +/** PiperResidualBank: mean over active branches of + * y2 = conv2(lrelu(y1)) + y1, y1 = conv1(lrelu(x)) + x. */ +core::TensorValue residual_bank( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & input, + const std::string & prefix, + const std::vector & branches) { + core::TensorValue sum; + for (const int64_t branch : branches) { + const std::string branch_prefix = + prefix + ".blocks." + std::to_string(branch); + const int64_t kernel = kBankKernels[branch]; + auto value = leaky_relu(ctx, input, kLeakyReluSlope); + value = conv1d_same( + ctx, + weights, + value, + branch_prefix + ".conv1", + input.shape.dims[1], + kernel, + kBankDilations1[branch]); + const auto y1 = add(ctx, value, input); + value = leaky_relu(ctx, y1, kLeakyReluSlope); + value = conv1d_same( + ctx, + weights, + value, + branch_prefix + ".conv2", + input.shape.dims[1], + kernel, + kBankDilations2[branch]); + const auto y2 = add(ctx, value, y1); + sum = sum.valid() ? add(ctx, sum, y2) : y2; + } + return scaled(ctx, sum, 1.0F / static_cast(branches.size())); +} + +core::TensorValue post_filter( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const SanoTtsPiperConfig & config, + const core::TensorValue & audio) { + auto value = conv1d_same( + ctx, + weights, + audio, + "decoder.post_filter.in_conv", + config.post_filter_channels, + kernel_of(weights, "decoder.post_filter.in_conv")); + for (int64_t layer = 0; layer < config.post_filter_layers; ++layer) { + const std::string prefix = + "decoder.post_filter.units." + std::to_string(layer); + auto branch = leaky_relu(ctx, value, kLeakyReluSlope); + branch = conv1d_same( + ctx, + weights, + branch, + prefix + ".conv1", + config.post_filter_channels, + kernel_of(weights, prefix + ".conv1"), + static_cast(1 + layer)); + branch = leaky_relu(ctx, branch, kLeakyReluSlope); + branch = conv1d_same( + ctx, + weights, + branch, + prefix + ".conv2", + config.post_filter_channels, + kernel_of(weights, prefix + ".conv2")); + const auto branch_source = contiguous(ctx, branch); + branch = core::wrap_tensor( + ggml_mul( + ctx.ggml, + branch_source.tensor, + weight(weights, prefix + ".scale").tensor), + branch.shape, + GGML_TYPE_F32); + value = add(ctx, value, branch); + } + auto correction = conv1d_same( + ctx, + weights, + value, + "decoder.post_filter.out_conv", + 1, + kernel_of(weights, "decoder.post_filter.out_conv")); + correction = scaled(ctx, correction, static_cast(config.post_filter_scale)); + auto mixed = add(ctx, audio, correction); + const auto mixed_source = contiguous(ctx, mixed); + return core::wrap_tensor( + ggml_tanh(ctx.ggml, mixed_source.tensor), + mixed.shape, + GGML_TYPE_F32); +} + +// ---- graphs -------------------------------------------------------------- + +struct DurationGraph : GraphResources { + int64_t token_count = 0; + ggml_tensor * tokens = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * log_duration = nullptr; +}; + +std::unique_ptr build_duration_graph( + const SanoTtsBackendWeights & weights, + const SanoTtsPiperConfig & config, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t token_count) { + auto out = std::make_unique(); + out->backend = backend; + out->token_count = token_count; + out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); + out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create duration graph contexts"); + } + core::ModuleBuildContext io_ctx{ + out->io_context.get(), + "sanotts.piper.duration.io", + backend_type, + }; + core::ModuleBuildContext ctx{ + out->graph_context.get(), + "sanotts.piper.duration", + backend_type, + }; + auto tokens = core::make_tensor( + io_ctx, + GGML_TYPE_I32, + core::TensorShape::from_dims({1, token_count})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, 3, token_count})); + ggml_set_input(tokens.tensor); + ggml_set_input(feats.tensor); + + auto hidden = embed_tokens( + ctx, + weights, + tokens, + "duration.embedding.weight", + config.duration_vocab, + config.duration_hidden); + hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); + hidden = conv1d( + ctx, + weights, + hidden, + "duration.input_proj", + config.duration_hidden, + 1, + 0); + for (int64_t block = 0; block < config.duration_depth; ++block) { + hidden = residual_block( + ctx, + weights, + hidden, + "duration.blocks." + std::to_string(block), + config.duration_hidden, + config.duration_kernel); + } + auto log_duration = conv1d(ctx, weights, hidden, "duration.output", 1, 1, 0); + log_duration = contiguous(ctx, log_duration); + out->tokens = tokens.tensor; + out->feats = feats.tensor; + out->log_duration = log_duration.tensor; + ggml_set_output(out->log_duration); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->log_duration); + allocate_graph(*out); + return out; +} + +struct TokenGraph : GraphResources { + int64_t token_count = 0; + ggml_tensor * tokens = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * context = nullptr; +}; + +std::unique_ptr build_token_graph( + const SanoTtsBackendWeights & weights, + const SanoTtsPiperConfig & config, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t token_count) { + auto out = std::make_unique(); + out->backend = backend; + out->token_count = token_count; + out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); + out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create token graph contexts"); + } + core::ModuleBuildContext io_ctx{ + out->io_context.get(), + "sanotts.piper.token.io", + backend_type, + }; + core::ModuleBuildContext ctx{ + out->graph_context.get(), + "sanotts.piper.token", + backend_type, + }; + auto tokens = core::make_tensor( + io_ctx, + GGML_TYPE_I32, + core::TensorShape::from_dims({1, token_count})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, 2, token_count})); + ggml_set_input(tokens.tensor); + ggml_set_input(feats.tensor); + + auto hidden = embed_tokens( + ctx, + weights, + tokens, + "acoustic.embedding.weight", + config.acoustic_vocab, + config.acoustic_hidden); + hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); + hidden = conv1d( + ctx, + weights, + hidden, + "acoustic.token_input_proj", + config.acoustic_hidden, + 1, + 0); + for (int64_t block = 0; block < config.acoustic_token_depth; ++block) { + hidden = residual_block( + ctx, + weights, + hidden, + "acoustic.token_blocks." + std::to_string(block), + config.acoustic_hidden, + config.acoustic_kernel); + } + hidden = contiguous(ctx, hidden); + out->tokens = tokens.tensor; + out->feats = feats.tensor; + out->context = hidden.tensor; + ggml_set_output(out->context); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->context); + allocate_graph(*out); + return out; +} + +struct DecoderGraph : GraphResources { + int64_t frames = 0; + ggml_tensor * context = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * waveform = nullptr; +}; + +/** Frame-stage acoustic blocks -> latent -> 3-stage ConvTranspose decoder + * with dilated residual banks -> tanh waveform (plus kristin's post + * filter when the config carries one). */ +std::unique_ptr build_decoder_graph( + const SanoTtsBackendWeights & weights, + const SanoTtsPiperConfig & config, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t frames) { + auto out = std::make_unique(); + out->backend = backend; + out->frames = frames; + out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); + out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create decoder graph contexts"); + } + core::ModuleBuildContext io_ctx{ + out->io_context.get(), + "sanotts.piper.decoder.io", + backend_type, + }; + core::ModuleBuildContext ctx{ + out->graph_context.get(), + "sanotts.piper.decoder", + backend_type, + }; + auto context = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, 3, frames})); + ggml_set_input(context.tensor); + ggml_set_input(feats.tensor); + + auto hidden = modules::ConcatModule({1}).build(ctx, context, feats); + hidden = conv1d( + ctx, + weights, + hidden, + "acoustic.frame_input_proj", + config.acoustic_hidden, + 1, + 0); + for (int64_t block = 0; block < config.acoustic_depth; ++block) { + hidden = residual_block( + ctx, + weights, + hidden, + "acoustic.frame_blocks." + std::to_string(block), + config.acoustic_hidden, + config.acoustic_kernel); + } + auto latent = conv1d( + ctx, + weights, + hidden, + "acoustic.output", + config.acoustic_out_channels, + 1, + 0); + + auto value = conv1d_same( + ctx, + weights, + latent, + "decoder.pre", + config.channels[0], + kernel_of(weights, "decoder.pre")); + for (size_t stage = 0; stage < 3; ++stage) { + const std::string up_name = "decoder.up" + std::to_string(stage); + value = leaky_relu(ctx, value, kLeakyReluSlope); + value = conv_transpose1d( + ctx, + weights, + value, + up_name, + config.channels[stage + 1], + kernel_of(weights, up_name), + kUpStrides[stage], + kUpPaddings[stage]); + value = residual_bank( + ctx, + weights, + value, + "decoder.res" + std::to_string(stage) + ".0", + config.stage_branches[stage]); + } + value = leaky_relu(ctx, value, 0.01F); + auto audio = conv1d_same( + ctx, + weights, + value, + "decoder.post", + 1, + kernel_of(weights, "decoder.post")); + { + const auto audio_source = contiguous(ctx, audio); + audio = core::wrap_tensor( + ggml_tanh(ctx.ggml, audio_source.tensor), + audio.shape, + GGML_TYPE_F32); + } + if (config.post_filter_channels > 0) { + audio = post_filter(ctx, weights, config, audio); + } + audio = contiguous(ctx, audio); + out->context = context.tensor; + out->feats = feats.tensor; + out->waveform = audio.tensor; + ggml_set_output(out->waveform); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->waveform); + allocate_graph(*out); + return out; +} + +} // namespace + +struct SanoTtsPiperRuntime::State { + struct BackendOwner { + ggml_backend_t value = nullptr; + ~BackendOwner() { + if (value != nullptr) { + ggml_backend_free(value); + } + } + }; + + State( + std::shared_ptr assets_in, + core::BackendConfig backend_config) + : assets(std::move(assets_in)), + threads(std::max(1, backend_config.threads)) { + if (assets == nullptr) { + throw std::runtime_error("sanoTTS piper runtime requires assets"); + } + backend_config.threads = threads; + backend.value = core::init_backend(backend_config); + backend_type = core::backend_type(backend.value); + core::set_backend_threads(backend.value, threads); + weights = load_weights( + assets, + backend.value, + backend_type, + expected_piper_tensor_count(assets->piper), + kWeightArenaBytes); + if (backend_type == core::BackendType::Cuda) { + // Durations round to integers and gate the whole frame layout; + // keep the tiny duration model on the host, as the nano runtime + // and inflect_v2 do. + core::BackendConfig duration_config{ + core::BackendType::Cpu, + 0, + threads, + }; + duration_backend.value = core::init_backend(duration_config); + core::set_backend_threads(duration_backend.value, threads); + duration_weights = load_weights( + assets, + duration_backend.value, + core::BackendType::Cpu, + expected_piper_tensor_count(assets->piper), + kWeightArenaBytes); + } + } + + DurationGraph & duration_graph(int64_t token_count) { + if (auto * found = duration_graphs.find(token_count)) { + engine::debug::trace_log_scalar("sanotts.duration_graph.cache_hit", true); + return **found; + } + engine::debug::trace_log_scalar("sanotts.duration_graph.cache_hit", false); + const auto backend_value = + duration_backend.value != nullptr ? duration_backend.value : backend.value; + const auto graph_backend_type = + duration_backend.value != nullptr ? core::BackendType::Cpu : backend_type; + const auto & selected_weights = + duration_weights != nullptr ? duration_weights : weights; + duration_graphs.put( + token_count, + build_duration_graph( + *selected_weights, + assets->piper, + backend_value, + graph_backend_type, + token_count)); + auto * created = duration_graphs.find(token_count); + if (created == nullptr) { + throw std::runtime_error("sanoTTS duration graph cache insert failed"); + } + return **created; + } + + TokenGraph & token_graph(int64_t token_count) { + if (auto * found = token_graphs.find(token_count)) { + engine::debug::trace_log_scalar("sanotts.token_graph.cache_hit", true); + return **found; + } + engine::debug::trace_log_scalar("sanotts.token_graph.cache_hit", false); + token_graphs.put( + token_count, + build_token_graph( + *weights, + assets->piper, + backend.value, + backend_type, + token_count)); + auto * created = token_graphs.find(token_count); + if (created == nullptr) { + throw std::runtime_error("sanoTTS token graph cache insert failed"); + } + return **created; + } + + DecoderGraph & decoder_graph(int64_t frames) { + if (auto * found = decoder_graphs.find(frames)) { + engine::debug::trace_log_scalar("sanotts.decoder_graph.cache_hit", true); + return **found; + } + engine::debug::trace_log_scalar("sanotts.decoder_graph.cache_hit", false); + decoder_graphs.put( + frames, + build_decoder_graph( + *weights, + assets->piper, + backend.value, + backend_type, + frames)); + auto * created = decoder_graphs.find(frames); + if (created == nullptr) { + throw std::runtime_error("sanoTTS decoder graph cache insert failed"); + } + return **created; + } + + std::shared_ptr assets; + int threads = 1; + core::BackendType backend_type = core::BackendType::Cpu; + BackendOwner backend; + std::shared_ptr weights; + BackendOwner duration_backend; + std::shared_ptr duration_weights; + runtime::CacheSlots> duration_graphs{4}; + runtime::CacheSlots> token_graphs{4}; + runtime::CacheSlots> decoder_graphs{2}; +}; + +SanoTtsPiperRuntime::SanoTtsPiperRuntime( + std::shared_ptr assets, + core::BackendConfig backend_config) + : state_(std::make_unique(std::move(assets), backend_config)) {} + +SanoTtsPiperRuntime::~SanoTtsPiperRuntime() = default; + +runtime::AudioBuffer SanoTtsPiperRuntime::synthesize( + const std::vector & token_ids, + const SanoTtsPiperGenerationOptions & options) { + const auto & config = state_->assets->piper; + const auto token_count = static_cast(token_ids.size()); + if (token_count <= 0) { + throw std::runtime_error("sanoTTS requires at least one phoneme token"); + } + const auto total_start = std::chrono::steady_clock::now(); + + // -- durations --------------------------------------------------------- + const auto duration_start = std::chrono::steady_clock::now(); + auto & duration = state_->duration_graph(token_count); + core::write_tensor_i32( + core::wrap_tensor( + duration.tokens, + core::TensorShape::from_dims({1, token_count}), + GGML_TYPE_I32), + clamp_ids_to_vocab(token_ids, config.duration_vocab)); + { + // [positions, length_hint, valid=1] rows; hints computed in double + // and cast, matching the reference implementation. + std::vector feats(static_cast(3 * token_count)); + linspace01(feats.data(), token_count); + const auto length_hint = static_cast( + std::log1p(static_cast(token_count)) / + std::log1p(static_cast(config.duration_max_tokens))); + std::fill_n(feats.begin() + token_count, token_count, length_hint); + std::fill_n(feats.begin() + 2 * token_count, token_count, 1.0F); + core::write_tensor_f32( + core::wrap_tensor( + duration.feats, + core::TensorShape::from_dims({1, 3, token_count}), + GGML_TYPE_F32), + feats); + } + compute_graph(duration, "sanoTTS piper duration"); + const auto log_duration = core::read_tensor_f32(duration.log_duration); + if (static_cast(log_duration.size()) != token_count) { + throw std::runtime_error("sanoTTS duration graph returned invalid output"); + } + // exp -> clamp_min(1) -> *scale -> round (ties to even) -> clamp, in + // double, exactly as the reference computes it. + const double scale = config.duration_length_scale * + static_cast(options.speaking_rate); + std::vector durations(static_cast(token_count)); + int64_t frames = 0; + for (int64_t token = 0; token < token_count; ++token) { + double value = std::exp( + static_cast(log_duration[static_cast(token)])); + if (!std::isfinite(value)) { + throw std::runtime_error("sanoTTS duration predictor produced a non-finite duration"); + } + if (value < 1.0) { + value = 1.0; + } + value = std::rint(value * scale); + if (value < 1.0) { + value = 1.0; + } + if (value > static_cast(config.duration_max_frames)) { + value = static_cast(config.duration_max_frames); + } + durations[static_cast(token)] = static_cast(value); + frames += durations[static_cast(token)]; + } + const int64_t max_frames = config.duration_max_tokens * config.duration_max_frames; + if (frames < 1 || frames > max_frames) { + throw std::runtime_error( + "sanoTTS expanded to " + std::to_string(frames) + + " frames, outside the supported range"); + } + engine::debug::timing_log_scalar( + "sanotts.duration_ms", + engine::debug::elapsed_ms(duration_start)); + + // -- token-stage acoustic context -------------------------------------- + const auto acoustic_start = std::chrono::steady_clock::now(); + auto & token_graph = state_->token_graph(token_count); + core::write_tensor_i32( + core::wrap_tensor( + token_graph.tokens, + core::TensorShape::from_dims({1, token_count}), + GGML_TYPE_I32), + clamp_ids_to_vocab(token_ids, config.acoustic_vocab)); + { + std::vector feats(static_cast(2 * token_count)); + linspace01(feats.data(), token_count); + double max_duration = 1.0; + for (int64_t token = 0; token < token_count; ++token) { + max_duration = std::max( + max_duration, + static_cast(durations[static_cast(token)])); + } + const double log_max_duration = std::log1p(max_duration); + for (int64_t token = 0; token < token_count; ++token) { + feats[static_cast(token_count + token)] = static_cast( + std::log1p(static_cast(durations[static_cast(token)])) / + log_max_duration); + } + core::write_tensor_f32( + core::wrap_tensor( + token_graph.feats, + core::TensorShape::from_dims({1, 2, token_count}), + GGML_TYPE_F32), + feats); + } + compute_graph(token_graph, "sanoTTS piper token context"); + const auto token_context = core::read_tensor_f32(token_graph.context); + if (static_cast(token_context.size()) != + config.acoustic_hidden * token_count) { + throw std::runtime_error("sanoTTS token graph returned invalid output"); + } + + // -- expand to frames -------------------------------------------------- + std::vector expanded(static_cast(config.acoustic_hidden * frames)); + for (int64_t channel = 0; channel < config.acoustic_hidden; ++channel) { + const float * row = token_context.data() + channel * token_count; + float * out_row = expanded.data() + channel * frames; + int64_t at = 0; + for (int64_t token = 0; token < token_count; ++token) { + const float value = row[token]; + for (int64_t j = 0; j < durations[static_cast(token)]; ++j) { + out_row[at++] = value; + } + } + } + std::vector frame_feats(static_cast(3 * frames)); + linspace01(frame_feats.data(), frames); + { + const int64_t denominator = token_count > 1 ? token_count - 1 : 1; + float * token_pos = frame_feats.data() + frames; + float * duration_pos = frame_feats.data() + 2 * frames; + int64_t at = 0; + for (int64_t token = 0; token < token_count; ++token) { + const int64_t count = durations[static_cast(token)]; + const auto position = static_cast( + static_cast(token) / static_cast(denominator)); + for (int64_t j = 0; j < count; ++j) { + token_pos[at] = position; + duration_pos[at] = count == 1 + ? 0.0F + : static_cast( + static_cast(j) / static_cast(count - 1)); + ++at; + } + } + } + engine::debug::timing_log_scalar( + "sanotts.acoustic_ms", + engine::debug::elapsed_ms(acoustic_start)); + + // -- frame stage + decoder --------------------------------------------- + const auto decoder_start = std::chrono::steady_clock::now(); + auto & decoder = state_->decoder_graph(frames); + core::write_tensor_f32( + core::wrap_tensor( + decoder.context, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames}), + GGML_TYPE_F32), + expanded); + core::write_tensor_f32( + core::wrap_tensor( + decoder.feats, + core::TensorShape::from_dims({1, 3, frames}), + GGML_TYPE_F32), + frame_feats); + compute_graph(decoder, "sanoTTS piper decoder"); + runtime::AudioBuffer out; + out.sample_rate = static_cast(config.sample_rate); + out.channels = 1; + out.samples = core::read_tensor_f32(decoder.waveform); + const auto expected_samples = static_cast(frames) * 256U; + if (out.samples.size() != expected_samples) { + throw std::runtime_error("sanoTTS decoder returned an unexpected sample count"); + } + engine::debug::timing_log_scalar( + "sanotts.decoder_ms", + engine::debug::elapsed_ms(decoder_start)); + engine::debug::trace_log_scalar("sanotts.token_count", token_count); + engine::debug::trace_log_scalar("sanotts.frames", frames); + engine::debug::trace_log_scalar( + "sanotts.output_samples", + static_cast(out.samples.size())); + engine::debug::timing_log_scalar( + "session.wall_ms", + engine::debug::elapsed_ms(total_start)); + return out; +} + +} // namespace engine::models::sanotts diff --git a/src/community_models/sanotts/runtime.cpp b/src/community_models/sanotts/runtime.cpp index 2bbf45611..47f2c7ddc 100644 --- a/src/community_models/sanotts/runtime.cpp +++ b/src/community_models/sanotts/runtime.cpp @@ -1,5 +1,7 @@ #include "engine/community_models/sanotts/runtime.h" +#include "graph_common.h" + #include "engine/framework/assets/tensor_source.h" #include "engine/framework/audio/istft_graph.h" #include "engine/framework/core/backend.h" @@ -34,6 +36,7 @@ namespace { namespace core = engine::core; namespace modules = engine::modules; +using namespace engine::models::sanotts::graph; // shared graph helpers constexpr size_t kIoArenaBytes = 8ULL * 1024ULL * 1024ULL; constexpr size_t kGraphArenaBytes = 128ULL * 1024ULL * 1024ULL; @@ -58,200 +61,12 @@ constexpr float kLayerNormEps = 1.0e-6F; constexpr float kDcBlockPole = 0.9973F; constexpr double kPi = 3.14159265358979323846; -struct GgmlContextDeleter { - void operator()(ggml_context * context) const noexcept { - if (context != nullptr) { - ggml_free(context); - } - } -}; - -core::TensorValue contiguous( - core::ModuleBuildContext & ctx, - const core::TensorValue & value) { - if (core::has_backend_addressable_layout(value.tensor)) { - return value; - } - return core::wrap_tensor( - ggml_cont(ctx.ggml, value.tensor), - value.shape, - value.type); -} - -core::TensorValue add( - core::ModuleBuildContext & ctx, - const core::TensorValue & lhs, - const core::TensorValue & rhs) { - return modules::AddModule().build(ctx, lhs, rhs); -} - -struct SanoTtsBackendWeights { - std::shared_ptr store; - std::unordered_map tensors; -}; - -const core::TensorValue & weight( - const SanoTtsBackendWeights & weights, - const std::string & name) { - const auto found = weights.tensors.find(name); - if (found == weights.tensors.end()) { - throw std::runtime_error("sanoTTS missing tensor: " + name); - } - return found->second; -} - -std::shared_ptr load_weights( - const std::shared_ptr & assets, - ggml_backend_t backend, - core::BackendType backend_type) { - auto out = std::make_shared(); - out->store = std::make_shared( - backend, - backend_type, - "sanotts.weights", - kWeightArenaBytes); - const auto metadata = assets->weights->tensors(); - const size_t expected = expected_tensor_count(assets->config); - if (metadata.size() != expected) { - throw std::runtime_error( - "sanoTTS expects exactly " + std::to_string(expected) + - " tensors for this config, found " + std::to_string(metadata.size())); - } - out->tensors.reserve(metadata.size()); - for (const auto & tensor : metadata) { - if (assets::ggml_type_for_tensor_dtype(tensor.dtype) != GGML_TYPE_F32) { - throw std::runtime_error( - "sanoTTS supports FP32 weights only: " + tensor.name); - } - out->tensors.emplace( - tensor.name, - out->store->load_tensor( - *assets->weights, - tensor.name, - assets::TensorStorageType::F32, - tensor.shape)); - } - out->store->upload(); - assets->weights->release_storage(); - return out; -} - // ---- graph builders ------------------------------------------------------ // // Values are carried channel-major as [1, C, T] (ggml ne0 = T) through the // convolutional front ends, and channel-last as [1, T, C] through the // ConvNeXt decoder blocks, matching the PyTorch modules they reproduce. -core::TensorValue conv1d( - core::ModuleBuildContext & ctx, - const SanoTtsBackendWeights & weights, - const core::TensorValue & input, - const std::string & prefix, - int64_t out_channels, - int64_t kernel, - int padding) { - const int64_t in_channels = input.shape.dims[1]; - const int64_t input_frames = input.shape.dims[2]; - const int64_t output_frames = input_frames + 2 * padding - (kernel - 1); - const auto source = contiguous(ctx, input); - auto * input_2d = ggml_reshape_2d( - ctx.ggml, - source.tensor, - input_frames, - in_channels); - auto * kernel_tensor = weight(weights, prefix + ".weight").tensor; - ggml_tensor * output = nullptr; - if (kernel == 1 && padding == 0) { - auto * kernel_2d = ggml_reshape_2d( - ctx.ggml, - kernel_tensor, - in_channels, - out_channels); - auto * input_channels_first = ggml_cont( - ctx.ggml, - ggml_permute(ctx.ggml, input_2d, 1, 0, 2, 3)); - auto * output_channels_first = - ggml_mul_mat(ctx.ggml, kernel_2d, input_channels_first); - output = ggml_reshape_2d( - ctx.ggml, - ggml_cont( - ctx.ggml, - ggml_permute(ctx.ggml, output_channels_first, 1, 0, 2, 3)), - output_frames, - out_channels); - } else { - auto * kernel_3d = ggml_reshape_3d( - ctx.ggml, - kernel_tensor, - kernel, - in_channels, - out_channels); - auto * input_3d = ggml_reshape_3d( - ctx.ggml, - input_2d, - input_frames, - in_channels, - 1); - auto * output_3d = ggml_conv_1d( - ctx.ggml, - kernel_3d, - input_3d, - 1, - padding, - 1); - output = ggml_reshape_2d( - ctx.ggml, - output_3d, - output_frames, - out_channels); - } - auto * bias = ggml_reshape_2d( - ctx.ggml, - weight(weights, prefix + ".bias").tensor, - 1, - out_channels); - output = ggml_add(ctx.ggml, output, bias); - return core::wrap_tensor( - ggml_reshape_3d(ctx.ggml, output, output_frames, out_channels, 1), - core::TensorShape::from_dims({1, out_channels, output_frames}), - GGML_TYPE_F32); -} - -/** x + scale * conv2(silu(conv1(x))) -- the front ends' ResidualConvBlock. - * `scale` is a learned one-element tensor, broadcast by ggml_mul. */ -core::TensorValue residual_block( - core::ModuleBuildContext & ctx, - const SanoTtsBackendWeights & weights, - const core::TensorValue & input, - const std::string & prefix, - int64_t hidden, - int64_t kernel) { - const int padding = static_cast(kernel / 2); - auto h = conv1d(ctx, weights, input, prefix + ".net.0", hidden, kernel, padding); - h = modules::SiluModule().build(ctx, h); - h = conv1d(ctx, weights, h, prefix + ".net.2", hidden, kernel, padding); - const auto h_source = contiguous(ctx, h); - const auto scaled_h = core::wrap_tensor( - ggml_mul(ctx.ggml, h_source.tensor, weight(weights, prefix + ".scale").tensor), - h.shape, - GGML_TYPE_F32); - return add(ctx, input, scaled_h); -} - -core::TensorValue embed_tokens( - core::ModuleBuildContext & ctx, - const SanoTtsBackendWeights & weights, - const core::TensorValue & tokens, - const std::string & name, - int64_t vocab, - int64_t hidden) { - auto embedded = modules::EmbeddingModule({vocab, hidden}).build( - ctx, - tokens, - weight(weights, name)); - return modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, embedded); -} - core::TensorValue channel_last_layer_norm( core::ModuleBuildContext & ctx, const SanoTtsBackendWeights & weights, @@ -267,60 +82,6 @@ core::TensorValue channel_last_layer_norm( }); } -struct GraphResources { - ~GraphResources() { - core::free_backend_graph_plan(backend, plan); - core::release_backend_graph_resources(backend, graph); - if (allocator != nullptr) { - ggml_gallocr_free(allocator); - } - if (io_buffer != nullptr) { - ggml_backend_buffer_free(io_buffer); - } - } - - std::unique_ptr io_context; - std::unique_ptr graph_context; - ggml_backend_buffer_t io_buffer = nullptr; - ggml_gallocr_t allocator = nullptr; - ggml_backend_t backend = nullptr; - ggml_backend_graph_plan_t plan = nullptr; - ggml_cgraph * graph = nullptr; -}; - -void allocate_graph(GraphResources & resources) { - resources.io_buffer = - ggml_backend_alloc_ctx_tensors(resources.io_context.get(), resources.backend); - if (resources.io_buffer == nullptr) { - throw std::runtime_error("sanoTTS failed to allocate graph input buffer"); - } - resources.allocator = - ggml_gallocr_new(ggml_backend_get_default_buffer_type(resources.backend)); - if (resources.allocator == nullptr || - !ggml_gallocr_reserve(resources.allocator, resources.graph) || - !ggml_gallocr_alloc_graph(resources.allocator, resources.graph)) { - throw std::runtime_error("sanoTTS failed to allocate backend graph"); - } - core::validate_backend_graph_supported( - resources.backend, - resources.graph, - "sanoTTS"); - resources.plan = - core::create_backend_graph_plan_if_host(resources.backend, resources.graph); -} - -void compute_graph(GraphResources & resources, const char * label) { - const auto status = core::compute_backend_graph( - resources.backend, - resources.graph, - resources.plan, - label); - ggml_backend_synchronize(resources.backend); - if (status != GGML_STATUS_SUCCESS) { - throw std::runtime_error(std::string(label) + " graph compute failed"); - } -} - // ---- ATen-compatible noise ---------------------------------------------- // // The decoder is noise-fed, so a rendering is only reproducible if the noise @@ -426,28 +187,6 @@ std::vector seeded_noise(uint64_t seed, int64_t channels, int64_t frames) return out; } -// ---- host float semantics shared with the reference front end ------------ - -/** torch.linspace(0, 1, n) with exact CPU-kernel float semantics: step in - * fp32, the first half filled as step*i, the second as fma(-step, n-1-i, 1). */ -void linspace01(float * dst, int64_t n) { - if (n <= 0) { - return; - } - if (n == 1) { - dst[0] = 0.0F; - return; - } - const auto step = 1.0F / static_cast(n - 1); - const int64_t half = n / 2; - for (int64_t i = 0; i < half; ++i) { - dst[i] = step * static_cast(i); - } - for (int64_t i = half; i < n; ++i) { - dst[i] = std::fma(-step, static_cast(n - 1 - i), 1.0F); - } -} - // ---- graphs -------------------------------------------------------------- struct DurationGraph : GraphResources { @@ -926,7 +665,12 @@ struct SanoTtsNativeRuntime::State { backend.value = core::init_backend(backend_config); backend_type = core::backend_type(backend.value); core::set_backend_threads(backend.value, threads); - weights = load_weights(assets, backend.value, backend_type); + weights = load_weights( + assets, + backend.value, + backend_type, + expected_tensor_count(assets->config), + kWeightArenaBytes); if (backend_type == core::BackendType::Cuda) { // Durations round to integers and gate the whole frame layout, so // small TF32 differences would move frame counts. Keep the tiny @@ -941,7 +685,9 @@ struct SanoTtsNativeRuntime::State { duration_weights = load_weights( assets, duration_backend.value, - core::BackendType::Cpu); + core::BackendType::Cpu, + expected_tensor_count(assets->config), + kWeightArenaBytes); } } diff --git a/src/community_models/sanotts/session.cpp b/src/community_models/sanotts/session.cpp index cc4b36c1d..68e50d23f 100644 --- a/src/community_models/sanotts/session.cpp +++ b/src/community_models/sanotts/session.cpp @@ -128,11 +128,21 @@ SanoTtsSession::SanoTtsSession( throw std::runtime_error("sanoTTS only supports offline TTS"); } validate_session_options(options, *contract_); - frontend_ = std::make_unique( - session_path(options, "sanotts.espeak_library_path"), - session_path(options, "sanotts.espeak_data_path"), - assets_->config.duration_max_tokens); - runtime_ = std::make_unique(assets_, options.backend); + if (assets_->graph == SanoTtsGraph::Nano) { + frontend_ = std::make_unique( + session_path(options, "sanotts.espeak_library_path"), + session_path(options, "sanotts.espeak_data_path"), + assets_->config.duration_max_tokens); + runtime_ = std::make_unique(assets_, options.backend); + } else { + piper_frontend_ = std::make_unique( + session_path(options, "sanotts.espeak_library_path"), + session_path(options, "sanotts.espeak_data_path"), + assets_->piper.espeak_voice, + assets_->piper.phoneme_id_map, + assets_->piper.duration_max_tokens); + piper_runtime_ = std::make_unique(assets_, options.backend); + } } SanoTtsSession::~SanoTtsSession() = default; @@ -154,11 +164,15 @@ runtime::TaskResult SanoTtsSession::run(const runtime::TaskRequest & request) { if (request.audio_input.has_value()) { throw std::runtime_error("sanoTTS does not accept audio input"); } + const std::string voice_language = + assets_->graph == SanoTtsGraph::Nano ? "en" : assets_->piper.language; if (!request.text_input->language.empty() && - request.text_input->language != "en" && - request.text_input->language != "en-us" && - request.text_input->language != "English") { - throw std::runtime_error("sanoTTS supports English only"); + request.text_input->language != voice_language && + !(voice_language == "en" && + (request.text_input->language == "en-us" || + request.text_input->language == "English"))) { + throw std::runtime_error( + "this sanoTTS voice supports language '" + voice_language + "' only"); } validate_chunk_mode(request); const int64_t chunk_size = chunk_size_from_request(request); @@ -177,7 +191,9 @@ runtime::TaskResult SanoTtsSession::run(const runtime::TaskRequest & request) { [&](const std::string & chunk, int depth) { SanoTtsEncoded encoded; try { - encoded = frontend_->encode(chunk); + encoded = frontend_ != nullptr + ? frontend_->encode(chunk) + : piper_frontend_->encode(chunk); } catch (const SanoTtsTooLongError &) { const size_t middle = chunk.size() / 2; size_t split = std::string::npos; @@ -202,18 +218,27 @@ runtime::TaskResult SanoTtsSession::run(const runtime::TaskRequest & request) { render_chunk(right, depth + 1); return; } - SanoTtsGenerationOptions chunk_options; - chunk_options.speaking_rate = request_options.speaking_rate; - // The default seed is derived from the chunk's own text -- the - // reference implementations' sha256(text)[:8] convention -- so a - // given sentence renders identically wherever it appears. An - // explicit seed advances per chunk instead, so long-form noise - // is not reused across chunks. - chunk_options.seed = request_options.seed_from_text - ? sanotts_text_seed(chunk) - : request_options.seed + rendered_chunks; + runtime::AudioBuffer audio; + if (runtime_ != nullptr) { + SanoTtsGenerationOptions chunk_options; + chunk_options.speaking_rate = request_options.speaking_rate; + // The default seed is derived from the chunk's own text -- + // the reference implementations' sha256(text)[:8] convention + // -- so a given sentence renders identically wherever it + // appears. An explicit seed advances per chunk instead, so + // long-form noise is not reused across chunks. + chunk_options.seed = request_options.seed_from_text + ? sanotts_text_seed(chunk) + : request_options.seed + rendered_chunks; + audio = runtime_->synthesize(encoded.token_ids, chunk_options); + } else { + // The piperlite decoder is deterministic; the seed option is + // documented as ignored for these voices. + SanoTtsPiperGenerationOptions chunk_options; + chunk_options.speaking_rate = request_options.speaking_rate; + audio = piper_runtime_->synthesize(encoded.token_ids, chunk_options); + } ++rendered_chunks; - auto audio = runtime_->synthesize(encoded.token_ids, chunk_options); runtime::append_audio_buffer(merged, audio); }; for (size_t index = 0; index < chunks.size(); ++index) { From ca2290932c34b765336844c215693754cd47167a Mon Sep 17 00:00:00 2001 From: ashish Date: Fri, 4 Sep 2026 17:32:07 +0545 Subject: [PATCH 7/7] sanotts: deduplicate the two runtimes through graph_common The duration and token stages run under identical tensor names in both lineages, so their four graph builders collapse into one shared build_front_graph over a small stage spec; the frame-level acoustic stage both decoders open with becomes acoustic_frame_stage. BackendState now carries the backend/weights/CUDA-duration-mirror boilerplate once, a cached_graph template replaces the six cache accessors, and the host-side feature builders and duration rounding are shared (rounding unified to double, which is what the numpy references both runtimes are gated against actually compute). All seven voices re-verified after the refactor: correlations unchanged (>= 0.99999996) at identical sample counts; unit test and the shared long-form case pass. Net -363 lines. Claude-Session: https://claude.ai/code/session_01P1iL37FdfJkGxdGrpjH1we --- src/community_models/sanotts/graph_common.h | 339 ++++++++++ .../sanotts/piper_runtime.cpp | 539 +++------------- src/community_models/sanotts/runtime.cpp | 577 ++++-------------- 3 files changed, 546 insertions(+), 909 deletions(-) diff --git a/src/community_models/sanotts/graph_common.h b/src/community_models/sanotts/graph_common.h index dd0b5a1c5..4dc0e24d1 100644 --- a/src/community_models/sanotts/graph_common.h +++ b/src/community_models/sanotts/graph_common.h @@ -13,9 +13,12 @@ #include "engine/framework/modules/lookup_modules.h" #include "engine/framework/modules/primitive_modules.h" #include "engine/framework/modules/structural_modules.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/runtime/cache_slots.h" #include "ggml-alloc.h" +#include #include #include #include @@ -293,4 +296,340 @@ inline void linspace01(float * dst, int64_t n) { } } + +// ---- shared front-end graphs --------------------------------------------- +// +// Both lineages run the same two token-level stages -- embed + positional +// features + input projection + residual blocks (+ optional 1x1 output conv) +// -- under the same tensor names; only the dimensions differ per package. + +struct FrontStageSpec { + const char * embedding = nullptr; // "duration.embedding.weight" + const char * input_proj = nullptr; // "duration.input_proj" + const char * block_prefix = nullptr; // "duration.blocks." + const char * output_conv = nullptr; // "duration.output", or nullptr + const char * label = nullptr; // graph label for contexts/tracing + int64_t feat_rows = 3; + int64_t vocab = 0; + int64_t hidden = 0; + int64_t depth = 0; + int64_t kernel = 0; + int64_t out_channels = 1; // used when output_conv is set +}; + +inline FrontStageSpec duration_stage_spec(int64_t vocab, int64_t hidden, int64_t depth, int64_t kernel) { + return {"duration.embedding.weight", "duration.input_proj", "duration.blocks.", + "duration.output", "sanotts.duration", 3, vocab, hidden, depth, kernel, 1}; +} + +inline FrontStageSpec token_stage_spec(int64_t vocab, int64_t hidden, int64_t depth, int64_t kernel) { + return {"acoustic.embedding.weight", "acoustic.token_input_proj", "acoustic.token_blocks.", + nullptr, "sanotts.token", 2, vocab, hidden, depth, kernel, 1}; +} + +struct FrontGraph : GraphResources { + int64_t token_count = 0; + ggml_tensor * tokens = nullptr; + ggml_tensor * feats = nullptr; + ggml_tensor * output = nullptr; // log-durations or token context +}; + +inline std::unique_ptr build_front_graph( + const SanoTtsBackendWeights & weights, + const FrontStageSpec & spec, + ggml_backend_t backend, + core::BackendType backend_type, + int64_t token_count, + size_t io_arena_bytes, + size_t graph_arena_bytes) { + auto out = std::make_unique(); + out->backend = backend; + out->token_count = token_count; + out->io_context.reset(ggml_init({io_arena_bytes, nullptr, true})); + out->graph_context.reset(ggml_init({graph_arena_bytes, nullptr, true})); + if (out->io_context == nullptr || out->graph_context == nullptr) { + throw std::runtime_error("sanoTTS failed to create graph contexts"); + } + core::ModuleBuildContext io_ctx{out->io_context.get(), spec.label, backend_type}; + core::ModuleBuildContext ctx{out->graph_context.get(), spec.label, backend_type}; + auto tokens = core::make_tensor( + io_ctx, + GGML_TYPE_I32, + core::TensorShape::from_dims({1, token_count})); + auto feats = core::make_tensor( + io_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, spec.feat_rows, token_count})); + ggml_set_input(tokens.tensor); + ggml_set_input(feats.tensor); + + auto hidden = embed_tokens(ctx, weights, tokens, spec.embedding, spec.vocab, spec.hidden); + hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); + hidden = conv1d(ctx, weights, hidden, spec.input_proj, spec.hidden, 1, 0); + for (int64_t block = 0; block < spec.depth; ++block) { + hidden = residual_block( + ctx, + weights, + hidden, + spec.block_prefix + std::to_string(block), + spec.hidden, + spec.kernel); + } + if (spec.output_conv != nullptr) { + hidden = conv1d(ctx, weights, hidden, spec.output_conv, spec.out_channels, 1, 0); + } + hidden = contiguous(ctx, hidden); + out->tokens = tokens.tensor; + out->feats = feats.tensor; + out->output = hidden.tensor; + ggml_set_output(out->output); + out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); + ggml_build_forward_expand(out->graph, out->output); + allocate_graph(*out); + return out; +} + +/** The frame-level acoustic stage both decoder graphs open with: expanded + * token context + [frame_pos, token_pos, duration_pos] -> residual blocks + * -> 1x1 output conv (mel-100 for nano, the 192-ch latent for piperlite). */ +inline core::TensorValue acoustic_frame_stage( + core::ModuleBuildContext & ctx, + const SanoTtsBackendWeights & weights, + const core::TensorValue & context, + const core::TensorValue & feats, + int64_t hidden, + int64_t depth, + int64_t kernel, + int64_t out_channels) { + auto value = modules::ConcatModule({1}).build(ctx, context, feats); + value = conv1d(ctx, weights, value, "acoustic.frame_input_proj", hidden, 1, 0); + for (int64_t block = 0; block < depth; ++block) { + value = residual_block( + ctx, + weights, + value, + "acoustic.frame_blocks." + std::to_string(block), + hidden, + kernel); + } + return conv1d(ctx, weights, value, "acoustic.output", out_channels, 1, 0); +} + +// ---- shared runtime state ------------------------------------------------ + +struct BackendOwner { + ggml_backend_t value = nullptr; + BackendOwner() = default; + BackendOwner(const BackendOwner &) = delete; + BackendOwner & operator=(const BackendOwner &) = delete; + ~BackendOwner() { + if (value != nullptr) { + ggml_backend_free(value); + } + } +}; + +/** Backend + uploaded weights, with the tiny duration model mirrored to the + * host on CUDA builds: durations round to integers and gate the whole frame + * layout, so small TF32 differences must not move them (the inflect_v2 + * rationale). */ +struct BackendState { + std::shared_ptr assets; + int threads = 1; + core::BackendType backend_type = core::BackendType::Cpu; + BackendOwner backend; + std::shared_ptr weights; + BackendOwner duration_backend; + std::shared_ptr duration_weights; + + BackendState( + std::shared_ptr assets_in, + core::BackendConfig backend_config, + size_t expected_tensors, + size_t weight_arena_bytes) + : assets(std::move(assets_in)), + threads(std::max(1, backend_config.threads)) { + if (assets == nullptr) { + throw std::runtime_error("sanoTTS runtime requires assets"); + } + backend_config.threads = threads; + backend.value = core::init_backend(backend_config); + backend_type = core::backend_type(backend.value); + core::set_backend_threads(backend.value, threads); + weights = load_weights( + assets, backend.value, backend_type, expected_tensors, weight_arena_bytes); + if (backend_type == core::BackendType::Cuda) { + core::BackendConfig duration_config{core::BackendType::Cpu, 0, threads}; + duration_backend.value = core::init_backend(duration_config); + core::set_backend_threads(duration_backend.value, threads); + duration_weights = load_weights( + assets, + duration_backend.value, + core::BackendType::Cpu, + expected_tensors, + weight_arena_bytes); + } + } + + ggml_backend_t duration_backend_value() const { + return duration_backend.value != nullptr ? duration_backend.value : backend.value; + } + core::BackendType duration_backend_type() const { + return duration_backend.value != nullptr ? core::BackendType::Cpu : backend_type; + } + const SanoTtsBackendWeights & duration_weights_ref() const { + return duration_weights != nullptr ? *duration_weights : *weights; + } +}; + +template +Graph & cached_graph( + runtime::CacheSlots> & slots, + int64_t key, + const char * trace_name, + Build && build) { + if (auto * found = slots.find(key)) { + engine::debug::trace_log_scalar(trace_name, true); + return **found; + } + engine::debug::trace_log_scalar(trace_name, false); + slots.put(key, build()); + auto * created = slots.find(key); + if (created == nullptr) { + throw std::runtime_error("sanoTTS graph cache insert failed"); + } + return **created; +} + +// ---- shared host-side pieces --------------------------------------------- +// +// Hints are computed in double and cast to fp32, matching the numpy +// references both runtimes are gated against. + +/** [positions, length_hint, valid=1] rows for the duration stage. */ +inline std::vector duration_features(int64_t token_count, int64_t max_tokens) { + std::vector feats(static_cast(3 * token_count)); + linspace01(feats.data(), token_count); + const auto length_hint = static_cast( + std::log1p(static_cast(token_count)) / + std::log1p(static_cast(max_tokens))); + std::fill_n(feats.begin() + token_count, token_count, length_hint); + std::fill_n(feats.begin() + 2 * token_count, token_count, 1.0F); + return feats; +} + +/** [token_pos, duration_hint] rows for the token stage. */ +inline std::vector token_features( + int64_t token_count, + const std::vector & durations) { + std::vector feats(static_cast(2 * token_count)); + linspace01(feats.data(), token_count); + double max_duration = 1.0; + for (const int64_t duration : durations) { + max_duration = std::max(max_duration, static_cast(duration)); + } + const double log_max_duration = std::log1p(max_duration); + for (int64_t token = 0; token < token_count; ++token) { + feats[static_cast(token_count + token)] = static_cast( + std::log1p(static_cast(durations[static_cast(token)])) / + log_max_duration); + } + return feats; +} + +/** exp -> clamp_min(1) -> *scale -> round (ties to even) -> clamp, in + * double, exactly as the references compute it. Returns per-token frame + * counts and validates the total against the trained limits. */ +inline std::vector round_durations( + const std::vector & log_duration, + double scale, + int64_t max_frames_per_token, + int64_t max_total_frames, + int64_t & total_frames) { + std::vector durations(log_duration.size()); + total_frames = 0; + for (size_t token = 0; token < log_duration.size(); ++token) { + double value = std::exp(static_cast(log_duration[token])); + if (!std::isfinite(value)) { + throw std::runtime_error("sanoTTS duration predictor produced a non-finite duration"); + } + if (value < 1.0) { + value = 1.0; + } + value = std::rint(value * scale); + if (value < 1.0) { + value = 1.0; + } + if (value > static_cast(max_frames_per_token)) { + value = static_cast(max_frames_per_token); + } + durations[token] = static_cast(value); + total_frames += durations[token]; + } + if (total_frames < 1 || total_frames > max_total_frames) { + throw std::runtime_error( + "sanoTTS expanded to " + std::to_string(total_frames) + + " frames, outside the supported range"); + } + return durations; +} + +/** Repeat each token's context vector across its own frames. */ +inline std::vector expand_context( + const std::vector & token_context, + const std::vector & durations, + int64_t hidden, + int64_t token_count, + int64_t frames) { + std::vector expanded(static_cast(hidden * frames)); + for (int64_t channel = 0; channel < hidden; ++channel) { + const float * row = token_context.data() + channel * token_count; + float * out_row = expanded.data() + channel * frames; + int64_t at = 0; + for (int64_t token = 0; token < token_count; ++token) { + const float value = row[token]; + for (int64_t j = 0; j < durations[static_cast(token)]; ++j) { + out_row[at++] = value; + } + } + } + return expanded; +} + +/** [frame_pos, token_pos, duration_pos] rows -- expand_features' documented + * float semantics: doubles cast to fp32, a duration of 1 contributing 0. */ +inline std::vector frame_features( + int64_t token_count, + const std::vector & durations, + int64_t frames) { + std::vector feats(static_cast(3 * frames)); + linspace01(feats.data(), frames); + const int64_t denominator = token_count > 1 ? token_count - 1 : 1; + float * token_pos = feats.data() + frames; + float * duration_pos = feats.data() + 2 * frames; + int64_t at = 0; + for (int64_t token = 0; token < token_count; ++token) { + const int64_t count = durations[static_cast(token)]; + const auto position = static_cast( + static_cast(token) / static_cast(denominator)); + for (int64_t j = 0; j < count; ++j) { + token_pos[at] = position; + duration_pos[at] = count == 1 + ? 0.0F + : static_cast( + static_cast(j) / static_cast(count - 1)); + ++at; + } + } + return feats; +} + +inline void write_i32_input(ggml_tensor * tensor, const core::TensorShape & shape, const std::vector & values) { + core::write_tensor_i32(core::wrap_tensor(tensor, shape, GGML_TYPE_I32), values); +} + +inline void write_f32_input(ggml_tensor * tensor, const core::TensorShape & shape, const std::vector & values) { + core::write_tensor_f32(core::wrap_tensor(tensor, shape, GGML_TYPE_F32), values); +} + } // namespace engine::models::sanotts::graph diff --git a/src/community_models/sanotts/piper_runtime.cpp b/src/community_models/sanotts/piper_runtime.cpp index 1e23c6d50..4b80ef346 100644 --- a/src/community_models/sanotts/piper_runtime.cpp +++ b/src/community_models/sanotts/piper_runtime.cpp @@ -2,9 +2,7 @@ #include "graph_common.h" -#include "engine/framework/debug/profiler.h" #include "engine/framework/modules/conv_modules.h" -#include "engine/framework/runtime/cache_slots.h" #include #include @@ -45,7 +43,7 @@ constexpr int64_t kUpPaddings[3] = {4, 4, 2}; * vocab-size mismatch (e.g. duration vocab 127 vs table ids to 156). */ constexpr int32_t kSchwaFallbackId = 59; -size_t expected_piper_tensor_count(const SanoTtsPiperConfig & config) { +size_t expected_tensor_count(const SanoTtsPiperConfig & config) { const auto duration = 5 + 5 * config.duration_depth; const auto acoustic = 7 + 5 * (config.acoustic_token_depth + config.acoustic_depth); @@ -280,165 +278,6 @@ core::TensorValue post_filter( GGML_TYPE_F32); } -// ---- graphs -------------------------------------------------------------- - -struct DurationGraph : GraphResources { - int64_t token_count = 0; - ggml_tensor * tokens = nullptr; - ggml_tensor * feats = nullptr; - ggml_tensor * log_duration = nullptr; -}; - -std::unique_ptr build_duration_graph( - const SanoTtsBackendWeights & weights, - const SanoTtsPiperConfig & config, - ggml_backend_t backend, - core::BackendType backend_type, - int64_t token_count) { - auto out = std::make_unique(); - out->backend = backend; - out->token_count = token_count; - out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); - out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); - if (out->io_context == nullptr || out->graph_context == nullptr) { - throw std::runtime_error("sanoTTS failed to create duration graph contexts"); - } - core::ModuleBuildContext io_ctx{ - out->io_context.get(), - "sanotts.piper.duration.io", - backend_type, - }; - core::ModuleBuildContext ctx{ - out->graph_context.get(), - "sanotts.piper.duration", - backend_type, - }; - auto tokens = core::make_tensor( - io_ctx, - GGML_TYPE_I32, - core::TensorShape::from_dims({1, token_count})); - auto feats = core::make_tensor( - io_ctx, - GGML_TYPE_F32, - core::TensorShape::from_dims({1, 3, token_count})); - ggml_set_input(tokens.tensor); - ggml_set_input(feats.tensor); - - auto hidden = embed_tokens( - ctx, - weights, - tokens, - "duration.embedding.weight", - config.duration_vocab, - config.duration_hidden); - hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); - hidden = conv1d( - ctx, - weights, - hidden, - "duration.input_proj", - config.duration_hidden, - 1, - 0); - for (int64_t block = 0; block < config.duration_depth; ++block) { - hidden = residual_block( - ctx, - weights, - hidden, - "duration.blocks." + std::to_string(block), - config.duration_hidden, - config.duration_kernel); - } - auto log_duration = conv1d(ctx, weights, hidden, "duration.output", 1, 1, 0); - log_duration = contiguous(ctx, log_duration); - out->tokens = tokens.tensor; - out->feats = feats.tensor; - out->log_duration = log_duration.tensor; - ggml_set_output(out->log_duration); - out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); - ggml_build_forward_expand(out->graph, out->log_duration); - allocate_graph(*out); - return out; -} - -struct TokenGraph : GraphResources { - int64_t token_count = 0; - ggml_tensor * tokens = nullptr; - ggml_tensor * feats = nullptr; - ggml_tensor * context = nullptr; -}; - -std::unique_ptr build_token_graph( - const SanoTtsBackendWeights & weights, - const SanoTtsPiperConfig & config, - ggml_backend_t backend, - core::BackendType backend_type, - int64_t token_count) { - auto out = std::make_unique(); - out->backend = backend; - out->token_count = token_count; - out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); - out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); - if (out->io_context == nullptr || out->graph_context == nullptr) { - throw std::runtime_error("sanoTTS failed to create token graph contexts"); - } - core::ModuleBuildContext io_ctx{ - out->io_context.get(), - "sanotts.piper.token.io", - backend_type, - }; - core::ModuleBuildContext ctx{ - out->graph_context.get(), - "sanotts.piper.token", - backend_type, - }; - auto tokens = core::make_tensor( - io_ctx, - GGML_TYPE_I32, - core::TensorShape::from_dims({1, token_count})); - auto feats = core::make_tensor( - io_ctx, - GGML_TYPE_F32, - core::TensorShape::from_dims({1, 2, token_count})); - ggml_set_input(tokens.tensor); - ggml_set_input(feats.tensor); - - auto hidden = embed_tokens( - ctx, - weights, - tokens, - "acoustic.embedding.weight", - config.acoustic_vocab, - config.acoustic_hidden); - hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); - hidden = conv1d( - ctx, - weights, - hidden, - "acoustic.token_input_proj", - config.acoustic_hidden, - 1, - 0); - for (int64_t block = 0; block < config.acoustic_token_depth; ++block) { - hidden = residual_block( - ctx, - weights, - hidden, - "acoustic.token_blocks." + std::to_string(block), - config.acoustic_hidden, - config.acoustic_kernel); - } - hidden = contiguous(ctx, hidden); - out->tokens = tokens.tensor; - out->feats = feats.tensor; - out->context = hidden.tensor; - ggml_set_output(out->context); - out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); - ggml_build_forward_expand(out->graph, out->context); - allocate_graph(*out); - return out; -} - struct DecoderGraph : GraphResources { int64_t frames = 0; ggml_tensor * context = nullptr; @@ -484,32 +323,15 @@ std::unique_ptr build_decoder_graph( ggml_set_input(context.tensor); ggml_set_input(feats.tensor); - auto hidden = modules::ConcatModule({1}).build(ctx, context, feats); - hidden = conv1d( + auto latent = acoustic_frame_stage( ctx, weights, - hidden, - "acoustic.frame_input_proj", + context, + feats, config.acoustic_hidden, - 1, - 0); - for (int64_t block = 0; block < config.acoustic_depth; ++block) { - hidden = residual_block( - ctx, - weights, - hidden, - "acoustic.frame_blocks." + std::to_string(block), - config.acoustic_hidden, - config.acoustic_kernel); - } - auto latent = conv1d( - ctx, - weights, - hidden, - "acoustic.output", - config.acoustic_out_channels, - 1, - 0); + config.acoustic_depth, + config.acoustic_kernel, + config.acoustic_out_channels); auto value = conv1d_same( ctx, @@ -568,132 +390,17 @@ std::unique_ptr build_decoder_graph( } // namespace -struct SanoTtsPiperRuntime::State { - struct BackendOwner { - ggml_backend_t value = nullptr; - ~BackendOwner() { - if (value != nullptr) { - ggml_backend_free(value); - } - } - }; - - State( - std::shared_ptr assets_in, - core::BackendConfig backend_config) - : assets(std::move(assets_in)), - threads(std::max(1, backend_config.threads)) { - if (assets == nullptr) { - throw std::runtime_error("sanoTTS piper runtime requires assets"); - } - backend_config.threads = threads; - backend.value = core::init_backend(backend_config); - backend_type = core::backend_type(backend.value); - core::set_backend_threads(backend.value, threads); - weights = load_weights( - assets, - backend.value, - backend_type, - expected_piper_tensor_count(assets->piper), - kWeightArenaBytes); - if (backend_type == core::BackendType::Cuda) { - // Durations round to integers and gate the whole frame layout; - // keep the tiny duration model on the host, as the nano runtime - // and inflect_v2 do. - core::BackendConfig duration_config{ - core::BackendType::Cpu, - 0, - threads, - }; - duration_backend.value = core::init_backend(duration_config); - core::set_backend_threads(duration_backend.value, threads); - duration_weights = load_weights( - assets, - duration_backend.value, - core::BackendType::Cpu, - expected_piper_tensor_count(assets->piper), - kWeightArenaBytes); - } - } - - DurationGraph & duration_graph(int64_t token_count) { - if (auto * found = duration_graphs.find(token_count)) { - engine::debug::trace_log_scalar("sanotts.duration_graph.cache_hit", true); - return **found; - } - engine::debug::trace_log_scalar("sanotts.duration_graph.cache_hit", false); - const auto backend_value = - duration_backend.value != nullptr ? duration_backend.value : backend.value; - const auto graph_backend_type = - duration_backend.value != nullptr ? core::BackendType::Cpu : backend_type; - const auto & selected_weights = - duration_weights != nullptr ? duration_weights : weights; - duration_graphs.put( - token_count, - build_duration_graph( - *selected_weights, - assets->piper, - backend_value, - graph_backend_type, - token_count)); - auto * created = duration_graphs.find(token_count); - if (created == nullptr) { - throw std::runtime_error("sanoTTS duration graph cache insert failed"); - } - return **created; - } - - TokenGraph & token_graph(int64_t token_count) { - if (auto * found = token_graphs.find(token_count)) { - engine::debug::trace_log_scalar("sanotts.token_graph.cache_hit", true); - return **found; - } - engine::debug::trace_log_scalar("sanotts.token_graph.cache_hit", false); - token_graphs.put( - token_count, - build_token_graph( - *weights, - assets->piper, - backend.value, - backend_type, - token_count)); - auto * created = token_graphs.find(token_count); - if (created == nullptr) { - throw std::runtime_error("sanoTTS token graph cache insert failed"); - } - return **created; - } - - DecoderGraph & decoder_graph(int64_t frames) { - if (auto * found = decoder_graphs.find(frames)) { - engine::debug::trace_log_scalar("sanotts.decoder_graph.cache_hit", true); - return **found; - } - engine::debug::trace_log_scalar("sanotts.decoder_graph.cache_hit", false); - decoder_graphs.put( - frames, - build_decoder_graph( - *weights, - assets->piper, - backend.value, - backend_type, - frames)); - auto * created = decoder_graphs.find(frames); - if (created == nullptr) { - throw std::runtime_error("sanoTTS decoder graph cache insert failed"); - } - return **created; - } - - std::shared_ptr assets; - int threads = 1; - core::BackendType backend_type = core::BackendType::Cpu; - BackendOwner backend; - std::shared_ptr weights; - BackendOwner duration_backend; - std::shared_ptr duration_weights; - runtime::CacheSlots> duration_graphs{4}; - runtime::CacheSlots> token_graphs{4}; +struct SanoTtsPiperRuntime::State : BackendState { + State(const std::shared_ptr & assets_in, + core::BackendConfig backend_config) + : BackendState( + assets_in, + backend_config, + assets_in == nullptr ? 0 : expected_tensor_count(assets_in->piper), + kWeightArenaBytes) {} + + runtime::CacheSlots> duration_graphs{4}; + runtime::CacheSlots> token_graphs{4}; runtime::CacheSlots> decoder_graphs{2}; }; @@ -716,161 +423,109 @@ runtime::AudioBuffer SanoTtsPiperRuntime::synthesize( // -- durations --------------------------------------------------------- const auto duration_start = std::chrono::steady_clock::now(); - auto & duration = state_->duration_graph(token_count); - core::write_tensor_i32( - core::wrap_tensor( - duration.tokens, - core::TensorShape::from_dims({1, token_count}), - GGML_TYPE_I32), + auto & duration = cached_graph( + state_->duration_graphs, + token_count, + "sanotts.duration_graph.cache_hit", + [&] { + return build_front_graph( + state_->duration_weights_ref(), + duration_stage_spec( + config.duration_vocab, + config.duration_hidden, + config.duration_depth, + config.duration_kernel), + state_->duration_backend_value(), + state_->duration_backend_type(), + token_count, + kIoArenaBytes, + kGraphArenaBytes); + }); + write_i32_input( + duration.tokens, + core::TensorShape::from_dims({1, token_count}), clamp_ids_to_vocab(token_ids, config.duration_vocab)); - { - // [positions, length_hint, valid=1] rows; hints computed in double - // and cast, matching the reference implementation. - std::vector feats(static_cast(3 * token_count)); - linspace01(feats.data(), token_count); - const auto length_hint = static_cast( - std::log1p(static_cast(token_count)) / - std::log1p(static_cast(config.duration_max_tokens))); - std::fill_n(feats.begin() + token_count, token_count, length_hint); - std::fill_n(feats.begin() + 2 * token_count, token_count, 1.0F); - core::write_tensor_f32( - core::wrap_tensor( - duration.feats, - core::TensorShape::from_dims({1, 3, token_count}), - GGML_TYPE_F32), - feats); - } + write_f32_input( + duration.feats, + core::TensorShape::from_dims({1, 3, token_count}), + duration_features(token_count, config.duration_max_tokens)); compute_graph(duration, "sanoTTS piper duration"); - const auto log_duration = core::read_tensor_f32(duration.log_duration); + const auto log_duration = core::read_tensor_f32(duration.output); if (static_cast(log_duration.size()) != token_count) { throw std::runtime_error("sanoTTS duration graph returned invalid output"); } - // exp -> clamp_min(1) -> *scale -> round (ties to even) -> clamp, in - // double, exactly as the reference computes it. - const double scale = config.duration_length_scale * - static_cast(options.speaking_rate); - std::vector durations(static_cast(token_count)); + // The user's speaking_rate multiplies the voice's tuned length scale. int64_t frames = 0; - for (int64_t token = 0; token < token_count; ++token) { - double value = std::exp( - static_cast(log_duration[static_cast(token)])); - if (!std::isfinite(value)) { - throw std::runtime_error("sanoTTS duration predictor produced a non-finite duration"); - } - if (value < 1.0) { - value = 1.0; - } - value = std::rint(value * scale); - if (value < 1.0) { - value = 1.0; - } - if (value > static_cast(config.duration_max_frames)) { - value = static_cast(config.duration_max_frames); - } - durations[static_cast(token)] = static_cast(value); - frames += durations[static_cast(token)]; - } - const int64_t max_frames = config.duration_max_tokens * config.duration_max_frames; - if (frames < 1 || frames > max_frames) { - throw std::runtime_error( - "sanoTTS expanded to " + std::to_string(frames) + - " frames, outside the supported range"); - } + const auto durations = round_durations( + log_duration, + config.duration_length_scale * static_cast(options.speaking_rate), + config.duration_max_frames, + config.duration_max_tokens * config.duration_max_frames, + frames); engine::debug::timing_log_scalar( "sanotts.duration_ms", engine::debug::elapsed_ms(duration_start)); // -- token-stage acoustic context -------------------------------------- const auto acoustic_start = std::chrono::steady_clock::now(); - auto & token_graph = state_->token_graph(token_count); - core::write_tensor_i32( - core::wrap_tensor( - token_graph.tokens, - core::TensorShape::from_dims({1, token_count}), - GGML_TYPE_I32), + auto & token_graph = cached_graph( + state_->token_graphs, + token_count, + "sanotts.token_graph.cache_hit", + [&] { + return build_front_graph( + *state_->weights, + token_stage_spec( + config.acoustic_vocab, + config.acoustic_hidden, + config.acoustic_token_depth, + config.acoustic_kernel), + state_->backend.value, + state_->backend_type, + token_count, + kIoArenaBytes, + kGraphArenaBytes); + }); + write_i32_input( + token_graph.tokens, + core::TensorShape::from_dims({1, token_count}), clamp_ids_to_vocab(token_ids, config.acoustic_vocab)); - { - std::vector feats(static_cast(2 * token_count)); - linspace01(feats.data(), token_count); - double max_duration = 1.0; - for (int64_t token = 0; token < token_count; ++token) { - max_duration = std::max( - max_duration, - static_cast(durations[static_cast(token)])); - } - const double log_max_duration = std::log1p(max_duration); - for (int64_t token = 0; token < token_count; ++token) { - feats[static_cast(token_count + token)] = static_cast( - std::log1p(static_cast(durations[static_cast(token)])) / - log_max_duration); - } - core::write_tensor_f32( - core::wrap_tensor( - token_graph.feats, - core::TensorShape::from_dims({1, 2, token_count}), - GGML_TYPE_F32), - feats); - } + write_f32_input( + token_graph.feats, + core::TensorShape::from_dims({1, 2, token_count}), + token_features(token_count, durations)); compute_graph(token_graph, "sanoTTS piper token context"); - const auto token_context = core::read_tensor_f32(token_graph.context); + const auto token_context = core::read_tensor_f32(token_graph.output); if (static_cast(token_context.size()) != config.acoustic_hidden * token_count) { throw std::runtime_error("sanoTTS token graph returned invalid output"); } - - // -- expand to frames -------------------------------------------------- - std::vector expanded(static_cast(config.acoustic_hidden * frames)); - for (int64_t channel = 0; channel < config.acoustic_hidden; ++channel) { - const float * row = token_context.data() + channel * token_count; - float * out_row = expanded.data() + channel * frames; - int64_t at = 0; - for (int64_t token = 0; token < token_count; ++token) { - const float value = row[token]; - for (int64_t j = 0; j < durations[static_cast(token)]; ++j) { - out_row[at++] = value; - } - } - } - std::vector frame_feats(static_cast(3 * frames)); - linspace01(frame_feats.data(), frames); - { - const int64_t denominator = token_count > 1 ? token_count - 1 : 1; - float * token_pos = frame_feats.data() + frames; - float * duration_pos = frame_feats.data() + 2 * frames; - int64_t at = 0; - for (int64_t token = 0; token < token_count; ++token) { - const int64_t count = durations[static_cast(token)]; - const auto position = static_cast( - static_cast(token) / static_cast(denominator)); - for (int64_t j = 0; j < count; ++j) { - token_pos[at] = position; - duration_pos[at] = count == 1 - ? 0.0F - : static_cast( - static_cast(j) / static_cast(count - 1)); - ++at; - } - } - } engine::debug::timing_log_scalar( "sanotts.acoustic_ms", engine::debug::elapsed_ms(acoustic_start)); // -- frame stage + decoder --------------------------------------------- const auto decoder_start = std::chrono::steady_clock::now(); - auto & decoder = state_->decoder_graph(frames); - core::write_tensor_f32( - core::wrap_tensor( - decoder.context, - core::TensorShape::from_dims({1, config.acoustic_hidden, frames}), - GGML_TYPE_F32), - expanded); - core::write_tensor_f32( - core::wrap_tensor( - decoder.feats, - core::TensorShape::from_dims({1, 3, frames}), - GGML_TYPE_F32), - frame_feats); + auto & decoder = cached_graph( + state_->decoder_graphs, + frames, + "sanotts.decoder_graph.cache_hit", + [&] { + return build_decoder_graph( + *state_->weights, + config, + state_->backend.value, + state_->backend_type, + frames); + }); + write_f32_input( + decoder.context, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames}), + expand_context(token_context, durations, config.acoustic_hidden, token_count, frames)); + write_f32_input( + decoder.feats, + core::TensorShape::from_dims({1, 3, frames}), + frame_features(token_count, durations, frames)); compute_graph(decoder, "sanoTTS piper decoder"); runtime::AudioBuffer out; out.sample_rate = static_cast(config.sample_rate); diff --git a/src/community_models/sanotts/runtime.cpp b/src/community_models/sanotts/runtime.cpp index 47f2c7ddc..64efa1ba0 100644 --- a/src/community_models/sanotts/runtime.cpp +++ b/src/community_models/sanotts/runtime.cpp @@ -2,32 +2,19 @@ #include "graph_common.h" -#include "engine/framework/assets/tensor_source.h" #include "engine/framework/audio/istft_graph.h" -#include "engine/framework/core/backend.h" -#include "engine/framework/core/backend_weight_store.h" -#include "engine/framework/debug/profiler.h" -#include "engine/framework/modules/activation_modules.h" #include "engine/framework/modules/linear_module.h" -#include "engine/framework/modules/lookup_modules.h" #include "engine/framework/modules/norm_modules.h" -#include "engine/framework/modules/primitive_modules.h" -#include "engine/framework/modules/structural_modules.h" -#include "engine/framework/runtime/cache_slots.h" - -#include "ggml-alloc.h" #include #include #include -#include #include #include #include #include #include #include -#include #include #include @@ -42,8 +29,17 @@ constexpr size_t kIoArenaBytes = 8ULL * 1024ULL * 1024ULL; constexpr size_t kGraphArenaBytes = 128ULL * 1024ULL * 1024ULL; constexpr size_t kWeightArenaBytes = 32ULL * 1024ULL * 1024ULL; -/** Tensor count implied by the config -- 103 for heart-nano, 115 for heart. - * Kept in lockstep with the inventory validate_tensors() builds. */ +// The decoder's norms are nn.LayerNorm(eps=1e-6), NOT torch's 1e-5 default. +// The difference compounds through the ConvNeXt blocks and is then amplified +// by the exp() in the magnitude head; the reference implementations document +// losing 0.06 of correlation and a third of the output amplitude to exactly +// this constant. +constexpr float kLayerNormEps = 1.0e-6F; +constexpr float kDcBlockPole = 0.9973F; +constexpr double kPi = 3.14159265358979323846; + +/** Tensor count implied by the config -- 103 for heart-nano, 117 for heart. + * Kept in lockstep with the inventory validate_nano_tensors() builds. */ size_t expected_tensor_count(const SanoTtsConfig & config) { const auto duration = 5 + 5 * config.duration_depth; const auto acoustic = @@ -52,21 +48,6 @@ size_t expected_tensor_count(const SanoTtsConfig & config) { return static_cast(duration + acoustic + decoder); } -// The decoder's norms are nn.LayerNorm(eps=1e-6), NOT torch's 1e-5 default. -// The difference compounds through the four ConvNeXt blocks and is then -// amplified by the exp() in the magnitude head; the reference implementations -// document losing 0.06 of correlation and a third of the output amplitude to -// exactly this constant. -constexpr float kLayerNormEps = 1.0e-6F; -constexpr float kDcBlockPole = 0.9973F; -constexpr double kPi = 3.14159265358979323846; - -// ---- graph builders ------------------------------------------------------ -// -// Values are carried channel-major as [1, C, T] (ggml ne0 = T) through the -// convolutional front ends, and channel-last as [1, T, C] through the -// ConvNeXt decoder blocks, matching the PyTorch modules they reproduce. - core::TensorValue channel_last_layer_norm( core::ModuleBuildContext & ctx, const SanoTtsBackendWeights & weights, @@ -187,164 +168,7 @@ std::vector seeded_noise(uint64_t seed, int64_t channels, int64_t frames) return out; } -// ---- graphs -------------------------------------------------------------- - -struct DurationGraph : GraphResources { - int64_t token_count = 0; - ggml_tensor * tokens = nullptr; - ggml_tensor * feats = nullptr; - ggml_tensor * log_duration = nullptr; -}; - -std::unique_ptr build_duration_graph( - const SanoTtsBackendWeights & weights, - const SanoTtsConfig & config, - ggml_backend_t backend, - core::BackendType backend_type, - int64_t token_count) { - auto out = std::make_unique(); - out->backend = backend; - out->token_count = token_count; - out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); - out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); - if (out->io_context == nullptr || out->graph_context == nullptr) { - throw std::runtime_error("sanoTTS failed to create duration graph contexts"); - } - core::ModuleBuildContext io_ctx{ - out->io_context.get(), - "sanotts.duration.io", - backend_type, - }; - core::ModuleBuildContext ctx{ - out->graph_context.get(), - "sanotts.duration", - backend_type, - }; - auto tokens = core::make_tensor( - io_ctx, - GGML_TYPE_I32, - core::TensorShape::from_dims({1, token_count})); - auto feats = core::make_tensor( - io_ctx, - GGML_TYPE_F32, - core::TensorShape::from_dims({1, 3, token_count})); - ggml_set_input(tokens.tensor); - ggml_set_input(feats.tensor); - - auto hidden = embed_tokens( - ctx, - weights, - tokens, - "duration.embedding.weight", - config.vocab_size, - config.duration_hidden); - hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); - hidden = conv1d( - ctx, - weights, - hidden, - "duration.input_proj", - config.duration_hidden, - 1, - 0); - for (int64_t block = 0; block < config.duration_depth; ++block) { - hidden = residual_block( - ctx, - weights, - hidden, - "duration.blocks." + std::to_string(block), - config.duration_hidden, - config.duration_kernel); - } - auto log_duration = conv1d(ctx, weights, hidden, "duration.output", 1, 1, 0); - log_duration = contiguous(ctx, log_duration); - out->tokens = tokens.tensor; - out->feats = feats.tensor; - out->log_duration = log_duration.tensor; - ggml_set_output(out->log_duration); - out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); - ggml_build_forward_expand(out->graph, out->log_duration); - allocate_graph(*out); - return out; -} - -struct TokenGraph : GraphResources { - int64_t token_count = 0; - ggml_tensor * tokens = nullptr; - ggml_tensor * feats = nullptr; - ggml_tensor * context = nullptr; -}; - -std::unique_ptr build_token_graph( - const SanoTtsBackendWeights & weights, - const SanoTtsConfig & config, - ggml_backend_t backend, - core::BackendType backend_type, - int64_t token_count) { - auto out = std::make_unique(); - out->backend = backend; - out->token_count = token_count; - out->io_context.reset(ggml_init({kIoArenaBytes, nullptr, true})); - out->graph_context.reset(ggml_init({kGraphArenaBytes, nullptr, true})); - if (out->io_context == nullptr || out->graph_context == nullptr) { - throw std::runtime_error("sanoTTS failed to create token graph contexts"); - } - core::ModuleBuildContext io_ctx{ - out->io_context.get(), - "sanotts.token.io", - backend_type, - }; - core::ModuleBuildContext ctx{ - out->graph_context.get(), - "sanotts.token", - backend_type, - }; - auto tokens = core::make_tensor( - io_ctx, - GGML_TYPE_I32, - core::TensorShape::from_dims({1, token_count})); - auto feats = core::make_tensor( - io_ctx, - GGML_TYPE_F32, - core::TensorShape::from_dims({1, 2, token_count})); - ggml_set_input(tokens.tensor); - ggml_set_input(feats.tensor); - - auto hidden = embed_tokens( - ctx, - weights, - tokens, - "acoustic.embedding.weight", - config.vocab_size, - config.acoustic_hidden); - hidden = modules::ConcatModule({1}).build(ctx, hidden, feats); - hidden = conv1d( - ctx, - weights, - hidden, - "acoustic.token_input_proj", - config.acoustic_hidden, - 1, - 0); - for (int64_t block = 0; block < config.acoustic_token_depth; ++block) { - hidden = residual_block( - ctx, - weights, - hidden, - "acoustic.token_blocks." + std::to_string(block), - config.acoustic_hidden, - config.acoustic_kernel); - } - hidden = contiguous(ctx, hidden); - out->tokens = tokens.tensor; - out->feats = feats.tensor; - out->context = hidden.tensor; - ggml_set_output(out->context); - out->graph = ggml_new_graph_custom(ctx.ggml, 16384, false); - ggml_build_forward_expand(out->graph, out->context); - allocate_graph(*out); - return out; -} +// ---- decoder graph ------------------------------------------------------- struct DecoderGraph : GraphResources { int64_t frames = 0; @@ -396,26 +220,15 @@ std::unique_ptr build_decoder_graph( ggml_set_input(feats.tensor); ggml_set_input(noise.tensor); - // Acoustic frame stage: expanded token context + positional features. - auto hidden = modules::ConcatModule({1}).build(ctx, context, feats); - hidden = conv1d( + auto mel = acoustic_frame_stage( ctx, weights, - hidden, - "acoustic.frame_input_proj", + context, + feats, config.acoustic_hidden, - 1, - 0); - for (int64_t block = 0; block < config.acoustic_depth; ++block) { - hidden = residual_block( - ctx, - weights, - hidden, - "acoustic.frame_blocks." + std::to_string(block), - config.acoustic_hidden, - config.acoustic_kernel); - } - auto mel = conv1d(ctx, weights, hidden, "acoustic.output", config.mels, 1, 0); + config.acoustic_depth, + config.acoustic_kernel, + config.mels); // ConvNeXt decoder. Noise-fed: the noise adapter's output is added to the // mel embedding before the first norm. @@ -643,132 +456,17 @@ uint64_t sanotts_text_seed(const std::string & text) { return (static_cast(h[0]) << 32) | static_cast(h[1]); } -struct SanoTtsNativeRuntime::State { - struct BackendOwner { - ggml_backend_t value = nullptr; - ~BackendOwner() { - if (value != nullptr) { - ggml_backend_free(value); - } - } - }; - - State( - std::shared_ptr assets_in, - core::BackendConfig backend_config) - : assets(std::move(assets_in)), - threads(std::max(1, backend_config.threads)) { - if (assets == nullptr) { - throw std::runtime_error("sanoTTS native runtime requires assets"); - } - backend_config.threads = threads; - backend.value = core::init_backend(backend_config); - backend_type = core::backend_type(backend.value); - core::set_backend_threads(backend.value, threads); - weights = load_weights( - assets, - backend.value, - backend_type, - expected_tensor_count(assets->config), - kWeightArenaBytes); - if (backend_type == core::BackendType::Cuda) { - // Durations round to integers and gate the whole frame layout, so - // small TF32 differences would move frame counts. Keep the tiny - // duration model on the host; the decoder stays on CUDA. - core::BackendConfig duration_config{ - core::BackendType::Cpu, - 0, - threads, - }; - duration_backend.value = core::init_backend(duration_config); - core::set_backend_threads(duration_backend.value, threads); - duration_weights = load_weights( - assets, - duration_backend.value, - core::BackendType::Cpu, - expected_tensor_count(assets->config), - kWeightArenaBytes); - } - } - - DurationGraph & duration_graph(int64_t token_count) { - if (auto * found = duration_graphs.find(token_count)) { - engine::debug::trace_log_scalar("sanotts.duration_graph.cache_hit", true); - return **found; - } - engine::debug::trace_log_scalar("sanotts.duration_graph.cache_hit", false); - const auto backend_value = - duration_backend.value != nullptr ? duration_backend.value : backend.value; - const auto graph_backend_type = - duration_backend.value != nullptr ? core::BackendType::Cpu : backend_type; - const auto & selected_weights = - duration_weights != nullptr ? duration_weights : weights; - duration_graphs.put( - token_count, - build_duration_graph( - *selected_weights, - assets->config, - backend_value, - graph_backend_type, - token_count)); - auto * created = duration_graphs.find(token_count); - if (created == nullptr) { - throw std::runtime_error("sanoTTS duration graph cache insert failed"); - } - return **created; - } - - TokenGraph & token_graph(int64_t token_count) { - if (auto * found = token_graphs.find(token_count)) { - engine::debug::trace_log_scalar("sanotts.token_graph.cache_hit", true); - return **found; - } - engine::debug::trace_log_scalar("sanotts.token_graph.cache_hit", false); - token_graphs.put( - token_count, - build_token_graph( - *weights, - assets->config, - backend.value, - backend_type, - token_count)); - auto * created = token_graphs.find(token_count); - if (created == nullptr) { - throw std::runtime_error("sanoTTS token graph cache insert failed"); - } - return **created; - } - - DecoderGraph & decoder_graph(int64_t frames) { - if (auto * found = decoder_graphs.find(frames)) { - engine::debug::trace_log_scalar("sanotts.decoder_graph.cache_hit", true); - return **found; - } - engine::debug::trace_log_scalar("sanotts.decoder_graph.cache_hit", false); - decoder_graphs.put( - frames, - build_decoder_graph( - *weights, - assets->config, - backend.value, - backend_type, - frames)); - auto * created = decoder_graphs.find(frames); - if (created == nullptr) { - throw std::runtime_error("sanoTTS decoder graph cache insert failed"); - } - return **created; - } - - std::shared_ptr assets; - int threads = 1; - core::BackendType backend_type = core::BackendType::Cpu; - BackendOwner backend; - std::shared_ptr weights; - BackendOwner duration_backend; - std::shared_ptr duration_weights; - runtime::CacheSlots> duration_graphs{4}; - runtime::CacheSlots> token_graphs{4}; +struct SanoTtsNativeRuntime::State : BackendState { + State(const std::shared_ptr & assets_in, + core::BackendConfig backend_config) + : BackendState( + assets_in, + backend_config, + assets_in == nullptr ? 0 : expected_tensor_count(assets_in->config), + kWeightArenaBytes) {} + + runtime::CacheSlots> duration_graphs{4}; + runtime::CacheSlots> token_graphs{4}; runtime::CacheSlots> decoder_graphs{2}; }; @@ -791,62 +489,44 @@ runtime::AudioBuffer SanoTtsNativeRuntime::synthesize( // -- durations --------------------------------------------------------- const auto duration_start = std::chrono::steady_clock::now(); - auto & duration = state_->duration_graph(token_count); - core::write_tensor_i32( - core::wrap_tensor( - duration.tokens, - core::TensorShape::from_dims({1, token_count}), - GGML_TYPE_I32), - token_ids); - { - // [positions, length_hint, valid=1] rows, the exact float semantics - // of the reference front end (mcu/src/snt_front_f32.c). - std::vector feats(static_cast(3 * token_count)); - linspace01(feats.data(), token_count); - const float length_hint = - std::log1p(static_cast(token_count)) / - static_cast(std::log1p(static_cast(config.duration_max_tokens))); - std::fill_n(feats.begin() + token_count, token_count, length_hint); - std::fill_n(feats.begin() + 2 * token_count, token_count, 1.0F); - core::write_tensor_f32( - core::wrap_tensor( - duration.feats, - core::TensorShape::from_dims({1, 3, token_count}), - GGML_TYPE_F32), - feats); - } + auto & duration = cached_graph( + state_->duration_graphs, + token_count, + "sanotts.duration_graph.cache_hit", + [&] { + return build_front_graph( + state_->duration_weights_ref(), + duration_stage_spec( + config.vocab_size, + config.duration_hidden, + config.duration_depth, + config.duration_kernel), + state_->duration_backend_value(), + state_->duration_backend_type(), + token_count, + kIoArenaBytes, + kGraphArenaBytes); + }); + write_i32_input( + duration.tokens, core::TensorShape::from_dims({1, token_count}), token_ids); + write_f32_input( + duration.feats, + core::TensorShape::from_dims({1, 3, token_count}), + duration_features(token_count, config.duration_max_tokens)); compute_graph(duration, "sanoTTS duration"); - const auto log_duration = core::read_tensor_f32(duration.log_duration); + const auto log_duration = core::read_tensor_f32(duration.output); if (static_cast(log_duration.size()) != token_count) { throw std::runtime_error("sanoTTS duration graph returned invalid output"); } - // predict_durations: exp -> clamp_min(1) -> *scale -> round -> clamp. - // rintf under the default FE_TONEAREST matches torch.round's ties-to-even. - std::vector durations(static_cast(token_count)); int64_t frames = 0; - for (int64_t token = 0; token < token_count; ++token) { - float value = std::exp(log_duration[static_cast(token)]); - if (!std::isfinite(value)) { - throw std::runtime_error("sanoTTS duration predictor produced a non-finite duration"); - } - if (value < 1.0F) { - value = 1.0F; - } - value = std::rint(value * options.speaking_rate); - if (value < 1.0F) { - value = 1.0F; - } - if (value > static_cast(config.duration_max_frames)) { - value = static_cast(config.duration_max_frames); - } - durations[static_cast(token)] = static_cast(value); - frames += durations[static_cast(token)]; - } - const int64_t max_frames = config.duration_max_tokens * config.duration_max_frames; - if (frames < 2 || frames > max_frames) { - throw std::runtime_error( - "sanoTTS expanded to " + std::to_string(frames) + - " frames, outside the supported range"); + const auto durations = round_durations( + log_duration, + static_cast(options.speaking_rate), + config.duration_max_frames, + config.duration_max_tokens * config.duration_max_frames, + frames); + if (frames < 2) { + throw std::runtime_error("sanoTTS duration predictor produced too few frames"); } engine::debug::timing_log_scalar( "sanotts.duration_ms", @@ -854,78 +534,36 @@ runtime::AudioBuffer SanoTtsNativeRuntime::synthesize( // -- token-stage acoustic context -------------------------------------- const auto acoustic_start = std::chrono::steady_clock::now(); - auto & token_graph = state_->token_graph(token_count); - core::write_tensor_i32( - core::wrap_tensor( - token_graph.tokens, - core::TensorShape::from_dims({1, token_count}), - GGML_TYPE_I32), - token_ids); - { - // [token_pos, duration_hint] rows. - std::vector feats(static_cast(2 * token_count)); - linspace01(feats.data(), token_count); - float max_duration = 1.0F; - for (int64_t token = 0; token < token_count; ++token) { - max_duration = std::max( - max_duration, - static_cast(durations[static_cast(token)])); - } - const float log_max_duration = std::log1p(max_duration); - for (int64_t token = 0; token < token_count; ++token) { - feats[static_cast(token_count + token)] = - std::log1p(static_cast(durations[static_cast(token)])) / - log_max_duration; - } - core::write_tensor_f32( - core::wrap_tensor( - token_graph.feats, - core::TensorShape::from_dims({1, 2, token_count}), - GGML_TYPE_F32), - feats); - } + auto & token_graph = cached_graph( + state_->token_graphs, + token_count, + "sanotts.token_graph.cache_hit", + [&] { + return build_front_graph( + *state_->weights, + token_stage_spec( + config.vocab_size, + config.acoustic_hidden, + config.acoustic_token_depth, + config.acoustic_kernel), + state_->backend.value, + state_->backend_type, + token_count, + kIoArenaBytes, + kGraphArenaBytes); + }); + write_i32_input( + token_graph.tokens, core::TensorShape::from_dims({1, token_count}), token_ids); + write_f32_input( + token_graph.feats, + core::TensorShape::from_dims({1, 2, token_count}), + token_features(token_count, durations)); compute_graph(token_graph, "sanoTTS token context"); - const auto token_context = core::read_tensor_f32(token_graph.context); + const auto token_context = core::read_tensor_f32(token_graph.output); if (static_cast(token_context.size()) != config.acoustic_hidden * token_count) { throw std::runtime_error("sanoTTS token graph returned invalid output"); } - - // -- expand to frames (host: pure copies plus the documented float - // semantics of expand_features: doubles cast to fp32) --------------- - std::vector expanded(static_cast(config.acoustic_hidden * frames)); - for (int64_t channel = 0; channel < config.acoustic_hidden; ++channel) { - const float * row = token_context.data() + channel * token_count; - float * out_row = expanded.data() + channel * frames; - int64_t at = 0; - for (int64_t token = 0; token < token_count; ++token) { - const float value = row[token]; - for (int64_t j = 0; j < durations[static_cast(token)]; ++j) { - out_row[at++] = value; - } - } - } - std::vector frame_feats(static_cast(3 * frames)); - linspace01(frame_feats.data(), frames); - { - const int64_t denominator = token_count > 1 ? token_count - 1 : 1; - float * token_pos = frame_feats.data() + frames; - float * duration_pos = frame_feats.data() + 2 * frames; - int64_t at = 0; - for (int64_t token = 0; token < token_count; ++token) { - const int64_t count = durations[static_cast(token)]; - const auto position = static_cast( - static_cast(token) / static_cast(denominator)); - for (int64_t j = 0; j < count; ++j) { - token_pos[at] = position; - duration_pos[at] = count == 1 - ? 0.0F - : static_cast( - static_cast(j) / static_cast(count - 1)); - ++at; - } - } - } const auto noise = seeded_noise(options.seed, config.noise_channels, frames); engine::debug::timing_log_scalar( "sanotts.acoustic_ms", @@ -933,24 +571,29 @@ runtime::AudioBuffer SanoTtsNativeRuntime::synthesize( // -- frame stage + decoder --------------------------------------------- const auto decoder_start = std::chrono::steady_clock::now(); - auto & decoder = state_->decoder_graph(frames); - core::write_tensor_f32( - core::wrap_tensor( - decoder.context, - core::TensorShape::from_dims({1, config.acoustic_hidden, frames}), - GGML_TYPE_F32), - expanded); - core::write_tensor_f32( - core::wrap_tensor( - decoder.feats, - core::TensorShape::from_dims({1, 3, frames}), - GGML_TYPE_F32), - frame_feats); - core::write_tensor_f32( - core::wrap_tensor( - decoder.noise, - core::TensorShape::from_dims({1, config.noise_channels, frames}), - GGML_TYPE_F32), + auto & decoder = cached_graph( + state_->decoder_graphs, + frames, + "sanotts.decoder_graph.cache_hit", + [&] { + return build_decoder_graph( + *state_->weights, + config, + state_->backend.value, + state_->backend_type, + frames); + }); + write_f32_input( + decoder.context, + core::TensorShape::from_dims({1, config.acoustic_hidden, frames}), + expand_context(token_context, durations, config.acoustic_hidden, token_count, frames)); + write_f32_input( + decoder.feats, + core::TensorShape::from_dims({1, 3, frames}), + frame_features(token_count, durations, frames)); + write_f32_input( + decoder.noise, + core::TensorShape::from_dims({1, config.noise_channels, frames}), noise); compute_graph(decoder, "sanoTTS decoder"); auto spectrum = core::read_tensor_f32(decoder.spectrum);