From a44ddf1f6e1bac2fbc23aad0631fcc9c1a87d2ab Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Sat, 31 Oct 2026 10:05:03 +0800 Subject: [PATCH 1/2] feat: add Kinyarwanda TN/ITN with raw-field tagging --- itn/kinyarwanda/__init__.py | 0 itn/kinyarwanda/data/numbers/digit.tsv | 9 + itn/kinyarwanda/data/numbers/digit_bi.tsv | 9 + itn/kinyarwanda/data/numbers/digit_e.tsv | 9 + itn/kinyarwanda/data/numbers/hundreds.tsv | 9 + itn/kinyarwanda/data/numbers/tens.tsv | 8 + itn/kinyarwanda/data/numbers/zero.tsv | 1 + itn/kinyarwanda/data/time/hours.tsv | 12 ++ itn/kinyarwanda/data/time/minutes.tsv | 59 ++++++ itn/kinyarwanda/data/whitelist.tsv | 18 ++ itn/kinyarwanda/inverse_normalizer.py | 53 +++++ itn/kinyarwanda/rules/__init__.py | 0 itn/kinyarwanda/rules/cardinal.py | 131 ++++++++++++ itn/kinyarwanda/rules/punctuation.py | 28 +++ itn/kinyarwanda/rules/time.py | 39 ++++ itn/kinyarwanda/rules/utils.py | 39 ++++ itn/kinyarwanda/rules/whitelist.py | 32 +++ itn/kinyarwanda/rules/word.py | 28 +++ itn/kinyarwanda/test/__init__.py | 0 itn/kinyarwanda/test/normalizer_test.py | 194 ++++++++++++++++++ tn/kinyarwanda/__init__.py | 0 tn/kinyarwanda/data/numbers/digit.tsv | 9 + tn/kinyarwanda/data/numbers/digit_bi.tsv | 9 + tn/kinyarwanda/data/numbers/digit_e.tsv | 9 + tn/kinyarwanda/data/numbers/hundreds.tsv | 9 + tn/kinyarwanda/data/numbers/tens.tsv | 8 + tn/kinyarwanda/data/numbers/zero.tsv | 1 + tn/kinyarwanda/data/time/hours.tsv | 12 ++ tn/kinyarwanda/data/time/minutes.tsv | 60 ++++++ tn/kinyarwanda/data/whitelist.tsv | 175 ++++++++++++++++ tn/kinyarwanda/normalizer.py | 54 +++++ tn/kinyarwanda/rules/__init__.py | 0 tn/kinyarwanda/rules/cardinal.py | 117 +++++++++++ tn/kinyarwanda/rules/punctuation.py | 29 +++ tn/kinyarwanda/rules/time.py | 40 ++++ tn/kinyarwanda/rules/utils.py | 43 ++++ tn/kinyarwanda/rules/whitelist.py | 28 +++ tn/kinyarwanda/rules/word.py | 29 +++ tn/kinyarwanda/test/__init__.py | 0 .../test/data/test_cases_cardinal.txt | 57 +++++ tn/kinyarwanda/test/data/test_cases_time.txt | 14 ++ .../test/data/test_cases_whitelist.txt | 3 + tn/kinyarwanda/test/data/test_cases_word.txt | 26 +++ tn/kinyarwanda/test/normalizer_test.py | 163 +++++++++++++++ 44 files changed, 1573 insertions(+) create mode 100644 itn/kinyarwanda/__init__.py create mode 100644 itn/kinyarwanda/data/numbers/digit.tsv create mode 100644 itn/kinyarwanda/data/numbers/digit_bi.tsv create mode 100644 itn/kinyarwanda/data/numbers/digit_e.tsv create mode 100644 itn/kinyarwanda/data/numbers/hundreds.tsv create mode 100644 itn/kinyarwanda/data/numbers/tens.tsv create mode 100644 itn/kinyarwanda/data/numbers/zero.tsv create mode 100644 itn/kinyarwanda/data/time/hours.tsv create mode 100644 itn/kinyarwanda/data/time/minutes.tsv create mode 100644 itn/kinyarwanda/data/whitelist.tsv create mode 100644 itn/kinyarwanda/inverse_normalizer.py create mode 100644 itn/kinyarwanda/rules/__init__.py create mode 100644 itn/kinyarwanda/rules/cardinal.py create mode 100644 itn/kinyarwanda/rules/punctuation.py create mode 100644 itn/kinyarwanda/rules/time.py create mode 100644 itn/kinyarwanda/rules/utils.py create mode 100644 itn/kinyarwanda/rules/whitelist.py create mode 100644 itn/kinyarwanda/rules/word.py create mode 100644 itn/kinyarwanda/test/__init__.py create mode 100644 itn/kinyarwanda/test/normalizer_test.py create mode 100644 tn/kinyarwanda/__init__.py create mode 100644 tn/kinyarwanda/data/numbers/digit.tsv create mode 100644 tn/kinyarwanda/data/numbers/digit_bi.tsv create mode 100644 tn/kinyarwanda/data/numbers/digit_e.tsv create mode 100644 tn/kinyarwanda/data/numbers/hundreds.tsv create mode 100644 tn/kinyarwanda/data/numbers/tens.tsv create mode 100644 tn/kinyarwanda/data/numbers/zero.tsv create mode 100644 tn/kinyarwanda/data/time/hours.tsv create mode 100644 tn/kinyarwanda/data/time/minutes.tsv create mode 100644 tn/kinyarwanda/data/whitelist.tsv create mode 100644 tn/kinyarwanda/normalizer.py create mode 100644 tn/kinyarwanda/rules/__init__.py create mode 100644 tn/kinyarwanda/rules/cardinal.py create mode 100644 tn/kinyarwanda/rules/punctuation.py create mode 100644 tn/kinyarwanda/rules/time.py create mode 100644 tn/kinyarwanda/rules/utils.py create mode 100644 tn/kinyarwanda/rules/whitelist.py create mode 100644 tn/kinyarwanda/rules/word.py create mode 100644 tn/kinyarwanda/test/__init__.py create mode 100644 tn/kinyarwanda/test/data/test_cases_cardinal.txt create mode 100644 tn/kinyarwanda/test/data/test_cases_time.txt create mode 100644 tn/kinyarwanda/test/data/test_cases_whitelist.txt create mode 100644 tn/kinyarwanda/test/data/test_cases_word.txt create mode 100644 tn/kinyarwanda/test/normalizer_test.py diff --git a/itn/kinyarwanda/__init__.py b/itn/kinyarwanda/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/kinyarwanda/data/numbers/digit.tsv b/itn/kinyarwanda/data/numbers/digit.tsv new file mode 100644 index 00000000..cd681e9e --- /dev/null +++ b/itn/kinyarwanda/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 rimwe +2 kabiri +3 gatatu +4 kane +5 gatanu +6 gatandatu +7 karindwi +8 umunani +9 icyenda diff --git a/itn/kinyarwanda/data/numbers/digit_bi.tsv b/itn/kinyarwanda/data/numbers/digit_bi.tsv new file mode 100644 index 00000000..4293c3ff --- /dev/null +++ b/itn/kinyarwanda/data/numbers/digit_bi.tsv @@ -0,0 +1,9 @@ +1 kimwe +2 bibiri +3 bitatu +4 bine +5 bitanu +6 bitandatu +7 birindwi +8 umunani +9 icyenda diff --git a/itn/kinyarwanda/data/numbers/digit_e.tsv b/itn/kinyarwanda/data/numbers/digit_e.tsv new file mode 100644 index 00000000..76c56b9d --- /dev/null +++ b/itn/kinyarwanda/data/numbers/digit_e.tsv @@ -0,0 +1,9 @@ +1 imwe +2 ebyiri +3 eshatu +4 enye +5 eshanu +6 esheshatu +7 zirindwi +8 umunani +9 icyenda diff --git a/itn/kinyarwanda/data/numbers/hundreds.tsv b/itn/kinyarwanda/data/numbers/hundreds.tsv new file mode 100644 index 00000000..10d00a82 --- /dev/null +++ b/itn/kinyarwanda/data/numbers/hundreds.tsv @@ -0,0 +1,9 @@ +1 ijana +2 magana abiri +3 magana atatu +4 magana ane +5 magana atanu +6 magana atandatu +7 magana arindwi +8 magana inani +9 magana cyenda diff --git a/itn/kinyarwanda/data/numbers/tens.tsv b/itn/kinyarwanda/data/numbers/tens.tsv new file mode 100644 index 00000000..66d40040 --- /dev/null +++ b/itn/kinyarwanda/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +2 makumyabiri +3 mirongo itatu +4 mirongo ine +5 mirongo itanu +6 mirongo itandatu +7 mirongo irindwi +8 mirongo inani +9 mirongo icyenda diff --git a/itn/kinyarwanda/data/numbers/zero.tsv b/itn/kinyarwanda/data/numbers/zero.tsv new file mode 100644 index 00000000..a0b17f6a --- /dev/null +++ b/itn/kinyarwanda/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 zeru diff --git a/itn/kinyarwanda/data/time/hours.tsv b/itn/kinyarwanda/data/time/hours.tsv new file mode 100644 index 00000000..53077822 --- /dev/null +++ b/itn/kinyarwanda/data/time/hours.tsv @@ -0,0 +1,12 @@ +saba 1 +munani 2 +cyenda 3 +cumi 4 +cumi n'imwe 5 +cumi n'ebyiri 6 +moya 7 +mbiri 8 +tatu 9 +ine 10 +tanu 11 +sita 12 diff --git a/itn/kinyarwanda/data/time/minutes.tsv b/itn/kinyarwanda/data/time/minutes.tsv new file mode 100644 index 00000000..bbacf984 --- /dev/null +++ b/itn/kinyarwanda/data/time/minutes.tsv @@ -0,0 +1,59 @@ +umunota umwe 01 +iminota ibiri 02 +iminota itatu 03 +iminota ine 04 +iminota itanu 05 +iminota itandatu 06 +iminota irindwi 07 +iminota umunani 08 +iminota icyenda 09 +iminota icumi 10 +iminota cumi n'umwe 11 +iminota cumi n'ibiri 12 +iminota cumi n'itatu 13 +iminota cumi n'ine 14 +iminota cumi n'itanu 15 +iminota cumi n'itandatu 16 +iminota cumi n'irindwi 17 +iminota cumi n'umunani 18 +iminota cumi n'icyenda 19 +iminota makumyabiri 20 +iminota makumyabiri na rimwe 21 +iminota makumyabiri n'ibiri 22 +iminota makumyabiri n'itatu 23 +iminota makumyabiri n'ine 24 +iminota makumyabiri n'itanu 25 +iminota makumyabiri n'itandatu 26 +iminota makumyabiri n'irindwi 27 +iminota makumyabiri n'umunani 28 +iminota makumyabiri n'icyenda 29 +iminota mirongo itatu 30 +iminota mirongo itatu n'umwe 31 +iminota mirongo itatu n'ibiri 32 +iminota mirongo itatu n'itatu 33 +iminota mirongo itatu n'ine 34 +iminota mirongo itatu n'itanu 35 +iminota mirongo itatu n'itandatu 36 +iminota mirongo itatu n'irindwi 37 +iminota mirongo itatu n'umunani 38 +iminota mirongo itatu n'icyenda 39 +iminota mirongo ine 40 +iminota mirongo ine n'umwe 41 +iminota mirongo ine n'ibiri 42 +iminota mirongo ine n'itatu 43 +iminota mirongo ine n'ine 44 +iminota mirongo ine n'itanu 45 +iminota mirongo ine n'itandatu 46 +iminota mirongo ine n'irindwi 47 +iminota mirongo ine n'umunani 48 +iminota mirongo ine n'icyenda 49 +iminota mirongo itanu 50 +iminota mirongo itanu n'umwe 51 +iminota mirongo itanu n'ibiri 52 +iminota mirongo itanu n'itatu 53 +iminota mirongo itanu n'ine 54 +iminota mirongo itanu n'itanu 55 +iminota mirongo itanu n'itandatu 56 +iminota mirongo itanu n'irindwi 57 +iminota mirongo itanu n'umunani 58 +iminota mirongo itanu n'icyenda 59 diff --git a/itn/kinyarwanda/data/whitelist.tsv b/itn/kinyarwanda/data/whitelist.tsv new file mode 100644 index 00000000..1abf5ce3 --- /dev/null +++ b/itn/kinyarwanda/data/whitelist.tsv @@ -0,0 +1,18 @@ +sidisi CDC +arabisi RBC +ade eni ADN +bibisi BBC +heci ayivi HIV +diyeniyi DNA +araneyi RNA +pisiyara PCR +ayisiyu ICU +arati-pisiyara RT-PCR +hashiseyeri HCR +efu emi FMI +aradibi RDB +arabidi RBD +aradiyefu RDF +vi o aye VOA +emu araneyi mRNA +wawawa www diff --git a/itn/kinyarwanda/inverse_normalizer.py b/itn/kinyarwanda/inverse_normalizer.py new file mode 100644 index 00000000..d30c3022 --- /dev/null +++ b/itn/kinyarwanda/inverse_normalizer.py @@ -0,0 +1,53 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import add_weight + +from itn.kinyarwanda.rules.cardinal import Cardinal +from itn.kinyarwanda.rules.punctuation import Punctuation +from itn.kinyarwanda.rules.time import Time +from itn.kinyarwanda.rules.whitelist import WhiteList +from itn.kinyarwanda.rules.word import Word +from tn.processor import Processor, RuleSpec + +# Every Kinyarwanda rule emits its fields in written order, so no token needs +# reordering before verbalization. +TOKEN_ORDERS = {} + + +class InverseNormalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="rw_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS) + self.build_fst("rw_itn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + punctuation = Punctuation() + word = Word() + rules = ( + RuleSpec(WhiteList(), 1.01), + RuleSpec(Time(), 1.05), + RuleSpec(Cardinal(), 1.1), + RuleSpec(word, 100), + RuleSpec(punctuation), + ) + classify = self.tagger_union(rules) + # Punctuation splits off only when attached to another token. + punct = add_weight(punctuation.tagger, 1.1) + token = punct.star + classify + punct.star + self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + verbalize = self.verbalizer_union(rules) + self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star diff --git a/itn/kinyarwanda/rules/__init__.py b/itn/kinyarwanda/rules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/kinyarwanda/rules/cardinal.py b/itn/kinyarwanda/rules/cardinal.py new file mode 100644 index 00000000..aee0c39c --- /dev/null +++ b/itn/kinyarwanda/rules/cardinal.py @@ -0,0 +1,131 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from itertools import product + +import pynini +from pynini import accep, cross, difference, invert, union +from pynini.lib.pynutil import add_weight, delete, insert + +from itn.kinyarwanda.rules.utils import string_map +from tn.processor import Processor + +VOWELS = "aeiou" +CONSONANTS = "bcdfghjklmnpqrstvwxyz" +# Ten has a standalone form and a form that is followed by its units. +TEN = "icumi" +TEN_BEFORE_UNITS = "cumi" +# Scale words: the bare word for exactly one, the class prefix otherwise, and +# the hand-written explicit form for one (``miliyoni imwe``), if any. +SCALES = ( + ("tiriyoni", "tiriyoni ", "tiriyoni imwe"), + ("miliyoni", "miliyoni ", "miliyoni imwe"), + ("igihumbi", "ibihumbi ", None), +) +# The explicit forms for one are accepted but never preferred over the bare +# scale word when several spellings are requested. +EXPLICIT_ONE_WEIGHT = 0.01 +# TN readings are not unique: ``ibihumbi ijana na mirongo itatu`` is written +# for both 130000 and 100030. Each part costs a little more the lower its +# scale group, so the reading that keeps parts under the larger scale wins. +GROUP_PART_WEIGHT = 0.001 + + +class Cardinal(Processor): + """Spoken cardinals, e.g. ``ijana na makumyabiri na gatatu`` -> ``123``. + + Accepts the readings written by Kinyarwanda TN: units agree with what + they count (``kabiri`` alone, ``ibihumbi bibiri``, ``miliyoni ebyiri``) + and parts are joined with ``na`` before a consonant and ``n'`` before a + vowel; ``tiriyoni`` names 10^9. Besides the bare ``miliyoni`` and + ``tiriyoni`` for exactly one, the explicit ``miliyoni imwe`` and + ``tiriyoni imwe`` are accepted. Digit-by-digit readings that start with + ``zeru`` (``zeru zeru kabiri``) are written as a digit string. + + The tagger keeps the spoken words (``integer`` or ``digits``); the + verbalizer writes the digits. + """ + + def __init__(self): + super().__init__(name="cardinal", ordertype="itn", token_orders={}) + digit = string_map("kinyarwanda/data/numbers/digit.tsv") + zero = string_map("kinyarwanda/data/numbers/zero.tsv") + integer = invert(zero | self.build_number()).optimize() + any_digit = invert(digit | zero) + digits = invert(zero) + (delete(" ") + any_digit).plus + + tagger = self.tag_field("integer", integer) | self.tag_field("digits", digits.optimize()) + self.tagger = self.add_tokens(tagger) + verbalizer = self.verbalize_field("integer", integer) | self.verbalize_field("digits", digits.optimize()) + self.verbalizer = self.delete_tokens(verbalizer) + + def connect(self, graph): + """Prefixes a part with ``na`` or ``n'`` depending on its first letter.""" + + vowel_initial = union(*VOWELS) + self.VSIGMA + consonant_initial = union(*CONSONANTS) + self.VSIGMA + return (insert(" n'") + (graph @ vowel_initial)) | (insert(" na ") + (graph @ consonant_initial)) + + def group(self, units, weight): + """Maps a three-digit group other than ``000`` to its reading in a unit class. + + Each spoken part of the group costs ``weight``. + """ + + hundreds = string_map("kinyarwanda/data/numbers/hundreds.tsv") + tens = string_map("kinyarwanda/data/numbers/tens.tsv") + options = [] + for has_hundreds, has_tens, has_units in product((False, True), repeat=3): + if not (has_hundreds or has_tens or has_units): + continue + ten = cross("1", TEN_BEFORE_UNITS if has_units else TEN) | tens + graph = accep("") + started = False + for present, part in ((has_hundreds, hundreds), (has_tens, ten), (has_units, units)): + if not present: + graph += delete("0") + continue + part = add_weight(part, weight) + graph += self.connect(part) if started else part + started = True + options.append(graph) + return union(*options).optimize() + + def build_number(self): + """Builds the digits -> words map; the caller inverts it.""" + + digit = string_map("kinyarwanda/data/numbers/digit.tsv") + digit_bi = string_map("kinyarwanda/data/numbers/digit_bi.tsv") + digit_e = string_map("kinyarwanda/data/numbers/digit_e.tsv") + not_one = difference(pynini.closure(self.DIGIT, 3, 3), "001").optimize() + + groups = [] + for index, ((one, prefix, explicit_one), units) in enumerate(zip(SCALES, (digit_e, digit_e, digit_bi))): + weight = index * GROUP_PART_WEIGHT + graph = add_weight(cross("001", one), weight) | (not_one @ (insert(prefix) + self.group(units, weight))) + if explicit_one is not None: + graph |= add_weight(cross("001", explicit_one), EXPLICIT_ONE_WEIGHT) + groups.append(graph.optimize()) + groups.append(self.group(digit, len(SCALES) * GROUP_PART_WEIGHT)) + + nonzero = difference(self.DIGIT, "0") + pad = insert("00") + nonzero | insert("0") + nonzero + self.DIGIT | nonzero + self.DIGIT + self.DIGIT + options = [] + for index, top in enumerate(groups): + graph = pad @ top + for lower in groups[index + 1:]: + graph += delete("000") | self.connect(lower) + options.append(graph) + return union(*options).optimize() diff --git a/itn/kinyarwanda/rules/punctuation.py b/itn/kinyarwanda/rules/punctuation.py new file mode 100644 index 00000000..b38f2d94 --- /dev/null +++ b/itn/kinyarwanda/rules/punctuation.py @@ -0,0 +1,28 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from tn.processor import Processor + + +class Punctuation(Processor): + """ASCII punctuation split off a neighbouring token, e.g. ``gatanu,``.""" + + def __init__(self): + super().__init__(name="punctuation", ordertype="itn", token_orders={}) + graph = pynini.union(*[pynini.escape(mark) for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~"]).optimize() + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/kinyarwanda/rules/time.py b/itn/kinyarwanda/rules/time.py new file mode 100644 index 00000000..caba5569 --- /dev/null +++ b/itn/kinyarwanda/rules/time.py @@ -0,0 +1,39 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import delete, insert + +from itn.kinyarwanda.rules.utils import string_map +from tn.processor import Processor + + +class Time(Processor): + """Rwandan clock times, e.g. ``saa cyenda n'iminota mirongo itatu`` -> ``3:30``. + + The tagger drops ``saa`` and the ``n'`` connector and keeps the hour + words and the minute phrase (``umunota umwe``, ``iminota ...``). A time + without minutes is written on the full hour. + """ + + def __init__(self): + super().__init__(name="time", ordertype="itn", token_orders={}) + hours = string_map("kinyarwanda/data/time/hours.tsv") + minutes = string_map("kinyarwanda/data/time/minutes.tsv") + tagger = delete("saa ") + self.tag_field("hours", hours) + tagger += (delete(" n'") + insert(" ") + self.tag_field("minutes", minutes)).ques + self.tagger = self.add_tokens(tagger) + verbalizer = self.verbalize_field("hours", hours) + insert(":") + verbalizer += self.verbalize_field("minutes", minutes, leading_space=True) | insert("00") + self.verbalizer = self.delete_tokens(verbalizer) diff --git a/itn/kinyarwanda/rules/utils.py b/itn/kinyarwanda/rules/utils.py new file mode 100644 index 00000000..7e47bdae --- /dev/null +++ b/itn/kinyarwanda/rules/utils.py @@ -0,0 +1,39 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from itn.utils import get_abs_path + + +def load_pairs(rel_path): + """Loads ``keyvalue`` rows of a Kinyarwanda data table in file order.""" + + pairs = [] + with open(get_abs_path(rel_path), encoding="utf-8") as stream: + for line in stream: + line = line.rstrip("\r\n") + if not line: + continue + key, value = line.split("\t")[:2] + pairs.append((key, value)) + return pairs + + +def string_map(rel_path): + """Builds a ``key -> value`` transducer from a table with unique keys.""" + + arcs = [pynini.cross(pynini.escape(key), pynini.escape(value)) for key, value in load_pairs(rel_path)] + return pynini.union(*arcs).optimize() diff --git a/itn/kinyarwanda/rules/whitelist.py b/itn/kinyarwanda/rules/whitelist.py new file mode 100644 index 00000000..0e437ec3 --- /dev/null +++ b/itn/kinyarwanda/rules/whitelist.py @@ -0,0 +1,32 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from itn.kinyarwanda.rules.utils import string_map +from tn.processor import Processor + + +class WhiteList(Processor): + """Spelled-out initialisms, e.g. ``bibisi`` -> ``BBC``. + + Only letter-by-letter spellings whose written form is an initialism are + listed. Ordinary transliterations (``televiziyo``, ``maroki``) are + the written Kinyarwanda words and pass through unchanged. + """ + + def __init__(self): + super().__init__(name="whitelist", ordertype="itn", token_orders={}) + graph = string_map("kinyarwanda/data/whitelist.tsv") + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/kinyarwanda/rules/word.py b/itn/kinyarwanda/rules/word.py new file mode 100644 index 00000000..cdadf2cc --- /dev/null +++ b/itn/kinyarwanda/rules/word.py @@ -0,0 +1,28 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import delete + +from tn.processor import Processor + + +class Word(Processor): + """Any other non-space run, kept verbatim.""" + + def __init__(self): + super().__init__(name="word", ordertype="itn", token_orders={}) + self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus)) + graph = self.NOT_SPACE.plus | delete(self.SPACE.plus) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/kinyarwanda/test/__init__.py b/itn/kinyarwanda/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/kinyarwanda/test/normalizer_test.py b/itn/kinyarwanda/test/normalizer_test.py new file mode 100644 index 00000000..d78c7a7b --- /dev/null +++ b/itn/kinyarwanda/test/normalizer_test.py @@ -0,0 +1,194 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path + +import pytest + +from itn.kinyarwanda.inverse_normalizer import InverseNormalizer + +PACKAGE = Path(__file__).resolve().parent.parent +# NeMo has no Kinyarwanda ITN fixtures; the TN fixtures are read reversed. +TN_DATA = PACKAGE.parent.parent / "tn" / "kinyarwanda" / "test" / "data" + +# Reversed TN pairs that cannot hold, with the output ITN gives instead. +EXCLUSIONS = { + # TN deletes the commas of a digit list; the separators are not recoverable. + "kabiri gatatu kane gatanu": "2 3 4 5", + # Transliterations are the written Kinyarwanda words and are not inverted. + "maroki": "maroki", + "televiziyo": "televiziyo", + # TN strips the surrounding whitespace that the written side keeps. + "no": "no", + "x": "x", +} + + +def reversed_cases(): + candidates = {} + for path in sorted(TN_DATA.glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for line in stream: + if line.endswith("\n"): + line = line[:-1] + written, spoken = line.split("~") + candidates.setdefault(spoken, []).append(written) + return candidates + + +REVERSED = reversed_cases() +CASES = [pytest.param(spoken, written, id=spoken or "empty") + for spoken, written in REVERSED.items() if spoken not in EXCLUSIONS] + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(InverseNormalizer) + + +def test_reversed_fixture_count(): + assert sum(len(written) for written in REVERSED.values()) == 100 + assert len(REVERSED) == 98 + assert len(CASES) == 93 + # Whole-hour readings end with a space and whitespace-only input reads as "". + assert REVERSED["saa saba "] == ["1:00"] + assert REVERSED[""] == ["", " "] + assert REVERSED["rimwe"] == ["1", "1"] + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_reversed(normalizer, spoken, written): + assert normalizer.normalize(spoken) in written + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_reversed_mapping(normalizer, spoken, written): + result = normalizer.normalize_with_mapping(spoken, include_identity=True) + assert result.output_text in written + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +@pytest.mark.parametrize("spoken, output", sorted(EXCLUSIONS.items())) +def test_exclusions(normalizer, spoken, output): + assert spoken in REVERSED + assert output not in REVERSED[spoken] + assert normalizer.normalize(spoken) == output + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for spoken in list(REVERSED) + list(EXCLUSIONS): + assert normalizer.verbalize(normalizer.tag(spoken)) == normalizer.normalize(spoken) + + +@pytest.mark.parametrize("spoken, kind, members, written", [ + ("ijana na makumyabiri na gatatu", "cardinal", {"integer": "ijana na makumyabiri na gatatu"}, "123"), + ("zeru", "cardinal", {"integer": "zeru"}, "0"), + ("zeru zeru kabiri", "cardinal", {"digits": "zeru zeru kabiri"}, "002"), + ("igihumbi", "cardinal", {"integer": "igihumbi"}, "1000"), + ("miliyoni n'igihumbi", "cardinal", {"integer": "miliyoni n'igihumbi"}, "1001000"), + ("miliyoni imwe", "cardinal", {"integer": "miliyoni imwe"}, "1000000"), + ("tiriyoni imwe", "cardinal", {"integer": "tiriyoni imwe"}, "1000000000"), + ("miliyoni makumyabiri n'imwe", "cardinal", {"integer": "miliyoni makumyabiri n'imwe"}, "21000000"), + ("saa cyenda n'iminota mirongo itatu", "time", {"hours": "cyenda", "minutes": "iminota mirongo itatu"}, + "3:30"), + ("saa munani n'umunota umwe", "time", {"hours": "munani", "minutes": "umunota umwe"}, "2:01"), + ("saa cumi n'imwe", "time", {"hours": "cumi n'imwe"}, "5:00"), + ("saa tanu", "time", {"hours": "tanu"}, "11:00"), + ("bibisi", "whitelist", {"value": "bibisi"}, "BBC"), + ("ade eni", "whitelist", {"value": "ade eni"}, "ADN"), + ("wawawa", "whitelist", {"value": "wawawa"}, "www"), + ("umwana", "word", {"value": "umwana"}, "umwana"), + ("televiziyo", "word", {"value": "televiziyo"}, "televiziyo"), +]) +def test_raw_fields(normalizer, spoken, kind, members, written): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == written + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken, written", [ + ("(gatanu)", "( 5 )"), + ("gatanu,", "5 ,"), + ("saa saba.", "1:00 ."), + ("ndi na makumyabiri", "ndi na 20"), + ("vi o aye na bibisi", "VOA na BBC"), + # TN writes both 130000 and 100030 this way; the larger scale keeps the parts. + ("ibihumbi ijana na mirongo itatu", "130000"), + ("ibihumbi ijana na makumyabiri na gatatu", "120003"), + ("rimwe kabiri", "1 2"), +]) +def test_additional_semantics(normalizer, spoken, written): + assert normalizer.normalize(spoken) == written + + +def test_ambiguous_reading_keeps_alternative(normalizer): + assert normalizer.normalize("ibihumbi ijana na mirongo itatu", nbest=2) == ["130000", "100030"] + + +@pytest.mark.parametrize("spoken", ["ijana na makumyabiri na gatatu", "saa sita n'iminota icyenda", "bibisi", + "zeru zeru kabiri", "(gatanu)"]) +def test_nbest_mapping_synchronization(normalizer, spoken): + outputs = normalizer.normalize(spoken, nbest=3) + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(spoken) + for result in mapped: + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + spoken = 'umwana\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(spoken)) + assert [token.members for token in parser.tokens] == [{"value": spoken}] + assert normalizer.normalize(spoken) == spoken + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("spoken", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, spoken): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": spoken} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(spoken, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) + + +def test_standalone_from_tn(): + tn_import = re.compile(r"^\s*(from|import)\s+tn\.kinyarwanda\b", re.MULTILINE) + for path in PACKAGE.rglob("*.py"): + assert not tn_import.search(path.read_text(encoding="utf-8")), path diff --git a/tn/kinyarwanda/__init__.py b/tn/kinyarwanda/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tn/kinyarwanda/data/numbers/digit.tsv b/tn/kinyarwanda/data/numbers/digit.tsv new file mode 100644 index 00000000..cd681e9e --- /dev/null +++ b/tn/kinyarwanda/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 rimwe +2 kabiri +3 gatatu +4 kane +5 gatanu +6 gatandatu +7 karindwi +8 umunani +9 icyenda diff --git a/tn/kinyarwanda/data/numbers/digit_bi.tsv b/tn/kinyarwanda/data/numbers/digit_bi.tsv new file mode 100644 index 00000000..4293c3ff --- /dev/null +++ b/tn/kinyarwanda/data/numbers/digit_bi.tsv @@ -0,0 +1,9 @@ +1 kimwe +2 bibiri +3 bitatu +4 bine +5 bitanu +6 bitandatu +7 birindwi +8 umunani +9 icyenda diff --git a/tn/kinyarwanda/data/numbers/digit_e.tsv b/tn/kinyarwanda/data/numbers/digit_e.tsv new file mode 100644 index 00000000..76c56b9d --- /dev/null +++ b/tn/kinyarwanda/data/numbers/digit_e.tsv @@ -0,0 +1,9 @@ +1 imwe +2 ebyiri +3 eshatu +4 enye +5 eshanu +6 esheshatu +7 zirindwi +8 umunani +9 icyenda diff --git a/tn/kinyarwanda/data/numbers/hundreds.tsv b/tn/kinyarwanda/data/numbers/hundreds.tsv new file mode 100644 index 00000000..10d00a82 --- /dev/null +++ b/tn/kinyarwanda/data/numbers/hundreds.tsv @@ -0,0 +1,9 @@ +1 ijana +2 magana abiri +3 magana atatu +4 magana ane +5 magana atanu +6 magana atandatu +7 magana arindwi +8 magana inani +9 magana cyenda diff --git a/tn/kinyarwanda/data/numbers/tens.tsv b/tn/kinyarwanda/data/numbers/tens.tsv new file mode 100644 index 00000000..66d40040 --- /dev/null +++ b/tn/kinyarwanda/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +2 makumyabiri +3 mirongo itatu +4 mirongo ine +5 mirongo itanu +6 mirongo itandatu +7 mirongo irindwi +8 mirongo inani +9 mirongo icyenda diff --git a/tn/kinyarwanda/data/numbers/zero.tsv b/tn/kinyarwanda/data/numbers/zero.tsv new file mode 100644 index 00000000..a0b17f6a --- /dev/null +++ b/tn/kinyarwanda/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 zeru diff --git a/tn/kinyarwanda/data/time/hours.tsv b/tn/kinyarwanda/data/time/hours.tsv new file mode 100644 index 00000000..fae6f089 --- /dev/null +++ b/tn/kinyarwanda/data/time/hours.tsv @@ -0,0 +1,12 @@ +1 saa saba +2 saa munani +3 saa cyenda +4 saa cumi +5 saa cumi n'imwe +6 saa cumi n'ebyiri +7 saa moya +8 saa mbiri +9 saa tatu +10 saa ine +11 saa tanu +12 saa sita \ No newline at end of file diff --git a/tn/kinyarwanda/data/time/minutes.tsv b/tn/kinyarwanda/data/time/minutes.tsv new file mode 100644 index 00000000..c3032710 --- /dev/null +++ b/tn/kinyarwanda/data/time/minutes.tsv @@ -0,0 +1,60 @@ +00 +01 n'umunota umwe +02 n'iminota ibiri +03 n'iminota itatu +04 n'iminota ine +05 n'iminota itanu +06 n'iminota itandatu +07 n'iminota irindwi +08 n'iminota umunani +09 n'iminota icyenda +10 n'iminota icumi +11 n'iminota cumi n'umwe +12 n'iminota cumi n'ibiri +13 n'iminota cumi n'itatu +14 n'iminota cumi n'ine +15 n'iminota cumi n'itanu +16 n'iminota cumi n'itandatu +17 n'iminota cumi n'irindwi +18 n'iminota cumi n'umunani +19 n'iminota cumi n'icyenda +20 n'iminota makumyabiri +21 n'iminota makumyabiri na rimwe +22 n'iminota makumyabiri n'ibiri +23 n'iminota makumyabiri n'itatu +24 n'iminota makumyabiri n'ine +25 n'iminota makumyabiri n'itanu +26 n'iminota makumyabiri n'itandatu +27 n'iminota makumyabiri n'irindwi +28 n'iminota makumyabiri n'umunani +29 n'iminota makumyabiri n'icyenda +30 n'iminota mirongo itatu +31 n'iminota mirongo itatu n'umwe +32 n'iminota mirongo itatu n'ibiri +33 n'iminota mirongo itatu n'itatu +34 n'iminota mirongo itatu n'ine +35 n'iminota mirongo itatu n'itanu +36 n'iminota mirongo itatu n'itandatu +37 n'iminota mirongo itatu n'irindwi +38 n'iminota mirongo itatu n'umunani +39 n'iminota mirongo itatu n'icyenda +40 n'iminota mirongo ine +41 n'iminota mirongo ine n'umwe +42 n'iminota mirongo ine n'ibiri +43 n'iminota mirongo ine n'itatu +44 n'iminota mirongo ine n'ine +45 n'iminota mirongo ine n'itanu +46 n'iminota mirongo ine n'itandatu +47 n'iminota mirongo ine n'irindwi +48 n'iminota mirongo ine n'umunani +49 n'iminota mirongo ine n'icyenda +50 n'iminota mirongo itanu +51 n'iminota mirongo itanu n'umwe +52 n'iminota mirongo itanu n'ibiri +53 n'iminota mirongo itanu n'itatu +54 n'iminota mirongo itanu n'ine +55 n'iminota mirongo itanu n'itanu +56 n'iminota mirongo itanu n'itandatu +57 n'iminota mirongo itanu n'irindwi +58 n'iminota mirongo itanu n'umunani +59 n'iminota mirongo itanu n'icyenda \ No newline at end of file diff --git a/tn/kinyarwanda/data/whitelist.tsv b/tn/kinyarwanda/data/whitelist.tsv new file mode 100644 index 00000000..e550214c --- /dev/null +++ b/tn/kinyarwanda/data/whitelist.tsv @@ -0,0 +1,175 @@ +Chris kiris +Hipkins Hipikineze +Dexamethasone dekisametazone +corticosteroid koritikositeroyide +immunosuppressant iminosupuresa +CDC sidisi +RBC arabisi +RISA risa +minisante minisante +sars sarisi +pfizer pifiza +BionTech biyoniteki +dollar dorari +ADN ade eni +BBC bibisi +Victoria vikitoria +Espagne esipanye +Nouvelle-Zélande nuveli zerandi +lopinavir lopinaviri +rotinavir rotinaviri +HIV heci ayivi +seychelles seyisheli +maroc maroki +sputnik siputinike +la crosse la korosi +paul pawulo +www wawawa +gov govu +rw rwa +http hecititipi +Berlin iberile +Remdesivir remudesivire +coranavirus koronavirusi +covid kovide +quarantine karantine +oms o e mesi +basketball basiketibalu +football futibolu +cholera kolera +radio radiyo +television televiziyo +service serivise +prof purofu +royal ruyolo +college koreji +health ubuzima +SARS-CoV-kabiri sarisi-kov-kabiri +recovery rekoveri +Dr dogiteri +protein puroteyine +spike sipiyike +victoria vigitoriya +technique tekinike +cell selile +electro erekitoro +sanitizer sanitayiza +Orthocoronavirinae oritocoronavirinaye +coronavirinae coronavirinaye +nidovirales nidoviralesi +Covs covuse +antibody antibodi +Hydroxychloroquine hidurokulorokine +company kompani +oxygen ogisijeni +Carolina karolina +jonathan jonatani +hyper hiperi +micro mikoro +microscope mikorosikope +microchip mikorocipu +glycoproteine gilicopuroteyine +sport siporo +lockdown lokidawuno +email imeli +japan japani +science siyansi +pubmed pubimedi +koica koyika +jica jika +DNA diyeniyi +RNA araneyi +wuhan wuhani +huanan hwanani +thermoregulation terimoregulashiyoni +alveolar aliviyola +hypoxemia hipokisemiya +PCR pisiyara +rapid-test rapidi-tesite +sepsis sepusisi +septique seputike +pulmonary pirimonari +extra egisitura +Real riyo +Time tayimu +Polymerase porimerase +poly pori +Chain ceyini +Reaction reyakishoni +hypoxic hipokisike +ICU ayisiyu +ambulance amburansi +antibiotic antibiyotike +vaccine vagisine +MEDAIR medayire +guardian garidiyani +covax covagise +paris parisi +transplant turansipulanti +laboratoire laboratuwari +Tedros tewodurosi +Ghebreyesus gebureyesusi +polybasic poribazike +china chinwa +RT-PCR arati-pisiyara +UNICEF yunicefu +HCR hashiseyeri +UNESCO yunesico +UN oni +World woridi +bank banki +FMI efu emi +new-york nuyoriki +times tayimuze +MERS merise +electron erekituronu +RDB aradibi +Platelet-fibrin puratele-fibirini +arterial ariteriyo +coagulopathie kowagulopati +RBD arabidi +RDF aradiyefu +module modile +Oxford ogisiforudu +AstraZeneca asutarazeneka +Astra-Zeneca asutarazeneka +astra asutara +zeneca zeneka +chlorine kulorakine +acide aside +peroxyacetic perukisiyatike +chlorhexidine kulorekidine +chloroform kuloroforume +disinfectant dezenkifekita +carbon kariboni +Hopkins hopikinze +communist komunisite +Tanzania tanzaniya +Africa afurika +VOA vi o aye +Jean yohana +Marie mariya +Vianney viyane +chimiotherapie kimyoterapi +sinopharm sinofarume +bus busi +ventilator ventirata +ventilators ventirataze +mRNA emu araneyi +Favipiravir favipiravire +command komandi +center santire +app apu +phone fone +telephone terefone +clinical kiliniko +clinique kilinike +lymphocytes lemfosite +twitter tuwita +youtube yutubi +facebook fasibuki +google gugoli +com komu +Antibodies antibodize +COVID-CUMI kovide-cumi +COVID-CUMI-N'ICYENDA kovide-cumi-n'icyenda diff --git a/tn/kinyarwanda/normalizer.py b/tn/kinyarwanda/normalizer.py new file mode 100644 index 00000000..859f00a6 --- /dev/null +++ b/tn/kinyarwanda/normalizer.py @@ -0,0 +1,54 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import add_weight + +from tn.kinyarwanda.rules.cardinal import Cardinal +from tn.kinyarwanda.rules.punctuation import Punctuation +from tn.kinyarwanda.rules.time import Time +from tn.kinyarwanda.rules.whitelist import WhiteList +from tn.kinyarwanda.rules.word import Word +from tn.processor import Processor, RuleSpec + +# Every Kinyarwanda rule emits its fields in spoken order, so no token needs +# reordering before verbalization. +TOKEN_ORDERS = {} + + +class Normalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="rw_normalizer", token_orders=TOKEN_ORDERS) + self.build_fst("rw_tn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + punctuation = Punctuation() + word = Word() + rules = ( + RuleSpec(WhiteList(), 1.01), + RuleSpec(Time(), 1.05), + RuleSpec(Cardinal(), 1.1), + RuleSpec(word, 100), + RuleSpec(punctuation), + ) + classify = self.tagger_union(rules) + # Punctuation splits off only when attached to another token. + punct = add_weight(punctuation.tagger, 1.1) + token = punct.star + classify + punct.star + self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + verbalize = self.verbalizer_union(rules) + self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star diff --git a/tn/kinyarwanda/rules/__init__.py b/tn/kinyarwanda/rules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tn/kinyarwanda/rules/cardinal.py b/tn/kinyarwanda/rules/cardinal.py new file mode 100644 index 00000000..5bb73398 --- /dev/null +++ b/tn/kinyarwanda/rules/cardinal.py @@ -0,0 +1,117 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from itertools import product + +import pynini +from pynini import accep, cross, difference, union +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.kinyarwanda.rules.utils import string_map +from tn.processor import Processor + +VOWELS = "aeiou" +CONSONANTS = "bcdfghjklmnpqrstvwxyz" +# Ten has a standalone form and a form that is followed by its units. +TEN = "icumi" +TEN_BEFORE_UNITS = "cumi" +# Scale words: the bare word for exactly one, the class prefix otherwise. +SCALES = ( + ("tiriyoni", "tiriyoni "), + ("miliyoni", "miliyoni "), + ("igihumbi", "ibihumbi "), +) + + +class Cardinal(Processor): + """Cardinal numbers, e.g. ``123`` -> ``ijana na makumyabiri na gatatu``. + + Every number word agrees with the noun it counts: plain units use the + counting class (``kabiri``), units of ``ibihumbi`` the bi-class + (``bibiri``), units of ``miliyoni``/``tiriyoni`` the zi-class + (``ebyiri``). Parts are joined with ``na`` before a consonant and ``n'`` + before a vowel. As in NeMo, ``tiriyoni`` names 10^9 and numbers up to + twelve digits are read as a whole; zero-padded, comma-separated or + longer digit strings are read digit by digit. + + The tagger keeps the written digits (``integer`` for whole numbers, + ``digits`` for digit-by-digit strings); the verbalizer spells them out. + """ + + def __init__(self): + super().__init__("cardinal", token_orders={}) + self.number = self.build_number() + digit = string_map("kinyarwanda/data/numbers/digit.tsv") + zero = string_map("kinyarwanda/data/numbers/zero.tsv") + integer = (zero | self.number).optimize() + any_digit = (digit | zero).optimize() + digits = (any_digit + (delete(",").ques + insert(" ") + any_digit).star).optimize() + + tagger = self.tag_field("integer", integer) | add_weight(self.tag_field("digits", digits), 0.1) + self.tagger = self.add_tokens(tagger) + verbalizer = self.verbalize_field("integer", integer) | self.verbalize_field("digits", digits) + self.verbalizer = self.delete_tokens(verbalizer) + + def connect(self, graph): + """Prefixes a part with ``na`` or ``n'`` depending on its first letter.""" + + vowel_initial = union(*VOWELS) + self.VSIGMA + consonant_initial = union(*CONSONANTS) + self.VSIGMA + return (insert(" n'") + (graph @ vowel_initial)) | (insert(" na ") + (graph @ consonant_initial)) + + def group(self, units): + """Spells a three-digit group other than ``000`` with the given unit class.""" + + hundreds = string_map("kinyarwanda/data/numbers/hundreds.tsv") + tens = string_map("kinyarwanda/data/numbers/tens.tsv") + options = [] + for has_hundreds, has_tens, has_units in product((False, True), repeat=3): + if not (has_hundreds or has_tens or has_units): + continue + ten = cross("1", TEN_BEFORE_UNITS if has_units else TEN) | tens + parts = [(has_hundreds, hundreds), (has_tens, ten), (has_units, units)] + graph = accep("") + started = False + for present, part in parts: + if not present: + graph += delete("0") + continue + graph += self.connect(part) if started else part + started = True + options.append(graph) + return union(*options).optimize() + + def build_number(self): + digit = string_map("kinyarwanda/data/numbers/digit.tsv") + digit_bi = string_map("kinyarwanda/data/numbers/digit_bi.tsv") + digit_e = string_map("kinyarwanda/data/numbers/digit_e.tsv") + three = pynini.closure(self.DIGIT, 3, 3) + not_one = difference(three, "001").optimize() + + scaled = [] + for (one, prefix), units in zip(SCALES, (digit_e, digit_e, digit_bi)): + scaled.append((cross("001", one) | (not_one @ (insert(prefix) + self.group(units)))).optimize()) + groups = scaled + [self.group(digit)] + + nonzero = difference(self.DIGIT, "0") + pad = insert("00") + nonzero | insert("0") + nonzero + self.DIGIT | nonzero + self.DIGIT + self.DIGIT + options = [] + for index, top in enumerate(groups): + graph = pad @ top + for lower in groups[index + 1:]: + graph += delete("000") | self.connect(lower) + options.append(graph) + return union(*options).optimize() diff --git a/tn/kinyarwanda/rules/punctuation.py b/tn/kinyarwanda/rules/punctuation.py new file mode 100644 index 00000000..827d9906 --- /dev/null +++ b/tn/kinyarwanda/rules/punctuation.py @@ -0,0 +1,29 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from tn.processor import Processor + + +class Punctuation(Processor): + """ASCII punctuation split off a neighbouring token, e.g. ``(5)``.""" + + def __init__(self): + super().__init__("punctuation", token_orders={}) + graph = pynini.union(*[pynini.escape(mark) for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~"]).optimize() + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/tn/kinyarwanda/rules/time.py b/tn/kinyarwanda/rules/time.py new file mode 100644 index 00000000..9b301359 --- /dev/null +++ b/tn/kinyarwanda/rules/time.py @@ -0,0 +1,40 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import delete, insert + +from tn.kinyarwanda.rules.utils import string_map +from tn.processor import Processor + + +class Time(Processor): + """Clock times, e.g. ``3:30`` -> ``saa cyenda n'iminota mirongo itatu``. + + Hours follow the Rwandan clock (``7`` is ``saa moya``). As in NeMo's + minute table, every minute reading carries its own leading space and + ``00`` reads as a bare space, which the original fixtures expect. + """ + + def __init__(self): + super().__init__("time", token_orders={}) + hours = string_map("kinyarwanda/data/time/hours.tsv") + minutes = string_map("kinyarwanda/data/time/minutes.tsv") + tagger = self.tag_field("hours", hours) + delete(":") + insert(" ") + tagger += self.tag_field("minutes", minutes) + self.tagger = self.add_tokens(tagger) + verbalizer = self.verbalize_field("hours", hours) + verbalizer += self.verbalize_field("minutes", minutes, leading_space=True) + self.verbalizer = self.delete_tokens(verbalizer) diff --git a/tn/kinyarwanda/rules/utils.py b/tn/kinyarwanda/rules/utils.py new file mode 100644 index 00000000..09bc035a --- /dev/null +++ b/tn/kinyarwanda/rules/utils.py @@ -0,0 +1,43 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from tn.utils import get_abs_path + + +def load_pairs(rel_path): + """Loads ``keyvalue`` rows of a Kinyarwanda data table in file order. + + Values are kept verbatim: the minute table relies on a leading space. + """ + + pairs = [] + with open(get_abs_path(rel_path), encoding="utf-8") as stream: + for line in stream: + line = line.rstrip("\r\n") + if not line: + continue + key, value = line.split("\t")[:2] + pairs.append((key, value)) + return pairs + + +def string_map(rel_path): + """Builds a ``key -> value`` transducer from a table with unique keys.""" + + arcs = [pynini.cross(pynini.escape(key), pynini.escape(value)) for key, value in load_pairs(rel_path)] + return pynini.union(*arcs).optimize() diff --git a/tn/kinyarwanda/rules/whitelist.py b/tn/kinyarwanda/rules/whitelist.py new file mode 100644 index 00000000..df905ff5 --- /dev/null +++ b/tn/kinyarwanda/rules/whitelist.py @@ -0,0 +1,28 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tn.kinyarwanda.rules.utils import string_map +from tn.processor import Processor + + +class WhiteList(Processor): + """Kinyarwanda transliterations, e.g. ``television`` -> ``televiziyo``.""" + + def __init__(self): + super().__init__("whitelist", token_orders={}) + graph = string_map("kinyarwanda/data/whitelist.tsv") + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/tn/kinyarwanda/rules/word.py b/tn/kinyarwanda/rules/word.py new file mode 100644 index 00000000..4bb36db1 --- /dev/null +++ b/tn/kinyarwanda/rules/word.py @@ -0,0 +1,29 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import delete + +from tn.processor import Processor + + +class Word(Processor): + """Any other non-space run, kept verbatim.""" + + def __init__(self): + super().__init__("word", token_orders={}) + self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus)) + graph = self.NOT_SPACE.plus | delete(self.SPACE.plus) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/tn/kinyarwanda/test/__init__.py b/tn/kinyarwanda/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tn/kinyarwanda/test/data/test_cases_cardinal.txt b/tn/kinyarwanda/test/data/test_cases_cardinal.txt new file mode 100644 index 00000000..baca7cbe --- /dev/null +++ b/tn/kinyarwanda/test/data/test_cases_cardinal.txt @@ -0,0 +1,57 @@ +1~rimwe +2~kabiri +3~gatatu +4~kane +5~gatanu +6~gatandatu +7~karindwi +8~umunani +9~icyenda +10~icumi +002~zeru zeru kabiri +11~cumi na rimwe +12~cumi na kabiri +13~cumi na gatatu +2,3,4,5~kabiri gatatu kane gatanu +14~cumi na kane +15~cumi na gatanu +16~cumi na gatandatu +17~cumi na karindwi +18~cumi n'umunani +19~cumi n'icyenda +20~makumyabiri +89~mirongo inani n'icyenda +123~ijana na makumyabiri na gatatu +730~magana arindwi na mirongo itatu +100~ijana +2004~ibihumbi bibiri na kane +9041~ibihumbi icyenda na mirongo ine na rimwe +5324~ibihumbi bitanu na magana atatu na makumyabiri na kane +8567~ibihumbi umunani na magana atanu na mirongo itandatu na karindwi +10000~ibihumbi icumi +14000~ibihumbi cumi na bine +24404~ibihumbi makumyabiri na bine na magana ane na kane +9000~ibihumbi icyenda +9700~ibihumbi icyenda na magana arindwi +250~magana abiri na mirongo itanu +367~magana atatu na mirongo itandatu na karindwi +90104~ibihumbi mirongo icyenda n'ijana na kane +111001~ibihumbi ijana na cumi na kimwe na rimwe +10999~ibihumbi icumi na magana cyenda na mirongo icyenda n'icyenda +100000~ibihumbi ijana +200000~ibihumbi magana abiri +101000~ibihumbi ijana na kimwe +130000~ibihumbi ijana na mirongo itatu +531000~ibihumbi magana atanu na mirongo itatu na kimwe +2200345~miliyoni ebyiri n'ibihumbi magana abiri na magana atatu na mirongo ine na gatanu +7000000~miliyoni zirindwi +9101100~miliyoni icyenda n'ibihumbi ijana na kimwe n'ijana +19034004~miliyoni cumi n'icyenda n'ibihumbi mirongo itatu na bine na kane +29000000~miliyoni makumyabiri n'icyenda +40000000~miliyoni mirongo ine +400000000~miliyoni magana ane +100000001~miliyoni ijana na rimwe +340000000~miliyoni magana atatu na mirongo ine +783100000~miliyoni magana arindwi na mirongo inani n'eshatu n'ibihumbi ijana +340010010~miliyoni magana atatu na mirongo ine n'ibihumbi icumi n'icumi +9374514510~tiriyoni icyenda na miliyoni magana atatu na mirongo irindwi n'enye n'ibihumbi magana atanu na cumi na bine na magana atanu n'icumi diff --git a/tn/kinyarwanda/test/data/test_cases_time.txt b/tn/kinyarwanda/test/data/test_cases_time.txt new file mode 100644 index 00000000..4a4ec27b --- /dev/null +++ b/tn/kinyarwanda/test/data/test_cases_time.txt @@ -0,0 +1,14 @@ +1:00~saa saba +2:01~saa munani n'umunota umwe +3:30~saa cyenda n'iminota mirongo itatu +4:21~saa cumi n'iminota makumyabiri na rimwe +5:12~saa cumi n'imwe n'iminota cumi n'ibiri +6:49~saa cumi n'ebyiri n'iminota mirongo ine n'icyenda +7:05~saa moya n'iminota itanu +8:23~saa mbiri n'iminota makumyabiri n'itatu +9:47~saa tatu n'iminota mirongo ine n'irindwi +10:56~saa ine n'iminota mirongo itanu n'itandatu +11:00~saa tanu +12:09~saa sita n'iminota icyenda +1:59~saa saba n'iminota mirongo itanu n'icyenda +12:31~saa sita n'iminota mirongo itatu n'umwe \ No newline at end of file diff --git a/tn/kinyarwanda/test/data/test_cases_whitelist.txt b/tn/kinyarwanda/test/data/test_cases_whitelist.txt new file mode 100644 index 00000000..b9b59793 --- /dev/null +++ b/tn/kinyarwanda/test/data/test_cases_whitelist.txt @@ -0,0 +1,3 @@ +www~wawawa +maroc~maroki +television~televiziyo \ No newline at end of file diff --git a/tn/kinyarwanda/test/data/test_cases_word.txt b/tn/kinyarwanda/test/data/test_cases_word.txt new file mode 100644 index 00000000..1c97057a --- /dev/null +++ b/tn/kinyarwanda/test/data/test_cases_word.txt @@ -0,0 +1,26 @@ +~ + ~ + no~no +x ~x +X!~X! +—~— +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aalem~aalem +a'ali~a'ali +aaliyan's~aaliyan's +mother-in-law~mother-in-law +1~rimwe +mar~mar +umwangavu~umwangavu diff --git a/tn/kinyarwanda/test/normalizer_test.py b/tn/kinyarwanda/test/normalizer_test.py new file mode 100644 index 00000000..87a7fad2 --- /dev/null +++ b/tn/kinyarwanda/test/normalizer_test.py @@ -0,0 +1,163 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path + +import pytest + +from tn.kinyarwanda.normalizer import Normalizer + +PACKAGE = Path(__file__).resolve().parent.parent + + +def original_cases(): + cases = [] + for path in sorted((Path(__file__).parent / "data").glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for index, line in enumerate(stream, 1): + if line.endswith("\n"): + line = line[:-1] + written, spoken = line.split("~") + cases.append(pytest.param(written, spoken, id="{}-{}".format(path.stem, index))) + return cases + + +CASES = original_cases() + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(Normalizer) + + +def test_original_fixture_count(): + assert len(CASES) == 100 + # A whole hour keeps NeMo's trailing space from the "00" minute reading. + assert CASES[57].values == ("1:00", "saa saba ") + # The time and whitelist fixtures have no trailing newline. + assert CASES[70].values == ("12:31", "saa sita n'iminota mirongo itatu n'umwe") + assert CASES[73].values == ("television", "televiziyo") + assert CASES[74].values == ("", "") + assert CASES[75].values == (" ", "") + + +@pytest.mark.parametrize("written, spoken", CASES) +def test_original(normalizer, written, spoken): + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, spoken", CASES) +def test_original_mapping(normalizer, written, spoken): + result = normalizer.normalize_with_mapping(written, include_identity=True) + assert result.output_text == spoken + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert spoken[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for param in CASES: + written, spoken = param.values + assert normalizer.verbalize(normalizer.tag(written)) == spoken + + +@pytest.mark.parametrize("written, kind, members, spoken", [ + ("123", "cardinal", {"integer": "123"}, "ijana na makumyabiri na gatatu"), + ("0", "cardinal", {"integer": "0"}, "zeru"), + ("002", "cardinal", {"digits": "002"}, "zeru zeru kabiri"), + ("2,3,4,5", "cardinal", {"digits": "2,3,4,5"}, "kabiri gatatu kane gatanu"), + ("1000", "cardinal", {"integer": "1000"}, "igihumbi"), + ("1001000", "cardinal", {"integer": "1001000"}, "miliyoni n'igihumbi"), + ("21000000", "cardinal", {"integer": "21000000"}, "miliyoni makumyabiri n'imwe"), + ("1000000000000", "cardinal", {"digits": "1000000000000"}, + "rimwe zeru zeru zeru zeru zeru zeru zeru zeru zeru zeru zeru zeru"), + ("3:30", "time", {"hours": "3", "minutes": "30"}, "saa cyenda n'iminota mirongo itatu"), + ("11:00", "time", {"hours": "11", "minutes": "00"}, "saa tanu "), + ("maroc", "whitelist", {"value": "maroc"}, "maroki"), + ("la crosse", "whitelist", {"value": "la crosse"}, "la korosi"), + ("ADN", "whitelist", {"value": "ADN"}, "ade eni"), + ("umwana", "word", {"value": "umwana"}, "umwana"), + ("13:00", "word", {"value": "13:00"}, "13:00"), +]) +def test_raw_fields(normalizer, written, kind, members, spoken): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == spoken + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, spoken", [ + ("(5)", "( gatanu )"), + ("5,", "gatanu ,"), + ("12:31.", "saa sita n'iminota mirongo itatu n'umwe ."), + ("Chris na 3:05", "kiris na saa cyenda n'iminota itanu"), + ("2,345", "kabiri gatatu kane gatanu"), + ("X!", "X!"), +]) +def test_additional_semantics(normalizer, written, spoken): + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written", ["123", "2,3,4,5", "12:31", "maroc", "(5)"]) +def test_nbest_mapping_synchronization(normalizer, written): + outputs = normalizer.normalize(written, nbest=3) + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(written) + for result in mapped: + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + written = 'umwana\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(written)) + assert [token.members for token in parser.tokens] == [{"value": written}] + assert normalizer.normalize(written) == written + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("written", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, written): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": written} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(written, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) + + +def test_standalone_from_itn(): + itn_import = re.compile(r"^\s*(from|import)\s+itn\.kinyarwanda\b", re.MULTILINE) + for path in PACKAGE.rglob("*.py"): + assert not itn_import.search(path.read_text(encoding="utf-8")), path From 155d2f7d677b60048e36e03f59c51696e3dbc1db Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Sat, 31 Oct 2026 12:43:31 +0800 Subject: [PATCH 2/2] feat: register Kinyarwanda pipelines in CLI, packaging and README --- README.md | 7 +++-- pyproject.toml | 4 +++ tn/chinese/test/cache_test.py | 4 +++ tn/cli.py | 2 ++ tn/kinyarwanda/normalizer.py | 14 +++++---- tn/kinyarwanda/rules/punctuation.py | 6 ++-- tn/kinyarwanda/test/normalizer_test.py | 41 ++++++++++++++++++++++++-- tn/test/cli_test.py | 20 ++++++------- 8 files changed, 73 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 20fc20d2..966b76a4 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ WeTextProcessing converts between written and spoken text using weighted finite-state transducers (WFSTs), built with [Pynini](https://www.openfst.org/twiki/bin/view/GRM/Pynini) and [OpenFst](https://www.openfst.org/twiki/bin/view/FST/WebHome). - **TN and ITN** — normalize numbers, dates, times, currencies, units, and other structured expressions. -- **Multilingual rules** — Arabic, Armenian, Chinese, English, French, German, Hindi, Japanese, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese pipelines with language-specific grammars. +- **Multilingual rules** — language-specific grammars for every language in [Supported languages](#supported-languages). - **Explainable output** — inspect raw tagged fields, trace input/output spans, and request unique n-best results. - **Reusable graphs** — automatic, configuration-aware FST caching and a separate C++ deployment runtime. @@ -45,6 +45,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12 | German | `de` | Yes | Yes | `german` | | Hindi | `hi` | Yes | Yes | `hindi` | | Japanese | `ja` | Yes | Yes | `japanese` | +| Kinyarwanda | `rw` | Yes | Yes | `kinyarwanda` | | Korean | `ko` | Yes | Yes | `korean` | | Portuguese | `pt` | Yes | Yes | `portuguese` | | Russian | `ru` | Yes | Yes | `russian` | @@ -54,7 +55,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12 Chinese is the CLI default. Rule coverage and configurable options vary by language and direction. -Arabic, Armenian, French, German, Hindi, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese support are included in this source checkout. Install from source to use them before the next package release. +The published package includes Chinese, English, and Japanese. The other languages are included in this source checkout; install from source to use them before the next package release. ## Quick start @@ -66,7 +67,7 @@ Install the published package: pip install WeTextProcessing ``` -For the latest rules, including Arabic, Armenian, French, German, Hindi, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese, install from source: +For the latest rules, including every language beyond Chinese, English, and Japanese, install from source: ```bash git clone https://github.com/wenet-e2e/WeTextProcessing.git diff --git a/pyproject.toml b/pyproject.toml index c2efca0a..1809e6fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,8 @@ tn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "kinyarwanda/data/*.tsv", + "kinyarwanda/data/*/*.tsv", ] itn = [ "armenian/data/*.tsv", @@ -102,4 +104,6 @@ itn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "kinyarwanda/data/*.tsv", + "kinyarwanda/data/*/*.tsv", ] diff --git a/tn/chinese/test/cache_test.py b/tn/chinese/test/cache_test.py index 2d7ded86..5ccce03a 100644 --- a/tn/chinese/test/cache_test.py +++ b/tn/chinese/test/cache_test.py @@ -10,6 +10,7 @@ from itn.french.inverse_normalizer import InverseNormalizer as FrInverseNormalizer from itn.german.inverse_normalizer import InverseNormalizer as DeInverseNormalizer from itn.hindi.inverse_normalizer import InverseNormalizer as HiInverseNormalizer +from itn.kinyarwanda.inverse_normalizer import InverseNormalizer as RwInverseNormalizer from itn.japanese.inverse_normalizer import InverseNormalizer as JaInverseNormalizer from itn.korean.inverse_normalizer import InverseNormalizer as KoInverseNormalizer from itn.portuguese.inverse_normalizer import InverseNormalizer as PtInverseNormalizer @@ -24,6 +25,7 @@ from tn.french.normalizer import Normalizer as FrNormalizer from tn.german.normalizer import Normalizer as DeNormalizer from tn.hindi.normalizer import Normalizer as HiNormalizer +from tn.kinyarwanda.normalizer import Normalizer as RwNormalizer from tn.japanese.normalizer import Normalizer as JaNormalizer from tn.korean.normalizer import Normalizer as KoNormalizer from tn.portuguese.normalizer import Normalizer as PtNormalizer @@ -96,6 +98,7 @@ def _build_small_graphs(processor): (EsNormalizer, "es_tn", "tn", {}, {"deterministic": True}), (EsNormalizer, "es_tn", "tn", {"deterministic": False}, {"deterministic": False}), (HiNormalizer, "hi_tn", "tn", {}, {}), + (RwNormalizer, "rw_tn", "tn", {}, {}), ( ZhInverseNormalizer, "zh_itn", @@ -143,6 +146,7 @@ def _build_small_graphs(processor): (EsInverseNormalizer, "es_itn", "itn", {}, {"cased": False}), (EsInverseNormalizer, "es_itn", "itn", {"cased": True}, {"cased": True}), (HiInverseNormalizer, "hi_itn", "itn", {}, {}), + (RwInverseNormalizer, "rw_itn", "itn", {}, {}), ], ) def test_all_pipeline_configs_are_part_of_manifest( diff --git a/tn/cli.py b/tn/cli.py index 6f7be9a4..2f2a879a 100644 --- a/tn/cli.py +++ b/tn/cli.py @@ -69,6 +69,7 @@ class PipelineSpec: (BooleanOption("deterministic", True, "use deterministic Spanish readings"),), ), "hi": PipelineSpec("tn.hindi.normalizer", "Normalizer"), + "rw": PipelineSpec("tn.kinyarwanda.normalizer", "Normalizer"), "ja": PipelineSpec( "tn.japanese.normalizer", "Normalizer", @@ -107,6 +108,7 @@ class PipelineSpec: (BooleanOption("cased", False, 'also accept capitalized spoken words, e.g. "Doce Dólares"'),), ), "hi": PipelineSpec("itn.hindi.inverse_normalizer", "InverseNormalizer"), + "rw": PipelineSpec("itn.kinyarwanda.inverse_normalizer", "InverseNormalizer"), "ja": PipelineSpec( "itn.japanese.inverse_normalizer", "InverseNormalizer", diff --git a/tn/kinyarwanda/normalizer.py b/tn/kinyarwanda/normalizer.py index 859f00a6..fac86f74 100644 --- a/tn/kinyarwanda/normalizer.py +++ b/tn/kinyarwanda/normalizer.py @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pynini.lib.pynutil import add_weight +from pynini import cdrewrite +from pynini.lib.pynutil import add_weight, delete from tn.kinyarwanda.rules.cardinal import Cardinal from tn.kinyarwanda.rules.punctuation import Punctuation @@ -41,14 +42,15 @@ def build_tagger_and_verbalizer(self): RuleSpec(WhiteList(), 1.01), RuleSpec(Time(), 1.05), RuleSpec(Cardinal(), 1.1), - RuleSpec(word, 100), - RuleSpec(punctuation), + RuleSpec(word, 100, verbalize=False), + RuleSpec(punctuation, verbalize=False), ) classify = self.tagger_union(rules) - # Punctuation splits off only when attached to another token. punct = add_weight(punctuation.tagger, 1.1) token = punct.star + classify + punct.star self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) - verbalize = self.verbalizer_union(rules) - self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star + verbalize = self.join_tokens(self.verbalizer_union(rules), (word.verbalizer, punctuation.verbalizer)) + # Whole hours read with a trailing space (fixture "1:00~saa saba "); drop the joining space after it. + squeeze = cdrewrite(delete(" "), " ", self.NOT_SPACE, self.VSIGMA) + self.verbalizer = (verbalize @ squeeze).optimize() diff --git a/tn/kinyarwanda/rules/punctuation.py b/tn/kinyarwanda/rules/punctuation.py index 827d9906..e7eab60b 100644 --- a/tn/kinyarwanda/rules/punctuation.py +++ b/tn/kinyarwanda/rules/punctuation.py @@ -15,6 +15,7 @@ # limitations under the License. import pynini +from pynini.lib.pynutil import add_weight from tn.processor import Processor @@ -25,5 +26,6 @@ class Punctuation(Processor): def __init__(self): super().__init__("punctuation", token_orders={}) graph = pynini.union(*[pynini.escape(mark) for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~"]).optimize() - self.tagger = self.add_tokens(self.tag_field("value", graph)) - self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) + space = add_weight(self.SPACE, -0.0001).star + self.tagger = self.add_tokens(self.tag_field("value", space + graph + space)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.SPACE.star + graph + self.SPACE.star)) diff --git a/tn/kinyarwanda/test/normalizer_test.py b/tn/kinyarwanda/test/normalizer_test.py index 87a7fad2..e9edd28d 100644 --- a/tn/kinyarwanda/test/normalizer_test.py +++ b/tn/kinyarwanda/test/normalizer_test.py @@ -104,17 +104,52 @@ def test_raw_fields(normalizer, written, kind, members, spoken): @pytest.mark.parametrize("written, spoken", [ - ("(5)", "( gatanu )"), - ("5,", "gatanu ,"), - ("12:31.", "saa sita n'iminota mirongo itatu n'umwe ."), + ("(5)", "(gatanu)"), + ("5,", "gatanu,"), + ("12:31.", "saa sita n'iminota mirongo itatu n'umwe."), ("Chris na 3:05", "kiris na saa cyenda n'iminota itanu"), ("2,345", "kabiri gatatu kane gatanu"), ("X!", "X!"), + ("Nzaza 1:00 ejo", "Nzaza saa saba ejo"), + ("11:00, 12:00", "saa tanu , saa sita "), ]) def test_additional_semantics(normalizer, written, spoken): assert normalizer.normalize(written) == spoken +@pytest.mark.parametrize("written, spoken, punctuation", [ + ("(5)", "(gatanu)", ["(", ")"]), + ("5,", "gatanu,", [","]), + ("5 !", "gatanu !", [" !"]), + ("( 5 )", "( gatanu )", ["( ", " )"]), +]) +def test_cardinal_punctuation_spacing(normalizer, written, spoken, punctuation): + assert normalizer.normalize(written) == spoken + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert [token.members for token in parser.tokens if token.name == "cardinal"] == [{"integer": "5"}] + assert [token.members for token in parser.tokens if token.name == "punctuation"] == [ + {"value": value} for value in punctuation + ] + assert normalizer.verbalize(tagged) == spoken + mapped = normalizer.normalize_with_mapping(written, include_identity=True) + assert mapped.output_text == spoken + assert [mapping.input_text for mapping in mapped.mappings if mapping.token_type == "punctuation"] == punctuation + for mapping in mapped.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert spoken[mapping.output_start:mapping.output_end] == mapping.output_text + + +@pytest.mark.parametrize("written", ["Nzaza 1:00 ejo", "11:00, 12:00", "1:00"]) +def test_whole_hour_mapping_is_consistent(normalizer, written): + result = normalizer.normalize_with_mapping(written, include_identity=True) + assert result.output_text == normalizer.normalize(written) + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + @pytest.mark.parametrize("written", ["123", "2,3,4,5", "12:31", "maroc", "(5)"]) def test_nbest_mapping_synchronization(normalizer, written): outputs = normalizer.normalize(written, nbest=3) diff --git a/tn/test/cli_test.py b/tn/test/cli_test.py index 9acb99d5..60167201 100644 --- a/tn/test/cli_test.py +++ b/tn/test/cli_test.py @@ -60,6 +60,7 @@ def verbalize(self, tagged): ("tn", "sv", "tn.swedish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "es", "tn.spanish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "hi", "tn.hindi.normalizer", "Normalizer", [], {}), + ("tn", "rw", "tn.kinyarwanda.normalizer", "Normalizer", [], {}), ( "itn", "zh", @@ -94,6 +95,7 @@ def verbalize(self, tagged): ("itn", "sv", "itn.swedish.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "es", "itn.spanish.inverse_normalizer", "InverseNormalizer", ["--cased"], {"cased": True}), ("itn", "hi", "itn.hindi.inverse_normalizer", "InverseNormalizer", [], {}), + ("itn", "rw", "itn.kinyarwanda.inverse_normalizer", "InverseNormalizer", [], {}), ], ) def test_create_processor_uses_language_capabilities( @@ -163,20 +165,14 @@ def import_module(name): parse_args(other_direction, ["--language", "xx"]) -@pytest.mark.parametrize("direction", ["tn", "itn"]) -@pytest.mark.parametrize("language", [ - "zh", "en", "ja", "ko", "de", "hy", "ar", "fr", "pt", "vi", "ru", "sv", "es", "hi", +@pytest.mark.parametrize("direction, language", [ + (direction, language) for direction, languages in cli.PIPELINES.items() for language in languages ]) def test_cli_defaults_and_forwarding_match_processor_signature(monkeypatch, direction, language): args = parse_args(direction, ["--language", language]) - package = { - "zh": "chinese", "en": "english", "ja": "japanese", "ko": "korean", "de": "german", "hy": "armenian", - "ar": "arabic", "fr": "french", "pt": "portuguese", "vi": "vietnamese", "ru": "russian", - "sv": "swedish", "es": "spanish", "hi": "hindi", - }[language] - module_suffix, class_name = ("normalizer", "Normalizer") if direction == "tn" else ( - "inverse_normalizer", "InverseNormalizer") - module = __import__("{}.{}.{}".format(direction, package, module_suffix), fromlist=[class_name]) + spec = cli.PIPELINES[direction][language] + module = __import__(spec.module, fromlist=[spec.class_name]) + class_name = spec.class_name parameters = inspect.signature(getattr(module, class_name)).parameters expected = {name: parameter.default for name, parameter in parameters.items()} assert {name: getattr(args, name) for name in expected} == expected @@ -223,6 +219,8 @@ def __init__(self, **kwargs): ("itn", "es", "--cased", "--enable-million"), ("tn", "hi", "--cache-dir", "--deterministic"), ("itn", "hi", "--cache-dir", "--enable-million"), + ("tn", "rw", "--cache-dir", "--deterministic"), + ("itn", "rw", "--cache-dir", "--enable-million"), ], ) def test_help_only_exposes_selected_pipeline_options(direction, language, present, absent):