From e720854009d90f4623a82beb86cabbf7d82df9d0 Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Wed, 2 Sep 2026 13:06:09 +0530 Subject: [PATCH 01/15] feat(te): add Telugu cardinal text normalization Rebased onto staging/telugu_tn_v1 with TE-only changes for PR #447. Signed-off-by: Haridas Nishitha Co-authored-by: Cursor Signed-off-by: Haridas Nishitha --- Jenkinsfile | 30 ++ .../text_normalization/normalize.py | 6 +- .../text_normalization/run_evaluate.py | 2 +- .../text_normalization/te/__init__.py | 13 + .../text_normalization/te/data/__init__.py | 13 + .../te/data/numbers/__init__.py | 13 + .../te/data/numbers/digit.tsv | 18 + .../te/data/numbers/exact_power.tsv | 8 + .../te/data/numbers/hundred_prefix.tsv | 4 + .../te/data/numbers/hundreds_before_one.tsv | 2 + .../te/data/numbers/magnitudes.tsv | 12 + .../te/data/numbers/teens_and_ties.tsv | 20 + .../te/data/numbers/ties.tsv | 16 + .../te/data/numbers/ties_one_suffix.tsv | 2 + .../te/data/numbers/zero.tsv | 2 + .../text_normalization/te/graph_utils.py | 145 ++++++ .../text_normalization/te/taggers/__init__.py | 13 + .../text_normalization/te/taggers/cardinal.py | 424 ++++++++++++++++++ .../te/taggers/punctuation.py | 61 +++ .../te/taggers/tokenize_and_classify.py | 115 +++++ .../text_normalization/te/taggers/word.py | 34 ++ .../text_normalization/te/utils.py | 27 ++ .../te/verbalizers/__init__.py | 13 + .../te/verbalizers/cardinal.py | 41 ++ .../te/verbalizers/verbalize.py | 38 ++ .../te/verbalizers/verbalize_final.py | 75 ++++ .../text_normalization/te/verbalizers/word.py | 35 ++ tests/nemo_text_processing/te/__init__.py | 13 + .../test_cases_cardinal.txt | 89 ++++ .../nemo_text_processing/te/test_cardinal.py | 37 ++ .../te/test_sparrowhawk_normalization.sh | 32 ++ .../pynini_export.py | 6 + 32 files changed, 1357 insertions(+), 2 deletions(-) create mode 100644 nemo_text_processing/text_normalization/te/__init__.py create mode 100644 nemo_text_processing/text_normalization/te/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/__init__.py create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/digit.tsv create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/exact_power.tsv create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/hundred_prefix.tsv create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/hundreds_before_one.tsv create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/magnitudes.tsv create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties.tsv create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/ties.tsv create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/ties_one_suffix.tsv create mode 100644 nemo_text_processing/text_normalization/te/data/numbers/zero.tsv create mode 100644 nemo_text_processing/text_normalization/te/graph_utils.py create mode 100644 nemo_text_processing/text_normalization/te/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/te/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/te/taggers/punctuation.py create mode 100644 nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/te/taggers/word.py create mode 100644 nemo_text_processing/text_normalization/te/utils.py create mode 100644 nemo_text_processing/text_normalization/te/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/te/verbalizers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/te/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/te/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/text_normalization/te/verbalizers/word.py create mode 100644 tests/nemo_text_processing/te/__init__.py create mode 100644 tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/te/test_cardinal.py create mode 100644 tests/nemo_text_processing/te/test_sparrowhawk_normalization.sh diff --git a/Jenkinsfile b/Jenkinsfile index 38e05bb00..aab941076 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,6 +30,7 @@ pipeline { JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-5' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' + TE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-21-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { @@ -113,6 +114,30 @@ pipeline { } } + stage('L0: Create TE TN/ITN Grammars') { + when { + anyOf { + branch 'main' + branch 'staging/**' + branch 'staging_*' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L0: TE TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=te --text="౧" --cache_dir ${TE_TN_CACHE}' + } + } + // stage('L0: TE ITN grammars') { + // steps { + // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=te --text="ఒకటి" --cache_dir ${TE_TN_CACHE}' + // } + // } + } + } + stage('L0: Create DE/ES TN/ITN Grammars') { when { anyOf { @@ -408,6 +433,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hi/ -m "not pleasefixme" --cpu --tn_cache_dir ${HI_TN_CACHE}' } } + stage('L1: Run all TE TN/ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/te/ -m "not pleasefixme" --cpu --tn_cache_dir ${TE_TN_CACHE}' + } + } stage('L1: Run all Codeswitched ES/EN TN/ITN tests (restore grammars from cache)') { steps { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/es_en/ -m "not pleasefixme" --cpu --tn_cache_dir ${ES_EN_TN_CACHE}' diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index d8ebf2f4d..02d3de03e 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -166,6 +166,7 @@ def __init__( if post_process: self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) + elif lang == 'it': from nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst @@ -191,6 +192,9 @@ def __init__( elif lang == 'ko': from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'te': + from nemo_text_processing.text_normalization.te.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.te.verbalizers.verbalize_final import VerbalizeFinalFst else: raise NotImplementedError(f"Language {lang} has not been supported yet.") @@ -737,7 +741,7 @@ def parse_args(): parser.add_argument( "--language", help="language", - choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"], + choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "te", "ko", "vi", "pt"], default="en", type=str, ) diff --git a/nemo_text_processing/text_normalization/run_evaluate.py b/nemo_text_processing/text_normalization/run_evaluate.py index 3a1964bbd..1df194042 100644 --- a/nemo_text_processing/text_normalization/run_evaluate.py +++ b/nemo_text_processing/text_normalization/run_evaluate.py @@ -35,7 +35,7 @@ def parse_args(): parser.add_argument( "--lang", help="language", - choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt'], + choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'te', 'ko', 'vi', 'pt'], default="en", type=str, ) diff --git a/nemo_text_processing/text_normalization/te/__init__.py b/nemo_text_processing/text_normalization/te/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/te/data/__init__.py b/nemo_text_processing/text_normalization/te/data/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/te/data/numbers/__init__.py b/nemo_text_processing/text_normalization/te/data/numbers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/te/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/te/data/numbers/digit.tsv new file mode 100644 index 000000000..ab878ec99 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/digit.tsv @@ -0,0 +1,18 @@ +1 ఒకటి +2 రెండు +3 మూడు +4 నాలుగు +5 ఐదు +6 ఆరు +7 ఏడు +8 ఎనిమిది +9 తొమ్మిది +౧ ఒకటి +౨ రెండు +౩ మూడు +౪ నాలుగు +౫ ఐదు +౬ ఆరు +౭ ఏడు +౮ ఎనిమిది +౯ తొమ్మిది \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/te/data/numbers/exact_power.tsv b/nemo_text_processing/text_normalization/te/data/numbers/exact_power.tsv new file mode 100644 index 000000000..c43f22889 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/exact_power.tsv @@ -0,0 +1,8 @@ +100 వంద +౧౦౦ వంద +1000 వెయ్యి +౧౦౦౦ వెయ్యి +100000 లక్ష +౧౦౦౦౦౦ లక్ష +10000000 కోటి +౧౦౦౦౦౦౦౦ కోటి diff --git a/nemo_text_processing/text_normalization/te/data/numbers/hundred_prefix.tsv b/nemo_text_processing/text_normalization/te/data/numbers/hundred_prefix.tsv new file mode 100644 index 000000000..b08a6aa40 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/hundred_prefix.tsv @@ -0,0 +1,4 @@ +10 నూట +౧౦ నూట +1 నూట +౧ నూట diff --git a/nemo_text_processing/text_normalization/te/data/numbers/hundreds_before_one.tsv b/nemo_text_processing/text_normalization/te/data/numbers/hundreds_before_one.tsv new file mode 100644 index 000000000..f4198c9ce --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/hundreds_before_one.tsv @@ -0,0 +1,2 @@ +1 వందల ఒక +౧ వందల ఒక diff --git a/nemo_text_processing/text_normalization/te/data/numbers/magnitudes.tsv b/nemo_text_processing/text_normalization/te/data/numbers/magnitudes.tsv new file mode 100644 index 000000000..91413eb39 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/magnitudes.tsv @@ -0,0 +1,12 @@ +hundreds_plural వందలు +hundreds_before వందల +thousand వెయ్యి +thousands_plural వేలు +thousands_before వేల +lakh లక్ష +lakh_before_digit లక్షా +lakhs_plural లక్షలు +lakhs_before లక్షల +crore కోటి +crores_plural కోట్లు +crores_before కోట్ల diff --git a/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties.tsv b/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..69eadf9f7 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties.tsv @@ -0,0 +1,20 @@ +౧౦ పది +౧౧ పదకొండు +౧౨ పన్నెండు +౧౩ పదమూడు +౧౪ పద్నాలుగు +౧౫ పదిహేను +౧౬ పదహారు +౧౭ పదిహేడు +౧౮ పధ్ధెనిమిది +౧౯ పంతొమ్మిది +10 పది +11 పదకొండు +12 పన్నెండు +13 పదమూడు +14 పద్నాలుగు +15 పదిహేను +16 పదహారు +17 పదిహేడు +18 పధ్ధెనిమిది +19 పంతొమ్మిది diff --git a/nemo_text_processing/text_normalization/te/data/numbers/ties.tsv b/nemo_text_processing/text_normalization/te/data/numbers/ties.tsv new file mode 100644 index 000000000..a8d7d7227 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/ties.tsv @@ -0,0 +1,16 @@ +2 ఇరవై +3 ముప్పై +4 నలభై +5 యాభై +6 అరవై +7 డెబ్బై +8 ఎనభై +9 తొంభై +౨ ఇరవై +౩ ముప్పై +౪ నలభై +౫ యాభై +౬ అరవై +౭ డెబ్బై +౮ ఎనభై +౯ తొంభై diff --git a/nemo_text_processing/text_normalization/te/data/numbers/ties_one_suffix.tsv b/nemo_text_processing/text_normalization/te/data/numbers/ties_one_suffix.tsv new file mode 100644 index 000000000..86bff5e86 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/ties_one_suffix.tsv @@ -0,0 +1,2 @@ +1 ఒక +౧ ఒక diff --git a/nemo_text_processing/text_normalization/te/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/te/data/numbers/zero.tsv new file mode 100644 index 000000000..309aab75e --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/zero.tsv @@ -0,0 +1,2 @@ +0 సున్నా +౦ సున్నా \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/te/graph_utils.py b/nemo_text_processing/text_normalization/te/graph_utils.py new file mode 100644 index 000000000..1bb247162 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/graph_utils.py @@ -0,0 +1,145 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +NEMO_CHAR = utf8.VALID_UTF8_CHAR +NEMO_DIGIT = byte.DIGIT + +# Telugu native digits (౦, ౧, ౨, ౩, ౪, ౫, ౬, ౭, ౮, ౯) +NEMO_TE_DIGIT = pynini.union("౦", "౧", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯").optimize() + +# Combined Telugu and Arabic digits for graphs that need to accept both +NEMO_ALL_DIGIT = pynini.union(NEMO_TE_DIGIT, NEMO_DIGIT).optimize() +NEMO_ALL_ZERO = pynini.union("౦", "0").optimize() + +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") + +MIN_NEG_WEIGHT = -0.0001 + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/te/taggers/__init__.py b/nemo_text_processing/text_normalization/te/taggers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/te/taggers/cardinal.py b/nemo_text_processing/text_normalization/te/taggers/cardinal.py new file mode 100644 index 000000000..3d5a3255f --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/cardinal.py @@ -0,0 +1,424 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import ( + NEMO_ALL_DIGIT, + NEMO_ALL_ZERO, + NEMO_DIGIT, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.te.utils import get_abs_path, load_labels + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + -౨౩ -> cardinal { negative: "true" integer: "ఇరవై మూడు" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + teens = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")) + ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + exact_power = pynini.string_file(get_abs_path("data/numbers/exact_power.tsv")) + hundred_prefix = pynini.string_file(get_abs_path("data/numbers/hundred_prefix.tsv")) + ties_one_suffix = pynini.string_file(get_abs_path("data/numbers/ties_one_suffix.tsv")) + hundreds_before_one = pynini.string_file(get_abs_path("data/numbers/hundreds_before_one.tsv")) + magnitude = {k: v for k, v in load_labels(get_abs_path("data/numbers/magnitudes.tsv"))} + + te_digit = pynini.difference(NEMO_ALL_DIGIT, NEMO_DIGIT).optimize() + exact_hundred = pynini.compose(NEMO_ALL_DIGIT**3, exact_power).optimize() + exact_thousand = pynini.compose(NEMO_ALL_DIGIT**4, exact_power).optimize() + exact_lakh = pynini.compose(NEMO_ALL_DIGIT**6, exact_power).optimize() + exact_crore = pynini.compose(NEMO_ALL_DIGIT**8, exact_power).optimize() + hundred_prefix_ten = ( + pynini.compose(NEMO_DIGIT + NEMO_DIGIT, hundred_prefix) + | pynini.compose(te_digit + te_digit, hundred_prefix) + ).optimize() + hundred_prefix_one = ( + pynini.compose(NEMO_DIGIT, hundred_prefix) | pynini.compose(te_digit, hundred_prefix) + ).optimize() + + thousand = magnitude["thousand"] + lakh = magnitude["lakh"] + crore = magnitude["crore"] + + ins_hundreds_plural = pynutil.insert(" " + magnitude["hundreds_plural"]) + ins_hundreds_before = pynutil.insert(" " + magnitude["hundreds_before"]) + ins_thousand = pynutil.insert(thousand) + ins_thousand_spaced = pynutil.insert(" " + thousand) + ins_thousands_plural = pynutil.insert(" " + magnitude["thousands_plural"]) + ins_thousands_before = pynutil.insert(" " + magnitude["thousands_before"]) + ins_lakh = pynutil.insert(lakh) + ins_lakh_spaced = pynutil.insert(" " + lakh) + ins_lakha_digit = pynutil.insert(magnitude["lakh_before_digit"]) + ins_lakhs_plural = pynutil.insert(" " + magnitude["lakhs_plural"]) + ins_lakhs_before = pynutil.insert(" " + magnitude["lakhs_before"]) + ins_crore = pynutil.insert(crore) + ins_crore_spaced = pynutil.insert(" " + crore) + ins_crores_plural = pynutil.insert(" " + magnitude["crores_plural"]) + ins_crores_before = pynutil.insert(" " + magnitude["crores_before"]) + + teens_en = pynini.compose(NEMO_DIGIT + NEMO_DIGIT, teens).optimize() + teens_te = pynini.compose(te_digit + te_digit, teens).optimize() + digit_en = (NEMO_DIGIT @ digit).optimize() + digit_te = (te_digit @ digit).optimize() + ties_en = (NEMO_DIGIT @ ties).optimize() + ties_te = (te_digit @ ties).optimize() + + one_digit = pynini.union("1", "౧") + digit_except_one = (pynini.difference(NEMO_ALL_DIGIT, NEMO_ALL_ZERO | one_digit) @ digit).optimize() + digit_except_one_en = (pynini.difference(NEMO_DIGIT, pynini.union("0", "1")) @ digit).optimize() + digit_except_one_te = (pynini.difference(te_digit, pynini.union("౦", "౧")) @ digit).optimize() + + teens_ties_en = teens_en | (ties_en + pynutil.delete("0")) | (ties_en + insert_space + digit_en) + teens_ties_te = teens_te | (ties_te + pynutil.delete("౦")) | (ties_te + insert_space + digit_te) + teens_ties = pynini.union(teens_ties_te, teens_ties_en) + ties_one_suffix_en = (NEMO_DIGIT @ ties_one_suffix).optimize() + ties_one_suffix_te = (te_digit @ ties_one_suffix).optimize() + teens_ties_thousand = ((ties_en + ties_one_suffix_en) | (ties_te + ties_one_suffix_te)).optimize() + teens_ties_except_one = pynini.union( + teens_en | (ties_en + pynutil.delete("0")) | (ties_en + insert_space + digit_except_one_en), + teens_te | (ties_te + pynutil.delete("౦")) | (ties_te + insert_space + digit_except_one_te), + ).optimize() + + single_digit_graph = digit | zero + self.single_digits_graph = single_digit_graph + pynini.closure(insert_space + single_digit_graph) + + delete_zero = pynutil.delete(NEMO_ALL_ZERO) + + zero_pow = {0: pynini.accep("")} + for _n in range(1, 8): + zero_pow[_n] = (zero_pow[_n - 1] + delete_zero).optimize() + + def create_graph_suffix(digit_graph, suffix, zeros_counts): + if zeros_counts == 0: + return digit_graph + suffix + + return digit_graph + zero_pow[zeros_counts] + suffix + + def create_larger_number_graph(digit_graph, suffix, zeros_counts, sub_graph): + if zeros_counts == 0: + return digit_graph + suffix + insert_space + sub_graph + + return digit_graph + suffix + zero_pow[zeros_counts] + insert_space + sub_graph + + def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): + """Union over a "ladder" of (zeros_count, remainder_graph) rungs.""" + graph = create_graph_suffix(prefix, head_suffix, head_zeros) if head_suffix is not None else None + for zeros, sub in ladder: + rung = create_larger_number_graph(prefix, rung_suffix, zeros, sub) + graph = rung if graph is None else graph | rung + return graph + + one_prefix = pynutil.delete(one_digit) + + graph_hundreds = exact_hundred + graph_hundreds |= hundred_prefix_ten + digit + graph_hundreds |= hundred_prefix_one + teens_ties + graph_hundreds |= create_graph_suffix(digit_except_one, ins_hundreds_plural, 2) + graph_hundreds |= create_larger_number_graph(digit_except_one, ins_hundreds_before, 1, digit) + graph_hundreds |= create_larger_number_graph(digit_except_one, ins_hundreds_before, 0, teens_ties) + graph_hundreds = graph_hundreds.optimize() + + thousand_ladder = [ + (2, digit), + (1, teens_ties), + (0, graph_hundreds), + ] + + graph_thousands = exact_thousand + graph_thousands |= build_group(one_prefix, ins_thousand, thousand_ladder) + graph_thousands |= build_group( + digit_except_one, ins_thousands_before, thousand_ladder, head_suffix=ins_thousands_plural, head_zeros=3 + ) + graph_thousands = graph_thousands.optimize() + + graph_ten_thousands = build_group( + teens_ties_thousand, ins_thousands_before, thousand_ladder, head_suffix=ins_thousand_spaced, head_zeros=3 + ) + graph_ten_thousands |= build_group( + teens_ties_except_one, + ins_thousands_before, + thousand_ladder, + head_suffix=ins_thousands_plural, + head_zeros=3, + ) + graph_ten_thousands = graph_ten_thousands.optimize() + + lakh_ladder = [ + (4, digit), + (3, teens_ties), + (2, graph_hundreds), + (1, graph_thousands), + (0, graph_ten_thousands), + ] + + graph_lakhs = exact_lakh + graph_lakhs |= create_larger_number_graph(one_prefix, ins_lakha_digit, 4, digit) + graph_lakhs |= build_group(one_prefix, ins_lakh, lakh_ladder[1:]) + graph_lakhs |= build_group( + digit_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5 + ) + graph_lakhs = graph_lakhs.optimize() + + graph_ten_lakhs = build_group( + teens_ties_thousand, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakh_spaced, head_zeros=5 + ) + graph_ten_lakhs |= build_group( + teens_ties_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5 + ) + graph_ten_lakhs = graph_ten_lakhs.optimize() + + crore_ladder = [ + (6, digit), + (5, teens_ties), + (4, graph_hundreds), + (3, graph_thousands), + (2, graph_ten_thousands), + (1, graph_lakhs), + (0, graph_ten_lakhs), + ] + + graph_crores = exact_crore + graph_crores |= build_group(one_prefix, ins_crore, crore_ladder) + graph_crores |= build_group( + digit_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ) + graph_crores = graph_crores.optimize() + + graph_ten_crores = build_group( + teens_ties_thousand, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ) + graph_ten_crores |= build_group( + teens_ties_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ) + graph_ten_crores = graph_ten_crores.optimize() + + hundred_crore_prefix = ( + exact_hundred + | (hundred_prefix_ten + digit) + | (hundred_prefix_one + teens_ties) + | create_graph_suffix(digit_except_one, ins_hundreds_before, 2) + | create_larger_number_graph(digit_except_one, ins_hundreds_before, 1, digit_except_one) + | create_larger_number_graph(digit_except_one, ins_hundreds_before, 0, teens_ties) + ).optimize() + + hundred_one_crore_prefix = digit_except_one + pynutil.delete(NEMO_ALL_ZERO) + hundreds_before_one + + graph_arabs = build_group( + hundred_one_crore_prefix, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ) + graph_arabs |= build_group( + hundred_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ) + graph_arabs |= create_larger_number_graph(hundred_crore_prefix, ins_crores_before, 0, graph_crores) + graph_arabs = graph_arabs.optimize() + + thousand_crore_ladder = [ + (2, digit), + (1, teens_ties), + (0, hundred_crore_prefix), + ] + + thousand_crore_prefix = ( + exact_thousand + | build_group( + digit_except_one, + ins_thousands_before, + thousand_crore_ladder, + head_suffix=ins_thousands_before, + head_zeros=3, + ) + | build_group(one_prefix, ins_thousand, thousand_crore_ladder) + ).optimize() + + ten_thousand_crore_prefix = ( + build_group( + teens_ties_thousand, + ins_thousands_before, + thousand_crore_ladder, + head_suffix=ins_thousand_spaced, + head_zeros=3, + ) + | build_group( + teens_ties_except_one, + ins_thousands_before, + thousand_crore_ladder, + head_suffix=ins_thousands_before, + head_zeros=3, + ) + ).optimize() + + crore_count_prefix = (thousand_crore_prefix | ten_thousand_crore_prefix).optimize() + + graph_ten_arabs = build_group( + crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ).optimize() + + graph_kharabs = build_group( + ten_thousand_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ).optimize() + + lakh_crore_ladder = [ + (4, digit), + (3, teens_ties), + (2, graph_hundreds), + (1, graph_thousands), + (0, ten_thousand_crore_prefix), + ] + + lakh_crore_prefix = ( + exact_lakh + | build_group(one_prefix, ins_lakh, lakh_crore_ladder) + | build_group( + digit_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5 + ) + ).optimize() + + ten_lakh_crore_prefix = ( + build_group( + teens_ties_thousand, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakh_spaced, head_zeros=5 + ) + | build_group( + teens_ties_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5 + ) + ).optimize() + + lakh_crore_count_prefix = (lakh_crore_prefix | ten_lakh_crore_prefix).optimize() + + graph_ten_kharabs = build_group( + lakh_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ).optimize() + + ten_nil_lakh_remainder_before_kotlu = build_group( + teens_ties, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_before, head_zeros=5 + ).optimize() + + koti_ladder = [ + (6, digit), + (5, teens_ties), + (4, graph_hundreds), + (3, graph_thousands), + (2, graph_ten_thousands), + (1, graph_lakhs), + (0, ten_nil_lakh_remainder_before_kotlu), + ] + + ten_nil_crore_count_prefix = ( + graph_crores + | graph_ten_crores + | create_larger_number_graph(one_prefix, ins_crore, 0, ten_nil_lakh_remainder_before_kotlu) + | create_larger_number_graph(digit_except_one, ins_crores_before, 0, ten_nil_lakh_remainder_before_kotlu) + | create_larger_number_graph(teens_ties, ins_crores_before, 0, ten_nil_lakh_remainder_before_kotlu) + ).optimize() + + graph_ten_nils = build_group( + ten_nil_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ).optimize() + + padma_crore_count_prefix = ( + build_group(teens_ties_thousand, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7) + | build_group( + teens_ties_except_one, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ) + ).optimize() + + graph_padmas = build_group( + padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ).optimize() + + ten_padma_one_crore_count_prefix = create_graph_suffix( + hundred_one_crore_prefix, ins_crore_spaced, 7 + ).optimize() + + ten_padma_crore_count_prefix = ( + ten_padma_one_crore_count_prefix + | build_group( + hundred_crore_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ) + ).optimize() + + graph_ten_padmas = build_group( + ten_padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ).optimize() + + shankh_koti_count_prefix = build_group( + crore_count_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ).optimize() + + graph_shankhs = build_group( + shankh_koti_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ).optimize() + + def exact_digits(n, graph): + return pynini.compose(NEMO_ALL_DIGIT**n, graph) + + graph_without_leading_zeros = ( + digit + | zero + | teens_ties + | graph_hundreds + | graph_thousands + | graph_ten_thousands + | graph_lakhs + | graph_ten_lakhs + | graph_crores + | graph_ten_crores + | exact_digits(10, graph_arabs) + | exact_digits(11, graph_ten_arabs) + | exact_digits(12, graph_kharabs) + | exact_digits(13, graph_ten_kharabs) + | exact_digits(14, graph_ten_kharabs) + | exact_digits(15, graph_ten_nils) + | exact_digits(16, graph_padmas) + | exact_digits(17, graph_ten_padmas) + | exact_digits(18, graph_shankhs) + | exact_digits(19, graph_shankhs) + ) + + cardinal_with_leading_zeros = pynini.compose( + NEMO_ALL_ZERO + pynini.closure(NEMO_ALL_DIGIT), self.single_digits_graph + ) + delete_separator = pynutil.delete(",") + two_digits = NEMO_ALL_DIGIT + NEMO_ALL_DIGIT + three_digits = NEMO_ALL_DIGIT + NEMO_ALL_DIGIT + NEMO_ALL_DIGIT + indian_grouping = ( + pynini.closure(NEMO_ALL_DIGIT, 1, 2) + + pynini.closure(delete_separator + two_digits) + + delete_separator + + three_digits + ) + western_grouping = pynini.closure(NEMO_ALL_DIGIT, 1, 3) + pynini.closure(delete_separator + three_digits, 1) + strip_separators = (indian_grouping | western_grouping).optimize() + cardinal_with_separators = pynini.compose(strip_separators, graph_without_leading_zeros).optimize() + + final_graph = graph_without_leading_zeros | cardinal_with_leading_zeros | cardinal_with_separators + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + self.final_graph = final_graph.optimize() + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.final_graph + pynutil.insert("\"") + self.fst = self.add_tokens(final_graph) diff --git a/nemo_text_processing/text_normalization/te/taggers/punctuation.py b/nemo_text_processing/text_normalization/te/taggers/punctuation.py new file mode 100644 index 000000000..c5811c55a --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/punctuation.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + + punct_marks = punct_unicode + list(s) + + punct = pynini.union(*punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + pynini.union( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)), + (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)), + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.fst = (pynutil.insert("name: \"") + punct + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..c2b1d07fb --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import ( + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.te.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.te.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.te.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"te_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + + classify = pynutil.add_weight(cardinal_graph, 1.1) + + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).fst + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct), + ), + 1, + ) + + classify = pynini.union(classify, pynutil.add_weight(word_graph, 100)) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(NEMO_SPACE)) + + token + + pynini.closure(pynutil.insert(NEMO_SPACE) + punct) + ) + + graph = token_plus_punct + pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct + pynutil.insert(NEMO_SPACE)), + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph = pynini.union(graph, punct) + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/te/taggers/word.py b/nemo_text_processing/text_normalization/te/taggers/word.py new file mode 100644 index 000000000..5ef0b0778 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying Telugu words. + e.g. తెలుగు -> tokens { name: "తెలుగు" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/te/utils.py b/nemo_text_processing/text_normalization/te/utils.py new file mode 100644 index 000000000..2bcba780d --- /dev/null +++ b/nemo_text_processing/text_normalization/te/utils.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/text_normalization/te/verbalizers/__init__.py b/nemo_text_processing/text_normalization/te/verbalizers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/te/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/te/verbalizers/cardinal.py new file mode 100644 index 000000000..b70c957f3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/cardinal.py @@ -0,0 +1,41 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals, e.g. + cardinal { integer: "ఐదు" } -> ఐదు + cardinal { negative: "true" integer: "ఇరవై మూడు" } -> మైనస్ ఇరవై మూడు + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "మైనస్ "), 0, 1) + + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + integer = pynutil.delete("integer:") + pynutil.delete(" \"") + integer + pynutil.delete("\"") + + graph = optional_sign + integer + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/te/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/te/verbalizers/verbalize.py new file mode 100644 index 000000000..f2564e504 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/verbalize.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.te.graph_utils import GraphFst +from nemo_text_processing.text_normalization.te.verbalizers.cardinal import CardinalFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + graph = cardinal_graph + + self.fst = graph diff --git a/nemo_text_processing/text_normalization/te/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/te/verbalizers/verbalize_final.py new file mode 100644 index 000000000..beb01924b --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/verbalize_final.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.te.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.te.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"te_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + + if deterministic: + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + else: + graph = delete_space + types + delete_space + + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) + logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/te/verbalizers/word.py b/nemo_text_processing/text_normalization/te/verbalizers/word.py new file mode 100644 index 000000000..ece2e9b76 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/word.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import NEMO_CHAR, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing Telugu words. + e.g. tokens { name: "ఏడు" } -> ఏడు + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + self.fst = char.optimize() diff --git a/tests/nemo_text_processing/te/__init__.py b/tests/nemo_text_processing/te/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/tests/nemo_text_processing/te/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..a50d89ce6 --- /dev/null +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,89 @@ +4 నాలుగులు~నాలుగు నాలుగులు +౬ మంది ఆటగాళ్లు బయటకు~ఆరు మంది ఆటగాళ్లు బయటకు +4 ఓవర్లలో 17 పరుగులు~నాలుగు ఓవర్లలో పదిహేడు పరుగులు +౫ చాక్లెట్లు ౯ టాఫీలు~ఐదు చాక్లెట్లు తొమ్మిది టాఫీలు +5551 అరటిపండ్లు~ఐదు వేల ఐదు వందల యాభై ఒకటి అరటిపండ్లు +౫౫౫౧ అరటిపండ్లు~ఐదు వేల ఐదు వందల యాభై ఒకటి అరటిపండ్లు +4 మంది మృతి 18 మందికి గాయాలు~నాలుగు మంది మృతి పధ్ధెనిమిది మందికి గాయాలు +అధ్యాయం ౧౦ శ్లోకం ౨~అధ్యాయం పది శ్లోకం రెండు +ఆదాయం -5000 రూపాయలు~ఆదాయం మైనస్ ఐదు వేలు రూపాయలు +₹ -100~₹ మైనస్ వంద +10~పది +౧౦~పది +100~వంద +౧౦౦~వంద +101~నూట ఒకటి +111~నూట పదకొండు +130~నూట ముప్పై +200~రెండు వందలు +203~రెండు వందల మూడు +312~మూడు వందల పన్నెండు +425~నాలుగు వందల ఇరవై ఐదు +525~ఐదు వందల ఇరవై ఐదు +౫౨౫~ఐదు వందల ఇరవై ఐదు +85~ఎనభై ఐదు +౯౧~తొంభై ఒకటి +1001~వెయ్యి ఒకటి +10099~పది వేల తొంభై తొమ్మిది +100001~లక్షా ఒకటి +12346~పన్నెండు వేల మూడు వందల నలభై ఆరు +1345~వెయ్యి మూడు వందల నలభై ఐదు +123456~లక్ష ఇరవై మూడు వేల నాలుగు వందల యాభై ఆరు +౧౩౪౫౬~పదమూడు వేల నాలుగు వందల యాభై ఆరు +872987~ఎనిమిది లక్షల డెబ్బై రెండు వేల తొమ్మిది వందల ఎనభై ఏడు +9876789~తొంభై ఎనిమిది లక్షల డెబ్బై ఆరు వేల ఏడు వందల ఎనభై తొమ్మిది +2345567~ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +12345567~కోటి ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +12121212~కోటి ఇరవై ఒక లక్షల ఇరవై ఒక వేల రెండు వందల పన్నెండు +1122345567~నూట పన్నెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +1022345567~నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +11022345567~వెయ్యి నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +51022345567~ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +10000000000~వెయ్యి కోట్లు +100000000000~పది వేల కోట్లు +1000000000000~లక్ష కోట్లు +10000000000000~పది లక్షల కోట్లు +100000000000000~కోటి కోట్లు +999999999999999~తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +1000000000000000~పది కోటి కోట్లు +999999999999999999~తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోటి తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +9999999999999999999~తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోటి తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +జనాభా 999999999999999~జనాభా తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯~తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯~తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోటి తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +05~సున్నా ఐదు +01~సున్నా ఒకటి +073~సున్నా ఏడు మూడు +0001~సున్నా సున్నా సున్నా ఒకటి +000~సున్నా సున్నా సున్నా +-23~మైనస్ ఇరవై మూడు +-100~మైనస్ వంద +-12345567~మైనస్ కోటి ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +-51022345567~మైనస్ ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +1,23,456~లక్ష ఇరవై మూడు వేల నాలుగు వందల యాభై ఆరు +12,345~పన్నెండు వేల మూడు వందల నలభై ఐదు +1,00,00,000~కోటి +5,50,00,000~ఐదు కోట్ల యాభై లక్షలు +32,45,000~ముప్పై రెండు లక్షల నలభై ఐదు వేలు +5,56,320~ఐదు లక్షల యాభై ఆరు వేల మూడు వందల ఇరవై +32,450,000~మూడు కోట్ల ఇరవై నాలుగు లక్షల యాభై వేలు +4,99,99,000~నాలుగు కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేలు +11,220~పదకొండు వేల రెండు వందల ఇరవై +౧,౨౩,౪౫౬~లక్ష ఇరవై మూడు వేల నాలుగు వందల యాభై ఆరు +-1,23,456~మైనస్ లక్ష ఇరవై మూడు వేల నాలుగు వందల యాభై ఆరు +10050~పది వేల యాభై +10100~పది వేల వంద +100100~లక్ష వంద +101000~లక్ష వెయ్యి +100010~లక్ష పది +1005000~పది లక్షల ఐదు వేలు +1002003~పది లక్షల రెండు వేల మూడు +2050607~ఇరవై లక్షల యాభై వేల ఆరు వందల ఏడు +10000050~కోటి యాభై +10010000~కోటి పది వేలు +10100000~కోటి లక్ష +10001001~కోటి వెయ్యి ఒకటి +100000000001~పది వేల కోట్ల ఒకటి +1,00,100~లక్ష వంద +10,05,000~పది లక్షల ఐదు వేలు +1,00,01,001~కోటి వెయ్యి ఒకటి \ No newline at end of file diff --git a/tests/nemo_text_processing/te/test_cardinal.py b/tests/nemo_text_processing/te/test_cardinal.py new file mode 100644 index 000000000..aacab9485 --- /dev/null +++ b/tests/nemo_text_processing/te/test_cardinal.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer( + input_case='cased', + lang='te', + cache_dir=CACHE_DIR, + overwrite_cache=False, + post_process=False, + ) + + @parameterized.expand(parse_test_case_file('te/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/te/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/te/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..20c4df79a --- /dev/null +++ b/tests/nemo_text_processing/te/test_sparrowhawk_normalization.sh @@ -0,0 +1,32 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while IFS= read -r testcase; do + IFS='~' read -r written spoken <<< "$testcase" + + # Escape backslashes and replace non breaking space with breaking space + escaped_written=$(printf '%s' "$written" | sed 's/\\/\\\\/g') + denorm_pred=$(echo "$escaped_written" | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space and remove space before punctuation + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/ \([!?.]\)/\1/g')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/ \([!?.]\)/\1/g')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +testTNCardinal() { + input=$PROJECT_DIR/te/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 73a4fc138..9fa9ac8d8 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -109,6 +109,7 @@ def parse_args(): 'ja', 'rw', 'ko', + 'te', ], type=str, default='en', @@ -224,6 +225,11 @@ def parse_args(): PostProcessingFst as TNPostProcessingFst, ) from nemo_text_processing.text_normalization.hi.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'te': + from nemo_text_processing.text_normalization.te.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.te.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'hu': from nemo_text_processing.text_normalization.hu.taggers.tokenize_and_classify import ( ClassifyFst as TNClassifyFst, From 4d2ca7b03f0fd25cc95a6d98e82fa16e99f26ff2 Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Wed, 2 Sep 2026 13:24:20 +0530 Subject: [PATCH 02/15] fix(te): restore graph_utils, utils, and Jenkinsfile from rebase Use TSV-based digit definitions in graph_utils, restore load_labels in utils, and update Jenkinsfile with correct cache paths and TE CI stages. Signed-off-by: Haridas Nishitha Co-authored-by: Cursor Signed-off-by: Haridas Nishitha --- Jenkinsfile | 29 +++------------- .../text_normalization/te/graph_utils.py | 33 +++++-------------- .../text_normalization/te/utils.py | 14 ++++++++ 3 files changed, 28 insertions(+), 48 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index aab941076..e9b2a5939 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,8 +28,8 @@ pipeline { HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0' MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-5' - KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' + KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-0' TE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-21-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } @@ -113,28 +113,9 @@ pipeline { } } } - - stage('L0: Create TE TN/ITN Grammars') { - when { - anyOf { - branch 'main' - branch 'staging/**' - branch 'staging_*' - changeRequest target: 'main' - } - } - failFast true - parallel { - stage('L0: TE TN grammars') { - steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=te --text="౧" --cache_dir ${TE_TN_CACHE}' - } - } - // stage('L0: TE ITN grammars') { - // steps { - // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=te --text="ఒకటి" --cache_dir ${TE_TN_CACHE}' - // } - // } + stage('L0: TE TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=te --text="౧" --cache_dir ${TE_TN_CACHE}' } } diff --git a/nemo_text_processing/text_normalization/te/graph_utils.py b/nemo_text_processing/text_normalization/te/graph_utils.py index 1bb247162..6fdea0492 100644 --- a/nemo_text_processing/text_normalization/te/graph_utils.py +++ b/nemo_text_processing/text_normalization/te/graph_utils.py @@ -23,19 +23,20 @@ from pynini.export import export from pynini.lib import byte, pynutil, utf8 +from nemo_text_processing.text_normalization.te.utils import get_abs_path + NEMO_CHAR = utf8.VALID_UTF8_CHAR NEMO_DIGIT = byte.DIGIT -# Telugu native digits (౦, ౧, ౨, ౩, ౪, ౫, ౬, ౭, ౮, ౯) -NEMO_TE_DIGIT = pynini.union("౦", "౧", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯").optimize() - -# Combined Telugu and Arabic digits for graphs that need to accept both -NEMO_ALL_DIGIT = pynini.union(NEMO_TE_DIGIT, NEMO_DIGIT).optimize() -NEMO_ALL_ZERO = pynini.union("౦", "0").optimize() +NEMO_ALL_ZERO = pynini.project(pynini.string_file(get_abs_path("data/numbers/zero.tsv")), "input").optimize() +NEMO_ALL_DIGIT = pynini.union( + pynini.project(pynini.string_file(get_abs_path("data/numbers/digit.tsv")), "input"), + NEMO_ALL_ZERO, +).optimize() NEMO_NON_BREAKING_SPACE = u"\u00a0" NEMO_SPACE = " " -NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", NEMO_NON_BREAKING_SPACE).optimize() NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() NEMO_SIGMA = pynini.closure(NEMO_CHAR) @@ -44,8 +45,6 @@ insert_space = pynutil.insert(" ") delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") -MIN_NEG_WEIGHT = -0.0001 - def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): """ @@ -62,20 +61,6 @@ def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): logging.info(f'Created {file_name}') -def convert_space(fst) -> 'pynini.FstLike': - """ - Converts space to nonbreaking space. - Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" - This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. - - Args: - fst: input fst - - Returns output fst where breaking spaces are converted to non breaking spaces - """ - return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) - - class GraphFst: """ Base class for all grammar fsts. @@ -142,4 +127,4 @@ def delete_tokens(self, fst) -> 'pynini.FstLike': + delete_space + pynutil.delete("}") ) - return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + return res @ pynini.cdrewrite(pynini.cross(NEMO_NON_BREAKING_SPACE, " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/te/utils.py b/nemo_text_processing/text_normalization/te/utils.py index 2bcba780d..b137791e7 100644 --- a/nemo_text_processing/text_normalization/te/utils.py +++ b/nemo_text_processing/text_normalization/te/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import csv import os @@ -25,3 +26,16 @@ def get_abs_path(rel_path): Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + return list(csv.reader(label_tsv, delimiter="\t")) \ No newline at end of file From 3bf806ab3d058ba9cc72fbb2412d63f212d81065 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 07:55:22 +0000 Subject: [PATCH 03/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: Haridas Nishitha --- nemo_text_processing/text_normalization/te/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/te/utils.py b/nemo_text_processing/text_normalization/te/utils.py index b137791e7..5e7e96c27 100644 --- a/nemo_text_processing/text_normalization/te/utils.py +++ b/nemo_text_processing/text_normalization/te/utils.py @@ -38,4 +38,4 @@ def load_labels(abs_path): Returns dictionary of mappings """ with open(abs_path, encoding="utf-8") as label_tsv: - return list(csv.reader(label_tsv, delimiter="\t")) \ No newline at end of file + return list(csv.reader(label_tsv, delimiter="\t")) From 31dac735468aac45c77b276a77b89e26987959b1 Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Wed, 2 Sep 2026 13:52:36 +0530 Subject: [PATCH 04/15] fix(te): align WordFst call with current word tagger signature tokenize_and_classify passed a punctuation argument that WordFst no longer accepts after the staging rebase. Signed-off-by: Haridas Nishitha Co-authored-by: Cursor Signed-off-by: Haridas Nishitha --- .../text_normalization/te/taggers/tokenize_and_classify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py index c2b1d07fb..980275081 100644 --- a/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py @@ -78,7 +78,7 @@ def __init__( classify = pynutil.add_weight(cardinal_graph, 1.1) - word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).fst + word_graph = WordFst(deterministic=deterministic).fst punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") punct = pynini.closure( From c01437d686c5f44a9b77fb0fa9e0e7a31dd31bc8 Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Wed, 9 Sep 2026 12:01:32 +0530 Subject: [PATCH 05/15] fix(ci): restore HI/KO TN cache paths; keep TE only Signed-off-by: Haridas Nishitha --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e9b2a5939..e71338ded 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,8 +28,8 @@ pipeline { HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0' MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' - KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-5' + KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' TE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-21-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } From a8e360118730e36c255232dfb189b369785f9987 Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Wed, 9 Sep 2026 13:14:51 +0530 Subject: [PATCH 06/15] feat(te): Updated. Signed-off-by: Haridas Nishitha --- Jenkinsfile | 19 +- .../te/data/numbers/ties_one_suffix.tsv | 4 +- .../text_normalization/te/taggers/cardinal.py | 200 +++++++----------- .../test_cases_cardinal.txt | 70 +++++- .../test_cases_punctuation.txt | 80 +++++++ .../test_cases_word.txt | 98 +++++++++ .../te/test_punctuation.py | 37 ++++ tests/nemo_text_processing/te/test_word.py | 37 ++++ 8 files changed, 416 insertions(+), 129 deletions(-) create mode 100644 tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt create mode 100644 tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/te/test_punctuation.py create mode 100644 tests/nemo_text_processing/te/test_word.py diff --git a/Jenkinsfile b/Jenkinsfile index e71338ded..b85f9c8ce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -113,9 +113,22 @@ pipeline { } } } - stage('L0: TE TN grammars') { - steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=te --text="౧" --cache_dir ${TE_TN_CACHE}' + stage('L0: Create TE TN Grammars') { + when { + anyOf { + branch 'main' + branch 'staging/**' + branch 'staging_*' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L0: TE TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=te --text="౧" --cache_dir ${TE_TN_CACHE}' + } + } } } diff --git a/nemo_text_processing/text_normalization/te/data/numbers/ties_one_suffix.tsv b/nemo_text_processing/text_normalization/te/data/numbers/ties_one_suffix.tsv index 86bff5e86..d312af237 100644 --- a/nemo_text_processing/text_normalization/te/data/numbers/ties_one_suffix.tsv +++ b/nemo_text_processing/text_normalization/te/data/numbers/ties_one_suffix.tsv @@ -1,2 +1,2 @@ -1 ఒక -౧ ఒక +1 ఒక +౧ ఒక diff --git a/nemo_text_processing/text_normalization/te/taggers/cardinal.py b/nemo_text_processing/text_normalization/te/taggers/cardinal.py index 3d5a3255f..094ebf303 100644 --- a/nemo_text_processing/text_normalization/te/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/te/taggers/cardinal.py @@ -30,6 +30,9 @@ class CardinalFst(GraphFst): Finite state transducer for classifying cardinals, e.g. -౨౩ -> cardinal { negative: "true" integer: "ఇరవై మూడు" } + Covers numbers up to 19 digits (Indian scale through shankh / శంఖ, 10^17). + Larger magnitudes are verbalized by composing crore (కోటి) groups. + Args: deterministic: if True will provide a single transduction option, for False multiple transduction are generated (used for audio-based normalization) @@ -98,7 +101,7 @@ def __init__(self, deterministic: bool = True): teens_ties = pynini.union(teens_ties_te, teens_ties_en) ties_one_suffix_en = (NEMO_DIGIT @ ties_one_suffix).optimize() ties_one_suffix_te = (te_digit @ ties_one_suffix).optimize() - teens_ties_thousand = ((ties_en + ties_one_suffix_en) | (ties_te + ties_one_suffix_te)).optimize() + teens_ties_thousand = ((ties_en + insert_space + ties_one_suffix_en) | (ties_te + insert_space + ties_one_suffix_te)).optimize() teens_ties_except_one = pynini.union( teens_en | (ties_en + pynutil.delete("0")) | (ties_en + insert_space + digit_except_one_en), teens_te | (ties_te + pynutil.delete("౦")) | (ties_te + insert_space + digit_except_one_te), @@ -135,38 +138,30 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): one_prefix = pynutil.delete(one_digit) - graph_hundreds = exact_hundred - graph_hundreds |= hundred_prefix_ten + digit - graph_hundreds |= hundred_prefix_one + teens_ties - graph_hundreds |= create_graph_suffix(digit_except_one, ins_hundreds_plural, 2) - graph_hundreds |= create_larger_number_graph(digit_except_one, ins_hundreds_before, 1, digit) - graph_hundreds |= create_larger_number_graph(digit_except_one, ins_hundreds_before, 0, teens_ties) - graph_hundreds = graph_hundreds.optimize() + graph_hundreds = ( + exact_hundred + | hundred_prefix_ten + digit + | hundred_prefix_one + teens_ties + | create_graph_suffix(digit_except_one, ins_hundreds_plural, 2) + | create_larger_number_graph(digit_except_one, ins_hundreds_before, 1, digit) + | create_larger_number_graph(digit_except_one, ins_hundreds_before, 0, teens_ties) + ).optimize() thousand_ladder = [ (2, digit), (1, teens_ties), (0, graph_hundreds), ] + graph_thousands = ( + exact_thousand + | build_group(one_prefix, ins_thousand, thousand_ladder) + | build_group(digit_except_one, ins_thousands_before, thousand_ladder, head_suffix=ins_thousands_plural, head_zeros=3) + ).optimize() - graph_thousands = exact_thousand - graph_thousands |= build_group(one_prefix, ins_thousand, thousand_ladder) - graph_thousands |= build_group( - digit_except_one, ins_thousands_before, thousand_ladder, head_suffix=ins_thousands_plural, head_zeros=3 - ) - graph_thousands = graph_thousands.optimize() - - graph_ten_thousands = build_group( - teens_ties_thousand, ins_thousands_before, thousand_ladder, head_suffix=ins_thousand_spaced, head_zeros=3 - ) - graph_ten_thousands |= build_group( - teens_ties_except_one, - ins_thousands_before, - thousand_ladder, - head_suffix=ins_thousands_plural, - head_zeros=3, - ) - graph_ten_thousands = graph_ten_thousands.optimize() + graph_ten_thousands = ( + build_group(teens_ties_thousand, ins_thousands_before, thousand_ladder, head_suffix=ins_thousand_spaced, head_zeros=3) + | build_group(teens_ties_except_one, ins_thousands_before, thousand_ladder, head_suffix=ins_thousands_plural, head_zeros=3) + ).optimize() lakh_ladder = [ (4, digit), @@ -176,21 +171,17 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): (0, graph_ten_thousands), ] - graph_lakhs = exact_lakh - graph_lakhs |= create_larger_number_graph(one_prefix, ins_lakha_digit, 4, digit) - graph_lakhs |= build_group(one_prefix, ins_lakh, lakh_ladder[1:]) - graph_lakhs |= build_group( - digit_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5 - ) - graph_lakhs = graph_lakhs.optimize() + graph_lakhs = ( + exact_lakh + | create_larger_number_graph(one_prefix, ins_lakha_digit, 4, digit) + | build_group(one_prefix, ins_lakh, lakh_ladder[1:]) + | build_group(digit_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5) + ).optimize() - graph_ten_lakhs = build_group( - teens_ties_thousand, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakh_spaced, head_zeros=5 - ) - graph_ten_lakhs |= build_group( - teens_ties_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5 - ) - graph_ten_lakhs = graph_ten_lakhs.optimize() + graph_ten_lakhs = ( + build_group(teens_ties_thousand, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakh_spaced, head_zeros=5) + | build_group(teens_ties_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5) + ).optimize() crore_ladder = [ (6, digit), @@ -202,40 +193,36 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): (0, graph_ten_lakhs), ] - graph_crores = exact_crore - graph_crores |= build_group(one_prefix, ins_crore, crore_ladder) - graph_crores |= build_group( - digit_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ) - graph_crores = graph_crores.optimize() + graph_crores = ( + exact_crore + | build_group(one_prefix, ins_crore, crore_ladder) + | build_group(digit_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7) + ).optimize() - graph_ten_crores = build_group( - teens_ties_thousand, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ) - graph_ten_crores |= build_group( - teens_ties_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ) - graph_ten_crores = graph_ten_crores.optimize() + graph_ten_crores = ( + build_group(teens_ties_thousand, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7) + | build_group(teens_ties_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7) + ).optimize() hundred_crore_prefix = ( exact_hundred - | (hundred_prefix_ten + digit) + | (hundred_prefix_ten + digit_except_one) | (hundred_prefix_one + teens_ties) | create_graph_suffix(digit_except_one, ins_hundreds_before, 2) | create_larger_number_graph(digit_except_one, ins_hundreds_before, 1, digit_except_one) | create_larger_number_graph(digit_except_one, ins_hundreds_before, 0, teens_ties) ).optimize() - hundred_one_crore_prefix = digit_except_one + pynutil.delete(NEMO_ALL_ZERO) + hundreds_before_one + hundred_one_crore_prefix = ( + hundred_prefix_one + pynutil.delete(NEMO_ALL_ZERO) + (one_digit @ ties_one_suffix) + | digit_except_one + pynutil.delete(NEMO_ALL_ZERO) + hundreds_before_one + ).optimize() - graph_arabs = build_group( - hundred_one_crore_prefix, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ) - graph_arabs |= build_group( - hundred_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ) - graph_arabs |= create_larger_number_graph(hundred_crore_prefix, ins_crores_before, 0, graph_crores) - graph_arabs = graph_arabs.optimize() + graph_arabs = ( + build_group(hundred_one_crore_prefix, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7) + | build_group(hundred_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7) + | create_larger_number_graph(hundred_crore_prefix, ins_crores_before, 0, graph_crores) + ).optimize() thousand_crore_ladder = [ (2, digit), @@ -245,41 +232,21 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): thousand_crore_prefix = ( exact_thousand - | build_group( - digit_except_one, - ins_thousands_before, - thousand_crore_ladder, - head_suffix=ins_thousands_before, - head_zeros=3, - ) + | build_group(digit_except_one,ins_thousands_before,thousand_crore_ladder,head_suffix=ins_thousands_before,head_zeros=3,) | build_group(one_prefix, ins_thousand, thousand_crore_ladder) ).optimize() ten_thousand_crore_prefix = ( - build_group( - teens_ties_thousand, - ins_thousands_before, - thousand_crore_ladder, - head_suffix=ins_thousand_spaced, - head_zeros=3, - ) - | build_group( - teens_ties_except_one, - ins_thousands_before, - thousand_crore_ladder, - head_suffix=ins_thousands_before, - head_zeros=3, - ) + build_group(teens_ties_thousand,ins_thousands_before,thousand_crore_ladder,head_suffix=ins_thousand_spaced,head_zeros=3,) + | build_group(teens_ties_except_one, ins_thousands_before, thousand_crore_ladder, head_suffix=ins_thousands_before, head_zeros=3) ).optimize() crore_count_prefix = (thousand_crore_prefix | ten_thousand_crore_prefix).optimize() - graph_ten_arabs = build_group( - crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_ten_arabs = build_group(crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - graph_kharabs = build_group( - ten_thousand_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_kharabs = build_group(ten_thousand_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() lakh_crore_ladder = [ @@ -293,28 +260,20 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): lakh_crore_prefix = ( exact_lakh | build_group(one_prefix, ins_lakh, lakh_crore_ladder) - | build_group( - digit_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5 - ) + | build_group(digit_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5) ).optimize() ten_lakh_crore_prefix = ( - build_group( - teens_ties_thousand, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakh_spaced, head_zeros=5 - ) - | build_group( - teens_ties_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5 - ) + build_group(teens_ties_thousand, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakh_spaced, head_zeros=5) + | build_group(teens_ties_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5) ).optimize() lakh_crore_count_prefix = (lakh_crore_prefix | ten_lakh_crore_prefix).optimize() - graph_ten_kharabs = build_group( - lakh_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_ten_kharabs = build_group(lakh_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - ten_nil_lakh_remainder_before_kotlu = build_group( - teens_ties, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_before, head_zeros=5 + ten_nil_lakh_remainder_before_kotlu = build_group(teens_ties, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_before, head_zeros=5 ).optimize() koti_ladder = [ @@ -335,42 +294,41 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): | create_larger_number_graph(teens_ties, ins_crores_before, 0, ten_nil_lakh_remainder_before_kotlu) ).optimize() - graph_ten_nils = build_group( - ten_nil_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_ten_nils = build_group(ten_nil_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() padma_crore_count_prefix = ( build_group(teens_ties_thousand, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7) - | build_group( - teens_ties_except_one, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ) + | build_group(teens_ties_except_one, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7) ).optimize() - graph_padmas = build_group( - padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_padmas = build_group(padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - ten_padma_one_crore_count_prefix = create_graph_suffix( - hundred_one_crore_prefix, ins_crore_spaced, 7 + ten_padma_one_crore_count_prefix = create_graph_suffix(hundred_one_crore_prefix, ins_crore_spaced, 7 ).optimize() ten_padma_crore_count_prefix = ( ten_padma_one_crore_count_prefix - | build_group( - hundred_crore_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ) + | build_group(hundred_crore_prefix, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7) ).optimize() - graph_ten_padmas = build_group( - ten_padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_ten_padmas = build_group(ten_padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - shankh_koti_count_prefix = build_group( - crore_count_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + shankh_oka_crore_count_prefix = build_group(teens_ties_thousand,ins_thousands_before,thousand_crore_ladder,head_suffix=ins_thousand_spaced,head_zeros=3, + ).optimize() + shankh_other_crore_count_prefix = ( + thousand_crore_prefix + | build_group(teens_ties_except_one,ins_thousands_before,thousand_crore_ladder,head_suffix=ins_thousands_before,head_zeros=3) ).optimize() - graph_shankhs = build_group( - shankh_koti_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + shankh_koti_count_prefix = ( + build_group(shankh_oka_crore_count_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7) + | build_group(shankh_other_crore_count_prefix, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7) + ).optimize() + + graph_shankhs = build_group(shankh_koti_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() def exact_digits(n, graph): @@ -399,6 +357,9 @@ def exact_digits(n, graph): | exact_digits(19, graph_shankhs) ) + same_script_number = pynini.closure(NEMO_DIGIT, 1) | pynini.closure(te_digit, 1) + graph_without_leading_zeros = pynini.compose(same_script_number, graph_without_leading_zeros) + cardinal_with_leading_zeros = pynini.compose( NEMO_ALL_ZERO + pynini.closure(NEMO_ALL_DIGIT), self.single_digits_graph ) @@ -422,3 +383,4 @@ def exact_digits(n, graph): self.final_graph = final_graph.optimize() final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.final_graph + pynutil.insert("\"") self.fst = self.add_tokens(final_graph) + \ No newline at end of file diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt index a50d89ce6..2c2a1afca 100644 --- a/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt @@ -45,12 +45,12 @@ 10000000000000~పది లక్షల కోట్లు 100000000000000~కోటి కోట్లు 999999999999999~తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది -1000000000000000~పది కోటి కోట్లు -999999999999999999~తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోటి తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది -9999999999999999999~తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోటి తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +1000000000000000~పది కోట్ల కోట్లు +999999999999999999~తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +9999999999999999999~తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది జనాభా 999999999999999~జనాభా తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది ౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯~తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది -౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯~తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోటి తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯౯~తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది 05~సున్నా ఐదు 01~సున్నా ఒకటి 073~సున్నా ఏడు మూడు @@ -86,4 +86,64 @@ 100000000001~పది వేల కోట్ల ఒకటి 1,00,100~లక్ష వంద 10,05,000~పది లక్షల ఐదు వేలు -1,00,01,001~కోటి వెయ్యి ఒకటి \ No newline at end of file +1,00,01,001~కోటి వెయ్యి ఒకటి +50000000001~ఐదు వేల కోట్ల ఒకటి +2100000000000~రెండు లక్షల పది వేల కోట్లు +210000000000000~రెండు కోట్ల పది లక్షల కోట్లు +2200000000000000~ఇరవై రెండు కోట్ల కోట్లు +10000000000000000~వంద కోట్ల కోట్లు +10200000000000000~నూట రెండు కోట్ల కోట్లు +990000000000000000~తొమ్మిది వేల తొమ్మిది వందల కోట్ల కోట్లు +9999999990000000~తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్లు +99999999999999999~తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +12345678901234567~నూట ఇరవై మూడు కోట్ల నలభై ఐదు లక్షల అరవై ఏడు వేల ఎనిమిది వందల తొంభై కోట్ల పన్నెండు లక్షల ముప్పై నాలుగు వేల ఐదు వందల అరవై ఏడు +2100000000000000~ఇరవై ఒక కోటి కోట్లు +20100000000000000~రెండు వందల ఒక కోటి కోట్లు +99000000000000~తొంభై తొమ్మిది లక్షల కోట్లు +990000000000000~తొమ్మిది కోట్ల తొంభై లక్షల కోట్లు +1010000000~నూట ఒక కోటి +1012345567~నూట ఒక కోటి ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +1,01,00,00,000~నూట ఒక కోటి +2010000000~రెండు వందల ఒక కోటి +9010000000~తొమ్మిది వందల ఒక కోటి +1020000000~నూట రెండు కోట్లు +12౩45~12౩45 +1౨345~1౨345 +౧2345~౧2345 +1234౫~1234౫ +123౪56789~123౪56789 +14~పద్నాలుగు +15~పదిహేను +16~పదహారు +19~పంతొమ్మిది +114~నూట పద్నాలుగు +1015~వెయ్యి పదిహేను +216~రెండు వందల పదహారు +౧౪~పద్నాలుగు +౧౯~పంతొమ్మిది +999999999~తొంభై తొమ్మిది కోట్ల తొంభై తొమ్మిది లక్షల తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది +220000000~ఇరవై రెండు కోట్లు +210000000~ఇరవై ఒక కోటి +900000000~తొంభై కోట్లు +123456789~పన్నెండు కోట్ల ముప్పై నాలుగు లక్షల యాభై ఆరు వేల ఏడు వందల ఎనభై తొమ్మిది +1000~వెయ్యి +100000~లక్ష +10000000~కోటి +౧౦౦౦~వెయ్యి +౧౦౦౦౦౦~లక్ష +౧౦౦౦౦౦౦౦~కోటి +౧౦౧~నూట ఒకటి +౧౧౧~నూట పదకొండు +౧౩౦~నూట ముప్పై +21000~ఇరవై ఒక వెయ్యి +31000~ముప్పై ఒక వెయ్యి +91000~తొంభై ఒక వెయ్యి +21500~ఇరవై ఒక వేల ఐదు వందలు +౨౧౦౦౦~ఇరవై ఒక వెయ్యి +0~సున్నా +౦~సున్నా +-0~మైనస్ సున్నా +99999999999999999999~99999999999999999999 +1,0000~ఒకటి , సున్నా సున్నా సున్నా సున్నా +12,34~పన్నెండు , ముప్పై నాలుగు +1,2,3~ఒకటి , రెండు , మూడు \ No newline at end of file diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt new file mode 100644 index 000000000..dd8bb335d --- /dev/null +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt @@ -0,0 +1,80 @@ +.~. +,~, +!~! +?~? +;~; +:~: +।~। +॥~॥ +॰~॰ +—~— +–~– +…~… +‥~‥ +·~· +•~• +₹~₹ +§~§ +°~° +′~′ +″~″ +|~| +౷~౷ +౿~౿ +?,~?, +...~... +???~??? +హలో.~హలో. +హలో!~హలో! +హలో?~హలో? +హలో,~హలో, +హలో;~హలో; +హలో:~హలో: +హలో।~హలో। +హలో॥~హలో॥ +హలో…~హలో… +హలో—~హలో— +హలో!!~హలో!! +హలో?!~హలో?! +హలో .~హలో . +హలో !~హలో ! +ఫేస్‌బుక్!~ఫేస్‌బుక్! +హలో౷~హలో౷ +హలో౿~హలో౿ +శ్రీ॰~శ్రీ॰ +‘హలో’~‘హలో’ +“తెలుగు”~“తెలుగు” +"హలో"~"హలో" +'హలో'~'హలో' +«తెలుగు»~«తెలుగు» +‹హలో›~‹హలో› +„హలో“~„హలో“ +(పరీక్ష)~(పరీక్ష) +[పరీక్ష]~[పరీక్ష] +{పరీక్ష}~{పరీక్ష} +`కోట్`~`కోట్` +అవును, సరే.~అవును, సరే. +మీరు ఎలా ఉన్నారు?~మీరు ఎలా ఉన్నారు? +డా. రాము~డా. రాము +ఎ.పి.~ఎ.పి. +రూ. వంద~రూ. వంద +2!~రెండు ! +౧!~ఒకటి ! +౨౩?~ఇరవై మూడు ? +1!!!!~ఒకటి !!!! +1, ~ఒకటి , +-౫~మైనస్ ఐదు +(1)హలో~(1)హలో +జంతువులు: (1), (2), (3)~జంతువులు: ( ఒకటి ), ( రెండు ), ( మూడు ) +114...48~నూట పద్నాలుగు ... నలభై ఎనిమిది +1961–1965~వెయ్యి తొమ్మిది వందల అరవై ఒకటి – వెయ్యి తొమ్మిది వందల అరవై ఐదు +2006—2012~రెండు వేల ఆరు — రెండు వేల పన్నెండు +100%~వంద % +₹100~₹100 +హలో~హలో +test -~test - +-5 test -~మైనస్ ఐదు test - +?,లేదు~?,లేదు +హలో...ప్రపంచం~హలో...ప్రపంచం +హలో & అక్కడ~హలో & అక్కడ +• ఐటమ్~• ఐటమ్ \ No newline at end of file diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..37bbde58f --- /dev/null +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt @@ -0,0 +1,98 @@ +~ + ~ + no~no +x ~x +X!~X! +—~— +…~… +·~· +।~। +॥~॥ +|~| +₹~₹ +×~× +÷~÷ +yahoo!~yahoo! +యాహూ!~యాహూ! +mother-in-law~mother-in-law +కచ్చా-పక్కా~కచ్చా-పక్కా +ఆంధ్ర-తెలంగాణ~ఆంధ్ర-తెలంగాణ +అత్తా-మామ~అత్తా-మామ +ఎ-బి-సి~ఎ-బి-సి +aachen's~aachen's +a'ali~a'ali +మేరీ'స్~మేరీ'స్ +May I come in?~May I come in? +మీరు లోపలికి రావచ్చా?~మీరు లోపలికి రావచ్చా? +Yes you may.~Yes you may. +అవును మీరు రావచ్చు.~అవును మీరు రావచ్చు. +అవును, సరే.~అవును, సరే. +love him while we may,~love him while we may, +లోహం కా!~లోహం కా! +చాలా బాగుంది!~చాలా బాగుంది! +మీరు ఎలా ఉన్నారు?~మీరు ఎలా ఉన్నారు? +డా. రాము~డా. రాము +రూమ్ నం.~రూమ్ నం. +aaa~aaa +aabach~aabach +mar~mar +తెలుగు~తెలుగు +నిద్ర~నిద్ర +గ్రంథాలయం~గ్రంథాలయం +బాగుంది~బాగుంది +ఝ~ఝ +ఓం~ఓం +ఆఆఆ~ఆఆఆ +అః~అః +సంస్కృతి~సంస్కృతి +క్షత్రియ~క్షత్రియ +జ్ఞానం~జ్ఞానం +క్షమించండి~క్షమించండి +చైతన్య~చైతన్య +హైదరాబాద్~హైదరాబాద్ +ఫేస్‌బుక్~ఫేస్‌బుక్ +రూ.~రూ. +డా.~డా. +శ్రీ.~శ్రీ. +శ్రీమతి~శ్రీమతి +ఎ.పి.~ఎ.పి. +టి.వి.~టి.వి. +యు.ఎస్.ఏ.~యు.ఎస్.ఏ. +ఐ.ఐ.టి.~ఐ.ఐ.టి. +‘హలో’~‘హలో’ +“తెలుగు”~“తెలుగు” +(పరీక్ష)~(పరీక్ష) +ISRO~ISRO +hello~hello +hello తెలుగు~hello తెలుగు +తెలుగు hello!~తెలుగు hello! +ఎపిఎస్~ఎపిఎస్ +CSI పరీక్ష~CSI పరీక్ష +GOOGLE తో~GOOGLE తో +టెస్ట్_కేసు~టెస్ట్_కేసు +es3~es3 +తె3~తె3 +abc3~abc3 +పరీక్ష-1~పరీక్ష-1 +తెలుగు123~తెలుగు123 +2024లో~2024లో +1~ఒకటి + 1~ఒకటి +!1~! ఒకటి +20 !~ఇరవై ! +స్స్స్స 55~స్స్స్స యాభై ఐదు +i saw (22) ducklings~i saw ( ఇరవై రెండు ) ducklings +నేను (22) బాతు పిల్లలను చూశాను~నేను ( ఇరవై రెండు ) బాతు పిల్లలను చూశాను +$ and 5% or %~$ and ఐదు % or % +/$€₩£BB¥#%AA and $€₩£¥#%~/$€₩£BB¥#%AA and $€₩£¥#% +there -0.4kg~there -0.4kg +there- -0.4kg~there- -0.4kg +zwanzig !~zwanzig ! +ఇరవై !~ఇరవై ! +హిందువు~హిందువు +ముస్లిం~ముస్లిం +క్రైస్తవుడు~క్రైస్తవుడు +ఏమిటి?~ఏమిటి? +అవును.~అవును. +పుస్తకం,~పుస్తకం, +హలో…~హలో… \ No newline at end of file diff --git a/tests/nemo_text_processing/te/test_punctuation.py b/tests/nemo_text_processing/te/test_punctuation.py new file mode 100644 index 000000000..97b433728 --- /dev/null +++ b/tests/nemo_text_processing/te/test_punctuation.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestPunctuation: + normalizer = Normalizer( + input_case='cased', + lang='te', + cache_dir=CACHE_DIR, + overwrite_cache=False, + post_process=False, + ) + + @parameterized.expand(parse_test_case_file('te/data_text_normalization/test_cases_punctuation.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected \ No newline at end of file diff --git a/tests/nemo_text_processing/te/test_word.py b/tests/nemo_text_processing/te/test_word.py new file mode 100644 index 000000000..6d151d0d7 --- /dev/null +++ b/tests/nemo_text_processing/te/test_word.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWord: + normalizer = Normalizer( + input_case='cased', + lang='te', + cache_dir=CACHE_DIR, + overwrite_cache=False, + post_process=False, + ) + + @parameterized.expand(parse_test_case_file('te/data_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected \ No newline at end of file From 7b58ae25e06c64f36c731c63a8f0a6bf2cd56e1b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:46:39 +0000 Subject: [PATCH 07/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: Haridas Nishitha --- .../text_normalization/te/taggers/cardinal.py | 155 ++++++++++++++---- .../te/test_punctuation.py | 2 +- tests/nemo_text_processing/te/test_word.py | 2 +- 3 files changed, 123 insertions(+), 36 deletions(-) diff --git a/nemo_text_processing/text_normalization/te/taggers/cardinal.py b/nemo_text_processing/text_normalization/te/taggers/cardinal.py index 094ebf303..0935d143b 100644 --- a/nemo_text_processing/text_normalization/te/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/te/taggers/cardinal.py @@ -101,7 +101,9 @@ def __init__(self, deterministic: bool = True): teens_ties = pynini.union(teens_ties_te, teens_ties_en) ties_one_suffix_en = (NEMO_DIGIT @ ties_one_suffix).optimize() ties_one_suffix_te = (te_digit @ ties_one_suffix).optimize() - teens_ties_thousand = ((ties_en + insert_space + ties_one_suffix_en) | (ties_te + insert_space + ties_one_suffix_te)).optimize() + teens_ties_thousand = ( + (ties_en + insert_space + ties_one_suffix_en) | (ties_te + insert_space + ties_one_suffix_te) + ).optimize() teens_ties_except_one = pynini.union( teens_en | (ties_en + pynutil.delete("0")) | (ties_en + insert_space + digit_except_one_en), teens_te | (ties_te + pynutil.delete("౦")) | (ties_te + insert_space + digit_except_one_te), @@ -155,12 +157,26 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): graph_thousands = ( exact_thousand | build_group(one_prefix, ins_thousand, thousand_ladder) - | build_group(digit_except_one, ins_thousands_before, thousand_ladder, head_suffix=ins_thousands_plural, head_zeros=3) + | build_group( + digit_except_one, ins_thousands_before, thousand_ladder, head_suffix=ins_thousands_plural, head_zeros=3 + ) ).optimize() graph_ten_thousands = ( - build_group(teens_ties_thousand, ins_thousands_before, thousand_ladder, head_suffix=ins_thousand_spaced, head_zeros=3) - | build_group(teens_ties_except_one, ins_thousands_before, thousand_ladder, head_suffix=ins_thousands_plural, head_zeros=3) + build_group( + teens_ties_thousand, + ins_thousands_before, + thousand_ladder, + head_suffix=ins_thousand_spaced, + head_zeros=3, + ) + | build_group( + teens_ties_except_one, + ins_thousands_before, + thousand_ladder, + head_suffix=ins_thousands_plural, + head_zeros=3, + ) ).optimize() lakh_ladder = [ @@ -180,7 +196,9 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): graph_ten_lakhs = ( build_group(teens_ties_thousand, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakh_spaced, head_zeros=5) - | build_group(teens_ties_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5) + | build_group( + teens_ties_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5 + ) ).optimize() crore_ladder = [ @@ -196,12 +214,18 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): graph_crores = ( exact_crore | build_group(one_prefix, ins_crore, crore_ladder) - | build_group(digit_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7) + | build_group( + digit_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ) ).optimize() graph_ten_crores = ( - build_group(teens_ties_thousand, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7) - | build_group(teens_ties_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7) + build_group( + teens_ties_thousand, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ) + | build_group( + teens_ties_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ) ).optimize() hundred_crore_prefix = ( @@ -219,8 +243,12 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): ).optimize() graph_arabs = ( - build_group(hundred_one_crore_prefix, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7) - | build_group(hundred_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7) + build_group( + hundred_one_crore_prefix, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ) + | build_group( + hundred_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + ) | create_larger_number_graph(hundred_crore_prefix, ins_crores_before, 0, graph_crores) ).optimize() @@ -232,21 +260,41 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): thousand_crore_prefix = ( exact_thousand - | build_group(digit_except_one,ins_thousands_before,thousand_crore_ladder,head_suffix=ins_thousands_before,head_zeros=3,) + | build_group( + digit_except_one, + ins_thousands_before, + thousand_crore_ladder, + head_suffix=ins_thousands_before, + head_zeros=3, + ) | build_group(one_prefix, ins_thousand, thousand_crore_ladder) ).optimize() ten_thousand_crore_prefix = ( - build_group(teens_ties_thousand,ins_thousands_before,thousand_crore_ladder,head_suffix=ins_thousand_spaced,head_zeros=3,) - | build_group(teens_ties_except_one, ins_thousands_before, thousand_crore_ladder, head_suffix=ins_thousands_before, head_zeros=3) + build_group( + teens_ties_thousand, + ins_thousands_before, + thousand_crore_ladder, + head_suffix=ins_thousand_spaced, + head_zeros=3, + ) + | build_group( + teens_ties_except_one, + ins_thousands_before, + thousand_crore_ladder, + head_suffix=ins_thousands_before, + head_zeros=3, + ) ).optimize() crore_count_prefix = (thousand_crore_prefix | ten_thousand_crore_prefix).optimize() - graph_ten_arabs = build_group(crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_ten_arabs = build_group( + crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - graph_kharabs = build_group(ten_thousand_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_kharabs = build_group( + ten_thousand_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() lakh_crore_ladder = [ @@ -260,20 +308,28 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): lakh_crore_prefix = ( exact_lakh | build_group(one_prefix, ins_lakh, lakh_crore_ladder) - | build_group(digit_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5) + | build_group( + digit_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5 + ) ).optimize() ten_lakh_crore_prefix = ( - build_group(teens_ties_thousand, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakh_spaced, head_zeros=5) - | build_group(teens_ties_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5) + build_group( + teens_ties_thousand, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakh_spaced, head_zeros=5 + ) + | build_group( + teens_ties_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5 + ) ).optimize() lakh_crore_count_prefix = (lakh_crore_prefix | ten_lakh_crore_prefix).optimize() - graph_ten_kharabs = build_group(lakh_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_ten_kharabs = build_group( + lakh_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - ten_nil_lakh_remainder_before_kotlu = build_group(teens_ties, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_before, head_zeros=5 + ten_nil_lakh_remainder_before_kotlu = build_group( + teens_ties, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_before, head_zeros=5 ).optimize() koti_ladder = [ @@ -294,41 +350,73 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): | create_larger_number_graph(teens_ties, ins_crores_before, 0, ten_nil_lakh_remainder_before_kotlu) ).optimize() - graph_ten_nils = build_group(ten_nil_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_ten_nils = build_group( + ten_nil_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() padma_crore_count_prefix = ( build_group(teens_ties_thousand, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7) - | build_group(teens_ties_except_one, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7) + | build_group( + teens_ties_except_one, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7 + ) ).optimize() - graph_padmas = build_group(padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_padmas = build_group( + padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - ten_padma_one_crore_count_prefix = create_graph_suffix(hundred_one_crore_prefix, ins_crore_spaced, 7 + ten_padma_one_crore_count_prefix = create_graph_suffix( + hundred_one_crore_prefix, ins_crore_spaced, 7 ).optimize() ten_padma_crore_count_prefix = ( ten_padma_one_crore_count_prefix - | build_group(hundred_crore_prefix, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7) + | build_group( + hundred_crore_prefix, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7 + ) ).optimize() - graph_ten_padmas = build_group(ten_padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_ten_padmas = build_group( + ten_padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - shankh_oka_crore_count_prefix = build_group(teens_ties_thousand,ins_thousands_before,thousand_crore_ladder,head_suffix=ins_thousand_spaced,head_zeros=3, + shankh_oka_crore_count_prefix = build_group( + teens_ties_thousand, + ins_thousands_before, + thousand_crore_ladder, + head_suffix=ins_thousand_spaced, + head_zeros=3, ).optimize() shankh_other_crore_count_prefix = ( thousand_crore_prefix - | build_group(teens_ties_except_one,ins_thousands_before,thousand_crore_ladder,head_suffix=ins_thousands_before,head_zeros=3) + | build_group( + teens_ties_except_one, + ins_thousands_before, + thousand_crore_ladder, + head_suffix=ins_thousands_before, + head_zeros=3, + ) ).optimize() shankh_koti_count_prefix = ( - build_group(shankh_oka_crore_count_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7) - | build_group(shankh_other_crore_count_prefix, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7) - ).optimize() - - graph_shankhs = build_group(shankh_koti_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + build_group( + shankh_oka_crore_count_prefix, + ins_crore_spaced, + koti_ladder, + head_suffix=ins_crore_spaced, + head_zeros=7, + ) + | build_group( + shankh_other_crore_count_prefix, + ins_crores_before, + koti_ladder, + head_suffix=ins_crores_before, + head_zeros=7, + ) + ).optimize() + + graph_shankhs = build_group( + shankh_koti_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() def exact_digits(n, graph): @@ -383,4 +471,3 @@ def exact_digits(n, graph): self.final_graph = final_graph.optimize() final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.final_graph + pynutil.insert("\"") self.fst = self.add_tokens(final_graph) - \ No newline at end of file diff --git a/tests/nemo_text_processing/te/test_punctuation.py b/tests/nemo_text_processing/te/test_punctuation.py index 97b433728..09e5eca74 100644 --- a/tests/nemo_text_processing/te/test_punctuation.py +++ b/tests/nemo_text_processing/te/test_punctuation.py @@ -34,4 +34,4 @@ class TestPunctuation: @pytest.mark.unit def test_norm(self, test_input, expected): pred = self.normalizer.normalize(test_input, verbose=False) - assert pred == expected \ No newline at end of file + assert pred == expected diff --git a/tests/nemo_text_processing/te/test_word.py b/tests/nemo_text_processing/te/test_word.py index 6d151d0d7..2d9fe6476 100644 --- a/tests/nemo_text_processing/te/test_word.py +++ b/tests/nemo_text_processing/te/test_word.py @@ -34,4 +34,4 @@ class TestWord: @pytest.mark.unit def test_norm(self, test_input, expected): pred = self.normalizer.normalize(test_input, verbose=False) - assert pred == expected \ No newline at end of file + assert pred == expected From 0f44929b10eafefe4e698c23cf071b03814162de Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Wed, 9 Sep 2026 13:41:19 +0530 Subject: [PATCH 08/15] feat(te): Updated. Signed-off-by: Haridas Nishitha --- .../te/test_sparrowhawk_normalization.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/nemo_text_processing/te/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/te/test_sparrowhawk_normalization.sh index 20c4df79a..edb3a32f6 100644 --- a/tests/nemo_text_processing/te/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/te/test_sparrowhawk_normalization.sh @@ -28,5 +28,15 @@ testTNCardinal() { runtest $input } +testTNWord() { + input=$PROJECT_DIR/te/data_text_normalization/test_cases_word.txt + runtest $input +} + +testTNPunctuation() { + input=$PROJECT_DIR/te/data_text_normalization/test_cases_punctuation.txt + runtest $input +} + # Load shUnit2 . $PROJECT_DIR/../shunit2/shunit2 From 265622eed57779a4262b8173a251a6ed0bf8cdbc Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Wed, 9 Sep 2026 14:13:41 +0530 Subject: [PATCH 09/15] Updated. Signed-off-by: Haridas Nishitha --- .../te/data_text_normalization/test_cases_punctuation.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt index dd8bb335d..b803d1b79 100644 --- a/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt @@ -44,7 +44,6 @@ శ్రీ॰~శ్రీ॰ ‘హలో’~‘హలో’ “తెలుగు”~“తెలుగు” -"హలో"~"హలో" 'హలో'~'హలో' «తెలుగు»~«తెలుగు» ‹హలో›~‹హలో› From 17aa5242928c8293b93f47a72135ad44ea4560ef Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Wed, 9 Sep 2026 14:34:48 +0530 Subject: [PATCH 10/15] feat(te): Updated. Signed-off-by: Haridas Nishitha --- nemo_text_processing/text_normalization/normalize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index 02d3de03e..f365dfeb2 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -166,7 +166,6 @@ def __init__( if post_process: self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) - elif lang == 'it': from nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst From c152573d5445362d4e6c28d0f40ff9a5d14f7f60 Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Thu, 10 Sep 2026 11:10:28 +0530 Subject: [PATCH 11/15] feat(te): Updated. Signed-off-by: Haridas Nishitha --- .../text_normalization/te/taggers/cardinal.py | 328 ++++++++---------- .../test_cases_cardinal.txt | 35 +- .../test_cases_punctuation.txt | 91 ++--- .../test_cases_punctuation_match_input.txt | 16 + .../test_cases_word.txt | 109 ++---- .../te/test_punctuation.py | 11 +- 6 files changed, 254 insertions(+), 336 deletions(-) create mode 100644 tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation_match_input.txt diff --git a/nemo_text_processing/text_normalization/te/taggers/cardinal.py b/nemo_text_processing/text_normalization/te/taggers/cardinal.py index 0935d143b..5d9924770 100644 --- a/nemo_text_processing/text_normalization/te/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/te/taggers/cardinal.py @@ -13,12 +13,14 @@ # limitations under the License. import pynini +from pynini.examples import plurals from pynini.lib import pynutil from nemo_text_processing.text_normalization.te.graph_utils import ( NEMO_ALL_DIGIT, NEMO_ALL_ZERO, NEMO_DIGIT, + NEMO_SIGMA, GraphFst, insert_space, ) @@ -30,8 +32,8 @@ class CardinalFst(GraphFst): Finite state transducer for classifying cardinals, e.g. -౨౩ -> cardinal { negative: "true" integer: "ఇరవై మూడు" } - Covers numbers up to 19 digits (Indian scale through shankh / శంఖ, 10^17). - Larger magnitudes are verbalized by composing crore (కోటి) groups. + Covers numbers up to 19 digits by composing crore (కోటి) groups + (through hundred crore crores (వంద కోట్ల కోట్లు) / 10^17). Args: deterministic: if True will provide a single transduction option, @@ -64,23 +66,19 @@ def __init__(self, deterministic: bool = True): pynini.compose(NEMO_DIGIT, hundred_prefix) | pynini.compose(te_digit, hundred_prefix) ).optimize() - thousand = magnitude["thousand"] - lakh = magnitude["lakh"] - crore = magnitude["crore"] - ins_hundreds_plural = pynutil.insert(" " + magnitude["hundreds_plural"]) ins_hundreds_before = pynutil.insert(" " + magnitude["hundreds_before"]) - ins_thousand = pynutil.insert(thousand) - ins_thousand_spaced = pynutil.insert(" " + thousand) + ins_thousand = pynutil.insert(magnitude["thousand"]) + ins_thousand_spaced = pynutil.insert(" " + magnitude["thousand"]) ins_thousands_plural = pynutil.insert(" " + magnitude["thousands_plural"]) ins_thousands_before = pynutil.insert(" " + magnitude["thousands_before"]) - ins_lakh = pynutil.insert(lakh) - ins_lakh_spaced = pynutil.insert(" " + lakh) + ins_lakh = pynutil.insert(magnitude["lakh"]) + ins_lakh_spaced = pynutil.insert(" " + magnitude["lakh"]) ins_lakha_digit = pynutil.insert(magnitude["lakh_before_digit"]) ins_lakhs_plural = pynutil.insert(" " + magnitude["lakhs_plural"]) ins_lakhs_before = pynutil.insert(" " + magnitude["lakhs_before"]) - ins_crore = pynutil.insert(crore) - ins_crore_spaced = pynutil.insert(" " + crore) + ins_crore = pynutil.insert(magnitude["crore"]) + ins_crore_spaced = pynutil.insert(" " + magnitude["crore"]) ins_crores_plural = pynutil.insert(" " + magnitude["crores_plural"]) ins_crores_before = pynutil.insert(" " + magnitude["crores_before"]) @@ -95,6 +93,8 @@ def __init__(self, deterministic: bool = True): digit_except_one = (pynini.difference(NEMO_ALL_DIGIT, NEMO_ALL_ZERO | one_digit) @ digit).optimize() digit_except_one_en = (pynini.difference(NEMO_DIGIT, pynini.union("0", "1")) @ digit).optimize() digit_except_one_te = (pynini.difference(te_digit, pynini.union("౦", "౧")) @ digit).optimize() + one_as_oka = (one_digit @ ties_one_suffix).optimize() + one_prefix = pynutil.delete(one_digit) teens_ties_en = teens_en | (ties_en + pynutil.delete("0")) | (ties_en + insert_space + digit_en) teens_ties_te = teens_te | (ties_te + pynutil.delete("౦")) | (ties_te + insert_space + digit_te) @@ -113,32 +113,53 @@ def __init__(self, deterministic: bool = True): self.single_digits_graph = single_digit_graph + pynini.closure(insert_space + single_digit_graph) delete_zero = pynutil.delete(NEMO_ALL_ZERO) - zero_pow = {0: pynini.accep("")} for _n in range(1, 8): zero_pow[_n] = (zero_pow[_n - 1] + delete_zero).optimize() - def create_graph_suffix(digit_graph, suffix, zeros_counts): - if zeros_counts == 0: - return digit_graph + suffix - - return digit_graph + zero_pow[zeros_counts] + suffix + def create_graph_suffix(prefix, suffix, zeros_counts): + return prefix + suffix if zeros_counts == 0 else prefix + zero_pow[zeros_counts] + suffix - def create_larger_number_graph(digit_graph, suffix, zeros_counts, sub_graph): - if zeros_counts == 0: - return digit_graph + suffix + insert_space + sub_graph - - return digit_graph + suffix + zero_pow[zeros_counts] + insert_space + sub_graph + def create_larger_number_graph(prefix, suffix, zeros_counts, sub_graph): + mid = suffix if zeros_counts == 0 else suffix + zero_pow[zeros_counts] + return prefix + mid + insert_space + sub_graph def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): - """Union over a "ladder" of (zeros_count, remainder_graph) rungs.""" graph = create_graph_suffix(prefix, head_suffix, head_zeros) if head_suffix is not None else None for zeros, sub in ladder: rung = create_larger_number_graph(prefix, rung_suffix, zeros, sub) graph = rung if graph is None else graph | rung return graph - one_prefix = pynutil.delete(one_digit) + def prefer(primary, secondary): + return plurals._priority_union(primary, secondary, NEMO_SIGMA) + + def tie_pair(oka_suffix, other_suffix, ladder, head_oka, head_other, head_zeros): + return ( + build_group(teens_ties_thousand, oka_suffix, ladder, head_suffix=head_oka, head_zeros=head_zeros) + | build_group( + teens_ties_except_one, other_suffix, ladder, head_suffix=head_other, head_zeros=head_zeros + ) + ).optimize() + + def oka_count_prefix(singular_ins, before_ins, remainders): + """Build …01 count prefixes: singular/before magnitude + (zeros, remainder→ఒక).""" + pieces = [] + for zeros, rem in remainders: + body = zero_pow[zeros] + insert_space + rem + pieces.append(one_prefix + singular_ins + body) + for pref in (digit_except_one, teens_ties_thousand, teens_ties_except_one): + pieces.append(pref + before_ins + body) + return pynini.union(*pieces).optimize() + + def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): + ladder = crore_ladder if ladder is None else ladder + other_head = ins_crores_plural if other_head is None else other_head + oka = build_group( + oka_prefix, ins_crore_spaced, ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ) + other = build_group(other_prefix, ins_crores_before, ladder, head_suffix=other_head, head_zeros=7) + return prefer(oka, other).optimize() graph_hundreds = ( exact_hundred @@ -149,11 +170,7 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): | create_larger_number_graph(digit_except_one, ins_hundreds_before, 0, teens_ties) ).optimize() - thousand_ladder = [ - (2, digit), - (1, teens_ties), - (0, graph_hundreds), - ] + thousand_ladder = [(2, digit), (1, teens_ties), (0, graph_hundreds)] graph_thousands = ( exact_thousand | build_group(one_prefix, ins_thousand, thousand_ladder) @@ -161,23 +178,14 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): digit_except_one, ins_thousands_before, thousand_ladder, head_suffix=ins_thousands_plural, head_zeros=3 ) ).optimize() - - graph_ten_thousands = ( - build_group( - teens_ties_thousand, - ins_thousands_before, - thousand_ladder, - head_suffix=ins_thousand_spaced, - head_zeros=3, - ) - | build_group( - teens_ties_except_one, - ins_thousands_before, - thousand_ladder, - head_suffix=ins_thousands_plural, - head_zeros=3, - ) - ).optimize() + graph_ten_thousands = tie_pair( + ins_thousands_before, + ins_thousands_before, + thousand_ladder, + ins_thousand_spaced, + ins_thousands_plural, + 3, + ) lakh_ladder = [ (4, digit), @@ -186,20 +194,15 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): (1, graph_thousands), (0, graph_ten_thousands), ] - graph_lakhs = ( exact_lakh | create_larger_number_graph(one_prefix, ins_lakha_digit, 4, digit) | build_group(one_prefix, ins_lakh, lakh_ladder[1:]) | build_group(digit_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5) ).optimize() - - graph_ten_lakhs = ( - build_group(teens_ties_thousand, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakh_spaced, head_zeros=5) - | build_group( - teens_ties_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5 - ) - ).optimize() + graph_ten_lakhs = tie_pair( + ins_lakhs_before, ins_lakhs_before, lakh_ladder, ins_lakh_spaced, ins_lakhs_plural, 5 + ) crore_ladder = [ (6, digit), @@ -210,7 +213,6 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): (1, graph_lakhs), (0, graph_ten_lakhs), ] - graph_crores = ( exact_crore | build_group(one_prefix, ins_crore, crore_ladder) @@ -218,15 +220,9 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): digit_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ) ).optimize() - - graph_ten_crores = ( - build_group( - teens_ties_thousand, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ) - | build_group( - teens_ties_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ) - ).optimize() + graph_ten_crores = tie_pair( + ins_crore_spaced, ins_crores_before, crore_ladder, ins_crore_spaced, ins_crores_plural, 7 + ) hundred_crore_prefix = ( exact_hundred @@ -236,28 +232,16 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): | create_larger_number_graph(digit_except_one, ins_hundreds_before, 1, digit_except_one) | create_larger_number_graph(digit_except_one, ins_hundreds_before, 0, teens_ties) ).optimize() - hundred_one_crore_prefix = ( - hundred_prefix_one + pynutil.delete(NEMO_ALL_ZERO) + (one_digit @ ties_one_suffix) + hundred_prefix_one + pynutil.delete(NEMO_ALL_ZERO) + one_as_oka | digit_except_one + pynutil.delete(NEMO_ALL_ZERO) + hundreds_before_one ).optimize() - graph_arabs = ( - build_group( - hundred_one_crore_prefix, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ) - | build_group( - hundred_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ) - | create_larger_number_graph(hundred_crore_prefix, ins_crores_before, 0, graph_crores) - ).optimize() - - thousand_crore_ladder = [ - (2, digit), - (1, teens_ties), - (0, hundred_crore_prefix), - ] - + # digit_except_one on the units rung so …01 cannot become ఒకటి కోట్లు. + thousand_crore_ladder = [(2, digit_except_one), (1, teens_ties), (0, hundred_crore_prefix)] + thousand_one_crore_prefix = oka_count_prefix( + ins_thousand, ins_thousands_before, [(2, one_as_oka), (0, hundred_one_crore_prefix)] + ) thousand_crore_prefix = ( exact_thousand | build_group( @@ -269,42 +253,40 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): ) | build_group(one_prefix, ins_thousand, thousand_crore_ladder) ).optimize() + ten_thousand_crore_prefix = tie_pair( + ins_thousands_before, + ins_thousands_before, + thousand_crore_ladder, + ins_thousand_spaced, + ins_thousands_before, + 3, + ) + crore_count_prefix = (thousand_crore_prefix | ten_thousand_crore_prefix).optimize() - ten_thousand_crore_prefix = ( + graph_hundred_crores = ( build_group( - teens_ties_thousand, - ins_thousands_before, - thousand_crore_ladder, - head_suffix=ins_thousand_spaced, - head_zeros=3, + hundred_one_crore_prefix, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 ) | build_group( - teens_ties_except_one, - ins_thousands_before, - thousand_crore_ladder, - head_suffix=ins_thousands_before, - head_zeros=3, + hundred_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ) + | create_larger_number_graph(hundred_crore_prefix, ins_crores_before, 0, graph_crores) ).optimize() - - crore_count_prefix = (thousand_crore_prefix | ten_thousand_crore_prefix).optimize() - - graph_ten_arabs = build_group( - crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ).optimize() - - graph_kharabs = build_group( - ten_thousand_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ).optimize() + graph_thousand_crores = crore_graph(thousand_one_crore_prefix, crore_count_prefix) + graph_ten_thousand_crores = crore_graph(thousand_one_crore_prefix, ten_thousand_crore_prefix) lakh_crore_ladder = [ - (4, digit), + (4, digit_except_one), (3, teens_ties), - (2, graph_hundreds), + (2, hundred_crore_prefix), (1, graph_thousands), (0, ten_thousand_crore_prefix), ] - + lakh_one_crore_prefix = oka_count_prefix( + ins_lakh, + ins_lakhs_before, + [(5, one_as_oka), (2, hundred_one_crore_prefix), (0, thousand_one_crore_prefix)], + ) lakh_crore_prefix = ( exact_lakh | build_group(one_prefix, ins_lakh, lakh_crore_ladder) @@ -312,26 +294,14 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): digit_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5 ) ).optimize() + ten_lakh_crore_prefix = tie_pair( + ins_lakhs_before, ins_lakhs_before, lakh_crore_ladder, ins_lakh_spaced, ins_lakhs_before, 5 + ) + graph_lakh_crores = crore_graph(lakh_one_crore_prefix, lakh_crore_prefix | ten_lakh_crore_prefix) - ten_lakh_crore_prefix = ( - build_group( - teens_ties_thousand, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakh_spaced, head_zeros=5 - ) - | build_group( - teens_ties_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5 - ) - ).optimize() - - lakh_crore_count_prefix = (lakh_crore_prefix | ten_lakh_crore_prefix).optimize() - - graph_ten_kharabs = build_group( - lakh_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ).optimize() - - ten_nil_lakh_remainder_before_kotlu = build_group( - teens_ties, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_before, head_zeros=5 - ).optimize() - + ten_lakh_crore_lakh_remainder = tie_pair( + ins_lakhs_before, ins_lakhs_before, lakh_ladder, ins_lakhs_before, ins_lakhs_before, 5 + ) koti_ladder = [ (6, digit), (5, teens_ties), @@ -339,55 +309,48 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): (3, graph_thousands), (2, graph_ten_thousands), (1, graph_lakhs), - (0, ten_nil_lakh_remainder_before_kotlu), + (0, ten_lakh_crore_lakh_remainder), ] - - ten_nil_crore_count_prefix = ( + ten_lakh_crore_count_prefix = ( graph_crores | graph_ten_crores - | create_larger_number_graph(one_prefix, ins_crore, 0, ten_nil_lakh_remainder_before_kotlu) - | create_larger_number_graph(digit_except_one, ins_crores_before, 0, ten_nil_lakh_remainder_before_kotlu) - | create_larger_number_graph(teens_ties, ins_crores_before, 0, ten_nil_lakh_remainder_before_kotlu) - ).optimize() - - graph_ten_nils = build_group( - ten_nil_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + | create_larger_number_graph(one_prefix, ins_crore, 0, ten_lakh_crore_lakh_remainder) + | create_larger_number_graph(digit_except_one, ins_crores_before, 0, ten_lakh_crore_lakh_remainder) + | create_larger_number_graph(teens_ties_thousand, ins_crore_spaced, 0, ten_lakh_crore_lakh_remainder) + | create_larger_number_graph(teens_ties_except_one, ins_crores_before, 0, ten_lakh_crore_lakh_remainder) ).optimize() - - padma_crore_count_prefix = ( - build_group(teens_ties_thousand, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7) - | build_group( - teens_ties_except_one, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7 - ) + graph_ten_lakh_crores = build_group( + ten_lakh_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - graph_padmas = build_group( - padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ).optimize() - - ten_padma_one_crore_count_prefix = create_graph_suffix( - hundred_one_crore_prefix, ins_crore_spaced, 7 + crore_crore_count_prefix = tie_pair( + ins_crore_spaced, ins_crores_before, koti_ladder, ins_crore_spaced, ins_crores_before, 7 + ) + graph_crore_crores = build_group( + crore_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - ten_padma_crore_count_prefix = ( - ten_padma_one_crore_count_prefix + ten_crore_crore_count_prefix = prefer( + create_graph_suffix(hundred_one_crore_prefix, ins_crore_spaced, 7) | build_group( + hundred_one_crore_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ), + build_group( hundred_crore_prefix, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7 - ) + ), ).optimize() - - graph_ten_padmas = build_group( - ten_padma_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_ten_crore_crores = build_group( + ten_crore_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() - shankh_oka_crore_count_prefix = build_group( + hundred_crore_crore_oka_count_prefix = build_group( teens_ties_thousand, ins_thousands_before, thousand_crore_ladder, head_suffix=ins_thousand_spaced, head_zeros=3, ).optimize() - shankh_other_crore_count_prefix = ( + hundred_crore_crore_other_count_prefix = ( thousand_crore_prefix | build_group( teens_ties_except_one, @@ -397,26 +360,23 @@ def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): head_zeros=3, ) ).optimize() - - shankh_koti_count_prefix = ( + hundred_crore_crore_count_prefix = prefer( build_group( - shankh_oka_crore_count_prefix, - ins_crore_spaced, - koti_ladder, - head_suffix=ins_crore_spaced, - head_zeros=7, + thousand_one_crore_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 ) | build_group( - shankh_other_crore_count_prefix, + hundred_crore_crore_oka_count_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + ), + build_group( + hundred_crore_crore_other_count_prefix, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7, - ) + ), ).optimize() - - graph_shankhs = build_group( - shankh_koti_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + graph_hundred_crore_crores = build_group( + hundred_crore_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 ).optimize() def exact_digits(n, graph): @@ -433,23 +393,24 @@ def exact_digits(n, graph): | graph_ten_lakhs | graph_crores | graph_ten_crores - | exact_digits(10, graph_arabs) - | exact_digits(11, graph_ten_arabs) - | exact_digits(12, graph_kharabs) - | exact_digits(13, graph_ten_kharabs) - | exact_digits(14, graph_ten_kharabs) - | exact_digits(15, graph_ten_nils) - | exact_digits(16, graph_padmas) - | exact_digits(17, graph_ten_padmas) - | exact_digits(18, graph_shankhs) - | exact_digits(19, graph_shankhs) + | exact_digits(10, graph_hundred_crores) + | exact_digits(11, graph_thousand_crores) + | exact_digits(12, graph_ten_thousand_crores) + | exact_digits(13, graph_lakh_crores) + | exact_digits(14, graph_lakh_crores) + | exact_digits(15, graph_ten_lakh_crores) + | exact_digits(16, graph_crore_crores) + | exact_digits(17, graph_ten_crore_crores) + | exact_digits(18, graph_hundred_crore_crores) + | exact_digits(19, graph_hundred_crore_crores) ) - same_script_number = pynini.closure(NEMO_DIGIT, 1) | pynini.closure(te_digit, 1) graph_without_leading_zeros = pynini.compose(same_script_number, graph_without_leading_zeros) cardinal_with_leading_zeros = pynini.compose( - NEMO_ALL_ZERO + pynini.closure(NEMO_ALL_DIGIT), self.single_digits_graph + (pynini.closure("0", 1) + pynini.closure(NEMO_DIGIT)) + | (pynini.closure("౦", 1) + pynini.closure(te_digit)), + self.single_digits_graph, ) delete_separator = pynutil.delete(",") two_digits = NEMO_ALL_DIGIT + NEMO_ALL_DIGIT @@ -461,13 +422,12 @@ def exact_digits(n, graph): + three_digits ) western_grouping = pynini.closure(NEMO_ALL_DIGIT, 1, 3) + pynini.closure(delete_separator + three_digits, 1) - strip_separators = (indian_grouping | western_grouping).optimize() - cardinal_with_separators = pynini.compose(strip_separators, graph_without_leading_zeros).optimize() + cardinal_with_separators = pynini.compose( + (indian_grouping | western_grouping).optimize(), graph_without_leading_zeros + ).optimize() final_graph = graph_without_leading_zeros | cardinal_with_leading_zeros | cardinal_with_separators - optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) - self.final_graph = final_graph.optimize() final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.final_graph + pynutil.insert("\"") self.fst = self.add_tokens(final_graph) diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt index 2c2a1afca..32eea938a 100644 --- a/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt @@ -146,4 +146,37 @@ 99999999999999999999~99999999999999999999 1,0000~ఒకటి , సున్నా సున్నా సున్నా సున్నా 12,34~పన్నెండు , ముప్పై నాలుగు -1,2,3~ఒకటి , రెండు , మూడు \ No newline at end of file +1,2,3~ఒకటి , రెండు , మూడు +51010000000~ఐదు వేల నూట ఒక కోటి +34018285364~మూడు వేల నాలుగు వందల ఒక కోటి ఎనభై రెండు లక్షల ఎనభై ఐదు వేల మూడు వందల అరవై నాలుగు +711010000000~డెబ్బై ఒక వేల నూట ఒక కోటి +465017762066~నలభై ఆరు వేల ఐదు వందల ఒక కోటి డెబ్బై ఏడు లక్షల అరవై రెండు వేల అరవై ఆరు +3164010000000~మూడు లక్షల పదహారు వేల నాలుగు వందల ఒక కోటి +97931010000000~తొంభై ఏడు లక్షల తొంభై మూడు వేల నూట ఒక కోటి +90140000000000000~తొమ్మిది వందల ఒక కోటి నలభై లక్షల కోట్లు +340100000000000000~మూడు వేల నాలుగు వందల ఒక కోటి కోట్లు +340182853640000000~మూడు వేల నాలుగు వందల ఒక కోటి ఎనభై రెండు లక్షల ఎనభై ఐదు వేల మూడు వందల అరవై నాలుగు కోట్లు +4650177620660000000~నలభై ఆరు వేల ఐదు వందల ఒక కోటి డెబ్బై ఏడు లక్షల అరవై రెండు వేల అరవై ఆరు కోట్లు +51020000000~ఐదు వేల నూట రెండు కోట్లు +34028285364~మూడు వేల నాలుగు వందల రెండు కోట్ల ఎనభై రెండు లక్షల ఎనభై ఐదు వేల మూడు వందల అరవై నాలుగు +90240000000000000~తొమ్మిది వందల రెండు కోట్ల నలభై లక్షల కోట్లు +90100000000000000~తొమ్మిది వందల ఒక కోటి కోట్లు +10010000000~వెయ్యి ఒక కోటి +20010000000~రెండు వేల ఒక కోటి +50010000000~ఐదు వేల ఒక కోటి +10001010000000~పది లక్షల నూట ఒక కోటి +10000000101~వెయ్యి కోట్ల నూట ఒకటి +10000000201~వెయ్యి కోట్ల రెండు వందల ఒకటి +220000000000000~రెండు కోట్ల ఇరవై లక్షల కోట్లు +230000000000000~రెండు కోట్ల ముప్పై లక్షల కోట్లు +410000000000000~నాలుగు కోట్ల పది లక్షల కోట్లు +941302654706146~తొమ్మిది కోట్ల నలభై ఒక లక్షల ముప్పై వేల రెండు వందల అరవై ఐదు కోట్ల నలభై ఏడు లక్షల ఆరు వేల నూట నలభై ఆరు +851750346234890~ఎనిమిది కోట్ల యాభై ఒక లక్షల డెబ్బై ఐదు వేల ముప్పై నాలుగు కోట్ల అరవై రెండు లక్షల ముప్పై నాలుగు వేల ఎనిమిది వందల తొంభై +0౫~0౫ +౦5~౦5 +00౭~00౭ +0౦5~0౦5 +05~సున్నా ఐదు +౦౫~సున్నా ఐదు +007~సున్నా సున్నా ఏడు +౦౦౭~సున్నా సున్నా ఏడు \ No newline at end of file diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt index b803d1b79..1e2426eb7 100644 --- a/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation.txt @@ -1,79 +1,38 @@ -.~. -,~, -!~! -?~? -;~; -:~: -।~। -॥~॥ -॰~॰ -—~— -–~– -…~… -‥~‥ -·~· -•~• -₹~₹ -§~§ -°~° -′~′ -″~″ -|~| -౷~౷ -౿~౿ +ఒక `కోట్` ఉదాహరణ~ఒక `కోట్` ఉదాహరణ +ఒక `25` ఉదాహరణ~ఒక ` ఇరవై ఐదు ` ఉదాహరణ +సింగిల్ కోట్.'~సింగిల్ కోట్.' +సింగిల్ కోట్. '4~సింగిల్ కోట్. ' నాలుగు +పరీక్ష -~పరీక్ష - +-5 పరీక్ష -~మైనస్ ఐదు పరీక్ష - +సంక్షిప్తం యు.ఎస్.ఏ.~సంక్షిప్తం యు.ఎస్.ఏ. +డా. రాము~డా. రాము +ఎ.పి.~ఎ.పి. +1-4=5~ఒకటి - నాలుగు = ఐదు ?,~?, -...~... -???~??? -హలో.~హలో. -హలో!~హలో! -హలో?~హలో? -హలో,~హలో, -హలో;~హలో; -హలో:~హలో: +?,లేదు~?,లేదు +అవును, సరే.~అవును, సరే. +మీరు ఎలా ఉన్నారు?~మీరు ఎలా ఉన్నారు? +ఇది (12 జాతులు) మరియు...~ఇది ( పన్నెండు జాతులు) మరియు... +జంతువులు: (1), (2), (3)~జంతువులు: ( ఒకటి ), ( రెండు ), ( మూడు ) +జంతువులు: ``పిల్లి``, ``కుక్క``, ``సింహం``~జంతువులు: ``పిల్లి``, ``కుక్క``, ``సింహం`` +జంతువులు: ``4~జంతువులు: `` నాలుగు +‘హలో’~‘హలో’ +“తెలుగు”~“తెలుగు” +(పరీక్ష)~(పరీక్ష) హలో।~హలో। హలో॥~హలో॥ హలో…~హలో… హలో—~హలో— -హలో!!~హలో!! -హలో?!~హలో?! -హలో .~హలో . -హలో !~హలో ! -ఫేస్‌బుక్!~ఫేస్‌బుక్! -హలో౷~హలో౷ -హలో౿~హలో౿ -శ్రీ॰~శ్రీ॰ -‘హలో’~‘హలో’ -“తెలుగు”~“తెలుగు” -'హలో'~'హలో' -«తెలుగు»~«తెలుగు» -‹హలో›~‹హలో› -„హలో“~„హలో“ -(పరీక్ష)~(పరీక్ష) -[పరీక్ష]~[పరీక్ష] -{పరీక్ష}~{పరీక్ష} -`కోట్`~`కోట్` -అవును, సరే.~అవును, సరే. -మీరు ఎలా ఉన్నారు?~మీరు ఎలా ఉన్నారు? -డా. రాము~డా. రాము -ఎ.పి.~ఎ.పి. -రూ. వంద~రూ. వంద +114...48~నూట పద్నాలుగు ... నలభై ఎనిమిది +1961–1965~వెయ్యి తొమ్మిది వందల అరవై ఒకటి – వెయ్యి తొమ్మిది వందల అరవై ఐదు +1961–1965 and (2006–2012)~వెయ్యి తొమ్మిది వందల అరవై ఒకటి – వెయ్యి తొమ్మిది వందల అరవై ఐదు and ( రెండు వేల ఆరు – రెండు వేల పన్నెండు ) 2!~రెండు ! ౧!~ఒకటి ! -౨౩?~ఇరవై మూడు ? 1!!!!~ఒకటి !!!! 1, ~ఒకటి , --౫~మైనస్ ఐదు (1)హలో~(1)హలో -జంతువులు: (1), (2), (3)~జంతువులు: ( ఒకటి ), ( రెండు ), ( మూడు ) -114...48~నూట పద్నాలుగు ... నలభై ఎనిమిది -1961–1965~వెయ్యి తొమ్మిది వందల అరవై ఒకటి – వెయ్యి తొమ్మిది వందల అరవై ఐదు -2006—2012~రెండు వేల ఆరు — రెండు వేల పన్నెండు -100%~వంద % +123,000012~నూట ఇరవై మూడు , సున్నా సున్నా సున్నా సున్నా ఒకటి రెండు ₹100~₹100 హలో~హలో test -~test - --5 test -~మైనస్ ఐదు test - -?,లేదు~?,లేదు -హలో...ప్రపంచం~హలో...ప్రపంచం -హలో & అక్కడ~హలో & అక్కడ -• ఐటమ్~• ఐటమ్ \ No newline at end of file +-5 test -~మైనస్ ఐదు test - \ No newline at end of file diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation_match_input.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation_match_input.txt new file mode 100644 index 000000000..7eef277e7 --- /dev/null +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_punctuation_match_input.txt @@ -0,0 +1,16 @@ +2!~రెండు! +౧!~ఒకటి! +1!!!!~ఒకటి!!!! +1,~ఒకటి, +(1)హలో~(1)హలో +1!హలో~1!హలో +?,లేదు~?,లేదు +ఇది (12 జాతులు) మరియు...~ఇది (పన్నెండు జాతులు) మరియు... +జంతువులు: (1), (2), (3)~జంతువులు: (ఒకటి), (రెండు), (మూడు) +ఒక `25` ఉదాహరణ~ఒక `ఇరవై ఐదు` ఉదాహరణ +సింగిల్ కోట్. '4~సింగిల్ కోట్. 'నాలుగు +114...48~నూట పద్నాలుగు...నలభై ఎనిమిది +123,000012~నూట ఇరవై మూడు,సున్నా సున్నా సున్నా సున్నా ఒకటి రెండు +జంతువులు: ``4~జంతువులు: ``నాలుగు +1961–1965 and (2006–2012)~వెయ్యి తొమ్మిది వందల అరవై ఒకటి – వెయ్యి తొమ్మిది వందల అరవై ఐదు and (రెండు వేల ఆరు – రెండు వేల పన్నెండు) +హలో...ప్రపంచం~హలో...ప్రపంచం \ No newline at end of file diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt index 37bbde58f..5ca23dcb7 100644 --- a/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt @@ -1,98 +1,41 @@ ~ ~ - no~no + 1~ఒకటి + తెలుగు~తెలుగు +మీరు లోపలికి రావచ్చా?~మీరు లోపలికి రావచ్చా? +అవును మీరు రావచ్చు.~అవును మీరు రావచ్చు. +మార్.~మార్. +తె3~తె3 x ~x X!~X! —~— -…~… -·~· ।~। ॥~॥ -|~| ₹~₹ -×~× -÷~÷ -yahoo!~yahoo! -యాహూ!~యాహూ! -mother-in-law~mother-in-law -కచ్చా-పక్కా~కచ్చా-పక్కా -ఆంధ్ర-తెలంగాణ~ఆంధ్ర-తెలంగాణ -అత్తా-మామ~అత్తా-మామ -ఎ-బి-సి~ఎ-బి-సి -aachen's~aachen's -a'ali~a'ali -మేరీ'స్~మేరీ'స్ -May I come in?~May I come in? -మీరు లోపలికి రావచ్చా?~మీరు లోపలికి రావచ్చా? -Yes you may.~Yes you may. -అవును మీరు రావచ్చు.~అవును మీరు రావచ్చు. -అవును, సరే.~అవును, సరే. -love him while we may,~love him while we may, -లోహం కా!~లోహం కా! -చాలా బాగుంది!~చాలా బాగుంది! -మీరు ఎలా ఉన్నారు?~మీరు ఎలా ఉన్నారు? -డా. రాము~డా. రాము -రూమ్ నం.~రూమ్ నం. -aaa~aaa -aabach~aabach -mar~mar +…~… +·~· తెలుగు~తెలుగు నిద్ర~నిద్ర -గ్రంథాలయం~గ్రంథాలయం బాగుంది~బాగుంది -ఝ~ఝ -ఓం~ఓం -ఆఆఆ~ఆఆఆ -అః~అః -సంస్కృతి~సంస్కృతి -క్షత్రియ~క్షత్రియ -జ్ఞానం~జ్ఞానం -క్షమించండి~క్షమించండి -చైతన్య~చైతన్య -హైదరాబాద్~హైదరాబాద్ -ఫేస్‌బుక్~ఫేస్‌బుక్ -రూ.~రూ. -డా.~డా. -శ్రీ.~శ్రీ. -శ్రీమతి~శ్రీమతి -ఎ.పి.~ఎ.పి. -టి.వి.~టి.వి. -యు.ఎస్.ఏ.~యు.ఎస్.ఏ. -ఐ.ఐ.టి.~ఐ.ఐ.టి. +మేరీ'స్~మేరీ'స్ +కచ్చా-పక్కా~కచ్చా-పక్కా +ఆంధ్ర-తెలంగాణ~ఆంధ్ర-తెలంగాణ +$ and 5% or %~$ and ఐదు % or % + 1~ఒకటి +1~ఒకటి +!1~! ఒకటి +౧~ఒకటి +౧!~ఒకటి ! +మార్~మార్ +/$€₩£BB¥#%AA and $€₩£¥#%~/$€₩£BB¥#%AA and $€₩£¥#% +లోహం కా,~లోహం కా, +యాహూ!~యాహూ! ‘హలో’~‘హలో’ “తెలుగు”~“తెలుగు” (పరీక్ష)~(పరీక్ష) -ISRO~ISRO -hello~hello -hello తెలుగు~hello తెలుగు -తెలుగు hello!~తెలుగు hello! -ఎపిఎస్~ఎపిఎస్ -CSI పరీక్ష~CSI పరీక్ష -GOOGLE తో~GOOGLE తో -టెస్ట్_కేసు~టెస్ట్_కేసు -es3~es3 -తె3~తె3 -abc3~abc3 -పరీక్ష-1~పరీక్ష-1 -తెలుగు123~తెలుగు123 -2024లో~2024లో -1~ఒకటి - 1~ఒకటి -!1~! ఒకటి -20 !~ఇరవై ! -స్స్స్స 55~స్స్స్స యాభై ఐదు -i saw (22) ducklings~i saw ( ఇరవై రెండు ) ducklings +తెలుగు।~తెలుగు। +నమస్కారం॥~నమస్కారం॥ +హలో…~హలో… +రూ. 100~రూ. వంద నేను (22) బాతు పిల్లలను చూశాను~నేను ( ఇరవై రెండు ) బాతు పిల్లలను చూశాను -$ and 5% or %~$ and ఐదు % or % -/$€₩£BB¥#%AA and $€₩£¥#%~/$€₩£BB¥#%AA and $€₩£¥#% -there -0.4kg~there -0.4kg -there- -0.4kg~there- -0.4kg -zwanzig !~zwanzig ! -ఇరవై !~ఇరవై ! -హిందువు~హిందువు -ముస్లిం~ముస్లిం -క్రైస్తవుడు~క్రైస్తవుడు -ఏమిటి?~ఏమిటి? -అవును.~అవును. -పుస్తకం,~పుస్తకం, -హలో…~హలో… \ No newline at end of file +ఇరవై !~ఇరవై ! \ No newline at end of file diff --git a/tests/nemo_text_processing/te/test_punctuation.py b/tests/nemo_text_processing/te/test_punctuation.py index 09e5eca74..fc78711eb 100644 --- a/tests/nemo_text_processing/te/test_punctuation.py +++ b/tests/nemo_text_processing/te/test_punctuation.py @@ -33,5 +33,12 @@ class TestPunctuation: @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm(self, test_input, expected): - pred = self.normalizer.normalize(test_input, verbose=False) - assert pred == expected + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input} != {expected}" + + @parameterized.expand(parse_test_case_file('te/data_text_normalization/test_cases_punctuation_match_input.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_python_punct_post_process(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=True) + assert pred == expected, f"for input |{test_input}|: pred: |{pred}| != expected: |{expected}|" From 39c0ac91a8282ccc9369358c44fd3261a9d1e530 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 05:41:36 +0000 Subject: [PATCH 12/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/te/taggers/cardinal.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nemo_text_processing/text_normalization/te/taggers/cardinal.py b/nemo_text_processing/text_normalization/te/taggers/cardinal.py index 5d9924770..72041eb70 100644 --- a/nemo_text_processing/text_normalization/te/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/te/taggers/cardinal.py @@ -155,9 +155,7 @@ def oka_count_prefix(singular_ins, before_ins, remainders): def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): ladder = crore_ladder if ladder is None else ladder other_head = ins_crores_plural if other_head is None else other_head - oka = build_group( - oka_prefix, ins_crore_spaced, ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ) + oka = build_group(oka_prefix, ins_crore_spaced, ladder, head_suffix=ins_crore_spaced, head_zeros=7) other = build_group(other_prefix, ins_crores_before, ladder, head_suffix=other_head, head_zeros=7) return prefer(oka, other).optimize() @@ -365,7 +363,11 @@ def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): thousand_one_crore_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 ) | build_group( - hundred_crore_crore_oka_count_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 + hundred_crore_crore_oka_count_prefix, + ins_crore_spaced, + koti_ladder, + head_suffix=ins_crore_spaced, + head_zeros=7, ), build_group( hundred_crore_crore_other_count_prefix, @@ -376,7 +378,11 @@ def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): ), ).optimize() graph_hundred_crore_crores = build_group( - hundred_crore_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 + hundred_crore_crore_count_prefix, + ins_crores_before, + crore_ladder, + head_suffix=ins_crores_plural, + head_zeros=7, ).optimize() def exact_digits(n, graph): From 10f12d5e1a291712150aa6a6afe4d32f73b33228 Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Thu, 10 Sep 2026 11:40:00 +0530 Subject: [PATCH 13/15] Updated. Signed-off-by: Haridas Nishitha --- .../text_normalization/te/taggers/cardinal.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/nemo_text_processing/text_normalization/te/taggers/cardinal.py b/nemo_text_processing/text_normalization/te/taggers/cardinal.py index 72041eb70..2ab3c9f48 100644 --- a/nemo_text_processing/text_normalization/te/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/te/taggers/cardinal.py @@ -147,9 +147,15 @@ def oka_count_prefix(singular_ins, before_ins, remainders): pieces = [] for zeros, rem in remainders: body = zero_pow[zeros] + insert_space + rem - pieces.append(one_prefix + singular_ins + body) - for pref in (digit_except_one, teens_ties_thousand, teens_ties_except_one): - pieces.append(pref + before_ins + body) + digit_pref = pynini.union( + one_prefix + singular_ins + body, + digit_except_one + before_ins + body, + ) + ties_pref = pynini.union( + teens_ties_thousand + before_ins + body, + teens_ties_except_one + before_ins + body, + ) + pieces.append(prefer(digit_pref, ties_pref)) return pynini.union(*pieces).optimize() def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): @@ -235,7 +241,6 @@ def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): | digit_except_one + pynutil.delete(NEMO_ALL_ZERO) + hundreds_before_one ).optimize() - # digit_except_one on the units rung so …01 cannot become ఒకటి కోట్లు. thousand_crore_ladder = [(2, digit_except_one), (1, teens_ties), (0, hundred_crore_prefix)] thousand_one_crore_prefix = oka_count_prefix( ins_thousand, ins_thousands_before, [(2, one_as_oka), (0, hundred_one_crore_prefix)] @@ -309,13 +314,13 @@ def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): (1, graph_lakhs), (0, ten_lakh_crore_lakh_remainder), ] - ten_lakh_crore_count_prefix = ( - graph_crores - | graph_ten_crores - | create_larger_number_graph(one_prefix, ins_crore, 0, ten_lakh_crore_lakh_remainder) + + ten_lakh_crore_count_prefix = prefer( + create_larger_number_graph(one_prefix, ins_crore, 0, ten_lakh_crore_lakh_remainder) | create_larger_number_graph(digit_except_one, ins_crores_before, 0, ten_lakh_crore_lakh_remainder) | create_larger_number_graph(teens_ties_thousand, ins_crore_spaced, 0, ten_lakh_crore_lakh_remainder) - | create_larger_number_graph(teens_ties_except_one, ins_crores_before, 0, ten_lakh_crore_lakh_remainder) + | create_larger_number_graph(teens_ties_except_one, ins_crores_before, 0, ten_lakh_crore_lakh_remainder), + graph_crores | graph_ten_crores, ).optimize() graph_ten_lakh_crores = build_group( ten_lakh_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 From 309ea76708c7ef0ec74bab51b9dfca0a55379624 Mon Sep 17 00:00:00 2001 From: Haridas Nishitha Date: Fri, 11 Sep 2026 00:31:21 +0530 Subject: [PATCH 14/15] Updated. Signed-off-by: Haridas Nishitha --- .../text_normalization/te/taggers/cardinal.py | 570 ++++++++---------- .../test_cases_cardinal.txt | 24 +- .../test_cases_word.txt | 12 +- 3 files changed, 271 insertions(+), 335 deletions(-) diff --git a/nemo_text_processing/text_normalization/te/taggers/cardinal.py b/nemo_text_processing/text_normalization/te/taggers/cardinal.py index 2ab3c9f48..2a860a6ec 100644 --- a/nemo_text_processing/text_normalization/te/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/te/taggers/cardinal.py @@ -51,146 +51,135 @@ def __init__(self, deterministic: bool = True): hundred_prefix = pynini.string_file(get_abs_path("data/numbers/hundred_prefix.tsv")) ties_one_suffix = pynini.string_file(get_abs_path("data/numbers/ties_one_suffix.tsv")) hundreds_before_one = pynini.string_file(get_abs_path("data/numbers/hundreds_before_one.tsv")) - magnitude = {k: v for k, v in load_labels(get_abs_path("data/numbers/magnitudes.tsv"))} + mag = dict(load_labels(get_abs_path("data/numbers/magnitudes.tsv"))) te_digit = pynini.difference(NEMO_ALL_DIGIT, NEMO_DIGIT).optimize() - exact_hundred = pynini.compose(NEMO_ALL_DIGIT**3, exact_power).optimize() - exact_thousand = pynini.compose(NEMO_ALL_DIGIT**4, exact_power).optimize() - exact_lakh = pynini.compose(NEMO_ALL_DIGIT**6, exact_power).optimize() - exact_crore = pynini.compose(NEMO_ALL_DIGIT**8, exact_power).optimize() - hundred_prefix_ten = ( - pynini.compose(NEMO_DIGIT + NEMO_DIGIT, hundred_prefix) - | pynini.compose(te_digit + te_digit, hundred_prefix) - ).optimize() - hundred_prefix_one = ( - pynini.compose(NEMO_DIGIT, hundred_prefix) | pynini.compose(te_digit, hundred_prefix) - ).optimize() - - ins_hundreds_plural = pynutil.insert(" " + magnitude["hundreds_plural"]) - ins_hundreds_before = pynutil.insert(" " + magnitude["hundreds_before"]) - ins_thousand = pynutil.insert(magnitude["thousand"]) - ins_thousand_spaced = pynutil.insert(" " + magnitude["thousand"]) - ins_thousands_plural = pynutil.insert(" " + magnitude["thousands_plural"]) - ins_thousands_before = pynutil.insert(" " + magnitude["thousands_before"]) - ins_lakh = pynutil.insert(magnitude["lakh"]) - ins_lakh_spaced = pynutil.insert(" " + magnitude["lakh"]) - ins_lakha_digit = pynutil.insert(magnitude["lakh_before_digit"]) - ins_lakhs_plural = pynutil.insert(" " + magnitude["lakhs_plural"]) - ins_lakhs_before = pynutil.insert(" " + magnitude["lakhs_before"]) - ins_crore = pynutil.insert(magnitude["crore"]) - ins_crore_spaced = pynutil.insert(" " + magnitude["crore"]) - ins_crores_plural = pynutil.insert(" " + magnitude["crores_plural"]) - ins_crores_before = pynutil.insert(" " + magnitude["crores_before"]) - - teens_en = pynini.compose(NEMO_DIGIT + NEMO_DIGIT, teens).optimize() - teens_te = pynini.compose(te_digit + te_digit, teens).optimize() - digit_en = (NEMO_DIGIT @ digit).optimize() - digit_te = (te_digit @ digit).optimize() - ties_en = (NEMO_DIGIT @ ties).optimize() - ties_te = (te_digit @ ties).optimize() - one_digit = pynini.union("1", "౧") - digit_except_one = (pynini.difference(NEMO_ALL_DIGIT, NEMO_ALL_ZERO | one_digit) @ digit).optimize() - digit_except_one_en = (pynini.difference(NEMO_DIGIT, pynini.union("0", "1")) @ digit).optimize() - digit_except_one_te = (pynini.difference(te_digit, pynini.union("౦", "౧")) @ digit).optimize() - one_as_oka = (one_digit @ ties_one_suffix).optimize() one_prefix = pynutil.delete(one_digit) + one_as_oka = (one_digit @ ties_one_suffix).optimize() + digit_x1 = (pynini.difference(NEMO_ALL_DIGIT, NEMO_ALL_ZERO | one_digit) @ digit).optimize() - teens_ties_en = teens_en | (ties_en + pynutil.delete("0")) | (ties_en + insert_space + digit_en) - teens_ties_te = teens_te | (ties_te + pynutil.delete("౦")) | (ties_te + insert_space + digit_te) - teens_ties = pynini.union(teens_ties_te, teens_ties_en) - ties_one_suffix_en = (NEMO_DIGIT @ ties_one_suffix).optimize() - ties_one_suffix_te = (te_digit @ ties_one_suffix).optimize() - teens_ties_thousand = ( - (ties_en + insert_space + ties_one_suffix_en) | (ties_te + insert_space + ties_one_suffix_te) - ).optimize() - teens_ties_except_one = pynini.union( - teens_en | (ties_en + pynutil.delete("0")) | (ties_en + insert_space + digit_except_one_en), - teens_te | (ties_te + pynutil.delete("౦")) | (ties_te + insert_space + digit_except_one_te), - ).optimize() + def U(*parts): + return pynini.union(*parts).optimize() + + def exact_n(n, graph=exact_power): + return pynini.compose(NEMO_ALL_DIGIT**n, graph).optimize() - single_digit_graph = digit | zero - self.single_digits_graph = single_digit_graph + pynini.closure(insert_space + single_digit_graph) + def ins(key, space=True): + return pynutil.insert((" " if space else "") + mag[key]) + + i_thou, i_thou_sp = ins("thousand", False), ins("thousand") + i_thous, i_thous_pl = ins("thousands_before"), ins("thousands_plural") + i_lakh, i_lakh_sp = ins("lakh", False), ins("lakh") + i_lakha, i_lakhs, i_lakhs_pl = pynutil.insert(mag["lakh_before_digit"]), ins("lakhs_before"), ins("lakhs_plural") + i_koti, i_koti_sp = ins("crore", False), ins("crore") + i_kotlu, i_kotlu_pl = ins("crores_before"), ins("crores_plural") + i_vandalu, i_vandalu_pl = ins("hundreds_before"), ins("hundreds_plural") + + hp_ten = U( + pynini.compose(NEMO_DIGIT + NEMO_DIGIT, hundred_prefix), + pynini.compose(te_digit + te_digit, hundred_prefix), + ) + hp_one = U(pynini.compose(NEMO_DIGIT, hundred_prefix), pynini.compose(te_digit, hundred_prefix)) + + def teens_ties_of(d_cls, zero_ch, dig_map): + t = pynini.compose(d_cls + d_cls, teens) + ti = (d_cls @ ties).optimize() + return t | (ti + pynutil.delete(zero_ch)) | (ti + insert_space + dig_map) + + dig_en, dig_te = (NEMO_DIGIT @ digit).optimize(), (te_digit @ digit).optimize() + dig_x1_en = (pynini.difference(NEMO_DIGIT, pynini.union("0", "1")) @ digit).optimize() + dig_x1_te = (pynini.difference(te_digit, pynini.union("౦", "౧")) @ digit).optimize() + teens_ties = U(teens_ties_of(NEMO_DIGIT, "0", dig_en), teens_ties_of(te_digit, "౦", dig_te)) + teens_ties_x1 = U(teens_ties_of(NEMO_DIGIT, "0", dig_x1_en), teens_ties_of(te_digit, "౦", dig_x1_te)) + teens_ties_oka = U( + (NEMO_DIGIT @ ties) + insert_space + (NEMO_DIGIT @ ties_one_suffix), + (te_digit @ ties) + insert_space + (te_digit @ ties_one_suffix), + ) + + self.single_digits_graph = (digit | zero) + pynini.closure(insert_space + (digit | zero)) delete_zero = pynutil.delete(NEMO_ALL_ZERO) - zero_pow = {0: pynini.accep("")} - for _n in range(1, 8): - zero_pow[_n] = (zero_pow[_n - 1] + delete_zero).optimize() + z = {0: pynini.accep("")} + for n in range(1, 8): + z[n] = (z[n - 1] + delete_zero).optimize() - def create_graph_suffix(prefix, suffix, zeros_counts): - return prefix + suffix if zeros_counts == 0 else prefix + zero_pow[zeros_counts] + suffix + def suffix(prefix, suf, zeros): + return prefix + suf if zeros == 0 else prefix + z[zeros] + suf - def create_larger_number_graph(prefix, suffix, zeros_counts, sub_graph): - mid = suffix if zeros_counts == 0 else suffix + zero_pow[zeros_counts] - return prefix + mid + insert_space + sub_graph + def rung(prefix, suf, zeros, sub): + return prefix + (suf if zeros == 0 else suf + z[zeros]) + insert_space + sub - def build_group(prefix, rung_suffix, ladder, head_suffix=None, head_zeros=None): - graph = create_graph_suffix(prefix, head_suffix, head_zeros) if head_suffix is not None else None + def group(prefix, suf, ladder, head=None, head_z=None): + g = suffix(prefix, head, head_z) if head is not None else None for zeros, sub in ladder: - rung = create_larger_number_graph(prefix, rung_suffix, zeros, sub) - graph = rung if graph is None else graph | rung - return graph - - def prefer(primary, secondary): - return plurals._priority_union(primary, secondary, NEMO_SIGMA) - - def tie_pair(oka_suffix, other_suffix, ladder, head_oka, head_other, head_zeros): - return ( - build_group(teens_ties_thousand, oka_suffix, ladder, head_suffix=head_oka, head_zeros=head_zeros) - | build_group( - teens_ties_except_one, other_suffix, ladder, head_suffix=head_other, head_zeros=head_zeros - ) - ).optimize() + g = rung(prefix, suf, zeros, sub) if g is None else g | rung(prefix, suf, zeros, sub) + return g - def oka_count_prefix(singular_ins, before_ins, remainders): - """Build …01 count prefixes: singular/before magnitude + (zeros, remainder→ఒక).""" - pieces = [] - for zeros, rem in remainders: - body = zero_pow[zeros] + insert_space + rem - digit_pref = pynini.union( - one_prefix + singular_ins + body, - digit_except_one + before_ins + body, - ) - ties_pref = pynini.union( - teens_ties_thousand + before_ins + body, - teens_ties_except_one + before_ins + body, - ) - pieces.append(prefer(digit_pref, ties_pref)) - return pynini.union(*pieces).optimize() - - def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): - ladder = crore_ladder if ladder is None else ladder - other_head = ins_crores_plural if other_head is None else other_head - oka = build_group(oka_prefix, ins_crore_spaced, ladder, head_suffix=ins_crore_spaced, head_zeros=7) - other = build_group(other_prefix, ins_crores_before, ladder, head_suffix=other_head, head_zeros=7) - return prefer(oka, other).optimize() - - graph_hundreds = ( - exact_hundred - | hundred_prefix_ten + digit - | hundred_prefix_one + teens_ties - | create_graph_suffix(digit_except_one, ins_hundreds_plural, 2) - | create_larger_number_graph(digit_except_one, ins_hundreds_before, 1, digit) - | create_larger_number_graph(digit_except_one, ins_hundreds_before, 0, teens_ties) - ).optimize() + def prefer(a, b): + return plurals._priority_union(a, b, NEMO_SIGMA) - thousand_ladder = [(2, digit), (1, teens_ties), (0, graph_hundreds)] - graph_thousands = ( - exact_thousand - | build_group(one_prefix, ins_thousand, thousand_ladder) - | build_group( - digit_except_one, ins_thousands_before, thousand_ladder, head_suffix=ins_thousands_plural, head_zeros=3 + def ties_group(oka_suf, other_suf, ladder, head_oka, head_other, head_z): + return U( + group(teens_ties_oka, oka_suf, ladder, head_oka, head_z), + group(teens_ties_x1, other_suf, ladder, head_other, head_z), ) - ).optimize() - graph_ten_thousands = tie_pair( - ins_thousands_before, - ins_thousands_before, - thousand_ladder, - ins_thousand_spaced, - ins_thousands_plural, - 3, - ) + def band(exact, sg, before, head, spaced, zeros, ladder, one_ladder=None, extra=None, ten_oka=None): + """Digit magnitude (+ optional teens ties). head is plural (standalone) or before (crore count).""" + one_ladder = ladder if one_ladder is None else one_ladder + g = exact | group(one_prefix, sg, one_ladder) + if extra is not None: + g = g | extra + g = U(g, group(digit_x1, before, ladder, head, zeros)) + ten = ties_group(ten_oka or before, before, ladder, spaced, head, zeros) + return g, ten + + def oka_count(sg, before, remainders): + """…01 multipliers: 1+sg / N+before / ties+before + zeros + rem→ఒక.""" + + def side(use_digit): + parts = [] + for zeros, rem in remainders: + body = z[zeros] + insert_space + rem + if use_digit: + parts += [one_prefix + sg + body, digit_x1 + before + body] + else: + parts += [teens_ties_oka + before + body, teens_ties_x1 + before + body] + return U(*parts) + + return prefer(side(True), side(False)).optimize() + + def crore_of(oka, other, other_head=None): + """10–14: …01 → కోటి; else → కోట్లు.""" + return prefer( + group(oka, i_koti_sp, crore_ladder, i_koti_sp, 7), + group(other, i_kotlu, crore_ladder, other_head or i_kotlu_pl, 7), + ).optimize() + + def kotlu(count): + """15–19 outer unit: always కోట్లు.""" + return group(count, i_kotlu, crore_ladder, i_kotlu_pl, 7).optimize() + + def oka_koti(*prefs, bare=(), pad=()): + """ఒక → ఒక కోటి (+ koti_ladder / bare / padded zeros).""" + parts = [group(p, i_koti_sp, koti_ladder, i_koti_sp, 7) for p in prefs] + parts += [p + i_koti_sp for p in bare] + parts += [suffix(p, i_koti_sp, n) for p, n in pad] + return U(*parts) + + graph_hundreds = U( + exact_n(3), + hp_ten + digit, + hp_one + teens_ties, + suffix(digit_x1, i_vandalu_pl, 2), + rung(digit_x1, i_vandalu, 1, digit), + rung(digit_x1, i_vandalu, 0, teens_ties), + ) + thousand_ladder = [(2, digit), (1, teens_ties), (0, graph_hundreds)] + graph_thousands, graph_ten_thousands = band( + exact_n(4), i_thou, i_thous, i_thous_pl, i_thou_sp, 3, thousand_ladder + ) lakh_ladder = [ (4, digit), (3, teens_ties), @@ -198,16 +187,17 @@ def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): (1, graph_thousands), (0, graph_ten_thousands), ] - graph_lakhs = ( - exact_lakh - | create_larger_number_graph(one_prefix, ins_lakha_digit, 4, digit) - | build_group(one_prefix, ins_lakh, lakh_ladder[1:]) - | build_group(digit_except_one, ins_lakhs_before, lakh_ladder, head_suffix=ins_lakhs_plural, head_zeros=5) - ).optimize() - graph_ten_lakhs = tie_pair( - ins_lakhs_before, ins_lakhs_before, lakh_ladder, ins_lakh_spaced, ins_lakhs_plural, 5 + graph_lakhs, graph_ten_lakhs = band( + exact_n(6), + i_lakh, + i_lakhs, + i_lakhs_pl, + i_lakh_sp, + 5, + lakh_ladder, + one_ladder=lakh_ladder[1:], + extra=rung(one_prefix, i_lakha, 4, digit), ) - crore_ladder = [ (6, digit), (5, teens_ties), @@ -217,94 +207,51 @@ def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): (1, graph_lakhs), (0, graph_ten_lakhs), ] - graph_crores = ( - exact_crore - | build_group(one_prefix, ins_crore, crore_ladder) - | build_group( - digit_except_one, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ) - ).optimize() - graph_ten_crores = tie_pair( - ins_crore_spaced, ins_crores_before, crore_ladder, ins_crore_spaced, ins_crores_plural, 7 + graph_crores, graph_ten_crores = band( + exact_n(8), i_koti, i_kotlu, i_kotlu_pl, i_koti_sp, 7, crore_ladder, ten_oka=i_koti_sp ) - hundred_crore_prefix = ( - exact_hundred - | (hundred_prefix_ten + digit_except_one) - | (hundred_prefix_one + teens_ties) - | create_graph_suffix(digit_except_one, ins_hundreds_before, 2) - | create_larger_number_graph(digit_except_one, ins_hundreds_before, 1, digit_except_one) - | create_larger_number_graph(digit_except_one, ins_hundreds_before, 0, teens_ties) - ).optimize() - hundred_one_crore_prefix = ( - hundred_prefix_one + pynutil.delete(NEMO_ALL_ZERO) + one_as_oka - | digit_except_one + pynutil.delete(NEMO_ALL_ZERO) + hundreds_before_one - ).optimize() - - thousand_crore_ladder = [(2, digit_except_one), (1, teens_ties), (0, hundred_crore_prefix)] - thousand_one_crore_prefix = oka_count_prefix( - ins_thousand, ins_thousands_before, [(2, one_as_oka), (0, hundred_one_crore_prefix)] + hundred_crore = U( + exact_n(3), + hp_ten + digit_x1, + hp_one + teens_ties, + suffix(digit_x1, i_vandalu, 2), + rung(digit_x1, i_vandalu, 1, digit_x1), + rung(digit_x1, i_vandalu, 0, teens_ties), ) - thousand_crore_prefix = ( - exact_thousand - | build_group( - digit_except_one, - ins_thousands_before, - thousand_crore_ladder, - head_suffix=ins_thousands_before, - head_zeros=3, - ) - | build_group(one_prefix, ins_thousand, thousand_crore_ladder) - ).optimize() - ten_thousand_crore_prefix = tie_pair( - ins_thousands_before, - ins_thousands_before, - thousand_crore_ladder, - ins_thousand_spaced, - ins_thousands_before, - 3, + hundred_one = U( + hp_one + pynutil.delete(NEMO_ALL_ZERO) + one_as_oka, + digit_x1 + pynutil.delete(NEMO_ALL_ZERO) + hundreds_before_one, ) - crore_count_prefix = (thousand_crore_prefix | ten_thousand_crore_prefix).optimize() - - graph_hundred_crores = ( - build_group( - hundred_one_crore_prefix, ins_crore_spaced, crore_ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ) - | build_group( - hundred_crore_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ) - | create_larger_number_graph(hundred_crore_prefix, ins_crores_before, 0, graph_crores) - ).optimize() - graph_thousand_crores = crore_graph(thousand_one_crore_prefix, crore_count_prefix) - graph_ten_thousand_crores = crore_graph(thousand_one_crore_prefix, ten_thousand_crore_prefix) + thou_crore_ladder = [(2, digit_x1), (1, teens_ties), (0, hundred_crore)] + thousand_one = oka_count(i_thou, i_thous, [(2, one_as_oka), (0, hundred_one)]) + thousand_crore, ten_thousand_crore = band( + exact_n(4), i_thou, i_thous, i_thous, i_thou_sp, 3, thou_crore_ladder + ) + graph_hundred_crores = U( + crore_of(hundred_one, hundred_crore), rung(hundred_crore, i_kotlu, 0, graph_crores) + ) + graph_thousand_crores = crore_of(thousand_one, U(thousand_crore, ten_thousand_crore)) + graph_ten_thousand_crores = crore_of(thousand_one, ten_thousand_crore) lakh_crore_ladder = [ - (4, digit_except_one), + (4, digit_x1), (3, teens_ties), - (2, hundred_crore_prefix), + (2, hundred_crore), (1, graph_thousands), - (0, ten_thousand_crore_prefix), + (0, ten_thousand_crore), ] - lakh_one_crore_prefix = oka_count_prefix( - ins_lakh, - ins_lakhs_before, - [(5, one_as_oka), (2, hundred_one_crore_prefix), (0, thousand_one_crore_prefix)], + lakh_one = oka_count( + i_lakh, + i_lakhs, + [(4, one_as_oka), (2, hundred_one), (1, exact_n(4, thousand_one)), (0, exact_n(5, thousand_one))], ) - lakh_crore_prefix = ( - exact_lakh - | build_group(one_prefix, ins_lakh, lakh_crore_ladder) - | build_group( - digit_except_one, ins_lakhs_before, lakh_crore_ladder, head_suffix=ins_lakhs_before, head_zeros=5 - ) - ).optimize() - ten_lakh_crore_prefix = tie_pair( - ins_lakhs_before, ins_lakhs_before, lakh_crore_ladder, ins_lakh_spaced, ins_lakhs_before, 5 + lakh_crore, ten_lakh_crore = band( + exact_n(6), i_lakh, i_lakhs, i_lakhs, i_lakh_sp, 5, lakh_crore_ladder ) - graph_lakh_crores = crore_graph(lakh_one_crore_prefix, lakh_crore_prefix | ten_lakh_crore_prefix) + graph_lakh_crores = crore_of(lakh_one, U(lakh_crore, ten_lakh_crore)) - ten_lakh_crore_lakh_remainder = tie_pair( - ins_lakhs_before, ins_lakhs_before, lakh_ladder, ins_lakhs_before, ins_lakhs_before, 5 - ) + ten_lakh_rem = ties_group(i_lakhs, i_lakhs, lakh_ladder, i_lakhs, i_lakhs, 5) koti_ladder = [ (6, digit), (5, teens_ties), @@ -312,133 +259,108 @@ def crore_graph(oka_prefix, other_prefix, ladder=None, other_head=None): (3, graph_thousands), (2, graph_ten_thousands), (1, graph_lakhs), - (0, ten_lakh_crore_lakh_remainder), + (0, ten_lakh_rem), ] - - ten_lakh_crore_count_prefix = prefer( - create_larger_number_graph(one_prefix, ins_crore, 0, ten_lakh_crore_lakh_remainder) - | create_larger_number_graph(digit_except_one, ins_crores_before, 0, ten_lakh_crore_lakh_remainder) - | create_larger_number_graph(teens_ties_thousand, ins_crore_spaced, 0, ten_lakh_crore_lakh_remainder) - | create_larger_number_graph(teens_ties_except_one, ins_crores_before, 0, ten_lakh_crore_lakh_remainder), - graph_crores | graph_ten_crores, - ).optimize() - graph_ten_lakh_crores = build_group( - ten_lakh_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ).optimize() - - crore_crore_count_prefix = tie_pair( - ins_crore_spaced, ins_crores_before, koti_ladder, ins_crore_spaced, ins_crores_before, 7 + crore_one = oka_count( + i_koti, + i_kotlu, + [(6, one_as_oka), (4, hundred_one), (1, exact_n(6, lakh_one)), (0, exact_n(7, lakh_one))], ) - graph_crore_crores = build_group( - crore_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ).optimize() - - ten_crore_crore_count_prefix = prefer( - create_graph_suffix(hundred_one_crore_prefix, ins_crore_spaced, 7) - | build_group( - hundred_one_crore_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ), - build_group( - hundred_crore_prefix, ins_crores_before, koti_ladder, head_suffix=ins_crores_before, head_zeros=7 + except_one_ladder = [(2, digit_x1), (1, teens_ties_x1)] + hundred_amt = U( + exact_n(3), + hp_ten + digit_x1, + hp_one + teens_ties_x1, + suffix(digit_x1, i_vandalu, 2), + rung(digit_x1, i_vandalu, 1, digit_x1), + ) + thousand_amt = U(*band(exact_n(4), i_thou, i_thous, i_thous, i_thou_sp, 3, except_one_ladder)) + ten_thousand_amt = ties_group( + i_thous, i_thous, except_one_ladder + [(0, hundred_amt)], i_thou_sp, i_thous, 3 + ) + crore_one_10 = rung(hundred_amt, i_kotlu, 6, one_as_oka).optimize() + crore_one_11 = rung(thousand_amt, i_kotlu, 6, one_as_oka).optimize() + crore_one_12 = rung(ten_thousand_amt, i_kotlu, 6, one_as_oka).optimize() + + ten_lakh_crore_count = prefer( + U( + rung(one_prefix, i_koti, 0, ten_lakh_rem), + rung(digit_x1, i_kotlu, 0, ten_lakh_rem), + rung(teens_ties_oka, i_koti_sp, 0, ten_lakh_rem), + rung(teens_ties_x1, i_kotlu, 0, ten_lakh_rem), ), + U(graph_crores, graph_ten_crores), ).optimize() - graph_ten_crore_crores = build_group( - ten_crore_crore_count_prefix, ins_crores_before, crore_ladder, head_suffix=ins_crores_plural, head_zeros=7 - ).optimize() - - hundred_crore_crore_oka_count_prefix = build_group( - teens_ties_thousand, - ins_thousands_before, - thousand_crore_ladder, - head_suffix=ins_thousand_spaced, - head_zeros=3, - ).optimize() - hundred_crore_crore_other_count_prefix = ( - thousand_crore_prefix - | build_group( - teens_ties_except_one, - ins_thousands_before, - thousand_crore_ladder, - head_suffix=ins_thousands_before, - head_zeros=3, - ) - ).optimize() - hundred_crore_crore_count_prefix = prefer( - build_group( - thousand_one_crore_prefix, ins_crore_spaced, koti_ladder, head_suffix=ins_crore_spaced, head_zeros=7 - ) - | build_group( - hundred_crore_crore_oka_count_prefix, - ins_crore_spaced, - koti_ladder, - head_suffix=ins_crore_spaced, - head_zeros=7, - ), - build_group( - hundred_crore_crore_other_count_prefix, - ins_crores_before, - koti_ladder, - head_suffix=ins_crores_before, - head_zeros=7, - ), + graph_ten_lakh_crores = prefer( + kotlu(exact_n(8, crore_one) + i_koti_sp), kotlu(ten_lakh_crore_count) ).optimize() - graph_hundred_crore_crores = build_group( - hundred_crore_crore_count_prefix, - ins_crores_before, - crore_ladder, - head_suffix=ins_crores_plural, - head_zeros=7, + graph_crore_crores = prefer( + kotlu(exact_n(9, crore_one) + i_koti_sp), + kotlu(ties_group(i_koti_sp, i_kotlu, koti_ladder, i_koti_sp, i_kotlu, 7)), ).optimize() - def exact_digits(n, graph): - return pynini.compose(NEMO_ALL_DIGIT**n, graph) - - graph_without_leading_zeros = ( - digit - | zero - | teens_ties - | graph_hundreds - | graph_thousands - | graph_ten_thousands - | graph_lakhs - | graph_ten_lakhs - | graph_crores - | graph_ten_crores - | exact_digits(10, graph_hundred_crores) - | exact_digits(11, graph_thousand_crores) - | exact_digits(12, graph_ten_thousand_crores) - | exact_digits(13, graph_lakh_crores) - | exact_digits(14, graph_lakh_crores) - | exact_digits(15, graph_ten_lakh_crores) - | exact_digits(16, graph_crore_crores) - | exact_digits(17, graph_ten_crore_crores) - | exact_digits(18, graph_hundred_crore_crores) - | exact_digits(19, graph_hundred_crore_crores) + graph_ten_crore_crores = kotlu( + prefer( + oka_koti(hundred_one, crore_one_10, bare=(crore_one_10,)), + group(hundred_crore, i_kotlu, koti_ladder, i_kotlu, 7), + ).optimize() + ) + hcc_oka = group(teens_ties_oka, i_thous, thou_crore_ladder, i_thou_sp, 3) + hcc_other = U(thousand_crore, group(teens_ties_x1, i_thous, thou_crore_ladder, i_thous, 3)) + graph_hundred_crore_crores = kotlu( + prefer( + oka_koti( + thousand_one, + hcc_oka, + crore_one_12, + crore_one_11, + crore_one_10, + bare=(crore_one_12, crore_one_11), + pad=((crore_one_10, 1),), + ), + group(hcc_other, i_kotlu, koti_ladder, i_kotlu, 7), + ).optimize() + ) + + graph = U( + digit, + zero, + teens_ties, + graph_hundreds, + graph_thousands, + graph_ten_thousands, + graph_lakhs, + graph_ten_lakhs, + graph_crores, + graph_ten_crores, + exact_n(10, graph_hundred_crores), + exact_n(11, graph_thousand_crores), + exact_n(12, graph_ten_thousand_crores), + exact_n(13, graph_lakh_crores), + exact_n(14, graph_lakh_crores), + exact_n(15, graph_ten_lakh_crores), + exact_n(16, graph_crore_crores), + exact_n(17, graph_ten_crore_crores), + exact_n(18, graph_hundred_crore_crores), + exact_n(19, graph_hundred_crore_crores), ) - same_script_number = pynini.closure(NEMO_DIGIT, 1) | pynini.closure(te_digit, 1) - graph_without_leading_zeros = pynini.compose(same_script_number, graph_without_leading_zeros) + graph = pynini.compose(pynini.closure(NEMO_DIGIT, 1) | pynini.closure(te_digit, 1), graph) - cardinal_with_leading_zeros = pynini.compose( + leading_zeros = pynini.compose( (pynini.closure("0", 1) + pynini.closure(NEMO_DIGIT)) | (pynini.closure("౦", 1) + pynini.closure(te_digit)), self.single_digits_graph, ) - delete_separator = pynutil.delete(",") - two_digits = NEMO_ALL_DIGIT + NEMO_ALL_DIGIT - three_digits = NEMO_ALL_DIGIT + NEMO_ALL_DIGIT + NEMO_ALL_DIGIT - indian_grouping = ( - pynini.closure(NEMO_ALL_DIGIT, 1, 2) - + pynini.closure(delete_separator + two_digits) - + delete_separator - + three_digits - ) - western_grouping = pynini.closure(NEMO_ALL_DIGIT, 1, 3) + pynini.closure(delete_separator + three_digits, 1) - cardinal_with_separators = pynini.compose( - (indian_grouping | western_grouping).optimize(), graph_without_leading_zeros + sep, two, three = pynutil.delete(","), NEMO_ALL_DIGIT**2, NEMO_ALL_DIGIT**3 + grouped = pynini.compose( + U( + pynini.closure(NEMO_ALL_DIGIT, 1, 2) + pynini.closure(sep + two) + sep + three, + pynini.closure(NEMO_ALL_DIGIT, 1, 3) + pynini.closure(sep + three, 1), + ), + graph, ).optimize() - final_graph = graph_without_leading_zeros | cardinal_with_leading_zeros | cardinal_with_separators - optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) - self.final_graph = final_graph.optimize() - final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.final_graph + pynutil.insert("\"") - self.fst = self.add_tokens(final_graph) + final = U(graph, leading_zeros, grouped) + minus = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + self.final_graph = final.optimize() + self.fst = self.add_tokens(minus + pynutil.insert('integer: "') + self.final_graph + pynutil.insert('"')) diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt index 32eea938a..bfce7dabe 100644 --- a/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt @@ -179,4 +179,26 @@ 05~సున్నా ఐదు ౦౫~సున్నా ఐదు 007~సున్నా సున్నా ఏడు -౦౦౭~సున్నా సున్నా ఏడు \ No newline at end of file +౦౦౭~సున్నా సున్నా ఏడు +1000010000000~లక్ష ఒక కోటి +2000010000000~రెండు లక్షల ఒక కోటి +5000010000000~ఐదు లక్షల ఒక కోటి +9000010000000~తొమ్మిది లక్షల ఒక కోటి +11000010000000~పదకొండు లక్షల ఒక కోటి +99000010000000~తొంభై తొమ్మిది లక్షల ఒక కోటి +1000014261933~లక్ష ఒక కోటి నలభై రెండు లక్షల అరవై ఒక వేల తొమ్మిది వందల ముప్పై మూడు +10000010000000~పది లక్షల ఒక కోటి +50000010000000~యాభై లక్షల ఒక కోటి +90000010701028~తొంభై లక్షల ఒక కోటి ఏడు లక్షల వెయ్యి ఇరవై ఎనిమిది +1010010000000~లక్ష వెయ్యి ఒక కోటి +3010010000000~మూడు లక్షల వెయ్యి ఒక కోటి +5010010000000~ఐదు లక్షల వెయ్యి ఒక కోటి +1010012900770~లక్ష వెయ్యి ఒక కోటి ఇరవై తొమ్మిది లక్షల ఏడు వందల డెబ్బై +5010012900770~ఐదు లక్షల వెయ్యి ఒక కోటి ఇరవై తొమ్మిది లక్షల ఏడు వందల డెబ్బై +5001010000000~ఐదు లక్షల నూట ఒక కోటి +90410010000205~తొంభై లక్షల నలభై ఒక వేల ఒక కోటి రెండు వందల ఐదు +30920010793149~ముప్పై లక్షల తొంభై రెండు వేల ఒక కోటి ఏడు లక్షల తొంభై మూడు వేల నూట నలభై తొమ్మిది +10601010405114~పది లక్షల అరవై వేల నూట ఒక కోటి నాలుగు లక్షల ఐదు వేల నూట పద్నాలుగు +1000020000000~లక్ష రెండు కోట్లు +10000020000000~పది లక్షల రెండు కోట్లు +50000020000000~యాభై లక్షల రెండు కోట్లు \ No newline at end of file diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt index 5ca23dcb7..6ce3d5ef7 100644 --- a/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_word.txt @@ -1,33 +1,25 @@ ~ ~ - 1~ఒకటి - తెలుగు~తెలుగు +తెలుగు~తెలుగు మీరు లోపలికి రావచ్చా?~మీరు లోపలికి రావచ్చా? అవును మీరు రావచ్చు.~అవును మీరు రావచ్చు. మార్.~మార్. +మార్~మార్ తె3~తె3 -x ~x -X!~X! -—~— ।~। ॥~॥ ₹~₹ …~… ·~· -తెలుగు~తెలుగు నిద్ర~నిద్ర బాగుంది~బాగుంది -మేరీ'స్~మేరీ'స్ కచ్చా-పక్కా~కచ్చా-పక్కా ఆంధ్ర-తెలంగాణ~ఆంధ్ర-తెలంగాణ -$ and 5% or %~$ and ఐదు % or % 1~ఒకటి 1~ఒకటి !1~! ఒకటి ౧~ఒకటి ౧!~ఒకటి ! -మార్~మార్ -/$€₩£BB¥#%AA and $€₩£¥#%~/$€₩£BB¥#%AA and $€₩£¥#% లోహం కా,~లోహం కా, యాహూ!~యాహూ! ‘హలో’~‘హలో’ From fa45be36a207085fbc70dec02ab4ba017cb8f058 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:05:07 +0000 Subject: [PATCH 15/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/te/taggers/cardinal.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/nemo_text_processing/text_normalization/te/taggers/cardinal.py b/nemo_text_processing/text_normalization/te/taggers/cardinal.py index 2a860a6ec..5ef9ddaf5 100644 --- a/nemo_text_processing/text_normalization/te/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/te/taggers/cardinal.py @@ -71,7 +71,11 @@ def ins(key, space=True): i_thou, i_thou_sp = ins("thousand", False), ins("thousand") i_thous, i_thous_pl = ins("thousands_before"), ins("thousands_plural") i_lakh, i_lakh_sp = ins("lakh", False), ins("lakh") - i_lakha, i_lakhs, i_lakhs_pl = pynutil.insert(mag["lakh_before_digit"]), ins("lakhs_before"), ins("lakhs_plural") + i_lakha, i_lakhs, i_lakhs_pl = ( + pynutil.insert(mag["lakh_before_digit"]), + ins("lakhs_before"), + ins("lakhs_plural"), + ) i_koti, i_koti_sp = ins("crore", False), ins("crore") i_kotlu, i_kotlu_pl = ins("crores_before"), ins("crores_plural") i_vandalu, i_vandalu_pl = ins("hundreds_before"), ins("hundreds_plural") @@ -228,9 +232,7 @@ def oka_koti(*prefs, bare=(), pad=()): thousand_crore, ten_thousand_crore = band( exact_n(4), i_thou, i_thous, i_thous, i_thou_sp, 3, thou_crore_ladder ) - graph_hundred_crores = U( - crore_of(hundred_one, hundred_crore), rung(hundred_crore, i_kotlu, 0, graph_crores) - ) + graph_hundred_crores = U(crore_of(hundred_one, hundred_crore), rung(hundred_crore, i_kotlu, 0, graph_crores)) graph_thousand_crores = crore_of(thousand_one, U(thousand_crore, ten_thousand_crore)) graph_ten_thousand_crores = crore_of(thousand_one, ten_thousand_crore) @@ -246,9 +248,7 @@ def oka_koti(*prefs, bare=(), pad=()): i_lakhs, [(4, one_as_oka), (2, hundred_one), (1, exact_n(4, thousand_one)), (0, exact_n(5, thousand_one))], ) - lakh_crore, ten_lakh_crore = band( - exact_n(6), i_lakh, i_lakhs, i_lakhs, i_lakh_sp, 5, lakh_crore_ladder - ) + lakh_crore, ten_lakh_crore = band(exact_n(6), i_lakh, i_lakhs, i_lakhs, i_lakh_sp, 5, lakh_crore_ladder) graph_lakh_crores = crore_of(lakh_one, U(lakh_crore, ten_lakh_crore)) ten_lakh_rem = ties_group(i_lakhs, i_lakhs, lakh_ladder, i_lakhs, i_lakhs, 5) @@ -275,9 +275,7 @@ def oka_koti(*prefs, bare=(), pad=()): rung(digit_x1, i_vandalu, 1, digit_x1), ) thousand_amt = U(*band(exact_n(4), i_thou, i_thous, i_thous, i_thou_sp, 3, except_one_ladder)) - ten_thousand_amt = ties_group( - i_thous, i_thous, except_one_ladder + [(0, hundred_amt)], i_thou_sp, i_thous, 3 - ) + ten_thousand_amt = ties_group(i_thous, i_thous, except_one_ladder + [(0, hundred_amt)], i_thou_sp, i_thous, 3) crore_one_10 = rung(hundred_amt, i_kotlu, 6, one_as_oka).optimize() crore_one_11 = rung(thousand_amt, i_kotlu, 6, one_as_oka).optimize() crore_one_12 = rung(ten_thousand_amt, i_kotlu, 6, one_as_oka).optimize()