diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..ba38def21 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { } environment { AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-0' - DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0' + DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-26-0' EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-1' ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0' ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0' diff --git a/nemo_text_processing/inverse_text_normalization/de/data/__init__.py b/nemo_text_processing/inverse_text_normalization/de/data/__init__.py new file mode 100644 index 000000000..bc443be41 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/de/data/numbers/__init__.py new file mode 100644 index 000000000..7d200df57 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/de/data/numbers/digit.tsv new file mode 100644 index 000000000..68d94d596 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/digit.tsv @@ -0,0 +1,10 @@ +zwei 2 +zwo 2 +zwö 2 +drei 3 +vier 4 +fünf 5 +sechs 6 +sieben 7 +acht 8 +neun 9 diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/minus.tsv b/nemo_text_processing/inverse_text_normalization/de/data/numbers/minus.tsv new file mode 100644 index 000000000..29987bb0f --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/minus.tsv @@ -0,0 +1 @@ +minus - diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/nouns_forcing_denormalization.tsv b/nemo_text_processing/inverse_text_normalization/de/data/numbers/nouns_forcing_denormalization.tsv new file mode 100644 index 000000000..0570e529a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/nouns_forcing_denormalization.tsv @@ -0,0 +1,34 @@ +abbildung +abschnitt +absatz +anhang +anlage +artikel +bahnsteig +band +etage +flug +folge +formular +gleis +gruppe +halle +kapitel +klasse +linie +modell +paragraph +phase +platz +rang +raum +runde +saal +schritt +seite +stufe +tabelle +teil +version +zeile +zimmer diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/ones.tsv b/nemo_text_processing/inverse_text_normalization/de/data/numbers/ones.tsv new file mode 100644 index 000000000..97f0779bb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/ones.tsv @@ -0,0 +1,3 @@ +eine 1 +ein 1 +eins 1 diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/quantities.tsv b/nemo_text_processing/inverse_text_normalization/de/data/numbers/quantities.tsv new file mode 100644 index 000000000..3d2988d66 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/quantities.tsv @@ -0,0 +1,14 @@ +hundert 100 +tausend 1.000 +million 1.000.000 +millionen 1.000.000 +milliarde 1.000.000.000 +milliarden 1.000.000.000 +billion 1.000.000.000.000 +billionen 1.000.000.000.000 +billiarde 1.000.000.000.000.000 +billiarden 1.000.000.000.000.000 +trillion 1.000.000.000.000.000.000 +trillionen 1.000.000.000.000.000.000 +trilliarde 1.000.000.000.000.000.000.000 +trilliarden 1.000.000.000.000.000.000.000 diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/teen.tsv b/nemo_text_processing/inverse_text_normalization/de/data/numbers/teen.tsv new file mode 100644 index 000000000..f50ac3d37 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +zehn 10 +elf 11 +zwölf 12 +dreizehn 13 +vierzehn 14 +fünfzehn 15 +sechzehn 16 +siebzehn 17 +achtzehn 18 +neunzehn 19 diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/ties.tsv b/nemo_text_processing/inverse_text_normalization/de/data/numbers/ties.tsv new file mode 100644 index 000000000..70894adbe --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +zwanzig 2 +dreißig 3 +vierzig 4 +fünfzig 5 +sechzig 6 +siebzig 7 +achtzig 8 +neunzig 9 diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/und.tsv b/nemo_text_processing/inverse_text_normalization/de/data/numbers/und.tsv new file mode 100644 index 000000000..ee61ac994 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/und.tsv @@ -0,0 +1 @@ +und diff --git a/nemo_text_processing/inverse_text_normalization/de/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/de/data/numbers/zero.tsv new file mode 100644 index 000000000..973c00968 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/numbers/zero.tsv @@ -0,0 +1 @@ +null 0 diff --git a/nemo_text_processing/inverse_text_normalization/de/graph_utils.py b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py new file mode 100644 index 000000000..8dc9c6391 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py @@ -0,0 +1,319 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", + "c", + "d", + "f", + "g", + "h", + "j", + "k", + "l", + "m", + "n", + "p", + "q", + "r", + "s", + "t", + "v", + "w", + "x", + "y", + "z", +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, + plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), + NEMO_SIGMA, +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: "pynini.FstLike", + original_graph_weight: float = None, + capitalized_graph_weight: float = None, +) -> "pynini.FstLike": + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [ + written_capitalized, + spoken.capitalize(), + ], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logger.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [ + [written, spoken_no_space], + [written_capitalized, spoken_no_space.upper()], + ] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 46fdca4e3..81d778931 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -12,57 +12,324 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import defaultdict +from typing import List + import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path, load_labels +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + TO_LOWER, + GraphFst, + capitalized_input_graph, + delete_space, +) class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals. Numbers below ten are not converted. + Finite state transducer for classifying cardinal numbers, e.g. + minus eine billion fünfundsechzig milliarden vier millionen sechs -> cardinal { negative: "-" integer: "1.065.004.000.006" } + The transducer implements a period separator every three digits by default. + Numbers below thirteen are not converted. Allows both compound numeral strings or separated by whitespace. "und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number. - e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } } - e.g. minus dreiundzwanzig -> cardinal { integer: "23" } } - e.g. dreizehn -> cardinal { integer: "13" } } - e.g. ein hundert -> cardinal { integer: "100" } } - e.g. einhundert -> cardinal { integer: "100" } } - e.g. ein tausend -> cardinal { integer: "1000" } } - e.g. eintausend -> cardinal { integer: "1000" } } - e.g. ein tausend zwanzig -> cardinal { integer: "1020" } } + e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } + e.g. minus dreiundzwanzig -> cardinal { negative: "-" integer: "23" } + e.g. dreizehn -> cardinal { integer: "13" } + e.g. ein hundert -> cardinal { integer: "100" } + e.g. einhundert -> cardinal { integer: "100" } + e.g. ein tausend -> cardinal { integer: "1.000" } + e.g. eintausend -> cardinal { integer: "1.000" } + e.g. ein tausend zwanzig -> cardinal { integer: "1.020" } + e.g. kapitel drei -> cardinal { morphosyntactic_features: "kapitel" integer: "3" } Args: - tn_cardinal_tagger: TN cardinal tagger + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self, tn_cardinal_tagger: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + def __init__(self, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="cardinal", kind="classify") + self.input_case = input_case - # add_space_between_chars = pynini.cdrewrite(pynini.closure(insert_space, 0, 1), NEMO_CHAR, NEMO_CHAR, NEMO_SIGMA) - optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit_no_one = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_one = pynini.string_file(get_abs_path("data/numbers/ones.tsv")) + graph_digit = graph_digit_no_one | graph_one + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_minus = pynini.string_file(get_abs_path("data/numbers/minus.tsv")) + graph_quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) + and_words = [row[0] for row in load_labels(get_abs_path("data/numbers/und.tsv"))] - graph = (tn_cardinal_tagger.graph @ optional_delete_space).invert().optimize() - self.graph_hundred_component_at_least_one_none_zero_digit = ( - (tn_cardinal_tagger.graph_hundred_component_at_least_one_none_zero_digit @ optional_delete_space) - .invert() - .optimize() + # German speaks the ones before the tens, e.g. "einundzwanzig" is 21, + # so the two digits cannot be read off in the order they are spoken + graph_ties_digit = self.get_ties_digit( + [get_abs_path("data/numbers/digit.tsv"), get_abs_path("data/numbers/ones.tsv")], + get_abs_path("data/numbers/ties.tsv"), + and_words, ) + delete_und = self.delete_word(pynini.union(*and_words)).ques + + self.graph_two_digit = (graph_teen | (graph_ties + pynutil.insert("0")) | graph_ties_digit).optimize() + graph_two_digit = self.graph_two_digit + + # isolated subgraphs handed to the other semiotic classes + self.digits = graph_digit.optimize() + self.graph_double_digits = self.graph_two_digit + self.graph_single_and_double_digits = (graph_digit | graph_two_digit).optimize() + + delete_hundert = self.delete_word(self.get_quantity(graph_quantities, "100")) + multiplier = (graph_digit | pynutil.insert("1")) + delete_space + graph_hundred_component = ( + (multiplier + delete_hundert + delete_space + delete_und + delete_space + graph_two_digit) + | ( + multiplier + + delete_hundert + + pynutil.insert("0") + + delete_space + + delete_und + + delete_space + + graph_digit + ) + | (multiplier + delete_hundert + pynutil.insert("00")) + ) + + # digits are grouped in clusters of three, written right to left and period separated + graph_cluster = ( + graph_hundred_component + | (pynutil.insert("0") + graph_two_digit) + | (pynutil.insert("00") + graph_digit) + | pynutil.insert("000") + ) + graph_cluster_non_zero = ( + graph_hundred_component | (pynutil.insert("0") + graph_two_digit) | (pynutil.insert("00") + graph_digit) + ) + + non_zero_digits = pynini.difference(NEMO_DIGIT, "0") + chars_to_remove = pynini.accep("0") | pynini.accep(".") + remove_chars = pynutil.delete(pynini.closure(chars_to_remove)) + remove_leading_zeros = pynini.cdrewrite(remove_chars, "[BOS]", non_zero_digits, NEMO_SIGMA) + remove_period_separators = pynini.cdrewrite(pynutil.delete("."), "", "", NEMO_SIGMA) + + self.graph_hundred_component_at_least_one_none_zero_digit = ( + graph_cluster_non_zero @ remove_leading_zeros + ).optimize() + + def magnitude(written, lower_magnitudes, leading_cluster=graph_cluster, empty_multiplier=True): + """ + WFST grammar for one order of magnitude, e.g. "million" - self.graph_ties = (tn_cardinal_tagger.two_digit_non_zero @ optional_delete_space).invert().optimize() - # this is to make sure if there is an ambiguity with decimal, decimal is chosen, e.g. 1000000 vs. 1 million + Args: + written: written form of the quantity, e.g. 1.000.000 + lower_magnitudes: WFST grammar for the next magnitude down, e.g. "thousands" + leading_cluster: WFST grammar for the cluster multiplying the quantity + empty_multiplier: whether the magnitude may be skipped, e.g. "eine million drei" + """ + quantity = self.get_quantity(graph_quantities, written) + delete_quantity = self.delete_word(quantity) + multiplied = delete_quantity + pynutil.insert("1.") + delete_space + delete_und | ( + leading_cluster + delete_space + delete_quantity + pynutil.insert(".") + delete_und + ) + if empty_multiplier: + multiplied |= pynutil.insert("000.") + standalone = graph_quantities @ pynini.accep(written) + return standalone | (multiplied + delete_space + lower_magnitudes) + + magnitudes = self.get_magnitudes(get_abs_path("data/numbers/quantities.tsv")) + graph_magnitudes = [] + lower_magnitudes = graph_cluster + for written in magnitudes: + lower_magnitudes = magnitude(written, lower_magnitudes) + graph_magnitudes.append(lower_magnitudes) + + graph = pynini.union(*graph_magnitudes, graph_cluster, graph_zero) + graph = graph @ remove_leading_zeros + + if input_case == INPUT_CASED: + graph = capitalized_input_graph(graph) + graph_minus = capitalized_input_graph(graph_minus) + + # where a reading is ambiguous the more specific class wins, e.g. a decimal over 1.000.000 + # and a year over 2.020 graph = pynutil.add_weight(graph, weight=0.001) - self.graph_no_exception = graph - self.digit = pynini.arcmap(tn_cardinal_tagger.digit, map_type="rmweight").invert().optimize() - graph_exception = pynini.project(self.digit, 'input') - self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + + # only a standalone cardinal is rendered with period separators; every class built on top of + # this one embeds the plain digit string, so the two readings are kept apart by name + graph_with_separators = graph.optimize() + graph_without_separators = (graph @ remove_period_separators).optimize() + + self.graph_no_exception = graph_without_separators + # alias under the name the standalone German grammars use + self.graph_all_cardinals = graph_without_separators + + # years 0 - 9999, including the colloquial readings, e.g. "zwanzigvierundzwanzig" -> 2024 + first_millenium = graph_cluster_non_zero + second_tenth_millenium = magnitude( + magnitudes[0], graph_cluster, leading_cluster=graph_cluster_non_zero, empty_multiplier=False + ) + ten = pynini.project(graph_teen @ pynini.accep("10"), "input") + graph_11_99 = (pynini.project(graph_two_digit, "input") - ten) @ graph_two_digit + + # single digit year tails take a leading zero, e.g. "neunzehnhundertfünf" and + # "neunzehnhundertnullfünf" both denormalize to 1905 + single_digit_years = (pynutil.insert("0") + graph_digit) | (graph_zero + delete_space + graph_digit) + + years_exceptions = ( + graph_11_99 + + delete_space + + delete_hundert.ques + + delete_space + + (graph_two_digit | single_digit_years | pynutil.insert("00")) + ) + years = first_millenium | second_tenth_millenium | years_exceptions + self.graph_years = (years @ remove_leading_zeros @ remove_period_separators).optimize() + + # a bare cardinal from zero to twelve inclusive stays spelled out, e.g. "drei" -> "drei" + spelled_out = graph_zero | graph_digit | (graph_teen @ pynini.union("10", "11", "12")) + self.dozen = spelled_out.optimize() + + # the classes built on top of this one withhold only the single digits, so that an ordinal + # such as "zehnter" still denormalizes to "10." while "zweiter" stays "zweiter" + single_digits = graph_zero | graph_digit + + if input_case == INPUT_CASED: + spelled_out = capitalized_input_graph(spelled_out) + single_digits = capitalized_input_graph(single_digits) + spelled_out = pynini.project(spelled_out, "input") + single_digits = pynini.project(single_digits, "input") + + self.graph = ( + (pynini.project(graph_without_separators, "input") - single_digits.arcsort()) @ graph_without_separators + ).optimize() self.optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", "\"-\" "), 0, 1 + pynutil.insert('negative: "') + graph_minus + pynutil.delete(NEMO_SPACE) + pynutil.insert('" '), 0, 1 ) + # alias under the name the standalone German grammars use + self.optional_negative = self.optional_minus_graph + + # standalone readings, the only place the period separators are emitted - final_graph = self.optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + # fully denormalized, used where the context rules out spelling numbers out + self.forced_integer_graph_with_separators = ( + self.optional_minus_graph + pynutil.insert('integer: "') + graph_with_separators + pynutil.insert('"') + ).optimize() - final_graph = self.add_tokens(final_graph) + # canonical, leaving the first dozen spelled out + canonical_graph = spelled_out | ( + (pynini.project(graph_with_separators, "input") - spelled_out.arcsort()) @ graph_with_separators + ) + self.canonical_integer_graph_with_separators = ( + self.optional_minus_graph + pynutil.insert('integer: "') + canonical_graph + pynutil.insert('"') + ).optimize() + + # a noun such as "kapitel" in the determiner position forces the numeral to denormalize, + # e.g. "kapitel drei" -> "kapitel 3" even though a bare "drei" stays spelled out + nouns_forcing_denormalization = pynini.string_file( + get_abs_path("data/numbers/nouns_forcing_denormalization.tsv") + ) + if input_case == INPUT_CASED: + nouns_forcing_denormalization = pynini.project( + capitalized_input_graph(nouns_forcing_denormalization), "input" + ) + graph_forced_denormalization = ( + pynutil.insert('morphosyntactic_features: "') + + nouns_forcing_denormalization + + pynutil.insert('"') + + pynini.accep(NEMO_SPACE) + + self.forced_integer_graph_with_separators + ) + + final_graph = self.add_tokens(self.canonical_integer_graph_with_separators | graph_forced_denormalization) self.fst = final_graph.optimize() + + def delete_word(self, word: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Deletes an acceptor, also matching its capitalized form for `cased` input. + German capitalizes nouns mid-sentence, so quantity words such as "Millionen" need this + even when they are not sentence initial. + + Args: + word: acceptor for the spoken form(s) to delete + Returns: + res: fst deleting the word + """ + + if self.input_case == INPUT_CASED: + word |= pynini.project(pynini.compose(TO_LOWER + NEMO_SIGMA, word), "input") + return pynutil.delete(word).optimize() + + def get_ties_digit(self, digit_paths: List[str], tie_path: str, and_words: List[str]) -> 'pynini.FstLike': + """ + getting all denormalizations for numbers between 21 - 100 + + Args: + digit_paths: files to the digit tsvs, e.g. digit.tsv for 2 - 9 and ones.tsv for 1 + tie_path: file to tie tsv, e.g. 20, 30, etc. + and_words: connectors between the digit and the tie, e.g. ["und"] + Returns: + res: fst that converts the verbalization of a number to that number + """ + + digits = defaultdict(list) + ties = defaultdict(list) + for digit_path in digit_paths: + for k, v in load_labels(digit_path): + digits[v].append(k) + + for k, v in load_labels(tie_path): + ties[v].append(k) + + d = [] + for i in range(21, 100): + s = str(i) + if s[1] == "0": + continue + + for di in digits[s[1]]: + for ti in ties[s[0]]: + for and_word in and_words: + for before in ("", " "): + for after in ("", " "): + word = di + before + and_word + after + ti + d.append((word, s)) + + res = pynini.string_map(d) + return res + + def get_quantity(self, quantities: 'pynini.FstLike', written: str) -> 'pynini.FstLike': + """ + getting all spoken forms of a quantity, e.g. "million", "millionen" + + Args: + quantities: fst mapping the spoken forms of the quantities to their written forms + written: written form of the quantity, e.g. 1.000.000 + Returns: + res: acceptor for the spoken forms of that quantity + """ + + return pynini.project(quantities @ pynini.accep(written), "input").optimize() + + def get_magnitudes(self, quantities_path: str) -> List[str]: + """ + getting the written forms of the quantities that span whole groups of three digits, + ordered from the smallest to the largest + + Args: + quantities_path: file to the quantities tsv, mapping the spoken forms of the quantities to + their written forms, e.g. "million" -> 1.000.000 + Returns: + res: written forms of the quantities, sorted by the number of digits they span + """ + + written_forms = {written for _, written in load_labels(quantities_path)} + magnitudes = {written for written in written_forms if "." in written} + return sorted(magnitudes, key=lambda written: len(written.replace(".", ""))) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/date.py b/nemo_text_processing/inverse_text_normalization/de/taggers/date.py index 916b9413d..4bd45f935 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/date.py @@ -60,7 +60,12 @@ def __init__( month_as_number = pynutil.delete("month: \"") + itn_cardinal_tagger.graph_no_exception + pynutil.delete("\"") month_as_string = pynutil.delete("month: \"") + tn_date_tagger.month_abbr.invert() + pynutil.delete("\"") - convert_year = (tn_date_tagger.year @ optional_delete_space).invert().optimize() + convert_year = (tn_date_tagger.year @ optional_delete_space).invert() + # year readings the inverted TN graph cannot reach, e.g. the leading zero of + # "neunzehnhundertnullfünf"; restricted to four digits so that short numerals in the year + # position stay cardinals + convert_year |= pynutil.add_weight(itn_cardinal_tagger.graph_years @ (NEMO_DIGIT**4), weight=0.01) + convert_year = convert_year.optimize() delete_year_marker = ( pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") ) @ convert_year diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py index 1d60d071a..27fba4ec0 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py @@ -93,7 +93,7 @@ def __init__( tn_electronic_verbalizer = TNElectronicVerbalizer(deterministic=False) tn_whitelist_tagger = TNWhitelistTagger(input_case="cased", deterministic=False, input_file=whitelist) - cardinal = CardinalFst(tn_cardinal_tagger=tn_cardinal_tagger) + cardinal = CardinalFst(input_case=input_case) cardinal_graph = cardinal.fst ordinal = OrdinalFst(itn_cardinal_tagger=cardinal, tn_ordinal_verbalizer=tn_ordinal_verbalizer) diff --git a/nemo_text_processing/inverse_text_normalization/de/utils.py b/nemo_text_processing/inverse_text_normalization/de/utils.py new file mode 100644 index 000000000..144bd9208 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index b13382a8e..01375a749 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -15,22 +15,48 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space class CardinalFst(GraphFst): """ Finite state transducer for verbalizing cardinal e.g. cardinal { integer: "23" negative: "-" } -> -23 - - Args: - tn_cardinal_verbalizer: TN cardinal verbalizer + e.g. cardinal { integer: "1.000" } -> 1.000 + e.g. cardinal { morphosyntactic_features: "kapitel" integer: "3" } -> kapitel 3 """ - def __init__(self, tn_cardinal_verbalizer: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) - self.numbers = tn_cardinal_verbalizer.numbers - optional_sign = pynini.closure(pynutil.delete("negative: \"") + NEMO_NOT_QUOTE + pynutil.delete("\" "), 0, 1) - graph = optional_sign + self.numbers + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + optional_noun = pynini.closure( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + + delete_space + + pynutil.insert(" "), + 0, + 1, + ) + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete('"') + + NEMO_NOT_QUOTE + + pynutil.delete('"') + + delete_space, + 0, + 1, + ) + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + self.numbers = graph + graph = optional_noun + optional_sign + graph delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py index 9c921a63a..7aa670991 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py @@ -17,7 +17,6 @@ from nemo_text_processing.inverse_text_normalization.de.verbalizers.measure import MeasureFst from nemo_text_processing.inverse_text_normalization.de.verbalizers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.de.verbalizers.time import TimeFst -from nemo_text_processing.text_normalization.de.verbalizers.cardinal import CardinalFst as TNCardinalVerbalizer from nemo_text_processing.text_normalization.de.verbalizers.decimal import DecimalFst as TNDecimalVerbalizer from nemo_text_processing.text_normalization.en.graph_utils import GraphFst @@ -31,10 +30,9 @@ class VerbalizeFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) - tn_cardinal_verbalizer = TNCardinalVerbalizer(deterministic=False) tn_decimal_verbalizer = TNDecimalVerbalizer(deterministic=False) - cardinal = CardinalFst(tn_cardinal_verbalizer=tn_cardinal_verbalizer) + cardinal = CardinalFst() cardinal_graph = cardinal.fst decimal = DecimalFst(tn_decimal_verbalizer=tn_decimal_verbalizer) decimal_graph = decimal.fst diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt index 0b2064296..1049b5052 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt @@ -4,26 +4,26 @@ ein hundert und zwei~102 einhundertzwei~102 ein hundert und zwanzig~120 ein hundert und elf~111 -ein tausend~1000 -eintausend~1000 +ein tausend~1.000 +eintausend~1.000 ein hundert zwanzig~120 -ein tausend zwanzig~1020 -eintausendzwanzig~1020 -neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9789382536130 +ein tausend zwanzig~1.020 +eintausendzwanzig~1.020 +neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9.789.382.536.130 zwei hundert vier und fünfzig~254 -ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147451 -eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1156173 -eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1593072961 -sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97808264772792005 -zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10010000010100010 -zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10010000010100010 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünfundzwanzigtausendsiebenunddreißig~-25037 -eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1264301938000104 -eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1264301938000104 +ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147.451 +eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1.156.173 +eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1.593.072.961 +sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97.808.264.772.792.005 +zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10.010.000.010.100.010 +zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10.010.000.010.100.010 +minus fünf und zwanzig tausend sieben und dreißig~-25.037 +minus fünf und zwanzig tausend sieben und dreißig~-25.037 +minus fünfundzwanzigtausendsiebenunddreißig~-25.037 +eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1.264.301.938.000.104 +eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1.264.301.938.000.104 minus sechzig~-60 -sechs und vierzig tausend sechs hundert vier und sechzig~46664 +sechsundvierzig tausend sechshundert vierundsechzig~46.664 sechzig~60 null~null eins~eins @@ -31,10 +31,12 @@ ein~ein eine~eine einer~einer zwei~zwei +zwö~zwö +zwo~zwo neun~neun -zehn~10 -elf~11 -zwölf~12 +zehn~zehn +elf~elf +zwölf~zwölf dreizehn~13 vierzehn~14 fünfzehn~15 @@ -49,14 +51,16 @@ sechzig~60 siebzig~70 achtzig~80 neunzig~90 -zwei millionen drei~2000003 -ein tausend dreizehn~1013 -ein tausend eins~1001 -ein tausend ein hundert~1100 -ein tausend sechs und zwanzig~1026 -ein tausend ein hundert sechs und zwanzig~1126 -achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18450990 -achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18940722 -achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18690916 -achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18690916 -achtzehn tausend acht hundert achtzig~18880 +zwei millionen drei~2.000.003 +ein tausend dreizehn~1.013 +ein tausend eins~1.001 +ein tausend ein hundert~1.100 +ein tausend sechs und zwanzig~1.026 +ein tausend ein hundert sechs und zwanzig~1.126 +achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18.450.990 +achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18.940.722 +achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18.690.916 +achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18.690.916 +achtzehn tausend acht hundert achtzig~18.880 +einhunderteins~101 +ein tausend einhundert sechsundzwanzig~1.126 diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_date.txt index e994b936d..1f81708c8 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_date.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_date.txt @@ -19,4 +19,4 @@ neunzehn hundert vierundneunzig~1994 neunzehn hundert vier und neunzig~1994 neunzehn vierundneunzig~1994 zwei tausend drei~2003 -ein tausend acht~1008 +ein tausend acht~1.008