diff --git a/README.md b/README.md index cfba4c1..951bad7 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12 | German | `de` | Yes | Yes | `german` | | Hindi | `hi` | Yes | Yes | `hindi` | | Hindi–English (code-mixed) | `hi_en` | No | Yes | `hindi_english` | +| Italian | `it` | Yes | Yes | `italian` | | Japanese | `ja` | Yes | Yes | `japanese` | | Kinyarwanda | `rw` | Yes | Yes | `kinyarwanda` | | Korean | `ko` | Yes | Yes | `korean` | diff --git a/itn/italian/__init__.py b/itn/italian/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/itn/italian/data/electronic/domain.tsv b/itn/italian/data/electronic/domain.tsv new file mode 100644 index 0000000..303d2fb --- /dev/null +++ b/itn/italian/data/electronic/domain.tsv @@ -0,0 +1,14 @@ +.com com +.org org +.gov gov +.uk UK +.fr FR +.net net +.html html +.py python +.ru RU +.us US +.de DE +.it IT +.jpg jpeg +.edu edu diff --git a/itn/italian/data/electronic/server_name.tsv b/itn/italian/data/electronic/server_name.tsv new file mode 100644 index 0000000..9a0c756 --- /dev/null +++ b/itn/italian/data/electronic/server_name.tsv @@ -0,0 +1,11 @@ +gmail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast \ No newline at end of file diff --git a/itn/italian/data/electronic/symbols.tsv b/itn/italian/data/electronic/symbols.tsv new file mode 100644 index 0000000..56c9270 --- /dev/null +++ b/itn/italian/data/electronic/symbols.tsv @@ -0,0 +1,22 @@ +. punto +- trattino +_ trattino basso +! punto esclamativo +# cancelletto +$ dollaro +% percento +& e commerciale +' apostrofo +* asterisco ++ più +/ slash += uguale +? punto interrogativo +^ esponente +` accento +{ apri graffa +| barra verticale +} chiudi graffa +~ tilde +, virgola +: due punti diff --git a/itn/italian/data/measure/denominators.tsv b/itn/italian/data/measure/denominators.tsv new file mode 100644 index 0000000..797bbcc --- /dev/null +++ b/itn/italian/data/measure/denominators.tsv @@ -0,0 +1,4 @@ +s secondo +h ora +min minuto +d giorno diff --git a/itn/italian/data/measure/units.tsv b/itn/italian/data/measure/units.tsv new file mode 100644 index 0000000..b6f2c3b --- /dev/null +++ b/itn/italian/data/measure/units.tsv @@ -0,0 +1,51 @@ +km chilometro chilometri +m metro metri +cm centimetro centimetri +mm millimetro millimetri +dm decimetro decimetri +μm micrometro micrometri +nm nanometro nanometri +ha ettaro ettari +mi miglio miglia +m² metro quadrato metri quadrati +km² chilometro quadrato chilometri quadrati +m³ metro cubo metri cubi +g grammo grammi +mg milligrammo milligrammi +μg microgrammo microgrammi +kg chilogrammo chilogrammi +kg chilo chili +t tonnellata tonnellate +l litro litri +ml millilitro millilitri +cl centilitro centilitri +dl decilitro decilitri +ms millisecondo millisecondi +Hz hertz hertz +kHz kilohertz kilohertz +MHz megahertz megahertz +GHz gigahertz gigahertz +V volt volt +mV millivolt millivolt +kV kilovolt kilovolt +W watt watt +kW kilowatt kilowatt +MW megawatt megawatt +GW gigawatt gigawatt +TW terawatt terawatt +A ampere ampere +mA milliampere milliampere +Ω ohm ohm +dB decibel decibel +kB kilobyte kilobyte +MB megabyte megabyte +GB gigabyte gigabyte +TB terabyte terabyte +kcal chilocaloria chilocalorie +cal caloria calorie +bar bar bar +°C grado celsius gradi celsius +°F grado fahrenheit gradi fahrenheit +K kelvin kelvin +% percento percento +% per cento per cento diff --git a/itn/italian/data/money/currency.tsv b/itn/italian/data/money/currency.tsv new file mode 100644 index 0000000..781b979 --- /dev/null +++ b/itn/italian/data/money/currency.tsv @@ -0,0 +1,11 @@ +€ euro euro centesimo centesimi +$ dollaro dollari centesimo centesimi +£ sterlina sterline penny penny +£ sterlina sterline penny pence +¥ yen yen +₽ rublo rubli copeca copeche +₹ rupia rupie paisa paise +USD dollaro statunitense dollari statunitensi centesimo centesimi +CAD dollaro canadese dollari canadesi centesimo centesimi +HKD dollaro di hong kong dollari di hong kong centesimo centesimi +CHF franco svizzero franchi svizzeri centesimo centesimi diff --git a/itn/italian/data/numbers/digit.tsv b/itn/italian/data/numbers/digit.tsv new file mode 100644 index 0000000..78382a7 --- /dev/null +++ b/itn/italian/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 uno +2 due +3 tre +4 quattro +5 cinque +6 sei +7 sette +8 otto +9 nove diff --git a/itn/italian/data/numbers/hundreds.tsv b/itn/italian/data/numbers/hundreds.tsv new file mode 100644 index 0000000..3c83993 --- /dev/null +++ b/itn/italian/data/numbers/hundreds.tsv @@ -0,0 +1,9 @@ +1 cento +2 duecento +3 trecento +4 quattrocento +5 cinquecento +6 seicento +7 settecento +8 ottocento +9 novecento diff --git a/itn/italian/data/numbers/quantities.tsv b/itn/italian/data/numbers/quantities.tsv new file mode 100644 index 0000000..c33e026 --- /dev/null +++ b/itn/italian/data/numbers/quantities.tsv @@ -0,0 +1,4 @@ +milione +milioni +miliardo +miliardi diff --git a/itn/italian/data/numbers/teen.tsv b/itn/italian/data/numbers/teen.tsv new file mode 100644 index 0000000..f04eb60 --- /dev/null +++ b/itn/italian/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +10 dieci +11 undici +12 dodici +13 tredici +14 quattordici +15 quindici +16 sedici +17 diciassette +18 diciotto +19 diciannove diff --git a/itn/italian/data/numbers/tens.tsv b/itn/italian/data/numbers/tens.tsv new file mode 100644 index 0000000..e4fe964 --- /dev/null +++ b/itn/italian/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +2 venti +3 trenta +4 quaranta +5 cinquanta +6 sessanta +7 settanta +8 ottanta +9 novanta diff --git a/itn/italian/data/whitelist.tsv b/itn/italian/data/whitelist.tsv new file mode 100644 index 0000000..d19ff1a --- /dev/null +++ b/itn/italian/data/whitelist.tsv @@ -0,0 +1,19 @@ +dott. dottor +dott.ssa dottoressa +prof. professore +prof.ssa professoressa +sig.ra signora +sig.na signorina +avv. avvocato +arch. architetto +ing. ingegnere +rag. ragioniere +geom. geometra +on. onorevole +rev. reverendo +sen. senatore +ecc. eccetera +v.le viale +p.za piazza +dip. dipartimento +c.v.d. come volevasi dimostrare diff --git a/itn/italian/inverse_normalizer.py b/itn/italian/inverse_normalizer.py new file mode 100644 index 0000000..2c90fa7 --- /dev/null +++ b/itn/italian/inverse_normalizer.py @@ -0,0 +1,77 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight + +from itn.italian.rules import money +from itn.italian.rules.cardinal import Cardinal +from itn.italian.rules.decimal import Decimal +from itn.italian.rules.electronic import Electronic +from itn.italian.rules.measure import Measure +from itn.italian.rules.money import Money +from itn.italian.rules.punctuation import Punctuation +from itn.italian.rules.time import Time +from itn.italian.rules.whitelist import WhiteList +from itn.italian.rules.word import Word +from tn.processor import Processor, RuleSpec + +TOKEN_ORDERS = dict(money.TOKEN_ORDER) + +# Written punctuation that attaches to the previous / next token. +CLOSING = ".,;:!?)}" +OPENING = "({" + + +class InverseNormalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="it_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS) + self.build_fst("it_itn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + decimal = Decimal(cardinal) + punctuation = Punctuation() + word = Word() + rules = ( + RuleSpec(WhiteList(), 1.01), + RuleSpec(Time(), 1.05), + RuleSpec(Money(cardinal, decimal), 1.07), + RuleSpec(decimal, 1.08), + RuleSpec(Measure(cardinal, decimal), 1.08), + RuleSpec(cardinal, 1.1), + RuleSpec(Electronic(), 1.1), + RuleSpec(word, 100), + RuleSpec(punctuation, 1.1, verbalize=False), + ) + classify = self.tagger_union(rules) + # Punctuation splits off only when attached to another token. + punct = add_weight(punctuation.tagger, 1.1) + token = punct.star + classify + punct.star + self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + + def marks(chars): + graph = pynini.union(*[pynini.escape(char) for char in chars]) + return punctuation.delete_tokens(punctuation.verbalize_field("value", graph)) + + other_marks = [mark for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~" if mark not in CLOSING + OPENING] + closing = marks(CLOSING) + opening = marks(OPENING) + other = self.verbalizer_union(rules) | marks(other_marks) + # Written Italian attaches closing marks to the previous token and + # opening marks to the next one: ``( cinque ) .`` -> ``(5).``. + item = opening.star + other | add_weight(opening.plus, 1.0) + self.verbalizer = (item | closing) + (closing | self.INSERT_SPACE + item).star diff --git a/itn/italian/rules/__init__.py b/itn/italian/rules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/itn/italian/rules/cardinal.py b/itn/italian/rules/cardinal.py new file mode 100644 index 0000000..a77e1bc --- /dev/null +++ b/itn/italian/rules/cardinal.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.italian.rules.utils import load_rows, string_map, table_map +from tn.processor import Processor + + +def two_digit_pairs(): + """Returns ``(written, spoken)`` pairs for 1..99 with two-digit written forms. + + Tens elide their final vowel before ``uno``/``otto`` (``ventuno``, + ``ventotto``); compounds of ``tre`` accept both ``ventitre`` and + ``ventitré``. + """ + + units = {int(written): spoken for written, spoken in load_rows("italian/data/numbers/digit.tsv")} + pairs = [("0{}".format(value), spoken) for value, spoken in units.items()] + pairs += [(written, spoken) for written, spoken in load_rows("italian/data/numbers/teen.tsv")] + for written, ten in load_rows("italian/data/numbers/tens.tsv"): + pairs.append((written + "0", ten)) + for value, unit in units.items(): + stem = ten[:-1] if unit[0] in "uo" else ten + pairs.append((written + str(value), stem + unit)) + if unit == "tre": + pairs.append((written + str(value), stem + "tré")) + return pairs + + +class Cardinal(Processor): + """Cardinals, e.g. ``tremila cinquecento quarantaquattro`` -> ``3544``. + + The tagger keeps the spoken words; the verbalizer writes the digits. + Components may be joined (``centocinquantacinque``) or spaced (``cento + cinquantacinque``). ``miliardo`` is 10^9; NeMo's TN style of counting + thousands of millions (``tremila ... milioni``) is accepted too. + """ + + def __init__(self): + super().__init__("cardinal", token_orders={}) + self.build_graphs() + self.sign = pynini.cross("meno", "-") + sign_tag = (self.tag_field("negative", self.sign) + pynini.accep(" ")).ques + sign_verb = (self.verbalize_field("negative", self.sign) + delete(" ")).ques + self.tagger = self.add_tokens(sign_tag + self.tag_field("integer", self.graph)) + self.verbalizer = self.delete_tokens(sign_verb + self.verbalize_field("integer", self.graph)) + + def build_graphs(self): + digit = self.DIGIT + join = delete(" ").ques + space = delete(" ") + two = string_map(two_digit_pairs()) + hundreds = table_map("italian/data/numbers/hundreds.tsv") + # ``centotto``/``centottanta`` drop the final ``o`` of the hundred. + elided = string_map([(written, spoken[:-1]) for written, spoken in load_rows("italian/data/numbers/hundreds.tsv")]) + two_ott = ("ott" + self.VSIGMA) @ two + three = ( + insert("0") + two + | hundreds + insert("00") + | hundreds + join + two + | elided + two_ott + ).optimize() + three_no_one = (three @ (pynini.closure(digit, 3, 3) - "001")).optimize() + rest3 = insert("000") | join + three + six = ( + insert("000") + three + | pynini.cross("mille", "001") + rest3 + | three_no_one + join + delete("mila") + rest3 + ).optimize() + rest6 = insert("000000") | space + six + millions = pynini.cross("un milione", "001") | three_no_one + delete(" milioni") + billions = pynini.cross("un miliardo", "001") | three_no_one + delete(" miliardi") + thousand_millions = six @ ((pynini.closure(digit, 3, 3) - "000") + pynini.closure(digit, 3, 3)) + padded = ( + billions + (space + millions | insert("000")) + rest6 + | insert("000") + millions + rest6 + | insert("000000") + six + | thousand_millions + delete(" milioni") + rest6 + ) + strip = delete("0").star + (digit - "0") + digit.star + self.number = (pynini.cross("zero", "0") | padded @ strip).optimize() + self.graph = self.number + self.two = two + # Unpadded readings below one thousand, e.g. for decimal digit groups. + self.below_thousand = (pynini.cross("zero", "0") | three @ strip).optimize() + # Counting contexts (measures, money) also say ``un``/``una`` for 1. + self.counted = (self.number | pynini.cross(pynini.union("un", "una"), "1")).optimize() diff --git a/itn/italian/rules/decimal.py b/itn/italian/rules/decimal.py new file mode 100644 index 0000000..8e03f62 --- /dev/null +++ b/itn/italian/rules/decimal.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from itn.italian.rules.utils import load_rows +from tn.processor import Processor + +# Penalty per digit group, so ``trecento quarantadue`` reads as one group (342). +GROUP_WEIGHT = 0.1 + + +class Decimal(Processor): + """Decimals and quantities, e.g. ``quattro virgola cinquantasei milioni`` -> ``4,56 milioni``. + + The tagger keeps the spoken integer, the spoken fractional digit groups + (dropping ``virgola``) and the quantity word; the verbalizer writes the + digits around a decimal comma. + """ + + def __init__(self, cardinal): + super().__init__("decimal", token_orders={}) + self.cardinal = cardinal + self.integer = cardinal.number + group = add_weight(cardinal.below_thousand, GROUP_WEIGHT) + self.fractional = (group + (delete(" ") + group).star).optimize() + self.quantity = pynini.union(*[row[0] for row in load_rows("italian/data/numbers/quantities.tsv")]).optimize() + # Bare quantities count below one thousand, e.g. ``un milione`` -> ``1 milione``. + self.quantity_integer = (cardinal.below_thousand | pynini.cross("un", "1")).optimize() + + quantity_tag = pynini.accep(" ") + self.tag_field("quantity", self.quantity) + fraction_tag = self.tag_field("integer_part", self.integer) + pynini.cross(" virgola ", " ") + fraction_tag += self.tag_field("fractional_part", self.fractional) + whole_tag = self.tag_field("integer_part", self.quantity_integer) + quantity_tag + self.fields_tagger = (fraction_tag + quantity_tag.ques | whole_tag).optimize() + self.fraction_tagger = fraction_tag.optimize() + sign_tag = (self.tag_field("negative", cardinal.sign) + pynini.accep(" ")).ques + self.tagger = self.add_tokens(sign_tag + self.fields_tagger) + + quantity_verb = pynini.accep(" ") + self.verbalize_field("quantity", self.quantity) + fraction_verb = self.verbalize_field("integer_part", self.integer) + delete(" ") + insert(",") + fraction_verb += self.verbalize_field("fractional_part", self.fractional) + whole_verb = self.verbalize_field("integer_part", self.quantity_integer) + quantity_verb + self.fields_verbalizer = (fraction_verb + quantity_verb.ques | whole_verb).optimize() + self.fraction_verbalizer = fraction_verb.optimize() + sign_verb = (self.verbalize_field("negative", cardinal.sign) + delete(" ")).ques + self.verbalizer = self.delete_tokens(sign_verb + self.fields_verbalizer) diff --git a/itn/italian/rules/electronic.py b/itn/italian/rules/electronic.py new file mode 100644 index 0000000..65fece3 --- /dev/null +++ b/itn/italian/rules/electronic.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.italian.rules.utils import load_rows, string_map, table_map +from tn.processor import Processor + + +class Electronic(Processor): + """Spelled URLs, e-mail addresses and handles, e.g. ``chiocciola j e n s e n`` -> ``@jensen``. + + The tagger keeps the spoken protocol, user name and domain; the + verbalizer joins letters, digit words (``uno`` -> ``1``), symbol names + (``trattino basso`` -> ``_``), server names (``nvidia``) and top-level + domains (``punto com`` -> ``.com``). A bare domain needs a ``punto`` and + must end in a known top-level domain or a run of at least two letters. + """ + + def __init__(self): + super().__init__("electronic", token_orders={}) + sep = delete(" ") + digits = table_map("italian/data/numbers/digit.tsv") | pynini.cross("zero", "0") + content = (self.ALPHA | digits).optimize() + names = pynini.union(*[row[0] for row in load_rows("italian/data/electronic/server_name.tsv")]) + symbols = table_map("italian/data/electronic/symbols.tsv") + dot = pynini.cross("punto", ".") + others = (symbols @ (self.VSIGMA - ".")).optimize() + tlds = string_map([(row[0], "punto " + row[1]) for row in load_rows("italian/data/electronic/domain.tsv")]) + + start = content | names + segment = start + (sep + (start | others)).star + long_segment = start + (sep + (start | others)).plus | names + domain = segment + (sep + (dot + sep + segment | tlds)).star + sep + (tlds | dot + sep + long_segment) + domain = domain.optimize() + user = (content + (sep + (start | others | dot)).star).optimize() + handle_user = (content + (sep + (start | others | dot)).plus).optimize() + + slashes = " due punti slash slash" + scheme = pynini.cross("h t t p s" + slashes, "https://") | pynini.cross("h t t p" + slashes, "http://") + www = pynini.cross("w w w punto", "www.") + protocol = (scheme | www | scheme + sep + www).optimize() + path_item = start | others | dot | tlds + path = sep + (others @ pynini.accep("/")) + (sep + path_item).plus + # Prefer the ``www.`` reading of the protocol over spelling it in the domain. + url_domain = (self.VSIGMA - ("w w w punto" + self.VSIGMA).optimize()) @ (domain + path.ques) + + space = pynini.accep(" ") + url = self.tag_field("protocol", protocol) + space + self.tag_field("domain", url_domain) + email = self.tag_field("username", user) + delete(" chiocciola") + space + self.tag_field("domain", domain) + handle = delete("chiocciola ") + self.tag_field("username", handle_user) + self.tagger = self.add_tokens(url | email | handle | self.tag_field("domain", domain)) + + url_verb = self.verbalize_field("protocol", protocol) + sep + self.verbalize_field("domain", url_domain) + email_verb = self.verbalize_field("username", user) + sep + insert("@") + self.verbalize_field("domain", domain) + handle_verb = insert("@") + self.verbalize_field("username", handle_user) + bare_verb = self.verbalize_field("domain", domain) + self.verbalizer = self.delete_tokens(url_verb | email_verb | handle_verb | bare_verb) diff --git a/itn/italian/rules/measure.py b/itn/italian/rules/measure.py new file mode 100644 index 0000000..a05d829 --- /dev/null +++ b/itn/italian/rules/measure.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.italian.rules.utils import load_rows, string_map +from tn.processor import Processor + + +class Measure(Processor): + """Quantities with units, e.g. ``cinque chilometri per secondo`` -> ``5 km/s``. + + The tagger keeps the spoken number fields and unit words; the + verbalizer writes the number and the unit symbol (``%`` is attached). + Singular and plural unit names are accepted after any number. Time units + (``giorni``, ``ore``, ``minuti``, ``secondi``) are only converted as + denominators, since ``sette giorni`` normally stays in words. + """ + + def __init__(self, cardinal, decimal): + super().__init__("measure", token_orders={}) + rows = load_rows("italian/data/measure/units.tsv") + units = string_map([(row[0], spoken) for row in rows for spoken in row[1:]]) + per_units = [(row[0], row[1]) for row in rows] + per_units += [(row[0], row[1]) for row in load_rows("italian/data/measure/denominators.tsv")] + denominator = pynini.cross(" per ", "/") + string_map(per_units) + percent = (units @ "%").optimize() + others = (units @ (self.VSIGMA - "%")).optimize() + unit_graph = (percent | others + denominator.ques).optimize() + + sign_tag = (self.tag_field("negative", cardinal.sign) + pynini.accep(" ")).ques + number_tag = self.tag_field("integer", cardinal.counted) | decimal.fraction_tagger + units_tag = pynini.accep(" ") + self.tag_field("units", unit_graph) + self.tagger = self.add_tokens(sign_tag + number_tag + units_tag) + + sign_verb = (self.verbalize_field("negative", cardinal.sign) + delete(" ")).ques + number_verb = self.verbalize_field("integer", cardinal.counted) | decimal.fraction_verbalizer + units_verb = delete(" ") + insert(" ") + self.verbalize_field("units", others + denominator.ques) + percent_verb = delete(" ") + self.verbalize_field("units", percent) + self.verbalizer = self.delete_tokens(sign_verb + number_verb + (units_verb | percent_verb)) diff --git a/itn/italian/rules/money.py b/itn/italian/rules/money.py new file mode 100644 index 0000000..1477f34 --- /dev/null +++ b/itn/italian/rules/money.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.italian.rules.utils import load_rows, string_map +from tn.processor import Processor + +# The verbalizer writes the amount before the currency symbol. +TOKEN_ORDER = {"money": ["integer_part", "fractional_part", "currency", "minor"]} + + +class Money(Processor): + """Amounts of money, e.g. ``due rubli un copeca`` -> ``2,01 ₽``. + + The tagger keeps the spoken amount, the currency name and the minor + unit name (an optional ``e`` connector is dropped); the verbalizer writes + `` `` with two-digit minor units. Minor units alone + (``cinquanta centesimi``) are written as ``0,50`` of the main currency. + """ + + def __init__(self, cardinal, decimal): + super().__init__("money", token_orders=TOKEN_ORDER) + rows = load_rows("italian/data/money/currency.tsv") + majors = string_map([(row[0], spoken) for row in rows for spoken in row[1:3]]) + minor_symbols = {} + for row in rows: + for spoken in row[3:5]: + if spoken: + minor_symbols.setdefault(spoken, row[0]) + minors_alone = string_map([(symbol, spoken) for spoken, symbol in minor_symbols.items()]) + cents = (cardinal.two | pynini.cross(pynini.union("un", "una"), "01")).optimize() + + integer_tag = self.tag_field("integer_part", cardinal.counted) + space = pynini.accep(" ") + # The minor unit must belong to the named currency. + named = [] + for row in rows: + major = self.tag_field("currency", pynini.union(*row[1:3])) + names = [spoken for spoken in row[3:5] if spoken] + if names: + minor = space + self.tag_field("minor", pynini.union(*names)) + fraction = (delete(" e") | "") + space + self.tag_field("fractional_part", cents) + minor + major += fraction.ques + named.append(major) + any_major = self.tag_field("currency", majors) + minor_alone_tag = self.tag_field("fractional_part", cents) + space + self.tag_field("minor", minors_alone) + branches = [ + integer_tag + space + pynini.union(*named), + decimal.fraction_tagger + space + any_major, + minor_alone_tag, + ] + self.tagger = self.add_tokens(pynini.union(*branches)) + + integer_verb = self.verbalize_field("integer_part", cardinal.counted) + cents_verb = delete(" ") + insert(",") + self.verbalize_field("fractional_part", cents) + currency_verb = delete(" ") + insert(" ") + self.verbalize_field("currency", majors) + minor_verb = delete(" ") + self.verbalize_field("minor", delete(pynini.union(*minor_symbols))) + minor_alone = delete(" ") + insert(" ") + self.verbalize_field("minor", minors_alone) + verbalizer = pynini.union( + integer_verb + currency_verb, + decimal.fraction_verbalizer + currency_verb, + integer_verb + cents_verb + currency_verb + minor_verb, + insert("0,") + self.verbalize_field("fractional_part", cents) + minor_alone, + ) + self.verbalizer = self.delete_tokens(verbalizer) diff --git a/itn/italian/rules/punctuation.py b/itn/italian/rules/punctuation.py new file mode 100644 index 0000000..a7740fb --- /dev/null +++ b/itn/italian/rules/punctuation.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from tn.processor import Processor + + +class Punctuation(Processor): + """ASCII punctuation split off a neighbouring token, e.g. ``(5)``.""" + + def __init__(self): + super().__init__("punctuation", token_orders={}) + graph = pynini.union(*[pynini.escape(mark) for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~"]).optimize() + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/italian/rules/time.py b/itn/italian/rules/time.py new file mode 100644 index 0000000..1c5294a --- /dev/null +++ b/itn/italian/rules/time.py @@ -0,0 +1,84 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.italian.rules.cardinal import two_digit_pairs +from itn.italian.rules.utils import string_map +from tn.processor import Processor + +PREPOSITIONS = ("alle", "dalle") +IDIOMS = {"un quarto": 15, "mezza": 30, "mezzo": 30, "tre quarti": 45} + + +class Time(Processor): + """Clock times, e.g. ``cinque e un quarto`` -> ``05:15``. + + The tagger keeps the spoken hour, minutes (without ``minuti``) and + seconds (without ``secondi``), the ``meno`` relation and a leading + ``alle``/``dalle``, which also licenses a bare hour (``alle sedici`` -> + ``alle 16:00``); the verbalizer writes ``HH:MM[:SS]``. + """ + + def __init__(self): + super().__init__("time", token_orders={}) + spoken_numbers = {} + for written, spoken in two_digit_pairs(): + spoken_numbers.setdefault(int(written), []).append(spoken) + hour_pairs = [("00", "zero"), ("01", "una")] + hour_pairs += [("{:02d}".format(h), s) for h in range(1, 25) for s in spoken_numbers[h]] + hours = string_map(hour_pairs) + previous = string_map([("{:02d}".format(int(w) - 1), s) for w, s in hour_pairs if int(w) > 0]) + special = string_map([("12", "mezzogiorno"), ("00", "mezzanotte")]) + + def counted(unit, plural): + pairs = [("01", "un " + unit)] + pairs += [("{:02d}".format(m), s + " " + plural) for m in range(2, 60) for s in spoken_numbers[m]] + return pairs + + minute_pairs = counted("minuto", "minuti") + [("{:02d}".format(v), s) for s, v in IDIOMS.items()] + minutes = string_map(minute_pairs) + # ``meno un quarto`` / ``meno dieci minuti`` count back from the hour. + before = string_map([ + ("{:02d}".format(60 - int(w)), s) for w, s in minute_pairs if s not in ("mezza", "mezzo", "tre quarti") + ]) + bare_minutes = string_map([("{:02d}".format(m), s) for m in range(1, 60) for s in spoken_numbers[m]]) + seconds = string_map(counted("secondo", "secondi")) + + space = pynini.accep(" ") + preposition = pynini.union(*PREPOSITIONS) + prep_tag = self.tag_field("preposition", preposition) + space + hours_tag = self.tag_field("hours", hours) + hours_prev_tag = self.tag_field("hours", previous) + and_ = delete(" e") + space + seconds_tag = and_ + self.tag_field("seconds", seconds) + clock = hours_tag + and_ + self.tag_field("minutes", minutes) + clock |= hours_prev_tag + space + self.tag_field("relation", pynini.accep("meno")) + space + self.tag_field("minutes", before) + clock += seconds_tag.ques + loose = prep_tag + hours_tag + (and_ + self.tag_field("minutes", bare_minutes)).ques + noon = prep_tag.ques + self.tag_field("hours", special) + self.tagger = self.add_tokens(prep_tag.ques + clock | loose | noon) + + prep_verb = self.verbalize_field("preposition", preposition) + space + colon = delete(" ") + insert(":") + hours_verb = self.verbalize_field("hours", hours) + seconds_verb = colon + self.verbalize_field("seconds", seconds) + clock_verb = hours_verb + colon + self.verbalize_field("minutes", minutes) + clock_verb |= (self.verbalize_field("hours", previous) + delete(" ") + self.verbalize_field("relation", delete("meno")) + + colon + self.verbalize_field("minutes", before)) + clock_verb += seconds_verb.ques + loose_verb = prep_verb + hours_verb + (colon + self.verbalize_field("minutes", bare_minutes) | insert(":00")) + noon_verb = prep_verb.ques + self.verbalize_field("hours", special) + insert(":00") + self.verbalizer = self.delete_tokens(prep_verb.ques + clock_verb | loose_verb | noon_verb) diff --git a/itn/italian/rules/utils.py b/itn/italian/rules/utils.py new file mode 100644 index 0000000..baba96a --- /dev/null +++ b/itn/italian/rules/utils.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from itn.utils import get_abs_path + + +def load_rows(rel_path): + """Loads the tab-separated rows of an Italian ITN data table in file order.""" + + rows = [] + with open(get_abs_path(rel_path), encoding="utf-8") as stream: + for line in stream: + line = line.rstrip("\r\n") + if line: + rows.append(line.split("\t")) + return rows + + +def string_map(pairs): + """Maps each spoken form to its written form (``(written, spoken)`` pairs).""" + + arcs = [pynini.cross(pynini.escape(spoken), pynini.escape(written)) for written, spoken in pairs] + return pynini.union(*arcs).optimize() + + +def table_map(rel_path): + """Maps the spoken column of a ``writtenspoken`` table to the written one.""" + + return string_map([(row[0], row[1]) for row in load_rows(rel_path)]) + + +def capitalized(pairs): + """Adds capitalized variants, e.g. ``(dott., dottor)`` -> ``(Dott., Dottor)``.""" + + extra = [(written[:1].upper() + written[1:], spoken[:1].upper() + spoken[1:]) for written, spoken in pairs] + return list(dict.fromkeys(list(pairs) + extra)) diff --git a/itn/italian/rules/whitelist.py b/itn/italian/rules/whitelist.py new file mode 100644 index 0000000..669dfc8 --- /dev/null +++ b/itn/italian/rules/whitelist.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from itn.italian.rules.utils import capitalized, load_rows, string_map +from tn.processor import Processor + + +class WhiteList(Processor): + """Titles and address words, e.g. ``signorina`` -> ``sig.na``. + + The table is deliberately small: only words that are routinely + abbreviated in Italian writing (plus capitalized variants). + """ + + def __init__(self): + super().__init__("whitelist", token_orders={}) + rows = load_rows("italian/data/whitelist.tsv") + graph = string_map(capitalized([(row[0], row[1]) for row in rows])) + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/italian/rules/word.py b/itn/italian/rules/word.py new file mode 100644 index 0000000..43fc775 --- /dev/null +++ b/itn/italian/rules/word.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import delete + +from tn.processor import Processor + + +class Word(Processor): + """Any other non-space run, kept verbatim.""" + + def __init__(self): + super().__init__("word", token_orders={}) + self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus)) + graph = self.NOT_SPACE.plus | delete(self.SPACE.plus) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/italian/test/__init__.py b/itn/italian/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/itn/italian/test/normalizer_test.py b/itn/italian/test/normalizer_test.py new file mode 100644 index 0000000..eaa9ac4 --- /dev/null +++ b/itn/italian/test/normalizer_test.py @@ -0,0 +1,277 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path + +import pytest + +from itn.italian.inverse_normalizer import InverseNormalizer + +PACKAGE = Path(__file__).resolve().parent.parent +# NeMo has no Italian ITN fixtures; the TN fixtures are read in reverse. +TN_DATA = PACKAGE.parent.parent / "tn" / "italian" / "test" / "data" + +# Reversed TN pairs that cannot hold, keyed by spoken form. +EXCLUDED = { + # ``d`` for days is not written back: ``sette giorni`` (also a TN fixture) stays in words. + "cento cinquantacinque giorni": "155 d", + # Spacing only: ITN writes every currency symbol after a space (``3,23 €``). + "tre euro ventitre centesimi": "3,23€", + # TN reads ``4,2`` and ``4,20`` alike; ITN writes two-digit minor units (``4,20 £``). + "quattro sterline venti penny": "4,2 £", + # TN reads ``eur`` and ``€`` alike; ITN writes the symbol (``1 €``). + "un euro": "1 eur", + "un euro per il caffè": "1 eur per il caffè", +} + + +def reversed_cases(): + """Groups TN fixtures by spoken form; any written original is a correct answer.""" + + groups = {} + for path in sorted(TN_DATA.glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for line in stream: + if line.endswith("\n"): + line = line[:-1] + written, *spoken_forms = line.split("~") + for spoken in spoken_forms: + groups.setdefault(spoken, []).append(written) + return groups + + +GROUPS = reversed_cases() +CASES = [pytest.param(spoken, tuple(written), id=spoken) for spoken, written in GROUPS.items() if spoken not in EXCLUDED] + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(InverseNormalizer) + + +def test_reversed_fixture_count(): + # 52 TN lines, three of which list two spoken readings. + assert sum(len(written) for written in GROUPS.values()) == 55 + assert len(GROUPS) == 55 + assert len(CASES) == 50 + for spoken, written in EXCLUDED.items(): + assert GROUPS[spoken] == [written] + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_reversed(normalizer, spoken, written): + assert normalizer.normalize(spoken) in written + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_reversed_mapping(normalizer, spoken, written): + result = normalizer.normalize_with_mapping(spoken, include_identity=True) + assert result.output_text in written + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +@pytest.mark.parametrize("spoken, written", list(EXCLUDED.items())) +def test_excluded_pairs_have_a_canonical_reading(normalizer, spoken, written): + assert normalizer.normalize(spoken) != written + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for spoken in list(GROUPS) + [case[0] for case in HAND_WRITTEN]: + assert normalizer.verbalize(normalizer.tag(spoken)) == normalizer.normalize(spoken) + + +HAND_WRITTEN = [ + # cardinal + ("centocinquantacinque", "155"), + ("duemilaventisei", "2026"), + ("mille novecento ottantaquattro", "1984"), + ("ventitré", "23"), + ("centottanta", "180"), + ("ventunomila", "21000"), + ("tre miliardi quattrocento cinquantatre milioni", "3453000000"), + ("meno quarantotto", "-48"), + ("zero", "0"), + ("sette giorni", "7 giorni"), + # decimal + ("zero virgola zero cinque", "0,05"), + ("meno tre virgola cinque", "-3,5"), + ("dodici virgola cento uno", "12,101"), + ("due milioni", "2 milioni"), + ("un miliardo di euro", "1 miliardo di euro"), + ("sei virgola dodici trentaquattro", "6,1234"), + # measure + ("venti gradi celsius", "20 °C"), + ("cento chilometri per ora", "100 km/h"), + ("tre chilogrammi", "3 kg"), + ("cinque per cento", "5%"), + ("due metri quadrati", "2 m²"), + ("una tonnellata", "1 t"), + ("due virgola cinque litri", "2,5 l"), + # money + ("cinquanta centesimi", "0,50 €"), + ("tre euro e venti centesimi", "3,20 €"), + ("due virgola cinque euro", "2,5 €"), + ("dieci dollari", "10 $"), + ("una sterlina", "1 £"), + ("dieci franchi svizzeri", "10 CHF"), + ("venti rupie e cinquanta paise", "20,50 ₹"), + ("un dollaro e un centesimo", "1,01 $"), + # time + ("cinque e un quarto", "05:15"), + ("cinque meno un quarto", "04:45"), + ("nove meno dieci minuti", "08:50"), + ("alle dodici e trenta", "alle 12:30"), + ("dalle nove alle diciotto", "dalle 09:00 alle 18:00"), + ("a mezzogiorno", "a 12:00"), + ("tre e trenta minuti e un secondo", "03:30:01"), + ("otto e mezza", "08:30"), + # electronic + ("m a r i o punto r o s s i chiocciola gmail punto com", "mario.rossi@gmail.com"), + ("w w w punto g o o g l e punto IT", "www.google.it"), + ("h t t p due punti slash slash e x a m p l e punto org", "http://example.org"), + ("chiocciola m a r i o", "@mario"), + ("s i t o punto i t", "sito.it"), + ("i n f o trattino basso uno chiocciola s i t o punto net", "info_1@sito.net"), + ("a chiocciola b punto com e poi", "a@b.com e poi"), + ("w w w punto nvidia punto com e poi", "www.nvidia.com e poi"), + # whitelist + ("Dottor Rossi", "Dott. Rossi"), + ("professoressa Bianchi", "prof.ssa Bianchi"), + ("eccetera", "ecc."), + ("come volevasi dimostrare", "c.v.d."), + ("viale Mazzini", "v.le Mazzini"), + # word + ("un uomo", "un uomo"), + ("una settimana", "una settimana"), + ("il punto e la virgola", "il punto e la virgola"), + ("una chiocciola", "una chiocciola"), + ("sette e trenta", "7 e 30"), + ("perché", "perché"), + # punctuation + ("( cinque ) .", "(5)."), + ("ciao , come stai ?", "ciao, come stai?"), + ("{ uno }", "{1}"), + ("sei - sette", "6 - 7"), + ("evviva !", "evviva!"), +] + + +@pytest.mark.parametrize("spoken, written", HAND_WRITTEN) +def test_hand_written(normalizer, spoken, written): + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken, members, written", [ + ("a chiocciola b punto com e poi", {"username": "a", "domain": "b punto com"}, "a@b.com e poi"), + ("w w w punto nvidia punto com e poi", {"protocol": "w w w punto", "domain": "nvidia punto com"}, + "www.nvidia.com e poi"), +]) +def test_electronic_boundaries_and_mapping(normalizer, spoken, members, written): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert parser.tokens[0].name == "electronic" + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == written + results = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in results] == normalizer.normalize(spoken, nbest=3) + assert results[0].output_text == written + for result in results: + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +@pytest.mark.parametrize("spoken, kind, members, written", [ + ("cento cinquantacinque", "cardinal", {"integer": "cento cinquantacinque"}, "155"), + ("meno cinque", "cardinal", {"negative": "meno", "integer": "cinque"}, "-5"), + ("quattro virgola cinquantasei milioni", "decimal", + {"integer_part": "quattro", "fractional_part": "cinquantasei", "quantity": "milioni"}, "4,56 milioni"), + ("cinque chilometri per secondo", "measure", {"integer": "cinque", "units": "chilometri per secondo"}, "5 km/s"), + ("quattro virgola tre litri", "measure", {"integer_part": "quattro", "fractional_part": "tre", "units": "litri"}, + "4,3 l"), + ("due rubli un copeca", "money", + {"integer_part": "due", "currency": "rubli", "fractional_part": "un", "minor": "copeca"}, "2,01 ₽"), + ("cinquanta centesimi", "money", {"fractional_part": "cinquanta", "minor": "centesimi"}, "0,50 €"), + ("diciassette e quindici minuti e ventisei secondi", "time", + {"hours": "diciassette", "minutes": "quindici minuti", "seconds": "ventisei secondi"}, "17:15:26"), + ("cinque meno un quarto", "time", {"hours": "cinque", "relation": "meno", "minutes": "un quarto"}, "04:45"), + ("alle sedici", "time", {"preposition": "alle", "hours": "sedici"}, "alle 16:00"), + ("h t t p s due punti slash slash w w w punto nvidia punto com", "electronic", + {"protocol": "h t t p s due punti slash slash w w w punto", "domain": "nvidia punto com"}, + "https://www.nvidia.com"), + ("chiocciola j e n s e n", "electronic", {"username": "j e n s e n"}, "@jensen"), + ("signorina", "whitelist", {"value": "signorina"}, "sig.na"), + ("caffè", "word", {"value": "caffè"}, "caffè"), +]) +def test_raw_fields(normalizer, spoken, kind, members, written): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == written + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken", ["dodici e mezza", "due rubli un copeca", "( cinque ) .", "signorina Sonia"]) +def test_nbest_mapping_synchronization(normalizer, spoken): + outputs = normalizer.normalize(spoken, nbest=3) + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(spoken) + for result in mapped: + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + spoken = 'perché\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(spoken)) + assert [token.members for token in parser.tokens] == [{"value": spoken}] + assert normalizer.normalize(spoken) == spoken + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("spoken", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, spoken): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": spoken} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(spoken, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) + + +def test_standalone_from_tn(): + tn_import = re.compile(r"^\s*(from|import)\s+tn\.italian\b", re.MULTILINE) + for path in PACKAGE.rglob("*.py"): + assert not tn_import.search(path.read_text(encoding="utf-8")), path diff --git a/pyproject.toml b/pyproject.toml index 4abc551..4d0dc87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,8 @@ tn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "italian/data/*.tsv", + "italian/data/*/*.tsv", "marathi/data/*/*.tsv", "kinyarwanda/data/*.tsv", "kinyarwanda/data/*/*.tsv", @@ -105,6 +107,8 @@ itn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "italian/data/*.tsv", + "italian/data/*/*.tsv", "hindi_english/data/*.tsv", "marathi/data/*/*.tsv", "kinyarwanda/data/*.tsv", diff --git a/tn/chinese/test/cache_test.py b/tn/chinese/test/cache_test.py index f459294..7a09b5b 100644 --- a/tn/chinese/test/cache_test.py +++ b/tn/chinese/test/cache_test.py @@ -11,6 +11,7 @@ from itn.german.inverse_normalizer import InverseNormalizer as DeInverseNormalizer from itn.hindi.inverse_normalizer import InverseNormalizer as HiInverseNormalizer from itn.hindi_english.inverse_normalizer import InverseNormalizer as HiEnInverseNormalizer +from itn.italian.inverse_normalizer import InverseNormalizer as ItInverseNormalizer from itn.kinyarwanda.inverse_normalizer import InverseNormalizer as RwInverseNormalizer from itn.japanese.inverse_normalizer import InverseNormalizer as JaInverseNormalizer from itn.korean.inverse_normalizer import InverseNormalizer as KoInverseNormalizer @@ -27,6 +28,7 @@ from tn.french.normalizer import Normalizer as FrNormalizer from tn.german.normalizer import Normalizer as DeNormalizer from tn.hindi.normalizer import Normalizer as HiNormalizer +from tn.italian.normalizer import Normalizer as ItNormalizer from tn.kinyarwanda.normalizer import Normalizer as RwNormalizer from tn.japanese.normalizer import Normalizer as JaNormalizer from tn.korean.normalizer import Normalizer as KoNormalizer @@ -101,6 +103,7 @@ def _build_small_graphs(processor): (EsNormalizer, "es_tn", "tn", {}, {"deterministic": True}), (EsNormalizer, "es_tn", "tn", {"deterministic": False}, {"deterministic": False}), (HiNormalizer, "hi_tn", "tn", {}, {}), + (ItNormalizer, "it_tn", "tn", {}, {}), (MrNormalizer, "mr_tn", "tn", {}, {}), (RwNormalizer, "rw_tn", "tn", {}, {}), ( @@ -150,6 +153,7 @@ def _build_small_graphs(processor): (EsInverseNormalizer, "es_itn", "itn", {}, {"cased": False}), (EsInverseNormalizer, "es_itn", "itn", {"cased": True}, {"cased": True}), (HiInverseNormalizer, "hi_itn", "itn", {}, {}), + (ItInverseNormalizer, "it_itn", "itn", {}, {}), (HiEnInverseNormalizer, "hi_en_itn", "itn", {}, {"cased": False}), (HiEnInverseNormalizer, "hi_en_itn", "itn", {"cased": True}, {"cased": True}), (MrInverseNormalizer, "mr_itn", "itn", {}, {}), diff --git a/tn/cli.py b/tn/cli.py index 00c6a60..92c462d 100644 --- a/tn/cli.py +++ b/tn/cli.py @@ -69,6 +69,7 @@ class PipelineSpec: (BooleanOption("deterministic", True, "use deterministic Spanish readings"),), ), "hi": PipelineSpec("tn.hindi.normalizer", "Normalizer"), + "it": PipelineSpec("tn.italian.normalizer", "Normalizer"), "mr": PipelineSpec("tn.marathi.normalizer", "Normalizer"), "rw": PipelineSpec("tn.kinyarwanda.normalizer", "Normalizer"), "ja": PipelineSpec( @@ -109,6 +110,7 @@ class PipelineSpec: (BooleanOption("cased", False, 'also accept capitalized spoken words, e.g. "Doce Dólares"'),), ), "hi": PipelineSpec("itn.hindi.inverse_normalizer", "InverseNormalizer"), + "it": PipelineSpec("itn.italian.inverse_normalizer", "InverseNormalizer"), "hi_en": PipelineSpec( "itn.hindi_english.inverse_normalizer", "InverseNormalizer", (BooleanOption("cased", False, 'also accept capitalized English words, e.g. "July"'),), diff --git a/tn/italian/__init__.py b/tn/italian/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tn/italian/data/electronic/domain.tsv b/tn/italian/data/electronic/domain.tsv new file mode 100644 index 0000000..c5c26d7 --- /dev/null +++ b/tn/italian/data/electronic/domain.tsv @@ -0,0 +1,14 @@ +.com punto com +.org punto org +.gov punto gov +.uk punto UK +.fr punto FR +.net punto net +.html punto html +.py punto python +.ru punto RU +.us punto US +.de punto DE +.it punto IT +.jpg punto jpeg +.edu punto edu \ No newline at end of file diff --git a/tn/italian/data/electronic/server_name.tsv b/tn/italian/data/electronic/server_name.tsv new file mode 100644 index 0000000..9a0c756 --- /dev/null +++ b/tn/italian/data/electronic/server_name.tsv @@ -0,0 +1,11 @@ +gmail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast \ No newline at end of file diff --git a/tn/italian/data/electronic/symbols.tsv b/tn/italian/data/electronic/symbols.tsv new file mode 100644 index 0000000..56c9270 --- /dev/null +++ b/tn/italian/data/electronic/symbols.tsv @@ -0,0 +1,22 @@ +. punto +- trattino +_ trattino basso +! punto esclamativo +# cancelletto +$ dollaro +% percento +& e commerciale +' apostrofo +* asterisco ++ più +/ slash += uguale +? punto interrogativo +^ esponente +` accento +{ apri graffa +| barra verticale +} chiudi graffa +~ tilde +, virgola +: due punti diff --git a/tn/italian/data/measure/measurements.tsv b/tn/italian/data/measure/measurements.tsv new file mode 100644 index 0000000..93170ae --- /dev/null +++ b/tn/italian/data/measure/measurements.tsv @@ -0,0 +1,64 @@ +d giorno +f fahrenheit +c celsius +°C grado celsius +°F grado fahrenheit +K kelvin +km chilometro +m metro +cm centimetro +mm millimetro +μm micrometro +nm nanometro +dm decimetro +pm picometro +hm ettometro +ha ettaro +mi miglio +m² metro quadrato -0.0001 +km² chilometro quadrato -0.0001 +m2 metro quadrato +km2 chilometro quadrato +m3 metro cubo +ft foot +g grammo +µg microgrammo +mg milligrammo +kg chilogrammo +lb libbra +h ora +s secondo +min minuto +ms millisecondo +μs microsecondo +hz hertz +kw kilowatt +ghz gigahertz +khz kilohertz +mhz megahertz +v volt +mc megacoulomb +mA milliampere +A ampere +tw terawatt +mv millivolt +mw megawatt +gw gigawatt +ω ohm +db decibel +gb gigabyte +kb kilobit +pb petabit +mb megabyte +kB kilobyte +tb terabyte +kv kilovolt +MV megavolt +kn kilonewton +ml millilitro +l litro +dl decilitro +bar bar +kcal chilocaloria +cal caloria +% percento \ No newline at end of file diff --git a/tn/italian/data/measure/suppletive.tsv b/tn/italian/data/measure/suppletive.tsv new file mode 100644 index 0000000..6a6308d --- /dev/null +++ b/tn/italian/data/measure/suppletive.tsv @@ -0,0 +1,11 @@ +grado celsius gradi celsius +metro metri +ora ore +grammo grammi +secondo secondi +minuto minuti +chilometro chilometri +caloria calorie +miglio miglia +litro litri +giorno giorni diff --git a/tn/italian/data/money/currency_major.tsv b/tn/italian/data/money/currency_major.tsv new file mode 100644 index 0000000..93a991e --- /dev/null +++ b/tn/italian/data/money/currency_major.tsv @@ -0,0 +1,11 @@ +€ euro +$ dollaro +£ sterlina +usd dollaro statunitense +eur euro +cad dollaro canadese +chf franco svizzero +hkd dollaro di hong kong +¥ yen +rs rupia +₽ rublo diff --git a/tn/italian/data/money/currency_minor.tsv b/tn/italian/data/money/currency_minor.tsv new file mode 100644 index 0000000..59c91ff --- /dev/null +++ b/tn/italian/data/money/currency_minor.tsv @@ -0,0 +1,11 @@ +€ centesimo +$ centesimo +£ penny +usd centesimo +eur centesimo +cad centesimo +chf centesimo +hkd centesimo +¥ yen +rs paisa +₽ copeca diff --git a/tn/italian/data/money/currency_plural_fem.tsv b/tn/italian/data/money/currency_plural_fem.tsv new file mode 100644 index 0000000..bd0e813 --- /dev/null +++ b/tn/italian/data/money/currency_plural_fem.tsv @@ -0,0 +1,3 @@ +sterlina sterline +rupia rupie +paisa paise diff --git a/tn/italian/data/money/currency_plural_masc.tsv b/tn/italian/data/money/currency_plural_masc.tsv new file mode 100644 index 0000000..a288842 --- /dev/null +++ b/tn/italian/data/money/currency_plural_masc.tsv @@ -0,0 +1,11 @@ +centesimo centesimi +euro euro +dollaro dollari +penny penny +dollaro canadese dollari canadesi +franco svizzero franchi svizzeri +dollaro statunitense dollari statunitensi +rublo rubli +dollaro di hong kong dollari di hong kong +copeca copeche +yen yen diff --git a/tn/italian/data/numbers/digit.tsv b/tn/italian/data/numbers/digit.tsv new file mode 100644 index 0000000..4f2bf34 --- /dev/null +++ b/tn/italian/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +uno 1 +due 2 +tre 3 +quattro 4 +cinque 5 +sei 6 +sette 7 +otto 8 +nove 9 \ No newline at end of file diff --git a/tn/italian/data/numbers/hundreds.tsv b/tn/italian/data/numbers/hundreds.tsv new file mode 100644 index 0000000..20f4aa2 --- /dev/null +++ b/tn/italian/data/numbers/hundreds.tsv @@ -0,0 +1,8 @@ +duecento 2 +trecento 3 +quattrocento 4 +cinquecento 5 +seicento 6 +settecento 7 +ottocento 8 +novecento 9 \ No newline at end of file diff --git a/tn/italian/data/numbers/quantities.tsv b/tn/italian/data/numbers/quantities.tsv new file mode 100644 index 0000000..fac3867 --- /dev/null +++ b/tn/italian/data/numbers/quantities.tsv @@ -0,0 +1,2 @@ +milioni +miliardi \ No newline at end of file diff --git a/tn/italian/data/numbers/teen.tsv b/tn/italian/data/numbers/teen.tsv new file mode 100644 index 0000000..5c256ec --- /dev/null +++ b/tn/italian/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +dieci 10 +undici 11 +dodici 12 +tredici 13 +quattordici 14 +quindici 15 +sedici 16 +diciassette 17 +diciotto 18 +diciannove 19 \ No newline at end of file diff --git a/tn/italian/data/numbers/tens.tsv b/tn/italian/data/numbers/tens.tsv new file mode 100644 index 0000000..5637114 --- /dev/null +++ b/tn/italian/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +venti 2 +trenta 3 +quaranta 4 +cinquanta 5 +sessanta 6 +settanta 7 +ottanta 8 +novanta 9 \ No newline at end of file diff --git a/tn/italian/data/numbers/tens_eight.tsv b/tn/italian/data/numbers/tens_eight.tsv new file mode 100644 index 0000000..1cd5701 --- /dev/null +++ b/tn/italian/data/numbers/tens_eight.tsv @@ -0,0 +1,8 @@ +ventotto 28 +trentotto 38 +quarantotto 48 +cinquantotto 58 +sessantotto 68 +settantotto 78 +ottantotto 88 +novantotto 98 \ No newline at end of file diff --git a/tn/italian/data/numbers/tens_one.tsv b/tn/italian/data/numbers/tens_one.tsv new file mode 100644 index 0000000..3ffbcec --- /dev/null +++ b/tn/italian/data/numbers/tens_one.tsv @@ -0,0 +1,8 @@ +ventuno 21 +trentuno 31 +quarantuno 41 +cinquantuno 51 +sessantuno 61 +settantuno 71 +ottantuno 81 +novantuno 91 \ No newline at end of file diff --git a/tn/italian/data/numbers/zero.tsv b/tn/italian/data/numbers/zero.tsv new file mode 100644 index 0000000..c479272 --- /dev/null +++ b/tn/italian/data/numbers/zero.tsv @@ -0,0 +1 @@ +zero 0 \ No newline at end of file diff --git a/tn/italian/data/whitelist/whitelist.tsv b/tn/italian/data/whitelist/whitelist.tsv new file mode 100644 index 0000000..cabaadc --- /dev/null +++ b/tn/italian/data/whitelist/whitelist.tsv @@ -0,0 +1,65 @@ +dr dottor +dr. dottor +Dr. dottor +dott. dottor +Dott. dottor +dott.sa dottoressa +Dott.sa dottoressa +prof. professore +Prof. professore +proff. professori +Proff. professori +prof.ssa professoressa +Prof.ssa professoressa +Sig. signore +Sig.na signorina +sig. signore +sig.na signorina +sig signore +Sigg. signori +sigg. signori +Sig.ri signori +sig.ri signori +Sig.re signore +sig.re signore +Sig.ra signora +sig.ra signora +Sig.ne signorine +sig.ne signorine +Avv. avvocato +avv. avvocato +Arch. architetto +arch. architetto +Rag. ragioniere +rag. ragioniere +Geom. geometra +geom. geometra +Ing. ingegnere +ing. ingegnere +Ingg. ingegneri +ingg. ingegneri +Fam. famiglia +fam. famiglia +f.lli fratelli +F.lli fratelli +Rev. reverendo +On. onorevole +on. onorevole +Onn. onorevoli +onn. onorevoli +Min. ministro +Sen. senatore +c.v.d. come volevasi dimostrare +ecc. eccetera +ecc eccetera +v.le viale +p.za piazza +l.go largo +loc. località +ist. istituto +segr. segreteria +set. settore +uff. ufficio +dip. dipartimento +div. divisione ++ più diff --git a/tn/italian/normalizer.py b/tn/italian/normalizer.py new file mode 100644 index 0000000..66a6b78 --- /dev/null +++ b/tn/italian/normalizer.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import add_weight + +from tn.italian.rules import money +from tn.italian.rules.cardinal import Cardinal +from tn.italian.rules.decimal import Decimal +from tn.italian.rules.electronic import Electronic +from tn.italian.rules.measure import Measure +from tn.italian.rules.money import Money +from tn.italian.rules.punctuation import Punctuation +from tn.italian.rules.time import Time +from tn.italian.rules.whitelist import WhiteList +from tn.italian.rules.word import Word +from tn.processor import Processor, RuleSpec + +TOKEN_ORDERS = dict(money.TOKEN_ORDER) + + +class Normalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="it_normalizer", token_orders=TOKEN_ORDERS) + self.build_fst("it_tn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + decimal = Decimal(cardinal) + punctuation = Punctuation() + word = Word() + rules = ( + RuleSpec(WhiteList(), 1.01), + RuleSpec(Time(cardinal), 1.1), + RuleSpec(decimal, 1.1), + RuleSpec(Measure(cardinal, decimal), 0.9), + RuleSpec(cardinal, 1.1), + RuleSpec(Money(cardinal, decimal), 1.1), + RuleSpec(Electronic(cardinal), 1.1), + RuleSpec(word, 100, verbalize=False), + RuleSpec(punctuation, verbalize=False), + ) + classify = self.tagger_union(rules) + punct = add_weight(punctuation.tagger, 1.1) + token = punct.star + classify + punct.star + self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + self.verbalizer = self.join_tokens(self.verbalizer_union(rules), (word.verbalizer, punctuation.verbalizer)) diff --git a/tn/italian/rules/__init__.py b/tn/italian/rules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tn/italian/rules/cardinal.py b/tn/italian/rules/cardinal.py new file mode 100644 index 0000000..06e2365 --- /dev/null +++ b/tn/italian/rules/cardinal.py @@ -0,0 +1,99 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.italian.rules.utils import preferred_map +from tn.processor import Processor + + +class Cardinal(Processor): + """Cardinals, e.g. ``3544`` -> ``tremila cinquecento quarantaquattro``. + + The tagger keeps the written digits (including ``.``/space thousands + separators such as ``1.000.000``) and an optional ``-`` sign; the + verbalizer reads them. As in NeMo, ``milioni`` counts in groups of six + digits (``3453243534`` -> ``tremila quattrocento cinquantatre milioni + ...``), so ``miliardo`` is 10^12 and ``trilione`` is 10^18. + """ + + def __init__(self): + super().__init__("cardinal", token_orders={}) + self.build_graphs() + self.sign = pynini.cross("-", "meno") + sign_tag = (self.tag_field("negative", self.sign) + insert(" ")).ques + sign_verb = (self.verbalize_field("negative", self.sign) + delete(" ") + insert(" ")).ques + self.tagger = self.add_tokens(sign_tag + self.tag_field("integer", self.graph)) + self.verbalizer = self.delete_tokens(sign_verb + self.verbalize_field("integer", self.graph)) + + def build_graphs(self): + digit = self.DIGIT + nonzero = digit - "0" + space = insert(" ") + zero = pynini.cross("0", "zero") + digits = preferred_map("italian/data/numbers/digit.tsv", invert=True) + teens = preferred_map("italian/data/numbers/teen.tsv", invert=True) + tens = preferred_map("italian/data/numbers/tens.tsv", invert=True) + tens_one = preferred_map("italian/data/numbers/tens_one.tsv", invert=True) + tens_eight = preferred_map("italian/data/numbers/tens_eight.tsv", invert=True) + hundreds = preferred_map("italian/data/numbers/hundreds.tsv", invert=True) + + # ``ventuno``/``ventotto`` elide the final vowel of the ten. + plain_units = pynini.union(*"2345679") @ digits + two_digits = teens | tens + (delete("0") | plain_units) | tens_one | tens_eight + self.digits = digits.optimize() + self.two_digits = two_digits.optimize() + hundred = pynini.cross("1", "cento") | hundreds + three_digits = hundred + (delete("00") | space + two_digits | delete("0") + space + digits) + self.three_digits = three_digits.optimize() + self.up_to_hundreds = (digits | two_digits | three_digits).optimize() + + # Exact three-digit groups with value 1..999, optionally excluding 1. + group = three_digits | delete("0") + two_digits | delete("00") + digits + group_no_one = (pynini.closure(digit, 3, 3) - "001") @ group + rest3 = delete("000") | space + group + # Exact six-digit groups with value 1..999999. + six = ( + delete("000") + group + | group_no_one + insert("mila") + rest3 + | pynini.cross("001", "mille") + rest3 + ).optimize() + six_no_one = (pynini.closure(digit, 6, 6) - "000001") @ six + + def head(length): + # Unpadded leading digits (1..length) padded to a six-digit group. + padded = pynini.union(*[ + insert("0" * (length - size)) + nonzero + pynini.closure(digit, size - 1, size - 1) + for size in range(1, length + 1) + ]) + return padded.optimize() + + below_million = head(6) @ six + rest = delete("000000") | space + six + scales = [("un milione", " milioni"), ("un miliardo", " miliardi"), ("un trilione", " trilioni")] + graph = below_million + for one, many in scales: + block = pynini.cross("000001", one) | six_no_one + insert(many) + leading = head(6) @ block + graph |= leading + rest + rest = delete("000000") + rest | space + block + rest + rest = rest.optimize() + self.number = (zero | graph).optimize() + + # Thousands separators: ``.`` or a space between three-digit groups. + separator = delete(pynini.union(".", " ")) + grouped = pynini.closure(digit, 1, 3) + (separator + pynini.closure(digit, 3, 3)).plus + self.graph = ((digit.plus | grouped) @ self.number).optimize() diff --git a/tn/italian/rules/decimal.py b/tn/italian/rules/decimal.py new file mode 100644 index 0000000..76dcbc6 --- /dev/null +++ b/tn/italian/rules/decimal.py @@ -0,0 +1,82 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.italian.rules.utils import load_words +from tn.processor import Processor + + +class Decimal(Processor): + """Decimals and quantities, e.g. ``4,56 milioni`` -> ``quattro virgola cinquantasei milioni``. + + The tagger keeps the written integer digits, the fractional digits + (dropping the decimal comma) and the quantity word; the verbalizer reads + the parts around ``virgola``. + """ + + def __init__(self, cardinal): + super().__init__("decimal", token_orders={}) + self.cardinal = cardinal + self.build_graphs() + self.build_tagger() + self.build_verbalizer() + + def build_graphs(self): + cardinal = self.cardinal + space = insert(" ") + zero = pynini.cross("0", "zero") + digit = cardinal.digits | zero + zeros = (space + zero).star + tens = cardinal.two_digits + # NeMo's deterministic reading of the fractional digits: whole + # numbers up to the hundreds, then pairs, with explicit zeros. + fractional = pynini.union( + add_weight(digit + zeros, -0.00001), + add_weight(tens + zeros, -0.00002), + add_weight(cardinal.three_digits + zeros, 0.00001), + add_weight( + tens + (space + tens).plus + (space + zero).ques + ((space + digit).ques | zeros), + -0.00002, + ), + zero + zeros + (space + digit).star, + ) + self.fractional = fractional.optimize() + self.integer = cardinal.graph + self.quantity = pynini.union(*load_words("italian/data/numbers/quantities.tsv")).optimize() + # Up to six digits before a bare quantity; separators are dropped. + separator = delete(pynini.union(".", " ")) + digits = self.DIGIT + (separator.ques + self.DIGIT).star + self.quantity_integer = (digits @ pynini.closure(self.DIGIT, 1, 6) @ cardinal.number).optimize() + + def build_tagger(self): + sign = (self.tag_field("negative", self.cardinal.sign) + insert(" ")).ques + quantity = delete(" ") + insert(" ") + self.tag_field("quantity", self.quantity) + fractional = self.tag_field("integer_part", self.integer) + delete(",") + insert(" ") + fractional += self.tag_field("fractional_part", self.fractional) + quantity.ques + whole = self.tag_field("integer_part", self.quantity_integer) + quantity + self.fields_tagger = (fractional | whole).optimize() + self.tagger = self.add_tokens(sign + self.fields_tagger) + + def build_verbalizer(self): + sign = (self.verbalize_field("negative", self.cardinal.sign) + delete(" ") + insert(" ")).ques + quantity = insert(" ") + self.verbalize_field("quantity", self.quantity, leading_space=True) + fractional = self.verbalize_field("integer_part", self.integer) + insert(" virgola") + fractional += insert(" ") + self.verbalize_field("fractional_part", self.fractional, leading_space=True) + fractional += quantity.ques + whole = self.verbalize_field("integer_part", self.quantity_integer) + quantity + self.fields_verbalizer = (fractional | whole).optimize() + self.verbalizer = self.delete_tokens(sign + self.fields_verbalizer) diff --git a/tn/italian/rules/electronic.py b/tn/italian/rules/electronic.py new file mode 100644 index 0000000..a677d37 --- /dev/null +++ b/tn/italian/rules/electronic.py @@ -0,0 +1,80 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.italian.rules.utils import load_pairs, load_words, preferred_map +from tn.processor import Processor + +# Penalty for spelling a run that also has a whole-word reading. +SPELL_WEIGHT = 0.001 + + +class Electronic(Processor): + """URLs, e-mail addresses and handles, e.g. ``@jensen`` -> ``chiocciola j e n s e n``. + + The tagger keeps the written protocol, user name and domain; the + verbalizer spells them, naming symbols and digits, while known server + names (``nvidia``) and top-level domains (``.com`` -> ``punto com``) are + read as words when they cover a whole letter/digit run. + """ + + def __init__(self, cardinal): + super().__init__("electronic", token_orders={}) + letter = self.ALPHA + digit_words = pynini.cross("0", "zero") | cardinal.digits + symbol_pairs = load_pairs("italian/data/electronic/symbols.tsv") + symbol_words = preferred_map("italian/data/electronic/symbols.tsv") + symbols = pynini.union(*[pynini.escape(key) for key, _ in symbol_pairs]).optimize() + symbols_no_dot = pynini.difference(symbols, ".").optimize() + servers = pynini.union(*load_words("italian/data/electronic/server_name.tsv")).optimize() + domains = preferred_map("italian/data/electronic/domain.tsv") + + alnum = letter | self.DIGIT + accepted = (alnum | symbols_no_dot).plus + every = (alnum | symbols).plus + segment = "." + accepted + # A bare domain needs a letter in its last segment (``1.000`` is a number). + last = accepted @ (self.VSIGMA + letter + self.VSIGMA) + domain = accepted + segment.star + "." + last + url_domain = accepted + segment.plus + protocol = pynini.union("https://", "http://") + protocol = protocol | "www." | protocol + "www." + + space = insert(" ") + char = letter | digit_words | symbol_words + spelled = char + (space + char).star + word = add_weight(alnum.plus @ spelled, SPELL_WEIGHT) | servers + # Runs are maximal: a run is always followed by a symbol or the end. + unit = (word | domains) + (space + domains).star + item = symbol_words | unit + space + symbol_words + domain_verb = ((item + space).star + (item | unit)).optimize() + + terminal_dot = pynini.cross(".", " .").ques + domain_tag = self.tag_field("domain", (url_domain @ domain_verb) + terminal_dot) + url = self.tag_field("protocol", protocol) + insert(" ") + domain_tag + email = self.tag_field("username", every @ spelled) + delete("@") + insert(" ") + domain_tag + handle = delete("@") + self.tag_field("username", (accepted + segment.star) @ spelled) + bare = self.tag_field("domain", (domain @ domain_verb) + terminal_dot) + self.tagger = self.add_tokens(url | email | handle | bare) + + protocol_verb = self.verbalize_field("protocol", protocol @ spelled) + delete(" ") + insert(" ") + domain_field = self.verbalize_field("domain", (url_domain @ domain_verb) + terminal_dot) + username = self.verbalize_field("username", spelled) + verbalizer = protocol_verb.ques + domain_field + verbalizer |= username + delete(" ") + insert(" chiocciola ") + domain_field + verbalizer |= insert("chiocciola ") + username + self.verbalizer = self.delete_tokens(verbalizer) diff --git a/tn/italian/rules/measure.py b/tn/italian/rules/measure.py new file mode 100644 index 0000000..0eaedef --- /dev/null +++ b/tn/italian/rules/measure.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.italian.rules.utils import load_pairs, ranked_map +from tn.processor import Processor + + +def pluralize(name, suppletive): + """Applies the longest suppletive suffix, e.g. ``centimetro`` -> ``centimetri``. + + As in NeMo, units without a suppletive entry (``ampere``, ``percento``) + stay invariant. + """ + + for singular in sorted(suppletive, key=len, reverse=True): + if name.endswith(singular): + return name[:-len(singular)] + suppletive[singular] + return name + + +class Measure(Processor): + """Quantities with units, e.g. ``5 km/s`` -> ``cinque chilometri per secondo``. + + The tagger keeps the written number fields and the unit symbol (with an + optional ``/unit`` denominator); the verbalizer reads the number and + names the unit, singular after ``1`` and plural otherwise. + """ + + def __init__(self, cardinal, decimal): + super().__init__("measure", token_orders={}) + units = load_pairs("italian/data/measure/measurements.tsv") + suppletive = dict(load_pairs("italian/data/measure/suppletive.tsv")) + singular = ranked_map(units) + plural = ranked_map([(key, pluralize(name, suppletive)) for key, name in units]) + denominator = pynini.cross("/", " per ") + singular + self.singular_units = (singular + denominator.ques).optimize() + self.plural_units = (plural + denominator.ques).optimize() + units_graph = self.singular_units | self.plural_units + + sign = cardinal.sign + sign_tag = (self.tag_field("negative", sign) + insert(" ")).ques + sign_verb = (self.verbalize_field("negative", sign) + delete(" ") + insert(" ")).ques + integer_tag = self.tag_field("integer", cardinal.graph) + decimal_tag = self.tag_field("integer_part", decimal.integer) + delete(",") + insert(" ") + decimal_tag += self.tag_field("fractional_part", decimal.fractional) + units_tag = delete(" ").ques + insert(" ") + self.tag_field("units", units_graph) + self.tagger = self.add_tokens(sign_tag + (integer_tag | decimal_tag) + units_tag) + + one = self.verbalize_field("integer", pynini.cross("1", "un")) + many = self.verbalize_field("integer", (self.VSIGMA - "1") @ cardinal.graph) + fractional = self.verbalize_field("integer_part", decimal.integer) + insert(" virgola") + fractional += insert(" ") + self.verbalize_field("fractional_part", decimal.fractional, leading_space=True) + + def units_verb(graph): + return insert(" ") + self.verbalize_field("units", graph, leading_space=True) + + verbalizer = one + units_verb(self.singular_units) | (many | fractional) + units_verb(self.plural_units) + self.verbalizer = self.delete_tokens(sign_verb + verbalizer) diff --git a/tn/italian/rules/money.py b/tn/italian/rules/money.py new file mode 100644 index 0000000..873dbef --- /dev/null +++ b/tn/italian/rules/money.py @@ -0,0 +1,92 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.italian.rules.utils import load_pairs +from tn.processor import Processor + +# The money verbalizer reads the amount before naming the currency. +TOKEN_ORDER = {"money": ["integer_part", "fractional_part", "quantity", "currency"]} + + +class Money(Processor): + """Amounts with a currency, e.g. ``2,01 ₽`` -> ``due rubli un copeca``. + + The tagger keeps the written amount fields and the currency symbol in + either order (``€1`` or ``1 €``); the verbalizer names the major unit + after the integer and reads one- or two-digit fractions as minor units. + Longer fractions are read as decimals followed by the plural major unit. + """ + + def __init__(self, cardinal, decimal): + super().__init__("money", token_orders=TOKEN_ORDER) + majors = load_pairs("italian/data/money/currency_major.tsv") + minors = dict(load_pairs("italian/data/money/currency_minor.tsv")) + feminine = dict(load_pairs("italian/data/money/currency_plural_fem.tsv")) + plurals = dict(load_pairs("italian/data/money/currency_plural_masc.tsv")) + plurals.update(feminine) + + digit = self.DIGIT + currency = pynini.union(*[pynini.escape(symbol) for symbol, _ in majors]).optimize() + integer = cardinal.graph + cents = digit | digit + digit + quantity = decimal.quantity + + # Tagger: amount and currency in either order. + integer_tag = self.tag_field("integer_part", integer) + fraction_tag = delete(",") + insert(" ") + self.tag_field("fractional_part", digit.plus) + quantity_tag = delete(" ") + insert(" ") + self.tag_field("quantity", quantity) + amount_tag = integer_tag + fraction_tag.ques + quantity_tag.ques + currency_tag = self.tag_field("currency", currency) + space = delete(" ").ques + insert(" ") + self.tagger = self.add_tokens(amount_tag + space + currency_tag | currency_tag + space + amount_tag) + + # Verbalizer: one branch per currency, since the currency names both + # the major unit (after the integer) and the minor unit. + many = (self.VSIGMA - "1") @ integer + two = (digit + digit | digit + insert("0")) @ (pynini.closure(digit, 2, 2) - "00" - "01") + cent_value = two @ (cardinal.two_digits | delete("0") + cardinal.digits) + integer_verb = self.verbalize_field("integer_part", integer) + decimal_verb = integer_verb + insert(" virgola ") + delete(" ") + decimal_verb += self.verbalize_field("fractional_part", decimal.fractional) + quantity_verb = delete(" ") + insert(" ") + self.verbalize_field("quantity", quantity) + insert(" di") + branches = [] + for symbol, name in majors: + plural = plurals.get(name, name) + minor = minors[symbol] + one_major = pynini.cross("1", "una" if name in feminine else "un") + insert(" " + name) + major_graph = one_major | many + insert(" " + plural) + major = self.verbalize_field("integer_part", major_graph) + # ``0,50 €`` reads only the minor units. + nonzero_major = self.verbalize_field("integer_part", (self.VSIGMA - "0") @ major_graph) + minor_one = pynini.cross("01", "un") + insert(" " + minor) + # A one-digit fraction counts tens of minor units (``4,2`` -> twenty pence). + minor_many = cent_value + insert(" " + plurals.get(minor, minor)) + cents_verb = delete(" ") + self.verbalize_field("fractional_part", cents @ (minor_one | minor_many)) + no_cents = delete(" ") + self.verbalize_field("fractional_part", pynini.cross("00", "")) + zero_major = self.verbalize_field("integer_part", delete("0")) + amount = pynini.union( + major, + nonzero_major + insert(" ") + cents_verb, + major + no_cents, + zero_major + cents_verb, + add_weight(decimal_verb + insert(" " + plural), 0.01), + (integer_verb | decimal_verb) + quantity_verb + insert(" " + plural), + ) + symbol_verb = delete(" ") + self.verbalize_field("currency", pynini.cross(pynini.escape(symbol), "")) + branches.append(amount + symbol_verb) + self.verbalizer = self.delete_tokens(pynini.union(*branches).optimize()) diff --git a/tn/italian/rules/punctuation.py b/tn/italian/rules/punctuation.py new file mode 100644 index 0000000..a0f0145 --- /dev/null +++ b/tn/italian/rules/punctuation.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib.pynutil import add_weight + +from tn.processor import Processor + + +class Punctuation(Processor): + """ASCII punctuation split off a neighbouring token, e.g. ``(5)``.""" + + def __init__(self): + super().__init__("punctuation", token_orders={}) + graph = pynini.union(*[pynini.escape(mark) for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~"]).optimize() + space = add_weight(self.SPACE, -0.0001).star + self.tagger = self.add_tokens(self.tag_field("value", space + graph + space)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.SPACE.star + graph + self.SPACE.star)) diff --git a/tn/italian/rules/time.py b/tn/italian/rules/time.py new file mode 100644 index 0000000..56d33df --- /dev/null +++ b/tn/italian/rules/time.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.processor import Processor + +# Penalty for the idiomatic ``un quarto``/``mezza`` readings, kept for n-best. +IDIOM_WEIGHT = 0.01 + + +class Time(Processor): + """Clock times, e.g. ``17:15:26`` -> ``diciassette e quindici minuti e ventisei secondi``. + + The tagger keeps the written hour, minute and second digits; the + verbalizer reads them, dropping ``:00`` minutes when there are no + seconds. + """ + + def __init__(self, cardinal): + super().__init__("time", token_orders={}) + digit = self.DIGIT + number = pynini.cross("0", "zero") | cardinal.digits | cardinal.two_digits + hour_digits = pynini.union(*[str(hour) for hour in range(25)]) + hours = ((delete("0") + digit | (digit - "0") + digit | digit) @ hour_digits @ number).optimize() + minute_value = (delete("0") + digit | pynini.union(*"12345") + digit) @ number + two = pynini.closure(digit, 2, 2) + minutes = pynini.cross("01", "un minuto") | (two - "01") @ minute_value + insert(" minuti") + minutes |= add_weight(pynini.cross("15", "un quarto") | pynini.cross("30", "mezza"), IDIOM_WEIGHT) + minutes = minutes.optimize() + seconds = pynini.cross("01", "un secondo") | (two - "01") @ minute_value + insert(" secondi") + seconds = seconds.optimize() + + tagger = self.tag_field("hours", hours) + delete(":") + insert(" ") + tagger += self.tag_field("minutes", minutes) + tagger += (delete(":") + insert(" ") + self.tag_field("seconds", seconds)).ques + self.tagger = self.add_tokens(tagger) + + hours_verb = self.verbalize_field("hours", hours) + + def and_field(name, graph): + return insert(" e ") + self.verbalize_field(name, graph, leading_space=True) + + no_minutes = delete(" ") + self.verbalize_field("minutes", pynini.cross("00", "")) + minutes_only = and_field("minutes", (two - "00") @ minutes) + with_seconds = and_field("minutes", minutes) + and_field("seconds", seconds) + verbalizer = hours_verb + (no_minutes | minutes_only | with_seconds) + self.verbalizer = self.delete_tokens(verbalizer) diff --git a/tn/italian/rules/utils.py b/tn/italian/rules/utils.py new file mode 100644 index 0000000..3e22ea5 --- /dev/null +++ b/tn/italian/rules/utils.py @@ -0,0 +1,74 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight + +from tn.utils import get_abs_path + +# Penalty per rank among the readings of one written key. +RANK_WEIGHT = 0.01 + + +def load_pairs(rel_path, invert=False): + """Loads ``keyvalue`` rows of an Italian data table in file order. + + Extra columns (e.g. NeMo arc weights) are ignored. + """ + + pairs = [] + with open(get_abs_path(rel_path), encoding="utf-8") as stream: + for line in stream: + line = line.rstrip("\r\n") + if not line: + continue + key, value = line.split("\t")[:2] + pairs.append((value, key) if invert else (key, value)) + return pairs + + +def load_words(rel_path): + """Loads the first column of an Italian data table.""" + + words = [] + with open(get_abs_path(rel_path), encoding="utf-8") as stream: + for line in stream: + line = line.rstrip("\r\n") + if line: + words.append(line.split("\t")[0]) + return words + + +def preferred_map(rel_path, invert=False): + """Builds a weighted map from a table whose keys may repeat. + + The shortest reading wins (file order breaks ties); the others stay + available as weighted alternatives for n-best output. + """ + + return ranked_map(load_pairs(rel_path, invert)) + + +def ranked_map(pairs): + readings = {} + for index, (key, value) in enumerate(pairs): + candidates = readings.setdefault(key, []) + if value not in [candidate for _, _, candidate in candidates]: + candidates.append((len(value), index, value)) + arcs = [] + for key, candidates in readings.items(): + for rank, (_, _, value) in enumerate(sorted(candidates)): + arc = pynini.cross(pynini.escape(key), pynini.escape(value)) + arcs.append(add_weight(arc, rank * RANK_WEIGHT) if rank else arc) + return pynini.union(*arcs).optimize() diff --git a/tn/italian/rules/whitelist.py b/tn/italian/rules/whitelist.py new file mode 100644 index 0000000..970b755 --- /dev/null +++ b/tn/italian/rules/whitelist.py @@ -0,0 +1,27 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tn.italian.rules.utils import preferred_map +from tn.processor import Processor + + +class WhiteList(Processor): + """Abbreviations, e.g. ``sig.na`` -> ``signorina``.""" + + def __init__(self): + super().__init__("whitelist", token_orders={}) + graph = preferred_map("italian/data/whitelist/whitelist.tsv") + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/tn/italian/rules/word.py b/tn/italian/rules/word.py new file mode 100644 index 0000000..87ceb82 --- /dev/null +++ b/tn/italian/rules/word.py @@ -0,0 +1,28 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pynini.lib.pynutil import delete + +from tn.processor import Processor + + +class Word(Processor): + """Any other non-space run, kept verbatim.""" + + def __init__(self): + super().__init__("word", token_orders={}) + self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus)) + graph = self.NOT_SPACE.plus | delete(self.SPACE.plus) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/tn/italian/test/__init__.py b/tn/italian/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tn/italian/test/data/test_cases_cardinal.txt b/tn/italian/test/data/test_cases_cardinal.txt new file mode 100644 index 0000000..795ec89 --- /dev/null +++ b/tn/italian/test/data/test_cases_cardinal.txt @@ -0,0 +1,13 @@ +2~due +3~tre +4~quattro +5~cinque +15~quindici +155~cento cinquantacinque +369~trecento sessantanove +3544~tremila cinquecento quarantaquattro +1000~mille +1034500~un milione trentaquattromila cinquecento +3453243534~tremila quattrocento cinquantatre milioni duecento quarantatremila cinquecento trentaquattro +38~trentotto +7 giorni sono una settimana~sette giorni sono una settimana \ No newline at end of file diff --git a/tn/italian/test/data/test_cases_decimal.txt b/tn/italian/test/data/test_cases_decimal.txt new file mode 100644 index 0000000..6ae9425 --- /dev/null +++ b/tn/italian/test/data/test_cases_decimal.txt @@ -0,0 +1,4 @@ +33,32~trentatre virgola trentadue +5,342~cinque virgola trecento quarantadue +4,56 milioni~quattro virgola cinquantasei milioni +3,55 miliardi~tre virgola cinquantacinque miliardi \ No newline at end of file diff --git a/tn/italian/test/data/test_cases_electronic.txt b/tn/italian/test/data/test_cases_electronic.txt new file mode 100644 index 0000000..cf011e9 --- /dev/null +++ b/tn/italian/test/data/test_cases_electronic.txt @@ -0,0 +1,13 @@ +Trovateci su mail.university.edu.~Trovateci su m a i l punto u n i v e r s i t y punto edu . +Trovateci su mail.nasa.gov.~Trovateci su m a i l punto n a s a punto gov . +Trovateci su mail.nasa.mx.~Trovateci su m a i l punto n a s a punto m x . +https://www.nvidia.com~h t t p s due punti slash slash w w w punto nvidia punto com +https://www.nvidia.com/abc/ciao.html~h t t p s due punti slash slash w w w punto nvidia punto com slash a b c slash c i a o punto html +https://www.nvidia.it/abc/12df/like.py~h t t p s due punti slash slash w w w punto nvidia punto IT slash a b c slash uno due d f slash l i k e punto python +abc.def@23df.sd.IT~a b c punto d e f chiocciola due tre d f punto s d punto I T +abc.de!f@23d_f.sd.us~a b c punto d e punto esclamativo f chiocciola due tre d trattino basso f punto s d punto US +www.nvidia.file&TN.com~w w w punto nvidia punto f i l e e commerciale T N punto com +@jensen~chiocciola j e n s e n +@jensen.me~chiocciola j e n s e n punto m e +@wezyr1986~chiocciola w e z y r uno nove otto sei +visitate il sito https://www.nvidia.com.~visitate il sito h t t p s due punti slash slash w w w punto nvidia punto com . diff --git a/tn/italian/test/data/test_cases_measure.txt b/tn/italian/test/data/test_cases_measure.txt new file mode 100644 index 0000000..aa1e945 --- /dev/null +++ b/tn/italian/test/data/test_cases_measure.txt @@ -0,0 +1,7 @@ +1 m~un metro +2 m~due metri +4,3 l~quattro virgola tre litri +5 km/s~cinque chilometri per secondo +15 A~quindici ampere +155 d~cento cinquantacinque giorni +il 18% delle emissioni di carbonio~il diciotto percento delle emissioni di carbonio \ No newline at end of file diff --git a/tn/italian/test/data/test_cases_money.txt b/tn/italian/test/data/test_cases_money.txt new file mode 100644 index 0000000..f800465 --- /dev/null +++ b/tn/italian/test/data/test_cases_money.txt @@ -0,0 +1,5 @@ +2,01 ₽~due rubli un copeca +3,23€~tre euro ventitre centesimi +4,2 £~quattro sterline venti penny +1 eur~un euro +1 eur per il caffè~un euro per il caffè \ No newline at end of file diff --git a/tn/italian/test/data/test_cases_time.txt b/tn/italian/test/data/test_cases_time.txt new file mode 100644 index 0000000..cc8e766 --- /dev/null +++ b/tn/italian/test/data/test_cases_time.txt @@ -0,0 +1,6 @@ +12:30~dodici e trenta minuti~dodici e mezza +05:15~cinque e quindici minuti~cinque e un quarto +17:15:26~diciassette e quindici minuti e ventisei secondi~diciassette e un quarto e ventisei secondi +23:45~ventitre e quarantacinque minuti +03:38~tre e trentotto minuti +l'evento inizia alle 16:00~l'evento inizia alle sedici \ No newline at end of file diff --git a/tn/italian/test/data/test_cases_whitelist.txt b/tn/italian/test/data/test_cases_whitelist.txt new file mode 100644 index 0000000..0fa7bd5 --- /dev/null +++ b/tn/italian/test/data/test_cases_whitelist.txt @@ -0,0 +1,4 @@ +avv.~avvocato +dip. di Fisica~dipartimento di Fisica +sig.na Sonia~signorina Sonia +p.za Grande~piazza Grande \ No newline at end of file diff --git a/tn/italian/test/normalizer_test.py b/tn/italian/test/normalizer_test.py new file mode 100644 index 0000000..44f31c3 --- /dev/null +++ b/tn/italian/test/normalizer_test.py @@ -0,0 +1,209 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path + +import pytest + +from tn.italian.normalizer import Normalizer + +PACKAGE = Path(__file__).resolve().parent.parent + + +def original_cases(): + """Reads ``written~spoken[~spoken...]`` lines; any listed reading is correct.""" + + cases = [] + for path in sorted((Path(__file__).parent / "data").glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for index, line in enumerate(stream, 1): + if line.endswith("\n"): + line = line[:-1] + written, *spoken = line.split("~") + cases.append(pytest.param(written, tuple(spoken), id="{}-{}".format(path.stem, index))) + return cases + + +CASES = original_cases() + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(Normalizer) + + +def test_original_fixture_count(): + assert len(CASES) == 52 + # Files without a trailing newline still count their last line. + assert CASES[12].values == ("7 giorni sono una settimana", ("sette giorni sono una settimana",)) + # Time fixtures list alternative readings. + assert CASES[42].values == ("12:30", ("dodici e trenta minuti", "dodici e mezza")) + + +@pytest.mark.parametrize("written, spoken", CASES) +def test_original(normalizer, written, spoken): + assert normalizer.normalize(written) in spoken + + +@pytest.mark.parametrize("written, spoken", CASES) +def test_original_mapping(normalizer, written, spoken): + result = normalizer.normalize_with_mapping(written, include_identity=True) + assert result.output_text in spoken + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for param in CASES: + written, _ = param.values + assert normalizer.verbalize(normalizer.tag(written)) == normalizer.normalize(written) + + +@pytest.mark.parametrize("written, kind, members, spoken", [ + ("155", "cardinal", {"integer": "155"}, "cento cinquantacinque"), + ("1.000.000", "cardinal", {"integer": "1.000.000"}, "un milione"), + ("-21", "cardinal", {"negative": "-", "integer": "21"}, "meno ventuno"), + ("5,342", "decimal", {"integer_part": "5", "fractional_part": "342"}, "cinque virgola trecento quarantadue"), + ("3,55 miliardi", "decimal", {"integer_part": "3", "fractional_part": "55", "quantity": "miliardi"}, + "tre virgola cinquantacinque miliardi"), + ("5 km/s", "measure", {"integer": "5", "units": "km/s"}, "cinque chilometri per secondo"), + ("1 m", "measure", {"integer": "1", "units": "m"}, "un metro"), + ("4,3 l", "measure", {"integer_part": "4", "fractional_part": "3", "units": "l"}, "quattro virgola tre litri"), + ("18%", "measure", {"integer": "18", "units": "%"}, "diciotto percento"), + ("2,01 ₽", "money", {"integer_part": "2", "fractional_part": "01", "currency": "₽"}, "due rubli un copeca"), + ("€1", "money", {"currency": "€", "integer_part": "1"}, "un euro"), + ("1 £", "money", {"integer_part": "1", "currency": "£"}, "una sterlina"), + ("0,50 $", "money", {"integer_part": "0", "fractional_part": "50", "currency": "$"}, "cinquanta centesimi"), + ("2 milioni €", "money", {"integer_part": "2", "quantity": "milioni", "currency": "€"}, + "due milioni di euro"), + ("17:15:26", "time", {"hours": "17", "minutes": "15", "seconds": "26"}, + "diciassette e quindici minuti e ventisei secondi"), + ("16:00", "time", {"hours": "16", "minutes": "00"}, "sedici"), + ("05:01", "time", {"hours": "05", "minutes": "01"}, "cinque e un minuto"), + ("https://www.nvidia.com", "electronic", {"protocol": "https://www.", "domain": "nvidia.com"}, + "h t t p s due punti slash slash w w w punto nvidia punto com"), + ("abc.def@23df.sd.IT", "electronic", {"username": "abc.def", "domain": "23df.sd.IT"}, + "a b c punto d e f chiocciola due tre d f punto s d punto I T"), + ("@jensen", "electronic", {"username": "jensen"}, "chiocciola j e n s e n"), + ("mail.nasa.gov", "electronic", {"domain": "mail.nasa.gov"}, "m a i l punto n a s a punto gov"), + ("sig.na", "whitelist", {"value": "sig.na"}, "signorina"), + ("caffè", "word", {"value": "caffè"}, "caffè"), +]) +def test_raw_fields(normalizer, written, kind, members, spoken): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == spoken + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, spoken", [ + ("(5)", "(cinque)"), + ("21000", "ventunomila"), + ("1001", "mille uno"), + ("1000000000000", "un miliardo"), + ("3,00 €", "tre euro"), + ("£4,20", "quattro sterline venti penny"), + ("2,5 kg", "due virgola cinque chilogrammi"), + ("10 cm", "dieci centimetri"), + ("1.000", "mille"), + ("Dott. Rossi", "dottor Rossi"), + ("mario.rossi@gmail.com", "m a r i o punto r o s s i chiocciola gmail punto com"), +]) +def test_additional_semantics(normalizer, written, spoken): + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, spoken, fields", [ + ("(5)", "(cinque)", [("punctuation", {"value": "("}), ("cardinal", {"integer": "5"}), + ("punctuation", {"value": ")"})]), + ("5,", "cinque,", [("cardinal", {"integer": "5"}), ("punctuation", {"value": ","})]), + ("5 !", "cinque !", [("cardinal", {"integer": "5"}), ("punctuation", {"value": " !"})]), + ("( 5 )", "( cinque )", [("punctuation", {"value": "( "}), ("cardinal", {"integer": "5"}), + ("punctuation", {"value": " )"})]), +]) +def test_punctuation_spacing(normalizer, written, spoken, fields): + assert normalizer.normalize(written) == spoken + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert [(token.name, token.members) for token in parser.tokens] == fields + assert normalizer.verbalize(tagged) == spoken + outputs = normalizer.normalize(written, nbest=3) + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert outputs[0] == spoken + assert len(outputs) == len(set(outputs)) + assert [result.output_text for result in mapped] == outputs + for result in mapped: + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_idiomatic_times_are_alternatives(normalizer): + assert normalizer.normalize("12:30", nbest=2) == ["dodici e trenta minuti", "dodici e mezza"] + assert normalizer.normalize("05:15", nbest=2) == ["cinque e quindici minuti", "cinque e un quarto"] + + +@pytest.mark.parametrize("written", ["12:30", "2,01 ₽", "5 km/s", "https://www.nvidia.com", "sig.na"]) +def test_nbest_mapping_synchronization(normalizer, written): + outputs = normalizer.normalize(written, nbest=3) + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(written) + for result in mapped: + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + written = 'perché\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(written)) + assert [token.members for token in parser.tokens] == [{"value": written}] + assert normalizer.normalize(written) == written + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("written", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, written): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": written} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(written, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) + + +def test_standalone_from_itn(): + itn_import = re.compile(r"^\s*(from|import)\s+itn\.italian\b", re.MULTILINE) + for path in PACKAGE.rglob("*.py"): + assert not itn_import.search(path.read_text(encoding="utf-8")), path diff --git a/tn/test/cli_test.py b/tn/test/cli_test.py index f77994e..dddc571 100644 --- a/tn/test/cli_test.py +++ b/tn/test/cli_test.py @@ -60,6 +60,7 @@ def verbalize(self, tagged): ("tn", "sv", "tn.swedish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "es", "tn.spanish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "hi", "tn.hindi.normalizer", "Normalizer", [], {}), + ("tn", "it", "tn.italian.normalizer", "Normalizer", [], {}), ("tn", "mr", "tn.marathi.normalizer", "Normalizer", [], {}), ("tn", "rw", "tn.kinyarwanda.normalizer", "Normalizer", [], {}), ( @@ -96,6 +97,7 @@ def verbalize(self, tagged): ("itn", "sv", "itn.swedish.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "es", "itn.spanish.inverse_normalizer", "InverseNormalizer", ["--cased"], {"cased": True}), ("itn", "hi", "itn.hindi.inverse_normalizer", "InverseNormalizer", [], {}), + ("itn", "it", "itn.italian.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "hi_en", "itn.hindi_english.inverse_normalizer", "InverseNormalizer", ["--cased"], {"cased": True}), ("itn", "mr", "itn.marathi.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "rw", "itn.kinyarwanda.inverse_normalizer", "InverseNormalizer", [], {}), @@ -228,6 +230,8 @@ def test_itn_only_languages_are_rejected_for_tn(language): ("itn", "es", "--cased", "--enable-million"), ("tn", "hi", "--cache-dir", "--deterministic"), ("itn", "hi", "--cache-dir", "--enable-million"), + ("tn", "it", "--cache-dir", "--deterministic"), + ("itn", "it", "--cache-dir", "--enable-million"), ("itn", "hi_en", "--cased", "--enable-million"), ("tn", "mr", "--cache-dir", "--deterministic"), ("itn", "mr", "--cache-dir", "--enable-million"),