From 733c29eec22d11d14dd70a70bf6cc301578bfa95 Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Sat, 31 Oct 2026 10:07:21 +0800 Subject: [PATCH 1/3] wip: feat: add Spanish-English code-mixed ITN with raw-field tagging --- itn/spanish_english/__init__.py | 0 itn/spanish_english/data/en_whitelist.tsv | 416 ++++++++++++++++++ itn/spanish_english/data/es_whitelist.tsv | 16 + itn/spanish_english/inverse_normalizer.py | 107 +++++ itn/spanish_english/rules/__init__.py | 0 itn/spanish_english/rules/prefixed.py | 34 ++ itn/spanish_english/rules/time.py | 39 ++ itn/spanish_english/rules/whitelist.py | 38 ++ itn/spanish_english/test/__init__.py | 0 .../test/data/test_cases_cardinal.txt | 80 ++++ .../test/data/test_cases_cardinal_cased.txt | 60 +++ .../test/data/test_cases_date.txt | 44 ++ .../test/data/test_cases_date_cased.txt | 20 + .../test/data/test_cases_decimal.txt | 92 ++++ .../test/data/test_cases_decimal_cased.txt | 19 + .../test/data/test_cases_electronic.txt | 41 ++ .../test/data/test_cases_electronic_cased.txt | 14 + .../test/data/test_cases_fraction.txt | 12 + .../test/data/test_cases_measure.txt | 132 ++++++ .../test/data/test_cases_measure_cased.txt | 22 + .../test/data/test_cases_money.txt | 76 ++++ .../test/data/test_cases_money_cased.txt | 16 + .../test/data/test_cases_ordinal.txt | 64 +++ .../test/data/test_cases_ordinal_cased.txt | 21 + .../test/data/test_cases_telephone.txt | 32 ++ .../test/data/test_cases_telephone_cased.txt | 19 + .../test/data/test_cases_time.txt | 54 +++ .../test/data/test_cases_time_cased.txt | 19 + .../test/data/test_cases_whitelist.txt | 18 + .../test/data/test_cases_word.txt | 104 +++++ .../test/data/test_cases_word_cased.txt | 17 + itn/spanish_english/test/normalizer_test.py | 166 +++++++ 32 files changed, 1792 insertions(+) create mode 100644 itn/spanish_english/__init__.py create mode 100644 itn/spanish_english/data/en_whitelist.tsv create mode 100644 itn/spanish_english/data/es_whitelist.tsv create mode 100644 itn/spanish_english/inverse_normalizer.py create mode 100644 itn/spanish_english/rules/__init__.py create mode 100644 itn/spanish_english/rules/prefixed.py create mode 100644 itn/spanish_english/rules/time.py create mode 100644 itn/spanish_english/rules/whitelist.py create mode 100644 itn/spanish_english/test/__init__.py create mode 100644 itn/spanish_english/test/data/test_cases_cardinal.txt create mode 100644 itn/spanish_english/test/data/test_cases_cardinal_cased.txt create mode 100644 itn/spanish_english/test/data/test_cases_date.txt create mode 100644 itn/spanish_english/test/data/test_cases_date_cased.txt create mode 100644 itn/spanish_english/test/data/test_cases_decimal.txt create mode 100644 itn/spanish_english/test/data/test_cases_decimal_cased.txt create mode 100644 itn/spanish_english/test/data/test_cases_electronic.txt create mode 100644 itn/spanish_english/test/data/test_cases_electronic_cased.txt create mode 100644 itn/spanish_english/test/data/test_cases_fraction.txt create mode 100644 itn/spanish_english/test/data/test_cases_measure.txt create mode 100644 itn/spanish_english/test/data/test_cases_measure_cased.txt create mode 100644 itn/spanish_english/test/data/test_cases_money.txt create mode 100644 itn/spanish_english/test/data/test_cases_money_cased.txt create mode 100644 itn/spanish_english/test/data/test_cases_ordinal.txt create mode 100644 itn/spanish_english/test/data/test_cases_ordinal_cased.txt create mode 100644 itn/spanish_english/test/data/test_cases_telephone.txt create mode 100644 itn/spanish_english/test/data/test_cases_telephone_cased.txt create mode 100644 itn/spanish_english/test/data/test_cases_time.txt create mode 100644 itn/spanish_english/test/data/test_cases_time_cased.txt create mode 100644 itn/spanish_english/test/data/test_cases_whitelist.txt create mode 100644 itn/spanish_english/test/data/test_cases_word.txt create mode 100644 itn/spanish_english/test/data/test_cases_word_cased.txt create mode 100644 itn/spanish_english/test/normalizer_test.py diff --git a/itn/spanish_english/__init__.py b/itn/spanish_english/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/spanish_english/data/en_whitelist.tsv b/itn/spanish_english/data/en_whitelist.tsv new file mode 100644 index 00000000..78424367 --- /dev/null +++ b/itn/spanish_english/data/en_whitelist.tsv @@ -0,0 +1,416 @@ +10K ten k +1D one d +1G one g +1K one k +2.5G two point five g +2D two d +2G two g +2K two k +3D three d +3G three g +4D four d +4G four g +4K four k +5D five d +5G five g +6G six g +7-eleven seven eleven +7G seven g +8K eight k +AAA triple a +AC a c +ACL a c l +ADAS a das +AEB a e b +AES a e s +AGX a g x +AI a i +AI ai +AI ais +ALU a l u +AMD a m d +API a p i +API api +ARP a r p +ASAP a s a p +ASAP a sap +ASMC a s m c +ASR a s r +ASUS a sus +ASUS asus +AT&T a t and t +ATM a t m +AV1 a v one +AV1 av one +AVAS a v a s +AVAS a vas +AVRCP a v r c p +AWS a w s +AWS aws +AlphaFold alpha fold +AstraZeneca astra zeneca +AstraZeneca astrazeneca +Audio2Face audio to face +Audio2Face audio too face +BBC b b c +BCP b c p +BEV b e v +BGA b g a +BGP b g p +BIOS Bios +BIOS bios +BIS b i s +BSP b s p +BasePOD base pod +BasePOD basepod +Billion-X billion X +C# c sharp +C++ c plus plus +CAT5 cat five +CAT5e cat five e +CAT6 cat six +CAT6A cat six a +CAT7 cat seven +CAT8 cat eight +CBC c b c +CCS c c s +CCTV c c t v +CCTV c c tv +CDN c d n +CEO c e o +CES c e s +CFO c f o +CHAdeMO cha demo +CHAdeMO chai demo +CLI c l i +CMO c m o +CMS c m s +CNN c n n +COVID covid +COVID-19 covid nineteen +CPU c p u +CSO c s o +CSP c s p +CSS c s s +CV c v +CVT c v t +CalTech cal tech +CalTech caltech +CapEx cap ex +CapEx capex +ChatGPT chat g p t +CloudXR cloud x r +ConnectX connect x +Core 2 core two +DALI d a l i +DB d b +DC d c +DCDB d c d b +DDR d d r +DDoS d d o s +DDoS d dos +DGL d g l +DGX d g x -1.1 +DGX A100 d g x a hundred -1.1 +DGX A100 d g x a one hundred -1.1 +DGX Station d g x station -1.1 +DHCP d h c p +DLSS d l s s +DLSS 2 d l s s two +DLSS 3 d l s s three +DNA d n a +DNN d n n +DNS d n s +DP d p +DPDT d p d t +DPU d p u +DPX d p x +DeepMind deep mind +DeepMind deepmind +DeepStream deep stream +DeepStream deepstream +DevOps dev ops +Direct3D direct three d +DisplayPort display port +EBC e b c +EBS e b s +EC2 e c two +ECC e c c +ECS e c s +EFS e f s +EGX e g x +EKS e k s +ELB e l b +ELCB e l c b +EMR e m r +EOL e o l +EV e v +Earth-2 earth two +FIFO fee foe +FIFO fifo +FOMO foe moe +FOMO fomo +FPS f p s +FSA f s a +FST f s t +FTP f t p +FTPS f t p s +FX f x +G-SYNC g sync +GCC g c c +GCP g c p +GCS g c s +GFN G f n +GPU g p u +GRPC g r p c +GRU g r u +GSK g s k +GTC g t c +GTX g t x -1.1 +GUI g u i +GUI gui +GeForce geforce +GeForce NOW geforce now +H.264 H two six four +H.264 H two sixty four +H.265 H two six five +H.265 H two sixty five +H100 H one hundred +HDMI h d m i +HEVC h e v c +HGX h g x +HMD Hmd +HPC h p c +HQ h q +HSM h s m +HT h t +HTML h t m l +HTTP h t t p +HTTPS h t t p s +HuggingFace hugging face +I/O i o +ICMP i c m p +IEC i e c +IEEE i triple e +IGMP i g m p +IGX i g x +IISc i i s c +IIT i i t +INT8 int eight +IP i p +IPSec i p sec +ISC i s c +ISI i s i +InfiniBand inifiband +IoT i o t +Isaac SIM isaac sim +JSON j son +JavaScript java script +JavaScript javascript +KVM k v m +LFP lee fei po +LFP life po +LGA l g a -1.1 +LLC l l c +LLM l l m +LLMs l l ms +LLP l l p +LSTM l s t m +LT l t +LTT l t t +LinkedIn linked in +LinkedIn linkedin +MB m b +MCB m c b +MCCB m c c b +MCP m c p +MDM m d m +MFA m f a +MIT m i t +ML m l +MLPerf m l perf +MONAI monai +MWC m w c +MXNet m x net +MacBook mac book +MacBook macbook +MacBook Pro mac book pro +Max-Q max q +Million-X million x +MoCap mo cap +NC n c +NGC n g c +NLP n l p +NLU n l u +NMC n m c +NO n o +NTP n t p +NVDec n v dec +NVDec n v deck +NVEnc n v enc +NVEnc n v inc +NVEnc n v ink +NVMe n v m e +Ni-CD knee cad +NiMH n i m h +NoSQL no s q l +NoSQL no sql +Nvidia A100 nvidia a hundred +Nvidia A100 nvidia a one hundred +Nvidia A30 nvidia a thirty +Nvidia A40 nvidia a forty +OCI o c i +OEM o e m +OEMs o e ms +OLTC o l t c +OLTP o l t p +ONNX o n n x +OVX o v x +OpEx op ex +OpEx opex +OpenCL open c l +OpenGL open g l +PCI p c i +PCIe p c i e -1.1 +PDF p d f +PHEV p h e v +POP3 pop three +PSO p s o +Photoshop photo shop +PhysX fizz ex +PhysX fizz x +PhysX phys ex +PhysX phys x +PlayStation play station +Premiere Pro premiere pro +ProVis pro vis +ProVis provis +PyG pi g +PyG py g +PyTorch pi torch +QODA q o d a +QOS q o s +RCBO r c b o +RCCB r c c b +RDS r d s +RFP r f p +RISC risc +RL r l +RNN r n n +ROI r o i +RSS r s s +RT r t +RTP r t p +RTX r t x -1.1 +RTX ON r t x on +RTX Off r t x off +RacerX racer x +Ryzen 2 rye zen two +Ryzen 3 rye zen three +Ryzen 5 rye zen five +Ryzen 7 rye zen seven +Ryzen 9 rye zen nine +S&P s and p +S3 s three +SAE s a e +SDK s d k +SEO s e o +SFU s f u +SHIELD TV shield tv +SIGGRAPH sig graph +SIGGRAPH siggraph +SMPS s m p s +SMTP s m t p +SNMP s n m p +SNS s n s +SOL s o l +SPDT s p d t +SPN s p n +SPST s p s t +SQL s q l +SQS s q s +SSD s s d +SSH s s h +SSL s s l +SSO s s o +SSR s s r +SUV s u v +SWG s w g +SaaS saas +ServiceNow service now +SuperCloud super cloud +SuperPOD super pod +TCP t c p +TCP/IP t c p i p +TFTP t f t p +TJ t j +TOS t o s +TPM t p m +TPMS t p m s +TPN t p n +TSL t s l +TSMC t s m c +TTS t t s +Telnet tell net +TensorRT tensor r t +Tick-Tock tick tock +TikTok app tick tock app +TikTok is tick tock is +UDP u d p +UEFI u e f i +UFI u f i +UPS u p s +USB u s b +VAAPI v a a p i +VAAPI v a api +VDI v d i +VDPAU v d p a u +VDPAU v d paw +VDPAU v d pow +VFD v f d +VLAN v lan +VP v p +VPN v p n +Vcc v c c +Vdd v d d +VoIP v o i p +WFSA w f s a +WFST w f s t +Wi-Fi wi fi +Wi-Fi wifi +XFX x f x +XFX x fx +XML x m l +Zen 2 zen two +Zen 3 zen three +Zen 4 zen four +c# c sharp +c++ c plus plus +cuBLAS cue blah +cuBLAS cue blahs +cuBLAS q blahs +cuDNN c u d n n +cuDNN cue d n n +cuDNN q d n n +cuOpt cue opt +cuOpt q opt +cuQuantum cue quantum +cuQuantum q quantum +dr. doctor +e.g. for example +es3 e s three +ext4 e x t four +i3 i three +i5 i five +i7 i seven +i9 i nine +iOS ios +iTPMS i t p m s +mr. mister +mrs. misses +on TikTok on tick tock +sFTP s f t p +st. saint +x1 x one +x2 x two +x8 x eight diff --git a/itn/spanish_english/data/es_whitelist.tsv b/itn/spanish_english/data/es_whitelist.tsv new file mode 100644 index 00000000..cfa739c1 --- /dev/null +++ b/itn/spanish_english/data/es_whitelist.tsv @@ -0,0 +1,16 @@ +Ud. usted +Uds. ustedes +Vd. vosotros +Vds. vosotros +Dr. doctor +Dra. doctora +D. don +Da. doña +EE. UU. estados unidos +p.ej. por ejemplo +Prof. profesor +Profa. profesora +Sr. señor +Sra. señora +Srta. señorita +etc. etcétera \ No newline at end of file diff --git a/itn/spanish_english/inverse_normalizer.py b/itn/spanish_english/inverse_normalizer.py new file mode 100644 index 00000000..df6ef004 --- /dev/null +++ b/itn/spanish_english/inverse_normalizer.py @@ -0,0 +1,107 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from itn.english import inverse_normalizer as english +from itn.english.rules.cardinal import Cardinal as EnCardinal +from itn.english.rules.date import Date as EnDate +from itn.english.rules.decimal import Decimal as EnDecimal +from itn.english.rules.electronic import Electronic as EnElectronic +from itn.english.rules.measure import Measure as EnMeasure +from itn.english.rules.money import Money as EnMoney +from itn.english.rules.ordinal import Ordinal as EnOrdinal +from itn.english.rules.telephone import Telephone as EnTelephone +from itn.english.rules.time import Time as EnTime +from itn.spanish import inverse_normalizer as spanish +from itn.spanish.rules.cardinal import Cardinal +from itn.spanish.rules.date import Date +from itn.spanish.rules.decimal import Decimal +from itn.spanish.rules.electronic import Electronic +from itn.spanish.rules.fraction import Fraction +from itn.spanish.rules.measure import Measure +from itn.spanish.rules.money import Money +from itn.spanish.rules.ordinal import Ordinal +from itn.spanish.rules.punctuation import Punctuation +from itn.spanish.rules.telephone import Telephone +from itn.spanish.rules.word import Word +from itn.spanish_english.rules.prefixed import Prefixed +from itn.spanish_english.rules.time import Time +from itn.spanish_english.rules.whitelist import Whitelist +from tn.processor import Processor, RuleSpec + +PREFIX = "en_" +TOKEN_ORDERS = dict(spanish.TOKEN_ORDERS) +TOKEN_ORDERS.update({PREFIX + name: order for name, order in english.TOKEN_ORDERS.items()}) + + +class InverseNormalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False, cased=False): + """``cased=True`` also accepts capitalized spoken words ("Doce Dólares").""" + super().__init__(name="es_en_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS) + self.cased = cased + self.build_fst("es_en_itn", cache_dir, overwrite_cache, {"cased": cased}) + + def build_tagger_and_verbalizer(self): + cased = self.cased + cardinal = Cardinal(cased) + ordinal = Ordinal(cardinal, cased) + decimal = Decimal(cardinal, cased) + fraction = Fraction(cardinal, ordinal, cased) + + en_cardinal = EnCardinal() + en_ordinal = EnOrdinal(cardinal=en_cardinal) + en_decimal = EnDecimal(cardinal=en_cardinal) + + def en(rule): + return Prefixed(rule, PREFIX) + + punctuation = Punctuation() + word = Word() + rules = ( + # "doctor" is on both lists; the code-mixed grammar writes "dr.". + RuleSpec(Whitelist("whitelist", "es_whitelist.tsv"), 1.011), + RuleSpec(Whitelist(PREFIX + "whitelist", "en_whitelist.tsv"), 1.01), + RuleSpec(Time(cased), 1.08), + RuleSpec(en(EnTime(cardinal=en_cardinal)), 1.1), + RuleSpec(Date(cardinal, cased), 1.09), + RuleSpec(en(EnDate(cardinal=en_cardinal, ordinal=en_ordinal)), 1.09), + RuleSpec(decimal, 1.09), + RuleSpec(en(en_decimal), 1.09), + RuleSpec(fraction, 1.09), + RuleSpec(Measure(cardinal, decimal, fraction, cased), 1.6), + RuleSpec(en(EnMeasure(cardinal=en_cardinal, decimal=en_decimal)), 1.1), + RuleSpec(cardinal, 1.6), + RuleSpec(en(en_cardinal), 1.1), + RuleSpec(ordinal, 1.6), + RuleSpec(en(en_ordinal), 1.09), + RuleSpec(Money(cardinal, decimal, cased), 1.6), + RuleSpec(en(EnMoney(cardinal=en_cardinal, decimal=en_decimal)), 1.1), + RuleSpec(Telephone(cased), 1.6), + RuleSpec(en(EnTelephone(cardinal=en_cardinal)), 1.1), + RuleSpec(Electronic(cased), 2.96), + RuleSpec(en(EnElectronic()), 1.1), + RuleSpec(word, 100), + RuleSpec(punctuation), + ) + classify = self.tagger_union(rules) + # Punctuation may stick to either side of a token ("veinte!"). + punct = pynutil.add_weight(punctuation.tagger, 1.1).star + token = punct + classify + punct + self.tagger = self.DELETE_SPACE + token + (pynutil.delete(self.SPACE.plus) + token).star + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + verbalize = self.verbalizer_union(rules) + self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star diff --git a/itn/spanish_english/rules/__init__.py b/itn/spanish_english/rules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/spanish_english/rules/prefixed.py b/itn/spanish_english/rules/prefixed.py new file mode 100644 index 00000000..fe6ab968 --- /dev/null +++ b/itn/spanish_english/rules/prefixed.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from tn.processor import Processor + + +class Prefixed(Processor): + """Re-exposes an existing rule under a language-prefixed token name. + + Spanish and English rules share token names ("cardinal", "money", ...) + but not field layouts or verbalizers, so the English rules are renamed + ("en_cardinal") to keep each raw tag bound to its own verbalizer and + field order. Only the leading token name is rewritten; the raw fields + produced by the wrapped tagger are untouched. + """ + + def __init__(self, rule, prefix="en_"): + super().__init__(prefix + rule.name, ordertype="itn") + rename = pynini.cross(rule.name + " {", self.name + " {") + self.VSIGMA + self.tagger = (rule.tagger @ rename).optimize() + self.verbalizer = (pynini.invert(rename) @ rule.verbalizer).optimize() diff --git a/itn/spanish_english/rules/time.py b/itn/spanish_english/rules/time.py new file mode 100644 index 00000000..d018510a --- /dev/null +++ b/itn/spanish_english/rules/time.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from itn.spanish.rules.time import Time as SpanishTime + + +class Time(SpanishTime): + """Spanish clock times written the code-mixed way. + + The code-mixed grammar writes an article-less time ("dos y media de la + tarde") with the English two-digit hour ("02:30 p.m."); times with an + article keep the Spanish form ("las 2:00 p.m."). The raw tag is the + Spanish one; only the verbalizer output is padded. + """ + + def __init__(self, cased_input=False): + super().__init__(cased_input) + digit = self.DIGIT + pad = pynini.union( + pynutil.insert("0") + digit + ":", + digit + digit + ":", + pynini.difference(self.VCHAR, digit), + ) + self.VSIGMA + self.verbalizer = (self.verbalizer @ pad.optimize()).optimize() diff --git a/itn/spanish_english/rules/whitelist.py b/itn/spanish_english/rules/whitelist.py new file mode 100644 index 00000000..38beb5d2 --- /dev/null +++ b/itn/spanish_english/rules/whitelist.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from itn.utils import get_abs_path, load_labels +from tn.processor import Processor + + +class Whitelist(Processor): + """Raw whitelisted phrases ("usted"); V writes the listed form ("Ud."). + + The code-mixed grammar carries its own Spanish and English lists: the + English one drops "DOS -> dos" so the Spanish number "dos" survives. + """ + + def __init__(self, name, tsv): + super().__init__(name, ordertype="itn") + # Spoken form -> written form; the first written form of a phrase wins. + # An optional third column is a path weight, as in a pynini string file. + mapping = {} + for written, spoken, *weight in load_labels(get_abs_path("spanish_english/data/" + tsv)): + mapping.setdefault(spoken, (spoken, written, *weight)) + self.graph = pynini.string_map([mapping[spoken] for spoken in sorted(mapping)]).optimize() + self.tagger = self.add_tokens(self.tag_field("value", self.graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.graph)) diff --git a/itn/spanish_english/test/__init__.py b/itn/spanish_english/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/spanish_english/test/data/test_cases_cardinal.txt b/itn/spanish_english/test/data/test_cases_cardinal.txt new file mode 100644 index 00000000..525827dd --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_cardinal.txt @@ -0,0 +1,80 @@ +doscientos cincuenta y uno~251 +novecientos noventa y nueve millones novecientos noventa y nueve mil novecientos noventa y nueve~999999999 +cero~cero +uno~uno +una~una +dos~dos +nueve~nueve +diez~10 +uno~uno +diez~10 +menos veintitrés~-23 +cien~100 +ciento uno~101 +ciento un~101 +ciento una~101 +mil y uno~1001 +mil uno~1001 +nueve billones setecientos ochenta y nueve mil trescientos ochenta y dos millones quinientos treinta y seis mil ciento treinta~9789382536130 +doscientos cincuenta y cuatro~254 +ciento cuarenta y siete mil cuatrocientos cincuenta y uno~147451 +un millón ciento cincuenta y seis mil ciento setenta y tres~1156173 +mil quinientos noventa y tres millones setenta y dos mil novecientos sesenta y uno~1593072961 +noventa y siete mil ochocientos ocho billones doscientos sesenta y cuatro mil setecientos setenta y dos millones setecientos noventa y dos mil cinco~97808264772792005 +diecisiete mil ochocientos cincuenta y cinco trillones treinta y seis mil seiscientos cincuenta y siete billones siete mil quinientos noventa y seis millones ciento diez mil novecientos cuarenta y nueve~17855036657007596110949 +diez mil diez billones diez millones cien mil diez~10010000010100010 +menos veinticinco mil treinta y siete~-25037 +mil doscientos sesenta y cuatro billones trescientos un mil novecientos treinta y ocho millones ciento cuatro~1264301938000104 +menos sesenta~-60 +cuarenta y seis mil seiscientos sesenta y cuatro~46664 +sesenta~60 +dos millones tres~2000003 +mil trece~1013 +mil cien~1100 +mil veintiséis~1026 +mil ciento veintiséis~1126 +dieciocho millones cuatrocientos cincuenta mil novecientos noventa~18450990 +dieciocho millones novecientos cuarenta mil setecientos veintidós~18940722 +dieciocho millones seiscientos noventa mil novecientos dieciséis~18690916 +dieciocho mil ochocientos ochenta~18880 +un millardo uno~1000000001 +mil millones uno~1000000001 +mil millones ciento uno~1000000101 +mil millones mil ciento uno~1000001101 +mil millones diez mil ciento uno~1000010101 +mil un millón diez mil ciento uno~1001010101 +dos millardos cincuenta y dos~2000000052 +muchas millones~muchas millones +mil billones uno~1000000000000001 +mil trillones uno~1000000000000000000001 +veintiacuátro~veintiacuátro +entre dieciséis mil y dieciocho mil~entre 16000 y 18000 +nine trillion seven hundred eighty nine billion three hundred eighty two million five hundred thirty six thousand one hundred thirty~9789382536130 +two hundred and fifty four~254 +one hundred forty seven thousand four hundred fifty one~147451 +one million one hundred fifty six thousand one hundred seventy three~1156173 +one billion five hundred ninety three million seventy two thousand nine hundred sixty one~1593072961 +ninety seven quadrillion eight hundred eight trillion two hundred sixty four billion seven hundred seventy two million seven hundred ninety two thousand five~97808264772792005 +seventeen sextillion eight hundred fifty five quintillion thirty six quadrillion six hundred fifty seven trillion seven billion five hundred ninety six million one hundred ten thousand nine hundred forty nine~17855036657007596110949 +ten quadrillion ten trillion ten million one hundred thousand ten~10010000010100010 +minus twenty five thousand thirty seven~-25037 +one quadrillion two hundred sixty four trillion three hundred one billion nine hundred thirty eight million one hundred four~1264301938000104 +minus sixty~-60 +forty six thousand six hundred sixty four~46664 +sixty~60 +zero~zero +two million three~2000003 +one thousand thirteen~1013 +one thousand one~1001 +one thousand one hundred~1100 +one thousand twenty six~1026 +one thousand one hundred twenty six~1126 +eighteen million four hundred fifty thousand nine hundred ninety~18450990 +eighteen million nine hundred forty thousand seven hundred twenty two~18940722 +eighteen million six hundred ninety thousand nine hundred sixteen~18690916 +eighteen thousand eight hundred eighty~18880 +eleven hundred~1100 +twenty one hundred~2100 +twenty one hundred and eleven~2111 +eleven hundred twenty one~1121 +twenty one crore ninety eight lakh thirty six thousand five hundred and ninety three~219836593 diff --git a/itn/spanish_english/test/data/test_cases_cardinal_cased.txt b/itn/spanish_english/test/data/test_cases_cardinal_cased.txt new file mode 100644 index 00000000..d6fc7a85 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_cardinal_cased.txt @@ -0,0 +1,60 @@ +Doscientos cincuenta y uno~251 +Novecientos noventa y nueve millones novecientos noventa y nueve mil novecientos noventa y nueve~999999999 +Cero~Cero +Uno~Uno +una~una +dos~dos +Nueve~Nueve +Diez~10 +, uno~, uno +, diez~, 10 +Menos veintitrés~-23 +cien~100 +ciento uno~101 +ciento un~101 +ciento una~101 +mil y uno~1001 +Mil una~1001 +nueve billones setecientos ochenta y nueve mil trescientos ochenta y dos millones quinientos treinta y seis mil ciento treinta~9789382536130 +Doscientos cincuenta y cuatro~254 +ciento cuarenta y siete mil cuatrocientos cincuenta y uno~147451 +Un Millón ciento cincuenta y seis mil ciento setenta y tres~1156173 +Mil quinientos noventa y tres millones setenta y dos mil novecientos sesenta y uno~1593072961 +noventa y siete mil ochocientos ocho billones doscientos sesenta y cuatro mil setecientos setenta y dos millones setecientos noventa y dos mil cinco~97808264772792005 +diecisiete mil ochocientos cincuenta y cinco trillones treinta y seis mil seiscientos cincuenta y siete billones siete mil quinientos noventa y seis millones ciento diez mil novecientos cuarenta y nueve~17855036657007596110949 +diez mil diez billones diez millones cien mil diez~10010000010100010 +Menos veinticinco mil treinta y siete~-25037 +mil doscientos sesenta y cuatro billones trescientos un mil novecientos treinta y ocho millones ciento cuatro~1264301938000104 +menos sesenta~-60 +cuarenta y seis mil seiscientos sesenta y cuatro~46664 +sesenta~60 +Nine trillion seven hundred eighty nine billion three hundred eighty two million five hundred thirty six thousand one hundred thirty~9789382536130 +Two hundred and fifty four~254 +One hundred forty seven thousand four hundred fifty one~147451 +One million one hundred fifty six thousand one hundred seventy three~1156173 +One billion five hundred ninety three million seventy two thousand nine hundred sixty one~1593072961 +Ninety seven quadrillion eight hundred eight trillion two hundred sixty four billion seven hundred seventy two million seven hundred ninety two thousand five~97808264772792005 +Seventeen sextillion eight hundred fifty five quintillion thirty six quadrillion six hundred fifty seven trillion seven billion five hundred ninety six million one hundred ten thousand nine hundred forty nine~17855036657007596110949 +Ten quadrillion ten trillion ten million one hundred thousand ten~10010000010100010 +Minus twenty five thousand thirty seven~-25037 +One quadrillion two hundred sixty four trillion three hundred one billion nine hundred thirty eight million one hundred four~1264301938000104 +Minus sixty~-60 +Forty six thousand six hundred sixty four~46664 +Sixty~60 +Zero~Zero +Two million three~2000003 +One thousand thirteen~1013 +One thousand one~1001 +One thousand one hundred~1100 +One thousand twenty six~1026 +One thousand one hundred twenty six~1126 +Eighteen million four hundred fifty thousand nine hundred ninety~18450990 +Eighteen million nine hundred forty thousand seven hundred twenty two~18940722 +Eighteen million six hundred ninety thousand nine hundred sixteen~18690916 +Eighteen thousand eight hundred eighty~18880 +Eleven hundred~1100 +Twenty one hundred~2100 +Twenty one hundred and eleven~2111 +Eleven hundred twenty one~1121 +Nineteen~19 +Twelve~Twelve \ No newline at end of file diff --git a/itn/spanish_english/test/data/test_cases_date.txt b/itn/spanish_english/test/data/test_cases_date.txt new file mode 100644 index 00000000..1147a12c --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_date.txt @@ -0,0 +1,44 @@ +primero de enero~1 de enero +uno de enero~1 de enero +el uno de diciembre~el 1 de diciembre +el primero de diciembre~el 1 de diciembre +domingo veintiséis de octubre~domingo 26 de octubre +treinta y uno de diciembre de mil novecientos noventa y dos~31 de diciembre de 1992 +siglo diecinueve~siglo xix +doscientos tres antes de cristo~203 a. c. +july twenty fifth two thousand twelve~july 25 2012 +nineteen eighties~1980s +two thousand and twenty~2020 +two thousand and nine~2009 +the twenty fifth of july twenty twelve~25 july 2012 +the twenty fifth of july two thousand twelve~25 july 2012 +the twenty second of july twenty twelve~22 july 2012 +the fifteenth of january~15 january +the seventeenth of may twenty ten~17 may 2010 +january first~january 1 +july twenty second two thousand eight~july 22 2008 +june thirty~june 30 +july twenty fifth twenty twelve~july 25 2012 +nineteen seventeen~1917 +twenty twelve~2012 +march sixteen sixty five~march 1665 +sixteen sixty five~1665 +july two thousand twelve~july 2012 +october nineteen oh five~october 1905 +july fifteen o six~july 1506 +the twenty fifth of july twenty twelve~25 july 2012 +july twenty fifth twenty twelve~july 25 2012 +july twenty fifth two thousand twelve~july 25 2012 +july one thousand eight hundred seventy six~july 1876 +february twenty fifth twenty sixteen~february 25 2016 +november twenty fourth twenty fourteen~november 24 2014 +nineteen ninety four~1994 +two thousand three~2003 +one thousand eight~1008 +nineteen seventy six~1976 +june twentieth twenty fourteen~june 20 2014 +nineteen seventy three~1973 +nineteen seventy five~1975 +eleven fifty five~1155 +second quarter of twenty twenty two~Q2 2022 +seven fifty b c~750BC diff --git a/itn/spanish_english/test/data/test_cases_date_cased.txt b/itn/spanish_english/test/data/test_cases_date_cased.txt new file mode 100644 index 00000000..ab8358dd --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_date_cased.txt @@ -0,0 +1,20 @@ +Primero De Enero~1 de Enero +Uno de enero~1 de Enero +el uno de Diciembre~el 1 de Diciembre +El primero de diciembre~El 1 de diciembre +Domingo Veintiséis De Octubre~Domingo 26 de Octubre +treinta y uno de diciembre de mil novecientos noventa y dos~31 de diciembre de 1992 +Siglo diecinueve~Siglo xix +doscientos tres antes de Cristo~203 A. C. +January first~January 1 +July twenty second two thousand eight~July 22 2008 +June thirty~June 30 +July twenty fifth twenty twelve~July 25 2012 +Nineteen seventeen~1917 +Twenty twelve~2012 +March sixteen sixty five~March 1665 +Sixteen sixty five~1665 +July two thousand twelve~July 2012 +October nineteen oh five~October 1905 +July fifteen o six~July 1506 +The twenty fifth of july twenty twelve~25 july 2012 \ No newline at end of file diff --git a/itn/spanish_english/test/data/test_cases_decimal.txt b/itn/spanish_english/test/data/test_cases_decimal.txt new file mode 100644 index 00000000..2ac23805 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_decimal.txt @@ -0,0 +1,92 @@ +uno coma dos seis~1,26 +menos uno coma dos seis~-1,26 +uno coma veintiséis~1,26 +cero coma dos seis~0,26 +cero coma veintiséis~0,26 +tres coma ciento cuarenta y uno~3,141 +tres coma cero ciento cuarenta y uno~3,0141 +tres coma ciento cuarenta y uno cincuenta y nueve~3,14159 +tres coma catorce ciento cincuenta y nueve~3,14159 +tres coma catorce quince noventa y dos sesenta y cinco treinta y cinco~3,1415926535 +tres coma catorce quince cero noventa y dos sesenta y cinco treinta y cinco~3,14150926535 +tres coma catorce quince cero novecientos veintiséis cero quinientos treinta y cinco~3,141509260535 +cuatrocientos millones~400 millones +uno punto treinta y tres~1.33 +uno punto treinta y tres millones~1.33 millones +cero coma seis millones~0,6 millones +mil ochocientos veinticuatro millón~1824 millón +mil ochocientos veinticuatro millones~1824 millones +punto dos seis~.26 +un millón~1 millón +dos millones~2 millones +un millardo~1 millardo +dos millardos~2 millardos +un billón~1 billón +dos billones~2 billones +un trillón~1 trillón +dos trillones~2 trillones +un cuatrillón~1 cuatrillón +dos cuatrillones~2 cuatrillones +five point two million~5.2 million +one hundred sixty four point five eight thousand~164.58 thousand +four hundred million~400 million +fifty billion~50 billion +four hundred five billion~405 billion +four point eight five billion~4.85 billion +one hundred billion~100 billion +one hundred ten billion~110 billion +one hundred thirty two billion~132 billion +one point eight four billion~1.84 billion +one point eight one billion~1.81 billion +one point five nine billion~1.59 billion +one point four five three billion~1.453 billion +one point seven two billion~1.72 billion +one point two five billion~1.25 billion +thirteen billion~13 billion +thirty billion~30 billion +two thousand eight hundred five point eight seven three billion~2805.873 billion +seventy trillion~70 trillion +thirteen million~13 million +eighteen billion~18 billion +four hundred fifty million~450 million +one hundred thirty million~130 million +ten million~10 million +four hundred million~400 million +five million~5 million +five hundred million~500 million +twelve million~12 million +thirteen million~13 million +four million~4 million +forty five million~45 million +fifteen million~15 million +fifteen trillion~15 trillion +fifteen billion~15 billion +two million~2 million +eight million~8 million +point one two o five~.1205 +minus sixty point two four zero zero~-60.2400 +zero point two six~0.26 +point zero two~.02 +sixty point two~60.2 +eighteen~18 +eighteen point eight five~18.85 +eighteen point five o~18.50 +eighteen point five six~18.56 +eighteen point nine~18.9 +eighteen point o five~18.05 +eighteen point one two~18.12 +eighteen point o one~18.01 +eighteen point o o o~18.000 +eighteen point six~18.6 +eighteen point three o o~18.300 +eighteen point three six~18.36 +eighteen point two five~18.25 +eighteen point two two~18.22 +eight hundred eighteen point three o three~818.303 +eight hundred eight point eight~808.8 +eight hundred eight point zero~808.0 +eight hundred eighty eight point one~888.1 +eight hundred eighty four point three~884.3 +eight hundred eighty two point eight~882.8 +eight hundred eighty two point zero~882.0 +eight hundred forty five point nine four~845.94 diff --git a/itn/spanish_english/test/data/test_cases_decimal_cased.txt b/itn/spanish_english/test/data/test_cases_decimal_cased.txt new file mode 100644 index 00000000..17b91271 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_decimal_cased.txt @@ -0,0 +1,19 @@ +Uno coma dos seis~1,26 +Menos uno coma dos seis~-1,26 +Uno Coma Veintiséis~1,26 +Cero coma Dos seis~0,26 +cero coma veintiséis~0,26 +tres coma ciento cuarenta y uno~3,141 +One point two five billion~1.25 billion +Thirteen billion~13 billion +Thirty billion~30 billion +Thirty Billion~30 Billion +Two thousand eight hundred five point eight seven three billion~2805.873 billion +Eighteen~18 +Eighteen point eight five~18.85 +Eighteen point five o~18.50 +Eighteen point five six~18.56 +Eighteen point nine~18.9 +Eighteen point o five~18.05 +Eighteen point one two~18.12 +Eighteen point o one~18.01 diff --git a/itn/spanish_english/test/data/test_cases_electronic.txt b/itn/spanish_english/test/data/test_cases_electronic.txt new file mode 100644 index 00000000..3082a9f0 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_electronic.txt @@ -0,0 +1,41 @@ +a punto b c arroba g mail punto com~a.bc@gmail.com +c d f arroba a b c punto e d u~cdf@abc.edu +a b c arroba g mail punto a b c~abc@gmail.abc +a b c arroba a b c punto com~abc@abc.com +a s d f uno dos tres arroba a b c punto com~asdf123@abc.com +a uno b dos arroba a b c punto com~a1b2@abc.com +a b tres punto s d d punto tres arroba g mail punto com~ab3.sdd.3@gmail.com +hache te te pe ese dos puntos barra barra doble ve doble ve doble ve punto n vidia punto com~https://www.nvidia.com +doble ve doble ve doble ve punto n vidia punto com~www.nvidia.com +doble ve doble ve doble ve punto nvidia punto com~www.nvidia.com +w w w punto nvidia punto com~www.nvidia.com +doble ve doble ve doble ve punto a b c punto es barra e f g~www.abc.es/efg +doble ve doble ve doble ve punto a b c punto e s~www.abc.es +doble ve doble ve doble ve punto a b c punto es barra e f g signo de interrogación i d signo igual a b c~www.abc.es/efg?id=abc +doble ve doble ve doble ve punto a b c punto gob~www.abc.gob +doble ve doble ve doble ve punto a b c punto d e f~www.abc.def +a dot b c at g mail dot com~a.bc@gmail.com +a at gmail dot com~a@gmail.com +a at m s n dot fr~a@msn.fr +a at a o l dot com~a@aol.com +a at m s n dot com~a@msn.com +a at nvidia dot com~a@nvidia.com +a dot b c at nvidia dot com~a.bc@nvidia.com +c d f at a b c dot e d u~cdf@abc.edu +a b c at g mail dot a b c~abc@gmail.abc +a b c at a b c dot com~abc@abc.com +a s d f one two three at a b c dot com~asdf123@abc.com +a one b two at a b c dot com~a1b2@abc.com +a b three dot s d d dot three at g mail dot com~ab3.sdd.3@gmail.com +dot three at g mail dot com~dot 3@gmail.com +one three at g mail dot com~13@gmail.com +a b three hyphen s d d dash three at g mail dot com~ab3-sdd-3@gmail.com +h t t p colon slash slash w w w dot o u r d a i l y n e w s dot com dot s m~http://www.ourdailynews.com.sm +h t t p colon slash slash w w w dot c o m d a i l y n e w s dot a b dot s m~http://www.comdailynews.ab.sm +h t t p colon slash slash w w w dot c o m d a i l y n e w s dot a b slash s m~http://www.comdailynews.ab/sm +w w w dot c o m d a i l y n e w s dot a b slash s m~www.comdailynews.ab/sm +c o m d a i l y n e w s dot a b slash s m~comdailynews.ab/sm +n vidia dot com~nvidia.com +abc at gmail dot com~abc@gmail.com +athreed at gmail dot com~athreed@gmail.com +kore dot ai~kore.ai diff --git a/itn/spanish_english/test/data/test_cases_electronic_cased.txt b/itn/spanish_english/test/data/test_cases_electronic_cased.txt new file mode 100644 index 00000000..9131f4f9 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_electronic_cased.txt @@ -0,0 +1,14 @@ +A punto B C Arroba G mail punto com~A.BC@gmail.com +c d f Arroba a b c Punto e d u~cdf@abc.edu +W W W Punto N vidia Punto com~www.nvidia.com +Doble ve doble ve doble ve punto a b c punto es barra e f g~www.abc.es/efg +Doble Ve Doble Ve Doble Ve Punto a b c Punto e s~www.abc.es +N vidia dot com~nvidia.com +Abc at gmail dot com~Abc@gmail.com +Athreed at gmail dot com~Athreed@gmail.com +Kore dot ai~Kore.ai +NVIDIA dot com~NVIDIA.com +NVIDIA dot COM~NVIDIA.COM +WWW.A B C at A B C dot com~WWW.A BC@ABC.com +W W W. A B C dot com~www.ABC.com +w w w . o u r d a i l y n e w s dot com . s m~www.ourdailynews.com . s m \ No newline at end of file diff --git a/itn/spanish_english/test/data/test_cases_fraction.txt b/itn/spanish_english/test/data/test_cases_fraction.txt new file mode 100644 index 00000000..885ed7f9 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_fraction.txt @@ -0,0 +1,12 @@ +medio~medio +un cuarto~un cuarto +ocho tercios~8/3 +dos quintos~2/5 +diez treintavos~10/30 +tres vigésimos~3/20 +once cientounavos~11/101 +un décimo~1/10 +un cuarentiunavo~1/41 +dos y dos tercios~2 2/3 +menos cuatro y un quinto~-4 1/5 +menos diez veinteavos~-10/20 \ No newline at end of file diff --git a/itn/spanish_english/test/data/test_cases_measure.txt b/itn/spanish_english/test/data/test_cases_measure.txt new file mode 100644 index 00000000..01278697 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_measure.txt @@ -0,0 +1,132 @@ +doscientos metros~200 m +tres horas~3 h +una hora~1 h +doscientos cuarenta y cinco millas por hora~245 mph +dos kilos~2 kg +sesenta coma dos cuatro cero cero kilogramos~60,2400 kg +menos sesenta coma veinticuatro cero cero kilogramos~-60,2400 kg +ocho coma cinco dos por ciento~8,52 % +menos ocho coma cinco dos por ciento~-8,52 % +uno porciento~1 % +tres centímetros~3 cm +cuatro segundos~4 s +cinco litros~5 l +tres metros cúbicos~3 m³ +dos kilómetros por hora~2 kph +diez grados farenheit~10 ° F +dos metros y medio~2 1/2 m +tres quintos de metro~3/5 m +menos tres y medio metros por hora~-3 1/2 m/h +dos más dos es igual a cuatro~2 + 2 = 4 +two hundred meters~200 m +fifty six point three per square kilometer~56.3 /km² +two hundred kilometers per hour~200 km/h +twenty eight kilograms force per square centimeter~28 kgf/cm² +forty two thousand two hundred fifty nine per square meter~42259 /m² +minus two thousand twelve kilo liters~-2012 kl +minus sixty six kilograms~-66 kg +two kilo watt hours~2 kWh +one point o o o o two eight cubic deci meters~1.000028 dm³ +seven point five peta bytes~7.5 pb +three hours~3 h +one milli volt~1 mv +two cubic meters~2 m³ +ninety grams~90 g +one hundred twenty four point three lumens~124.3 lm +four hundred forty milliliters~440 ml +thirty one thousand four hundred eighty square feet~31480 sq ft +one thousand six hundred hours~1600 h +thirty one thousand four hundred eighty square feet~31480 sq ft +two square miles~2 sq mi +zero point one nine square miles~0.19 sq mi +one thousand five hundred thirty one c c~1531 cc +three hundred micrometers~300 μm +sixty five thousand square kilometers~65000 km² +two miles per hour~2 mph +two hundred forty five miles per hour~245 mph +one hundred fifty c c~150 cc +sixty point two four zero zero kilograms~60.2400 kg +zero feet~0 ft +zero foot~0 ft +two feet~2 ft +twenty foot~20 ft +point two meters~.2 m +two square meters~2 m² +eighteen feet~18 ft +eighteen mega siemens~18 ms +eighteen ounces~18 oz +eighteen point five kilometers~18.5 km +eighteen point five two square kilometers~18.52 km² +eighteen point nine one square kilometers~18.91 km² +eighteen point one four percent~18.14 % +eighteen point one six percent~18.16 % +eighteen point one square kilometers~18.1 km² +eighteen point six percent~18.6 % +eighteen point two two kilometers~18.22 km +eighteen point zero kilometers~18.0 km +eighteen point zero percent~18.0 % +eighteen square kilometers~18 km² +eighteen thousand eight hundred giga watt hours~18800 gWh +eighteen thousand seven hundred hectares~18700 ha +eight hectares~8 ha +eight hundred eighty five astronomical units~885 au +eight hundred eighty hectares~880 ha +eight hundred eighty kilobytes~880 kb +eight hundred eighty kilometers~880 km +eight hundred eighty nine feet~889 ft +eight hundred eighty six kilometers~886 km +eight hundred eighty two megawatts~882 mW +eight hundred feet~800 ft +eight hundred fifty five square kilometers~855 km² +eight hundred fifty megahertz~850 mhz +eight hundred fifty meters~850 m +eight hundred fifty nanometers~850 nm +eight hundred fifty one meters~851 m +eight hundred fifty seven square kilometers~857 km² +eight hundred fifty three meters~853 m +eight hundred fifty three point six meters~853.6 m +eight hundred five point four six square kilometers~805.46 km² +eight hundred forty two point nine meters~842.9 m +eight hundred forty two square kilometers~842 km² +eight hundred gigabytes~800 gb +eight hundred horsepower~800 hp +eight hundred kilograms~800 kg +eight hundred kilo watt hours~800 kWh +eight hundred kilowatts~800 kW +eight hundred megahertz~800 mhz +eight hundred ninety four c c~894 cc +eight hundred ninety kilowatts~890 kW +eight hundred ninety millimeters~890 mm +eight hundred ninety two square kilometers~892 km² +eight hundred seventy horsepower~870 hp +eight hundred seventy meters~870 m +eight hundred sixty kilograms~860 kg +eight hundred sixty kilometers~860 km +eight hundred sixty miles~860 mi +eight hundred sixty six feet~866 ft +eight hundred ten hectares~810 ha +eight hundred ten kilohertz~810 khz +eight hundred thirty eight point two millimeters~838.2 mm +eight hundred thirty five kilometers~835 km +eight hundred thirty kilohertz~830 khz +eight hundred thirty megawatts~830 mW +eight hundred thirty nine kilometers~839 km +eight hundred thirty six meters~836 m +eight hundred twenty feet~820 ft +eight hundred twenty kilometers~820 km +eight hundred twenty meters~820 m +eight hundred twenty one point zero feet~821.0 ft +eight hundred two point eight nine kilometers~802.89 km +eight hundred volts~800 v +eight kilobits~8 kb +eight kilograms~8 kg +eight million two hundred thousand feet~8200000 ft +eight point eight kilometers~8.8 km +eight point eight meters~8.8 m +eight point eight miles~8.8 mi +eight point five centimeters~8.5 cm +eight point five five percent~8.55 % +eight point five megawatts~8.5 mW +eight point five meters~8.5 m +eight point five two percent~8.52 % +eight point four four percent~8.44 % diff --git a/itn/spanish_english/test/data/test_cases_measure_cased.txt b/itn/spanish_english/test/data/test_cases_measure_cased.txt new file mode 100644 index 00000000..7073a7bf --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_measure_cased.txt @@ -0,0 +1,22 @@ +Doscientos metros~200 m +tres horas~3 h +una hora~1 h +Doscientos cuarenta y cinco Millas Por Hora~245 mph +Dos Kilos~2 kg +sesenta coma dos cuatro cero cero kilogramos~60,2400 kg +Menos sesenta coma veinticuatro cero cero kilogramos~-60,2400 kg +menos Ocho Coma Cinco Dos por ciento~-8,52 % +uno Porciento~1 % +tres centímetros~3 cm +dos más dos es igual a cuatro~2 + 2 = 4 +Eight point five megawatts~8.5 mW +Eight point five meters~8.5 m +Eight point five two percent~8.52 % +Eight point four four percent~8.44 % +one gigabit per second~1 gbps +nine gigabits per second~9 gbps +five degrees celsius~5 °C +seventy two degrees fahrenheit~72 °F +Seventy two Degrees Fahrenheit~72 °F +two hundred seventy three kelvin~273 K +Nine GigaBits per second~9 gbps \ No newline at end of file diff --git a/itn/spanish_english/test/data/test_cases_money.txt b/itn/spanish_english/test/data/test_cases_money.txt new file mode 100644 index 00000000..8ec57f01 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_money.txt @@ -0,0 +1,76 @@ +doce dólares y cinco centavos~$12,05 +doce dólares y cinco céntimos~$12,05 +setenta y cinco dólares sesenta y tres~$75,63 +setenta y cinco dólares y sesenta y tres centavos~$75,63 +setenta y cinco dólares con sesenta y tres centavos~$75,63 +setenta y cinco dólares con sesenta y tres~$75,63 +veintinueve dólares cincuenta centavos~$29,50 +un dólar~$1 +veinticinco centavos~$0,25 +veinticinco céntimos~€0,25 +doce euros y cinco centavos~€12,05 +doce dólares estadounidenses y cinco centavos~US$12,05 +doce dólares americanos y cinco centavos~US$12,05 +doce pesos y cinco centavos~$12,05 +doce yenes y cinco centavos~¥12,05 +dos dólares y sesenta y tres dólares~$2 y $63 +diez pesetas~₧10 +un colón~₡1 +un chon~₩0,01 +tres wones con veinte~₩3,20 +cien quetzales~q100 +nueve punto cinco millones de pesos~$9.5 millones +catorce millones quinientos mil pesos mexicanos~Mex$14500000 +diez pesos mexicanos~Mex$10 +two dollars~$2 +one cent~$0.01 +four united states dollars and sixty nine cents~$4.69 +seventy five dollars sixty three~$75.63 +twenty nine dollars fifty cents~$29.50 +eleven dollars and fifty one cents~$11.51 +nine hundred ninety three dollars and ninety two cents~$993.92 +four hundred sixty billion won~₩460 billion +thirty billion yen~¥30 billion +two point five billion dollars~$2.5 billion +forty five billion dollars~$45 billion +fifty million dollars~$50 million +fifty billion dollars~$50 billion +zero point two million dollars~$0.2 million +fifteen point two billion dollars~$15.2 billion +one point six nine billion yuan~1.69 billion yuan +one point four three six billion yuan~1.436 billion yuan +four million yuan~4 million yuan +one dollar~$1 +fifteen thousand dollars~$15000 +one dollars~one dollars +twenty dollar~$20 +twenty point five o six dollars~$20.506 +point five o six dollars~$.506 +eighteen dollars~$18 +eighteen million nine hundred twenty five thousand dollars~$18925000 +eighteen thousand eight hundred fifty four dollars~$18854 +eighteen thousand eight hundred one dollars~$18801 +eighteen thousand eight hundred seventy five dollars~$18875 +eighteen thousand eighty one dollars~$18081 +eighteen thousand fifty two dollars~$18052 +eighteen thousand five hundred forty two dollars~$18542 +eighteen thousand five hundred nineteen dollars~$18519 +eighteen thousand five hundred seventy dollars~$18570 +eighteen thousand five hundred seventy eight dollars~$18578 +eighteen thousand five hundred sixteen dollars~$18516 +eighteen thousand four hundred eighty two dollars~$18482 +eighteen thousand four hundred seventy eight dollars~$18478 +eighteen thousand four hundred sixty eight dollars~$18468 +eighteen thousand nine hundred three dollars~$18903 +eighteen thousand nine hundred twenty nine dollars~$18929 +eighteen thousand ninety five dollars~$18095 +eighteen thousand one hundred seventeen dollars~$18117 +eighteen thousand one hundred twenty eight dollars~$18128 +eighteen thousand one hundred twenty five dollars~$18125 +eighteen thousand one hundred twenty four dollars~$18124 +eighteen thousand one hundred twenty nine dollars~$18129 +one thousand fifty five dollars~$1055 +one fifty five dollars~$155 +fifteen hundred dollars~$1500 +ninety nine hundred dollars~$9900 +ninety nine hundred and fifteen dollars and one cent~$9915.01 diff --git a/itn/spanish_english/test/data/test_cases_money_cased.txt b/itn/spanish_english/test/data/test_cases_money_cased.txt new file mode 100644 index 00000000..613a822a --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_money_cased.txt @@ -0,0 +1,16 @@ +doce dólares y cinco centavos~$12,05 +Doce Dólares Y Cinco Céntimos~$12,05 +setenta y cinco Dólares sesenta y tres~$75,63 +Veintinueve dólares cincuenta centavos~$29,50 +Catorce millones quinientos mil Pesos mexicanos~Mex$14500000 +diez pesos Mexicanos~Mex$10 +Two dollars~$2 +One cent~$0.01 +Four united states dollars and sixty nine cents~$4.69 +Seventy five dollars sixty three~$75.63 +Twenty nine dollars fifty cents~$29.50 +Eleven dollars and fifty one cents~$11.51 +Nine hundred ninety three dollars and ninety two cents~$993.92 +Four hundred sixty billion won~₩460 billion +Thirty billion yen~¥30 billion +Two point five billion dollars~$2.5 billion diff --git a/itn/spanish_english/test/data/test_cases_ordinal.txt b/itn/spanish_english/test/data/test_cases_ordinal.txt new file mode 100644 index 00000000..3906822f --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_ordinal.txt @@ -0,0 +1,64 @@ +primero~primero +tercera~tercera +primer~primer +tercer~tercer +noveno~noveno +novena~novena +décimo~10.º +décima~10.ª +undécimo~11.º +undécima~11.ª +decimoprimero~11.º +décimo primero~11.º +decimoprimer~11.ᵉʳ +décimo primer~11.ᵉʳ +decimoprimera~11.ª +décima primera~11.ª +(technically ungrammatical) décimo primera~(technically ungrammatical) 11.ª +decimotercero~13.º +vigésimo primero~21.º +vigésima primera~21.ª +(technically ungrammatical) vigésimo primera~(technically ungrammatical) 21.ª +vigésimo primer~21.ᵉʳ +vigesimosegundo~22.º +vigésimo segundo~22.º +vigesimosegunda~22.ª +vigésima segunda~22.ª +vigésimo tercero~23.º +centésimo undécimo~111.º +centésimo trigésimo cuarto~134.º +vigesimoctavo~28.º +one hundredth~100th +twenty five thousand one hundred eleventh~25111th +second~2nd +zeroth~0th +first~1st +second~2nd +third~3rd +fourth~4th +eleventh~11th +twelfth~12th +thirteenth~13th +twenty first~21st +twenty third~23rd +one hundred eleventh~111th +one thousandth~1000th +one hundred twenty first~121st +eleven hundred twenty first~1121st +second~2nd +tenth~10th +sixth~6th +third~3rd +nineteenth~19th +third~3rd +twelfth~12th +forty eighth~48th +seventy first~71st +third~3rd +forty second~42nd +seventeenth~17th +twentieth~20th +twenty first~21st +seventh~7th +second~2nd +fifth~5th diff --git a/itn/spanish_english/test/data/test_cases_ordinal_cased.txt b/itn/spanish_english/test/data/test_cases_ordinal_cased.txt new file mode 100644 index 00000000..7b7df1ed --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_ordinal_cased.txt @@ -0,0 +1,21 @@ +primero~primero +Tercera~Tercera +Primer~Primer +tercer~tercer +Décima~10.ª +undécimo~11.º +Decimoprimer~11.ᵉʳ +Décimo primer~11.ᵉʳ +Décima Primera~11.ª +(technically ungrammatical) décimo primera~(technically ungrammatical) 11.ª +decimotercero~13.º +One hundredth~100th +Twenty five thousand one hundred eleventh~25111th +Second~2nd +Zeroth~0th +First~1st +Second~2nd +Third~3rd +Fourth~4th +Eleventh~11th +Twelfth~12th diff --git a/itn/spanish_english/test/data/test_cases_telephone.txt b/itn/spanish_english/test/data/test_cases_telephone.txt new file mode 100644 index 00000000..aae88910 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_telephone.txt @@ -0,0 +1,32 @@ +uno dos tres uno dos tres cinco seis siete ocho~123-123-5678 +uno veintitrés uno veintitrés cincuenta y seis setenta y ocho~123-123-5678 +uno dos tres cuatro cinco seis siete ocho nueve~123-456-789 +uno veintitrés cuatro cincuenta y seis siete ochenta y nueve~123-456-789 +uno dos tres cuatro cinco seis siete ocho~1234-5678 +doce treinta y cuatro cincuenta y seis setenta y ocho~1234-5678 +triple tres uno dos tres cinco seis siete ocho~333-123-5678 +más uno uno dos tres uno dos tres cinco seis siete ocho~+1-123-123-5678 +más cincuenta y cuatro uno dos tres uno dos tres cinco seis siete ocho extensión doce~+54-123-123-5678 ext. 12 +one two three one two three five six seven eight~123-123-5678 +plus nine one one two three one two three five six seven eight~+91 123-123-5678 +plus forty four one two three one two three five six seven eight~+44 123-123-5678 +four one two three one two three five six seven eight~4 123-123-5678 +zero two three one two three five six seven eight~023-123-5678 +o two three one two three five six seven eight~023-123-5678 +oh two three one two three five six seven eight~023-123-5678 +double oh three one two three five six seven eight~003-123-5678 +four three two double seven three two one four three two one four three double zero five~432 7732 143214 3005 +one two three dot one two three dot o dot four o~123.123.0.40 +one twenty three dot one two three dot o dot four o~123.123.0.40 +two two five dot double five dot o dot four o~225.55.0.40 +two two five dot double five dot o dot forty five~225.55.0.45 +ssn is seven double nine one two three double one three~ssn is 799-12-3113 +seven nine nine~799 +a b nine~ab9 +a b c~a b c +five w k r a three one~5wkra31 +x eighty six~x86 +x three eighty six~x386 +a thirty six~a 36 +a ten eighty p display~a 1080p display +r t x forty fifty t i~RTX 4050ti diff --git a/itn/spanish_english/test/data/test_cases_telephone_cased.txt b/itn/spanish_english/test/data/test_cases_telephone_cased.txt new file mode 100644 index 00000000..6b9613da --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_telephone_cased.txt @@ -0,0 +1,19 @@ +Uno dos tres uno dos tres cinco seis siete ocho~123-123-5678 +uno veintitrés uno veintitrés cincuenta y seis setenta y ocho~123-123-5678 +Uno Dos Tres Cuatro Cinco Seis Siete Ocho Nueve~123-456-789 +Triple tres uno dos tres cinco seis siete ocho~333-123-5678 +Más uno uno dos tres uno dos tres cinco seis siete ocho~+1-123-123-5678 +más cincuenta y cuatro uno dos tres uno dos tres cinco seis siete ocho Extensión doce~+54-123-123-5678 ext. 12 +One two three one two three five six seven eight~123-123-5678 +Plus nine one one two three one two three five six seven eight~+91 123-123-5678 +Plus forty four one two three one two three five six seven eight~+44 123-123-5678 +O two three one two three five six seven eight~023-123-5678 +Oh two three one two three five six seven eight~023-123-5678 +Double oh three one two three five six seven eight~003-123-5678 +Two two five dot double five dot o dot four o~225.55.0.40 +Two two five dot double five dot o dot forty five~225.55.0.45 +SSN is seven double nine one two three double one three~SSN is 799-12-3113 +Seven nine nine~799 +A b nine~Ab9 +A b c~A b c +Five w k r a three one~5wkra31 diff --git a/itn/spanish_english/test/data/test_cases_time.txt b/itn/spanish_english/test/data/test_cases_time.txt new file mode 100644 index 00000000..17113ad9 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_time.txt @@ -0,0 +1,54 @@ +las dieciséis cincuenta~las 16:50 +la una~la una +las dos~las dos +las tres personas~las tres personas +las dos a eme~las 2:00 a.m. +la una pe eme~la 1:00 p.m. +la una y diez~la 1:10 +la una y diez a eme~la 1:10 a.m. +la una y diez pe eme~la 1:10 p.m. +la una diez~la 1:10 +la una con diez~la 1:10 +la una y cuarto~la 1:15 +la una y media~la 1:30 +las dos menos veinte~la 1:40 +las dos menos cuarto~la 1:45 +cuarto para las dos~la 1:45 +un cuarto para las dos~la 1:45 +las veintitrés y media~las 23:30 +las veintitrés y cincuenta y nueve~las 23:59 +las dos de la tarde~las 2:00 p.m. +cuarto para las cero~las 23:45 +cuarto para las veinticuatro~las 23:45 +diez para las doce~las 11:50 +dos y media de la tarde~02:30 p.m. +la una de la tarde u t c más cuatro~la 1:00 p.m. UTC+4 +eight oclock g m t~08:00 gmt +seven a m e s t~07:00 a.m. est +two p m~02:00 p.m. +two thirty~02:30 +three o'clock~03:00 +quarter past one~01:15 +half past three~03:30 +eight fifty one~08:51 +eight fifty two~08:52 +eight forty~08:40 +eight nineteen~08:19 +eight o six~08:06 +eight thirty eight~08:38 +eight thirty two~08:32 +eight twenty nine~08:29 +eleven fifty five p m~11:55 p.m. +eleven fifty three p m~11:53 p.m. +eleven forty a m~11:40 a.m. +eleven forty five a m~11:45 a.m. +eleven forty p m~11:40 p.m. +eleven forty six a m~11:46 a.m. +eleven o six p m~11:06 p.m. +eleven thirteen a m~11:13 a.m. +half past twelve~12:30 +quarter past one~01:15 +quarter to one~12:45 +quarter to twelve~11:45 +set alarm at ten to eleven pm~set alarm at 10:50 p.m. +one min to one am~12:59 a.m. diff --git a/itn/spanish_english/test/data/test_cases_time_cased.txt b/itn/spanish_english/test/data/test_cases_time_cased.txt new file mode 100644 index 00000000..875ff97d --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_time_cased.txt @@ -0,0 +1,19 @@ +las dieciséis cincuenta~las 16:50 +la una~la una +Las dos~Las dos +Las tres personas~Las tres personas +Las Dos a eme~Las 2:00 a.m. +la una Pe Eme~la 1:00 P.M. +la una y diez~la 1:10 +la una y Diez a eme~la 1:10 a.m. +La Una Y Diez pe eme~La 1:10 p.m. +Eight oclock g m t~08:00 gmt +Seven a m e s t~07:00 a.m. est +Two p m~02:00 p.m. +Seven A M E S T~07:00 A.M. EST +Two P M~02:00 P.M. +Two thirty~02:30 +Set alarm at ten to eleven pm~Set alarm at 10:50 p.m. +One min to one am~12:59 a.m. +eleven Forty six A M~11:46 A.M. +eleven forty six AM~11:46 A.M. \ No newline at end of file diff --git a/itn/spanish_english/test/data/test_cases_whitelist.txt b/itn/spanish_english/test/data/test_cases_whitelist.txt new file mode 100644 index 00000000..90c024aa --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_whitelist.txt @@ -0,0 +1,18 @@ +usted~Ud. +ustedes~Uds. +habla usted español~habla Ud. español +hablan ustedes español~hablan Uds. español +estados unidos~EE. UU. +doctor dao~dr. dao +misses smith~mrs. smith +mister dao~mr. dao +saint george~st. george +i like for example ice cream~i like e.g. ice cream +s and p five hundred~S&P 500 +seven eleven stores~7-eleven stores +r t x~RTX +cat five e~CAT5e +nvidia a one hundred~Nvidia A100 +c u d n n~cuDNN +p c i e x eight~PCIe x8 +l g a eleven fifty~LGA 1150 diff --git a/itn/spanish_english/test/data/test_cases_word.txt b/itn/spanish_english/test/data/test_cases_word.txt new file mode 100644 index 00000000..937dc581 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_word.txt @@ -0,0 +1,104 @@ +~ +yahoo!~yahoo! +veinte!~20 ! +x ~x +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aagnes~aagnes +aagomoni~aagomoni +aagon~aagon +aagoo~aagoo +aagot~aagot +aahar~aahar +aahh~aahh +aahperd~aahperd +aaibinterstate~aaibinterstate +aajab~aajab +aakasa~aakasa +aakervik~aakervik +aakirkeby~aakirkeby +aalam~aalam +aalbaek~aalbaek +aaldiu~aaldiu +aalem~aalem +a'ali~a'ali +aalilaassamthey~aalilaassamthey +aalin~aalin +aaliyan~aaliyan +aaliyan's~aaliyan's +aamadu~aamadu +aamara~aamara +aambala~aambala +aamera~aamera +aamer's~aamer's +aamina~aamina +aaminah~aaminah +aamjiwnaang~aamjiwnaang +~ +, one~, one +, one , two , three , four~, one , two , three , four +e s three~es3 +yahoo!~yahoo! +twenty!~20 ! +x ~x +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aagnes~aagnes +aagomoni~aagomoni +aagon~aagon +aagoo~aagoo +aagot~aagot +aahar~aahar +aahh~aahh +aahperd~aahperd +aaibinterstate~aaibinterstate +aajab~aajab +aakasa~aakasa +aakervik~aakervik +aakirkeby~aakirkeby +aalam~aalam +aalbaek~aalbaek +aaldiu~aaldiu +aalem~aalem +a'ali~a'ali +aalilaassamthey~aalilaassamthey +aalin~aalin +aaliyan~aaliyan +aaliyan's~aaliyan's +aamadu~aamadu +aamara~aamara +aambala~aambala +aamera~aamera +aamer's~aamer's +aamina~aamina +aaminah~aaminah +aamjiwnaang~aamjiwnaang +~ +, one~, one +, one , two , three , four~, one , two , three , four diff --git a/itn/spanish_english/test/data/test_cases_word_cased.txt b/itn/spanish_english/test/data/test_cases_word_cased.txt new file mode 100644 index 00000000..ebe0c086 --- /dev/null +++ b/itn/spanish_english/test/data/test_cases_word_cased.txt @@ -0,0 +1,17 @@ +~ +Yahoo!~Yahoo! +Veinte!~20 ! +X ~X +—~— +AAA~AAA +Aabach~Aabach +aabenraa~aabenraa +Aachen's~Aachen's +Aadri~Aadri +Aaliyan's~Aaliyan's +Aahar~Aahar +Aahh~Aahh +Aahperd~Aahperd +Aaibinterstate~Aaibinterstate +Aajab~Aajab +Aakasa~Aakasa \ No newline at end of file diff --git a/itn/spanish_english/test/normalizer_test.py b/itn/spanish_english/test/normalizer_test.py new file mode 100644 index 00000000..5a78e2e5 --- /dev/null +++ b/itn/spanish_english/test/normalizer_test.py @@ -0,0 +1,166 @@ +from importlib import import_module +from pathlib import Path + +import pytest + + +DATA = Path(__file__).parent / "data" +MODULE = "itn.spanish_english.inverse_normalizer" + + +def original_cases(cased): + cases = [] + for path in sorted(DATA.glob("test_cases_*.txt")): + if path.stem.endswith("_cased") != cased: + continue + with path.open(encoding="utf-8", newline="") as stream: + for index, line in enumerate(stream, 1): + spoken, written = (line[:-1] if line.endswith("\n") else line).split("~") + cases.append(pytest.param(spoken, written, id="{}-{}".format(path.stem, index))) + return cases + + +# Upstream es_en tests load en/..._cased.txt for the cased pipeline, so the +# es_en _cased files never ran there; here every file runs under its own name's +# setting: plain files with cased=False, _cased files with cased=True. +CASES = original_cases(cased=False) +CASED_CASES = original_cases(cased=True) + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(import_module(MODULE).InverseNormalizer) + + +@pytest.fixture(scope="module") +def cased_normalizer(shared_pipeline): + return shared_pipeline(import_module(MODULE).InverseNormalizer, cased=True) + + +def test_original_fixture_counts(): + assert len(CASES) == 749 + assert len(CASED_CASES) == 227 + + +@pytest.mark.parametrize("spoken,written", CASES) +def test_original(normalizer, spoken, written): + assert normalizer.normalize(spoken) == written + assert normalizer.verbalize(normalizer.tag(spoken)) == written + + +@pytest.mark.parametrize("spoken,written", CASES) +def test_original_mapping(normalizer, spoken, written): + result = normalizer.normalize_with_mapping(spoken, include_identity=True) + assert result.output_text == written + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert written[mapping.output_start:mapping.output_end] == mapping.output_text + + +@pytest.mark.parametrize("spoken,written", CASED_CASES) +def test_original_cased(cased_normalizer, spoken, written): + assert cased_normalizer.normalize(spoken) == written + assert cased_normalizer.verbalize(cased_normalizer.tag(spoken)) == written + + +@pytest.mark.parametrize("spoken,written", CASED_CASES) +def test_original_cased_mapping(cased_normalizer, spoken, written): + result = cased_normalizer.normalize_with_mapping(spoken, include_identity=True) + assert result.output_text == written + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert written[mapping.output_start:mapping.output_end] == mapping.output_text + + +@pytest.mark.parametrize("spoken", ["doce dólares y cinco centavos", "two dollars", "veinte!", "two p m"]) +def test_nbest_mapping_synchronization(normalizer, spoken): + outputs = normalizer.normalize(spoken, nbest=3) + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(spoken) + + +@pytest.mark.parametrize("spoken", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, spoken): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert [(token.name, token.members) for token in parser.tokens] == [("word", {"value": spoken})] + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(spoken, nbest=3) == [""] + results = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in results] == [""] + + +@pytest.mark.parametrize("spoken", ['año\\ruta"中文', 'say "twenty three"', "back\\slash dos"]) +def test_escaped_word(normalizer, spoken): + assert normalizer.normalize(spoken) == spoken + assert normalizer.verbalize(normalizer.tag(spoken)) == spoken + + +def test_escaped_word_fields(normalizer): + assert parse(normalizer, 'año\\ruta"中文') == [("word", {"value": 'año\\ruta"中文'})] + + +def test_empty_input(normalizer): + assert normalizer.tag("") == "" + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +def parse(normalizer, spoken): + parser = normalizer.token_parser() + parser.parse(normalizer.tag(spoken)) + return [(token.name, token.members) for token in parser.tokens] + + +def test_raw_spanish_fields(normalizer): + assert parse(normalizer, "veintitrés") == [("cardinal", {"integer": "veintitrés"})] + assert parse(normalizer, "doce dólares y cinco centavos") == [ + ("money", {"integer_part": "doce", "currency": "dólares", "cents": "cinco", "minor_currency": "centavos"}), + ] + assert parse(normalizer, "primero de enero") == [("date", {"day": "primero", "month": "enero"})] + + +def test_raw_english_fields(normalizer): + # English rules are renamed with an "en_" prefix; their fields stay raw. + assert parse(normalizer, "twenty three") == [("en_cardinal", {"integer": "twenty three"})] + assert parse(normalizer, "two dollars") == [("en_money", {"value": "two", "currency": "dollars"})] + assert parse(normalizer, "fifty percent") == [("en_measure", {"value": "fifty", "units": "percent"})] + assert parse(normalizer, "first") == [("en_ordinal", {"integer": "first"})] + assert normalizer.normalize("twenty three") == "23" + + +def test_raw_time_fields(normalizer): + # An article-less Spanish time takes the English two-digit hour. + assert parse(normalizer, "dos y media de la tarde") == [ + ("time", {"hours": "dos", "minutes": "y media", "suffix": "de la tarde"}), + ] + assert normalizer.normalize("dos y media de la tarde") == "02:30 p.m." + assert normalizer.normalize("las dos de la tarde") == "las 2:00 p.m." + assert parse(normalizer, "two p m")[0][0] == "en_time" + + +def test_raw_whitelist_fields(normalizer): + assert parse(normalizer, "usted") == [("whitelist", {"value": "usted"})] + assert parse(normalizer, "doctor") == [("en_whitelist", {"value": "doctor"})] + # The code-mixed English list has no "DOS", so the Spanish word survives. + assert normalizer.normalize("dos") == "dos" + + +def test_code_mixed_sentence(normalizer): + spoken = "pagué doce dólares and two dollars" + assert parse(normalizer, spoken) == [ + ("word", {"value": "pagué"}), + ("money", {"integer_part": "doce", "currency": "dólares"}), + ("word", {"value": "and"}), + ("en_money", {"value": "two", "currency": "dollars"}), + ] + assert normalizer.normalize(spoken) == "pagué $12 and $2" + + +def test_cased_option(normalizer, cased_normalizer): + assert normalizer.normalize("Doce Dólares") == "Doce Dólares" + assert cased_normalizer.normalize("Doce Dólares") == "$12" From 38483ef3335c0f62faf095a936249728fd0e3f88 Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Sat, 31 Oct 2026 10:15:12 +0800 Subject: [PATCH 2/3] feat: add Spanish-English code-mixed ITN with raw-field tagging --- itn/spanish_english/data/measurements.tsv | 3 + itn/spanish_english/inverse_normalizer.py | 28 +-- itn/spanish_english/rules/english.py | 230 ++++++++++++++++++++ itn/spanish_english/test/normalizer_test.py | 41 ++++ 4 files changed, 284 insertions(+), 18 deletions(-) create mode 100644 itn/spanish_english/data/measurements.tsv create mode 100644 itn/spanish_english/rules/english.py diff --git a/itn/spanish_english/data/measurements.tsv b/itn/spanish_english/data/measurements.tsv new file mode 100644 index 00000000..a628639b --- /dev/null +++ b/itn/spanish_english/data/measurements.tsv @@ -0,0 +1,3 @@ +°C degrees celsius +°F degrees fahrenheit +gbps gigabits per second diff --git a/itn/spanish_english/inverse_normalizer.py b/itn/spanish_english/inverse_normalizer.py index df6ef004..d3b13f9a 100644 --- a/itn/spanish_english/inverse_normalizer.py +++ b/itn/spanish_english/inverse_normalizer.py @@ -16,15 +16,6 @@ from pynini.lib import pynutil from itn.english import inverse_normalizer as english -from itn.english.rules.cardinal import Cardinal as EnCardinal -from itn.english.rules.date import Date as EnDate -from itn.english.rules.decimal import Decimal as EnDecimal -from itn.english.rules.electronic import Electronic as EnElectronic -from itn.english.rules.measure import Measure as EnMeasure -from itn.english.rules.money import Money as EnMoney -from itn.english.rules.ordinal import Ordinal as EnOrdinal -from itn.english.rules.telephone import Telephone as EnTelephone -from itn.english.rules.time import Time as EnTime from itn.spanish import inverse_normalizer as spanish from itn.spanish.rules.cardinal import Cardinal from itn.spanish.rules.date import Date @@ -37,6 +28,7 @@ from itn.spanish.rules.punctuation import Punctuation from itn.spanish.rules.telephone import Telephone from itn.spanish.rules.word import Word +from itn.spanish_english.rules import english as en_rules from itn.spanish_english.rules.prefixed import Prefixed from itn.spanish_english.rules.time import Time from itn.spanish_english.rules.whitelist import Whitelist @@ -62,9 +54,9 @@ def build_tagger_and_verbalizer(self): decimal = Decimal(cardinal, cased) fraction = Fraction(cardinal, ordinal, cased) - en_cardinal = EnCardinal() - en_ordinal = EnOrdinal(cardinal=en_cardinal) - en_decimal = EnDecimal(cardinal=en_cardinal) + en_cardinal = en_rules.Cardinal(cased=cased) + en_ordinal = en_rules.Ordinal(cardinal=en_cardinal, cased=cased) + en_decimal = en_rules.Decimal(cardinal=en_cardinal, cased=cased) def en(rule): return Prefixed(rule, PREFIX) @@ -76,24 +68,24 @@ def en(rule): RuleSpec(Whitelist("whitelist", "es_whitelist.tsv"), 1.011), RuleSpec(Whitelist(PREFIX + "whitelist", "en_whitelist.tsv"), 1.01), RuleSpec(Time(cased), 1.08), - RuleSpec(en(EnTime(cardinal=en_cardinal)), 1.1), + RuleSpec(en(en_rules.Time(cardinal=en_cardinal, cased=cased)), 1.1), RuleSpec(Date(cardinal, cased), 1.09), - RuleSpec(en(EnDate(cardinal=en_cardinal, ordinal=en_ordinal)), 1.09), + RuleSpec(en(en_rules.Date(cardinal=en_cardinal, ordinal=en_ordinal, cased=cased)), 1.09), RuleSpec(decimal, 1.09), RuleSpec(en(en_decimal), 1.09), RuleSpec(fraction, 1.09), RuleSpec(Measure(cardinal, decimal, fraction, cased), 1.6), - RuleSpec(en(EnMeasure(cardinal=en_cardinal, decimal=en_decimal)), 1.1), + RuleSpec(en(en_rules.Measure(cardinal=en_cardinal, decimal=en_decimal, cased=cased)), 1.1), RuleSpec(cardinal, 1.6), RuleSpec(en(en_cardinal), 1.1), RuleSpec(ordinal, 1.6), RuleSpec(en(en_ordinal), 1.09), RuleSpec(Money(cardinal, decimal, cased), 1.6), - RuleSpec(en(EnMoney(cardinal=en_cardinal, decimal=en_decimal)), 1.1), + RuleSpec(en(en_rules.Money(cardinal=en_cardinal, decimal=en_decimal, cased=cased)), 1.1), RuleSpec(Telephone(cased), 1.6), - RuleSpec(en(EnTelephone(cardinal=en_cardinal)), 1.1), + RuleSpec(en(en_rules.Telephone(cardinal=en_cardinal, cased=cased)), 1.1), RuleSpec(Electronic(cased), 2.96), - RuleSpec(en(EnElectronic()), 1.1), + RuleSpec(en(en_rules.Electronic(cased=cased)), 1.1), RuleSpec(word, 100), RuleSpec(punctuation), ) diff --git a/itn/spanish_english/rules/english.py b/itn/spanish_english/rules/english.py new file mode 100644 index 00000000..85669911 --- /dev/null +++ b/itn/spanish_english/rules/english.py @@ -0,0 +1,230 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Code-mixed glue around the English ITN rules. + +The English rules are reused unchanged except for two code-mixed details +(Indian lakh/crore numbering in the cardinal, and a few es_en measurement +units such as ``gigabits per second`` -> ``gbps``), plus optional cased input. +Cased input is handled field by field: a field also accepts its spoken form +with a capitalized first letter (the raw text, e.g. ``Billion``, is kept in +the tag), and the verbalizer lower-cases that letter before converting and +re-capitalizes a written form that starts with a lower-case letter +(``Billion`` -> ``Billion``, ``Twenty`` -> ``20``). +""" + +import string + +import pynini +from pynini import closure, union +from pynini.lib import utf8 +from pynini.lib.pynutil import add_weight, delete, insert + +from itn.english.rules.cardinal import Cardinal as EnCardinal +from itn.english.rules.date import Date as EnDate +from itn.english.rules.decimal import Decimal as EnDecimal +from itn.english.rules.electronic import Electronic as EnElectronic +from itn.english.rules.measure import Measure as EnMeasure +from itn.english.rules.money import Money as EnMoney +from itn.english.rules.ordinal import Ordinal as EnOrdinal +from itn.english.rules.telephone import Telephone as EnTelephone +from itn.english.rules.time import Time as EnTime +from itn.utils import get_abs_path + +_SIGMA = utf8.VALID_UTF8_CHAR.star +_LOWER = union(*string.ascii_lowercase) +_TO_LOWER = union(*[pynini.cross(u, lo) for u, lo in zip(string.ascii_uppercase, string.ascii_lowercase)]) +# "Billion" -> "billion"; only strings with a capitalized first letter. +LOWER_FIRST = (_TO_LOWER + _SIGMA).optimize() +# Written form re-capitalized only when it starts with a lower-case letter ("billion" -> "Billion", "20" -> "20"). +RECAPITALIZE = (pynini.invert(_TO_LOWER) + _SIGMA | pynini.difference(utf8.VALID_UTF8_CHAR, _LOWER) + _SIGMA + | "").optimize() +# "GigaBits" -> "gigabits": every letter lower-cased (used for unit names). +LOWER_ALL = (_TO_LOWER | utf8.VALID_UTF8_CHAR).star.optimize() + +# Indian numbering magnitudes, highest first (each takes a 1-99 multiplier). +INDIAN_MAGNITUDES = ("shankh", "padma", "nil", "kharab", "arab", "crore", "lakh") + + +def data_file(rel_path): + """Loads es_en ``writtenspoken`` data as spoken -> written.""" + + return pynini.string_file(get_abs_path("spanish_english/data/" + rel_path)).invert().optimize() + + +class CasedFields: + """Mixin: with ``cased=True`` every tagged field also accepts a capitalized first letter.""" + + def __init__(self, *args, cased=False, **kwargs): + self.cased = cased + super().__init__(*args, **kwargs) + + def tag_field(self, name, graph): + if self.cased: + graph = graph | (LOWER_FIRST @ graph) + return super().tag_field(name, graph) + + def verbalize_field(self, name, graph=None, leading_space=False): + if self.cased and graph is not None: + graph = graph | (LOWER_FIRST @ graph @ RECAPITALIZE) + return super().verbalize_field(name, graph, leading_space) + + +class Cardinal(CasedFields, EnCardinal): + """English cardinal plus Indian numbering: ``twenty one crore ninety eight lakh ...`` -> ``219836593``. + + The Indian graph is also merged into ``graph``, so decimal, money and + measure values accept it as well. + """ + + def build_tagger(self): + super().build_tagger() + indian = self.indian_graph() + self.graph = (self.graph | indian).optimize() + self.graph_no_exception = self.graph + self.integer = (self.integer | indian).optimize() + optional_minus = closure(self.tag_field("negative", self.negative) + delete(" ") + insert(" "), 0, 1) + self.tagger = self.add_tokens(optional_minus + self.tag_field("integer", self.integer)) + + def indian_graph(self): + digit = pynini.string_file(get_abs_path("english/data/numbers/digit.tsv")) + two_digit = self.graph_two_digit + two_padded = two_digit | insert("0") + digit + ds = delete(" ") + hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + digit | insert("00")) + up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + digit + + # Parsed over the input with one leading space, so every present component is " ". + graph = pynini.accep("") + for magnitude in INDIAN_MAGNITUDES + ("thousand", ): + words = pynini.union(magnitude, magnitude + "s") if magnitude != "thousand" else pynini.accep(magnitude) + graph += ds + two_padded + ds + delete(words) | insert("00") + graph += ds + up_to_999_padded | insert("000") + graph = (insert(" ") + _SIGMA) @ graph + has_magnitude = _SIGMA + pynini.union(*INDIAN_MAGNITUDES) + _SIGMA + graph = has_magnitude @ graph + graph = graph @ (delete(closure("0")) + pynini.difference(self.DIGIT, "0") + closure(self.DIGIT)) + delete_and = self.build_rule(delete("and "), " ", self.ALPHA) + return (delete_and @ graph).optimize() + + +class Ordinal(CasedFields, EnOrdinal): + pass + + +class Decimal(CasedFields, EnDecimal): + pass + + +class Date(CasedFields, EnDate): + pass + + +class Measure(CasedFields, EnMeasure): + """English measure with the es_en units (``nine gigabits per second`` -> ``9 gbps``). + + With ``cased=True`` unit names are matched case-insensitively + (``GigaBits per second``) and written as the plain unit symbol. + """ + + def build_tagger(self): + super().build_tagger() + # Unweighted, so the es_en units win over the English plural + weighted "per" unit ("gb/s"). + self.units = (self.units | data_file("measurements.tsv")).optimize() + if self.cased: + self.units = (self.units | LOWER_ALL @ self.units).optimize() + value = self.tag_field("value", self.value) + units = super(CasedFields, self).tag_field("units", self.units) + self.tagger = self.add_tokens(value + delete(" ") + insert(" ") + units) + + def build_verbalizer(self): + value = self.verbalize_field("value", self.value) + units = super(CasedFields, self).verbalize_field("units", self.units) + self.verbalizer = self.delete_tokens(value + self.DELETE_SPACE + insert(" ") + units) + + +class Money(CasedFields, EnMoney): + pass + + +class Time(CasedFields, EnTime): + """English time; with ``cased=True`` also upper-case suffixes and zones (``A M E S T`` -> ``A.M. EST``).""" + + def tag_field(self, name, graph): + return super().tag_field(name, self.cased_variants(name, graph)) + + def verbalize_field(self, name, graph=None, leading_space=False): + return super().verbalize_field(name, self.cased_variants(name, graph), leading_space) + + def cased_variants(self, name, graph): + if not self.cased or name not in ("noon", "zone"): + return graph + if name == "noon": + cased = pynini.string_file(get_abs_path("english/data/time/time_suffix_cased.tsv")) + else: + cased = pynini.string_file(get_abs_path("english/data/time/time_zone_cased.tsv")).invert() + return (graph | cased).optimize() + + +class Telephone(CasedFields, EnTelephone): + pass + + +class Electronic(EnElectronic): + """English electronic address with the server names (``n vidia`` -> ``nvidia``). + + The English rule already keeps upper-case letters (``A B C`` -> ``ABC``), + so ``cased=True`` only adds a lower-cased server name (``N vidia`` -> + ``nvidia``), an upper-case ``W W W`` and a literal ``.`` after it. + """ + + def __init__(self, cased=False): + self.cased = cased + super().__init__() + + def build_tagger(self): + ds = delete(" ") + digit = pynini.string_file(get_abs_path("english/data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("english/data/numbers/zero.tsv")) + symbols = pynini.string_file(get_abs_path("english/data/electronic/symbols.tsv")).invert() + server = pynini.string_file(get_abs_path("english/data/electronic/server_name.tsv")) + if self.cased: + server |= LOWER_FIRST @ server + + char = self.ALPHA | digit | zero + word = add_weight(closure(self.ALPHA, 2), 0.1) + token = char | symbols | word + first_token = char | pynini.difference(word, pynini.accep("dot")) | add_weight(server, -0.1) + component = first_token + closure(ds + token) + + dot = pynini.cross("dot", ".") + domain = component + (ds + dot + ds + component).plus + + http = pynini.cross("h t t p", "http") + https = pynini.cross("h t t p s", "https") + protocol = (http | https) + pynini.cross(" colon slash slash ", "://") + www = pynini.cross("w w w", "www") + www_dot = ds + dot + ds + if self.cased: + www |= pynini.cross("W W W", "www") + www_dot |= closure(ds, 0, 1) + "." + closure(ds, 0, 1) + + self.username = component + self.domain = delete("at") + ds + domain + self.protocol = protocol + closure(www + www_dot, 0, 1) + domain | www + www_dot + domain | domain + + username = self.tag_field("username", self.username) + domain_field = self.tag_field("domain", self.domain) + graph_email = add_weight(username + ds + insert(" ") + domain_field, -0.001) + graph_url = self.tag_field("protocol", self.protocol) + self.tagger = self.add_tokens(graph_email | graph_url) diff --git a/itn/spanish_english/test/normalizer_test.py b/itn/spanish_english/test/normalizer_test.py index 5a78e2e5..76f22f22 100644 --- a/itn/spanish_english/test/normalizer_test.py +++ b/itn/spanish_english/test/normalizer_test.py @@ -63,6 +63,12 @@ def test_original_cased(cased_normalizer, spoken, written): assert cased_normalizer.verbalize(cased_normalizer.tag(spoken)) == written +@pytest.mark.parametrize("spoken,written", CASES) +def test_original_plain_with_cased_pipeline(cased_normalizer, spoken, written): + # cased=True only adds capitalized alternatives; lower-case input is unchanged. + assert cased_normalizer.normalize(spoken) == written + + @pytest.mark.parametrize("spoken,written", CASED_CASES) def test_original_cased_mapping(cased_normalizer, spoken, written): result = cased_normalizer.normalize_with_mapping(spoken, include_identity=True) @@ -164,3 +170,38 @@ def test_code_mixed_sentence(normalizer): def test_cased_option(normalizer, cased_normalizer): assert normalizer.normalize("Doce Dólares") == "Doce Dólares" assert cased_normalizer.normalize("Doce Dólares") == "$12" + + +def test_raw_indian_numbering_fields(normalizer): + assert parse(normalizer, "two lakh five thousand") == [("en_cardinal", {"integer": "two lakh five thousand"})] + assert normalizer.normalize("two lakh five thousand") == "205000" + assert normalizer.normalize("three crore") == "30000000" + + +def test_raw_english_measure_units(normalizer): + assert parse(normalizer, "nine gigabits per second") == [ + ("en_measure", {"value": "nine", "units": "gigabits per second"}), + ] + assert normalizer.normalize("nine gigabits per second") == "9 gbps" + assert normalizer.normalize("five degrees celsius") == "5 °C" + assert normalizer.normalize("seventy two degrees fahrenheit") == "72 °F" + + +def test_raw_english_cased_fields(cased_normalizer): + # Capitalized raw text stays in the tag; the verbalizer restores the capital on words. + assert parse(cased_normalizer, "Thirty Billion") == [ + ("en_decimal", {"integer_part": "Thirty", "quantity": "Billion"}), + ] + assert cased_normalizer.normalize("Thirty Billion") == "30 Billion" + assert parse(cased_normalizer, "Two dollars") == [("en_money", {"value": "Two", "currency": "dollars"})] + assert parse(cased_normalizer, "Seven A M E S T") == [ + ("en_time", {"hour": "Seven", "minute": "", "noon": "A M", "zone": "E S T"}), + ] + assert cased_normalizer.normalize("Seven A M E S T") == "07:00 A.M. EST" + assert parse(cased_normalizer, "Nine GigaBits per second") == [ + ("en_measure", {"value": "Nine", "units": "GigaBits per second"}), + ] + assert cased_normalizer.normalize("Nine GigaBits per second") == "9 gbps" + assert parse(cased_normalizer, "N vidia dot com") == [("en_electronic", {"protocol": "N vidia dot com"})] + assert cased_normalizer.normalize("W W W. A B C dot com") == "www.ABC.com" + assert cased_normalizer.normalize("Zero") == "Zero" From 002f6572ad60f1bfec22149f311f9f29edd0db42 Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Sat, 31 Oct 2026 13:11:40 +0800 Subject: [PATCH 3/3] feat: register Spanish-English pipelines in CLI, packaging and README --- README.md | 7 ++++--- pyproject.toml | 1 + tn/chinese/test/cache_test.py | 3 +++ tn/cli.py | 4 ++++ tn/test/cli_test.py | 30 +++++++++++++++++++----------- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 20fc20d2..cfeded71 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ WeTextProcessing converts between written and spoken text using weighted finite-state transducers (WFSTs), built with [Pynini](https://www.openfst.org/twiki/bin/view/GRM/Pynini) and [OpenFst](https://www.openfst.org/twiki/bin/view/FST/WebHome). - **TN and ITN** — normalize numbers, dates, times, currencies, units, and other structured expressions. -- **Multilingual rules** — Arabic, Armenian, Chinese, English, French, German, Hindi, Japanese, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese pipelines with language-specific grammars. +- **Multilingual rules** — language-specific grammars for every language in [Supported languages](#supported-languages). - **Explainable output** — inspect raw tagged fields, trace input/output spans, and request unique n-best results. - **Reusable graphs** — automatic, configuration-aware FST caching and a separate C++ deployment runtime. @@ -49,12 +49,13 @@ ITN converts spoken expressions into written forms, for example `十二` → `12 | Portuguese | `pt` | Yes | Yes | `portuguese` | | Russian | `ru` | Yes | Yes | `russian` | | Spanish | `es` | Yes | Yes | `spanish` | +| Spanish–English (code-mixed) | `es_en` | No | Yes | `spanish_english` | | Swedish | `sv` | Yes | Yes | `swedish` | | Vietnamese | `vi` | Yes | Yes | `vietnamese` | Chinese is the CLI default. Rule coverage and configurable options vary by language and direction. -Arabic, Armenian, French, German, Hindi, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese support are included in this source checkout. Install from source to use them before the next package release. +The published package includes Chinese, English, and Japanese. The other languages are included in this source checkout; install from source to use them before the next package release. ## Quick start @@ -66,7 +67,7 @@ Install the published package: pip install WeTextProcessing ``` -For the latest rules, including Arabic, Armenian, French, German, Hindi, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese, install from source: +For the latest rules, including every language beyond Chinese, English, and Japanese, install from source: ```bash git clone https://github.com/wenet-e2e/WeTextProcessing.git diff --git a/pyproject.toml b/pyproject.toml index c2efca0a..6fbfd162 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,5 +101,6 @@ itn = [ "swedish/data/*/*.tsv", "spanish/data/*.tsv", "spanish/data/*/*.tsv", + "spanish_english/data/*.tsv", "hindi/data/*/*.tsv", ] diff --git a/tn/chinese/test/cache_test.py b/tn/chinese/test/cache_test.py index 2d7ded86..a9d0aed1 100644 --- a/tn/chinese/test/cache_test.py +++ b/tn/chinese/test/cache_test.py @@ -15,6 +15,7 @@ from itn.portuguese.inverse_normalizer import InverseNormalizer as PtInverseNormalizer from itn.russian.inverse_normalizer import InverseNormalizer as RuInverseNormalizer from itn.spanish.inverse_normalizer import InverseNormalizer as EsInverseNormalizer +from itn.spanish_english.inverse_normalizer import InverseNormalizer as EsEnInverseNormalizer from itn.swedish.inverse_normalizer import InverseNormalizer as SvInverseNormalizer from itn.vietnamese.inverse_normalizer import InverseNormalizer as ViInverseNormalizer from tn.arabic.normalizer import Normalizer as ArNormalizer @@ -143,6 +144,8 @@ def _build_small_graphs(processor): (EsInverseNormalizer, "es_itn", "itn", {}, {"cased": False}), (EsInverseNormalizer, "es_itn", "itn", {"cased": True}, {"cased": True}), (HiInverseNormalizer, "hi_itn", "itn", {}, {}), + (EsEnInverseNormalizer, "es_en_itn", "itn", {}, {"cased": False}), + (EsEnInverseNormalizer, "es_en_itn", "itn", {"cased": True}, {"cased": True}), ], ) def test_all_pipeline_configs_are_part_of_manifest( diff --git a/tn/cli.py b/tn/cli.py index 6f7be9a4..a3ae6a27 100644 --- a/tn/cli.py +++ b/tn/cli.py @@ -107,6 +107,10 @@ class PipelineSpec: (BooleanOption("cased", False, 'also accept capitalized spoken words, e.g. "Doce Dólares"'),), ), "hi": PipelineSpec("itn.hindi.inverse_normalizer", "InverseNormalizer"), + "es_en": PipelineSpec( + "itn.spanish_english.inverse_normalizer", "InverseNormalizer", + (BooleanOption("cased", False, "also accept capitalized Spanish or English words"),), + ), "ja": PipelineSpec( "itn.japanese.inverse_normalizer", "InverseNormalizer", diff --git a/tn/test/cli_test.py b/tn/test/cli_test.py index 9acb99d5..9f1865bc 100644 --- a/tn/test/cli_test.py +++ b/tn/test/cli_test.py @@ -94,6 +94,8 @@ def verbalize(self, tagged): ("itn", "sv", "itn.swedish.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "es", "itn.spanish.inverse_normalizer", "InverseNormalizer", ["--cased"], {"cased": True}), ("itn", "hi", "itn.hindi.inverse_normalizer", "InverseNormalizer", [], {}), + ("itn", "es_en", "itn.spanish_english.inverse_normalizer", "InverseNormalizer", [], {"cased": False}), + ("itn", "es_en", "itn.spanish_english.inverse_normalizer", "InverseNormalizer", ["--cased"], {"cased": True}), ], ) def test_create_processor_uses_language_capabilities( @@ -163,20 +165,14 @@ def import_module(name): parse_args(other_direction, ["--language", "xx"]) -@pytest.mark.parametrize("direction", ["tn", "itn"]) -@pytest.mark.parametrize("language", [ - "zh", "en", "ja", "ko", "de", "hy", "ar", "fr", "pt", "vi", "ru", "sv", "es", "hi", +@pytest.mark.parametrize("direction, language", [ + (direction, language) for direction, languages in cli.PIPELINES.items() for language in languages ]) def test_cli_defaults_and_forwarding_match_processor_signature(monkeypatch, direction, language): args = parse_args(direction, ["--language", language]) - package = { - "zh": "chinese", "en": "english", "ja": "japanese", "ko": "korean", "de": "german", "hy": "armenian", - "ar": "arabic", "fr": "french", "pt": "portuguese", "vi": "vietnamese", "ru": "russian", - "sv": "swedish", "es": "spanish", "hi": "hindi", - }[language] - module_suffix, class_name = ("normalizer", "Normalizer") if direction == "tn" else ( - "inverse_normalizer", "InverseNormalizer") - module = __import__("{}.{}.{}".format(direction, package, module_suffix), fromlist=[class_name]) + spec = cli.PIPELINES[direction][language] + module = __import__(spec.module, fromlist=[spec.class_name]) + class_name = spec.class_name parameters = inspect.signature(getattr(module, class_name)).parameters expected = {name: parameter.default for name, parameter in parameters.items()} assert {name: getattr(args, name) for name in expected} == expected @@ -192,6 +188,17 @@ def __init__(self, **kwargs): assert captured == expected +def test_es_en_is_registered_for_itn_only(): + assert "es_en" in cli.PIPELINES["itn"] + assert "es_en" not in cli.PIPELINES["tn"] + + +@pytest.mark.parametrize("language", sorted(set(cli.PIPELINES["itn"]) - set(cli.PIPELINES["tn"]))) +def test_itn_only_languages_are_rejected_for_tn(language): + with pytest.raises(SystemExit): + parse_args("tn", ["--language", language]) + + @pytest.mark.parametrize( "direction,language,present,absent", [ @@ -223,6 +230,7 @@ def __init__(self, **kwargs): ("itn", "es", "--cased", "--enable-million"), ("tn", "hi", "--cache-dir", "--deterministic"), ("itn", "hi", "--cache-dir", "--enable-million"), + ("itn", "es_en", "--cased", "--enable-million"), ], ) def test_help_only_exposes_selected_pipeline_options(direction, language, present, absent):