feat: locale packs so non-English calls are not silent passes

fixtures/misheard_es.json

@@ -0,0 +1,13 @@
+{
+  "id": "refund-misheard-cincuenta",
+  "language": "es",
+  "policy": {"max_refund": 50},
+  "completed": true,
+  "turns": [
+    {"speaker": "agent", "text": "¿En qué puedo ayudarle?", "start_s": 0.0, "end_s": 1.4},
+    {"speaker": "user", "text": "Necesito un reembolso de cincuenta dólares.",
+     "truth": "Necesito un reembolso de quince dólares.", "start_s": 1.8, "end_s": 5.0},
+    {"speaker": "agent", "text": "De acuerdo, reembolso cincuenta dólares ahora.", "start_s": 5.4, "end_s": 7.4,
+     "actions": [{"name": "refund", "args": {"amount": 50}, "consequential": true}]}
+  ]
+}

pyproject.toml

@@ -21,6 +21,9 @@ dev = ["pytest>=8.3", "ruff>=0.7"]
 [tool.setuptools]
 packages = ["voiceeval"]
 
+[tool.setuptools.package-data]
+voiceeval = ["locale_packs/*.json"]
+
 [project.scripts]
 voiceeval = "voiceeval.cli:main"
 

tests/test_voiceeval.py

@@ -50,6 +50,41 @@ def test_misheard_call_fails_the_suite():
     assert score(inter).passed is False
 
 
+def test_spanish_locale_pack_catches_quince_cincuenta():
+    inter = load(FIXTURES / "misheard_es.json")
+    findings = analyse(inter)
+    assert any(f.check == "misheard_number" for f in findings)
+    assert "locale_unavailable" not in {f.check for f in findings}
+
+
+def test_missing_locale_pack_is_not_a_silent_pass():
+    inter = load(FIXTURES / "good_call.json")
+    inter.language = "hi"
+    findings = analyse(inter)
+    assert findings and findings[0].check == "locale_unavailable"
+    assert score(inter).passed is False
+
+
+def test_custom_locale_pack_overrides_confirmation():
+    from voiceeval.locales import LocalePack
+
+    pack = LocalePack(
+        language="en",
+        confusable_pairs=( (r"\bfifteen\b", r"\bfifty\b"), ),
+        confirmation_phrases=("please double-check",),
+    )
+    inter = Interaction(
+        id="custom",
+        turns=[
+            _t("user", "refund fifteen", 0, 1, truth="refund fifteen"),
+            _t("agent", "Just to confirm, fifteen?", 1.2, 2),
+            _t("agent", "Done.", 2.2, 3, actions=[Action("refund", {"amount": 15}, True)]),
+        ],
+    )
+    assert any(f.check == "no_confirmation" for f in analyse(inter, locale_pack=pack))
+    assert "no_confirmation" not in {f.check for f in analyse(inter)}
+
+
 def test_a_clean_call_passes_with_no_findings():
     """Guards against the checker crying wolf. The good call confirms before refunding, replies
     promptly, and the STT matches ground truth."""

voiceeval/checks.py

@@ -21,6 +21,7 @@
 import re
 from dataclasses import dataclass
 
+from .locales import LocalePack, load_builtin_packs
 from .turns import Interaction, Turn
 
 
@@ -32,30 +33,31 @@ class Finding:
     turn_index: int | None = None
 
 
-# Numbers that STT reliably confuses. The -teen/-ty pairs are the classic: one unstressed syllable
-# apart, and both are plausible amounts, so neither the model nor a human reviewer notices.
-_CONFUSABLE = [
-    (r"\bfifteen\b", r"\bfifty\b"),
-    (r"\bsixteen\b", r"\bsixty\b"),
-    (r"\bseventeen\b", r"\bseventy\b"),
-    (r"\beighteen\b", r"\beighty\b"),
-    (r"\bnineteen\b", r"\bninety\b"),
-    (r"\bthirteen\b", r"\bthirty\b"),
-    (r"\bfourteen\b", r"\bforty\b"),
-]
+# English default, kept as the built-in pack so existing callers stay identical.
+# Override via locale packs (see locales.py) — a missing pack is not a pass.
+_DEFAULT_PACKS = load_builtin_packs()
+_CONFUSABLE = list(_DEFAULT_PACKS["en"].confusable_pairs)
 
 _NUMERIC = re.compile(r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|"
                       r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|"
                       r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b", re.I)
 
-_CONFIRM = re.compile(
-    r"\b(?:just to confirm|confirm|did you say|is that right|correct\?|to be clear|"
-    r"you'?d like|shall I|should I|can I go ahead)\b",
-    re.I,
-)
+_CONFIRM = _DEFAULT_PACKS["en"].confirm_re
+
+
+def _resolve_pack(
+    inter: Interaction,
+    *,
+    packs: dict[str, LocalePack] | None = None,
+    locale_pack: LocalePack | None = None,
+) -> LocalePack | None:
+    if locale_pack is not None:
+        return locale_pack
+    table = packs if packs is not None else _DEFAULT_PACKS
+    return table.get(inter.language)
 
 
-def check_misheard(inter: Interaction) -> list[Finding]:
+def check_misheard(inter: Interaction, pack: LocalePack | None = None) -> list[Finding]:
     """STT heard something different from what was said.
 
     Only detectable when you have ground truth, which is exactly why scripted test calls are worth
@@ -70,7 +72,15 @@ def check_misheard(inter: Interaction) -> list[Finding]:
 
         heard_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.text))
         said_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.truth))
-        if heard_nums != said_nums:
+        pair = False
+        if pack is not None:
+            for left, right in pack.confusable_pairs:
+                if (re.search(left, t.text, re.I) and re.search(right, t.truth, re.I)) or (
+                    re.search(right, t.text, re.I) and re.search(left, t.truth, re.I)
+                ):
+                    pair = True
+                    break
+        if heard_nums != said_nums or pair:
             out.append(
                 Finding(
                     "misheard_number",
@@ -87,7 +97,7 @@ def check_misheard(inter: Interaction) -> list[Finding]:
     return out
 
 
-def check_acted_without_confirming(inter: Interaction) -> list[Finding]:
+def check_acted_without_confirming(inter: Interaction, pack: LocalePack | None = None) -> list[Finding]:
     """A consequential action with no confirmation anywhere before it.
 
     In text, a misunderstanding costs one turn. In voice, it costs the refund.
@@ -97,8 +107,9 @@ def check_acted_without_confirming(inter: Interaction) -> list[Finding]:
         consequential = [a for a in t.actions if a.consequential]
         if not consequential:
             continue
+        confirm_re = pack.confirm_re if pack is not None else _CONFIRM
         confirmed = any(
-            _CONFIRM.search(p.text) for p in inter.turns[:i] if p.speaker == "agent"
+            confirm_re.search(p.text) for p in inter.turns[:i] if p.speaker == "agent"
         )
         if not confirmed:
             names = ", ".join(a.name for a in consequential)
@@ -213,10 +224,28 @@ def check_incomplete(inter: Interaction) -> list[Finding]:
 ]
 
 
-def analyse(inter: Interaction) -> list[Finding]:
+def analyse(
+    inter: Interaction,
+    *,
+    packs: dict[str, LocalePack] | None = None,
+    locale_pack: LocalePack | None = None,
+) -> list[Finding]:
+    pack = _resolve_pack(inter, packs=packs, locale_pack=locale_pack)
+    if pack is None:
+        return [
+            Finding(
+                "locale_unavailable",
+                "high",
+                f"No locale pack for language {inter.language!r}; "
+                "mishearing and confirmation checks did not run.",
+            )
+        ]
     out: list[Finding] = []
     for check in CHECKS:
-        out.extend(check(inter))
+        if check in (check_misheard, check_acted_without_confirming):
+            out.extend(check(inter, pack))
+        else:
+            out.extend(check(inter))
     order = {"high": 0, "medium": 1, "low": 2}
     out.sort(key=lambda f: (order.get(f.severity, 9), f.turn_index if f.turn_index is not None else -1))
     return out

voiceeval/locale_packs/en.json

@@ -0,0 +1,24 @@
+{
+  "language": "en",
+  "confusable_pairs": [
+    ["\\bfifteen\\b", "\\bfifty\\b"],
+    ["\\bsixteen\\b", "\\bsixty\\b"],
+    ["\\bseventeen\\b", "\\bseventy\\b"],
+    ["\\beighteen\\b", "\\beighty\\b"],
+    ["\\bnineteen\\b", "\\bninety\\b"],
+    ["\\bthirteen\\b", "\\bthirty\\b"],
+    ["\\bfourteen\\b", "\\bforty\\b"]
+  ],
+  "confirmation_phrases": [
+    "just to confirm",
+    "confirm",
+    "did you say",
+    "is that right",
+    "correct\\?",
+    "to be clear",
+    "you'?d like",
+    "shall I",
+    "should I",
+    "can I go ahead"
+  ]
+}

voiceeval/locale_packs/es.json

@@ -0,0 +1,16 @@
+{
+  "language": "es",
+  "confusable_pairs": [
+    ["\\bquince\\b", "\\bcincuenta\\b"],
+    ["\\bdieciséis\\b", "\\bsesenta\\b"],
+    ["\\bdiecisiete\\b", "\\bsetenta\\b"]
+  ],
+  "confirmation_phrases": [
+    "para confirmar",
+    "confirmar",
+    "dijiste",
+    "es correcto",
+    "puedo proceder",
+    "debo proceder"
+  ]
+}

voiceeval/locales.py

@@ -0,0 +1,40 @@
+"""Locale packs for mishearing and confirmation checks.
+
+English is the default. A missing pack is a failed check, not a silent pass.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+PACK_DIR = Path(__file__).parent / "locale_packs"
+
+
+@dataclass(frozen=True)
+class LocalePack:
+    language: str
+    confusable_pairs: tuple[tuple[str, str], ...]
+    confirmation_phrases: tuple[str, ...]
+
+    @property
+    def confirm_re(self) -> re.Pattern[str]:
+        joined = "|".join(self.confirmation_phrases)
+        return re.compile(rf"\b(?:{joined})\b", re.I)
+
+
+def load_pack(path: Path) -> LocalePack:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    pairs = tuple((str(a), str(b)) for a, b in data["confusable_pairs"])
+    phrases = tuple(str(p) for p in data["confirmation_phrases"])
+    return LocalePack(language=str(data["language"]), confusable_pairs=pairs, confirmation_phrases=phrases)
+
+
+def load_builtin_packs() -> dict[str, LocalePack]:
+    packs: dict[str, LocalePack] = {}
+    for path in sorted(PACK_DIR.glob("*.json")):
+        pack = load_pack(path)
+        packs[pack.language] = pack
+    return packs

voiceeval/turns.py

@@ -61,6 +61,8 @@ class Interaction:
     # hardcoding thresholds, because "what is allowed" is a business decision, not a library one.
     policy: dict = field(default_factory=dict)
     completed: bool = True  # did the call reach its goal, or did it just end
+    # BCP-47-ish tag used to select a locale pack. Default English.
+    language: str = "en"
 
     def pairs(self) -> list[tuple[Turn, Turn]]:
         """(user turn, the agent turn that answered it). Where response latency lives."""
@@ -111,4 +113,5 @@ def from_dict(data: dict) -> Interaction:
         turns=turns,
         policy=data.get("policy", {}),
         completed=bool(data.get("completed", True)),
+        language=str(data.get("language") or "en"),
     )