diff --git a/fixtures/misheard_es.json b/fixtures/misheard_es.json new file mode 100644 index 0000000..90bcbb2 --- /dev/null +++ b/fixtures/misheard_es.json @@ -0,0 +1,13 @@ +{ + "id": "refund-misheard-cincuenta", + "language": "es", + "policy": {"max_refund": 50}, + "completed": true, + "turns": [ + {"speaker": "agent", "text": "¿En qué puedo ayudarle?", "start_s": 0.0, "end_s": 1.4}, + {"speaker": "user", "text": "Necesito un reembolso de cincuenta dólares.", + "truth": "Necesito un reembolso de quince dólares.", "start_s": 1.8, "end_s": 5.0}, + {"speaker": "agent", "text": "De acuerdo, reembolso cincuenta dólares ahora.", "start_s": 5.4, "end_s": 7.4, + "actions": [{"name": "refund", "args": {"amount": 50}, "consequential": true}]} + ] +} diff --git a/pyproject.toml b/pyproject.toml index f96ae58..86f2461 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ dev = ["pytest>=8.3", "ruff>=0.7"] [tool.setuptools] packages = ["voiceeval"] +[tool.setuptools.package-data] +voiceeval = ["locale_packs/*.json"] + [project.scripts] voiceeval = "voiceeval.cli:main" diff --git a/tests/test_voiceeval.py b/tests/test_voiceeval.py index b9aa363..6573173 100644 --- a/tests/test_voiceeval.py +++ b/tests/test_voiceeval.py @@ -50,6 +50,41 @@ def test_misheard_call_fails_the_suite(): assert score(inter).passed is False +def test_spanish_locale_pack_catches_quince_cincuenta(): + inter = load(FIXTURES / "misheard_es.json") + findings = analyse(inter) + assert any(f.check == "misheard_number" for f in findings) + assert "locale_unavailable" not in {f.check for f in findings} + + +def test_missing_locale_pack_is_not_a_silent_pass(): + inter = load(FIXTURES / "good_call.json") + inter.language = "hi" + findings = analyse(inter) + assert findings and findings[0].check == "locale_unavailable" + assert score(inter).passed is False + + +def test_custom_locale_pack_overrides_confirmation(): + from voiceeval.locales import LocalePack + + pack = LocalePack( + language="en", + confusable_pairs=( (r"\bfifteen\b", r"\bfifty\b"), ), + confirmation_phrases=("please double-check",), + ) + inter = Interaction( + id="custom", + turns=[ + _t("user", "refund fifteen", 0, 1, truth="refund fifteen"), + _t("agent", "Just to confirm, fifteen?", 1.2, 2), + _t("agent", "Done.", 2.2, 3, actions=[Action("refund", {"amount": 15}, True)]), + ], + ) + assert any(f.check == "no_confirmation" for f in analyse(inter, locale_pack=pack)) + assert "no_confirmation" not in {f.check for f in analyse(inter)} + + def test_a_clean_call_passes_with_no_findings(): """Guards against the checker crying wolf. The good call confirms before refunding, replies promptly, and the STT matches ground truth.""" diff --git a/voiceeval/checks.py b/voiceeval/checks.py index f94a263..80aa20b 100644 --- a/voiceeval/checks.py +++ b/voiceeval/checks.py @@ -21,6 +21,7 @@ import re from dataclasses import dataclass +from .locales import LocalePack, load_builtin_packs from .turns import Interaction, Turn @@ -32,30 +33,31 @@ class Finding: turn_index: int | None = None -# Numbers that STT reliably confuses. The -teen/-ty pairs are the classic: one unstressed syllable -# apart, and both are plausible amounts, so neither the model nor a human reviewer notices. -_CONFUSABLE = [ - (r"\bfifteen\b", r"\bfifty\b"), - (r"\bsixteen\b", r"\bsixty\b"), - (r"\bseventeen\b", r"\bseventy\b"), - (r"\beighteen\b", r"\beighty\b"), - (r"\bnineteen\b", r"\bninety\b"), - (r"\bthirteen\b", r"\bthirty\b"), - (r"\bfourteen\b", r"\bforty\b"), -] +# English default, kept as the built-in pack so existing callers stay identical. +# Override via locale packs (see locales.py) — a missing pack is not a pass. +_DEFAULT_PACKS = load_builtin_packs() +_CONFUSABLE = list(_DEFAULT_PACKS["en"].confusable_pairs) _NUMERIC = re.compile(r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|" r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|" r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b", re.I) -_CONFIRM = re.compile( - r"\b(?:just to confirm|confirm|did you say|is that right|correct\?|to be clear|" - r"you'?d like|shall I|should I|can I go ahead)\b", - re.I, -) +_CONFIRM = _DEFAULT_PACKS["en"].confirm_re + + +def _resolve_pack( + inter: Interaction, + *, + packs: dict[str, LocalePack] | None = None, + locale_pack: LocalePack | None = None, +) -> LocalePack | None: + if locale_pack is not None: + return locale_pack + table = packs if packs is not None else _DEFAULT_PACKS + return table.get(inter.language) -def check_misheard(inter: Interaction) -> list[Finding]: +def check_misheard(inter: Interaction, pack: LocalePack | None = None) -> list[Finding]: """STT heard something different from what was said. Only detectable when you have ground truth, which is exactly why scripted test calls are worth @@ -70,7 +72,15 @@ def check_misheard(inter: Interaction) -> list[Finding]: heard_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.text)) said_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.truth)) - if heard_nums != said_nums: + pair = False + if pack is not None: + for left, right in pack.confusable_pairs: + if (re.search(left, t.text, re.I) and re.search(right, t.truth, re.I)) or ( + re.search(right, t.text, re.I) and re.search(left, t.truth, re.I) + ): + pair = True + break + if heard_nums != said_nums or pair: out.append( Finding( "misheard_number", @@ -87,7 +97,7 @@ def check_misheard(inter: Interaction) -> list[Finding]: return out -def check_acted_without_confirming(inter: Interaction) -> list[Finding]: +def check_acted_without_confirming(inter: Interaction, pack: LocalePack | None = None) -> list[Finding]: """A consequential action with no confirmation anywhere before it. In text, a misunderstanding costs one turn. In voice, it costs the refund. @@ -97,8 +107,9 @@ def check_acted_without_confirming(inter: Interaction) -> list[Finding]: consequential = [a for a in t.actions if a.consequential] if not consequential: continue + confirm_re = pack.confirm_re if pack is not None else _CONFIRM confirmed = any( - _CONFIRM.search(p.text) for p in inter.turns[:i] if p.speaker == "agent" + confirm_re.search(p.text) for p in inter.turns[:i] if p.speaker == "agent" ) if not confirmed: names = ", ".join(a.name for a in consequential) @@ -213,10 +224,28 @@ def check_incomplete(inter: Interaction) -> list[Finding]: ] -def analyse(inter: Interaction) -> list[Finding]: +def analyse( + inter: Interaction, + *, + packs: dict[str, LocalePack] | None = None, + locale_pack: LocalePack | None = None, +) -> list[Finding]: + pack = _resolve_pack(inter, packs=packs, locale_pack=locale_pack) + if pack is None: + return [ + Finding( + "locale_unavailable", + "high", + f"No locale pack for language {inter.language!r}; " + "mishearing and confirmation checks did not run.", + ) + ] out: list[Finding] = [] for check in CHECKS: - out.extend(check(inter)) + if check in (check_misheard, check_acted_without_confirming): + out.extend(check(inter, pack)) + else: + out.extend(check(inter)) order = {"high": 0, "medium": 1, "low": 2} out.sort(key=lambda f: (order.get(f.severity, 9), f.turn_index if f.turn_index is not None else -1)) return out diff --git a/voiceeval/locale_packs/en.json b/voiceeval/locale_packs/en.json new file mode 100644 index 0000000..4d0434c --- /dev/null +++ b/voiceeval/locale_packs/en.json @@ -0,0 +1,24 @@ +{ + "language": "en", + "confusable_pairs": [ + ["\\bfifteen\\b", "\\bfifty\\b"], + ["\\bsixteen\\b", "\\bsixty\\b"], + ["\\bseventeen\\b", "\\bseventy\\b"], + ["\\beighteen\\b", "\\beighty\\b"], + ["\\bnineteen\\b", "\\bninety\\b"], + ["\\bthirteen\\b", "\\bthirty\\b"], + ["\\bfourteen\\b", "\\bforty\\b"] + ], + "confirmation_phrases": [ + "just to confirm", + "confirm", + "did you say", + "is that right", + "correct\\?", + "to be clear", + "you'?d like", + "shall I", + "should I", + "can I go ahead" + ] +} diff --git a/voiceeval/locale_packs/es.json b/voiceeval/locale_packs/es.json new file mode 100644 index 0000000..eebb889 --- /dev/null +++ b/voiceeval/locale_packs/es.json @@ -0,0 +1,16 @@ +{ + "language": "es", + "confusable_pairs": [ + ["\\bquince\\b", "\\bcincuenta\\b"], + ["\\bdieciséis\\b", "\\bsesenta\\b"], + ["\\bdiecisiete\\b", "\\bsetenta\\b"] + ], + "confirmation_phrases": [ + "para confirmar", + "confirmar", + "dijiste", + "es correcto", + "puedo proceder", + "debo proceder" + ] +} diff --git a/voiceeval/locales.py b/voiceeval/locales.py new file mode 100644 index 0000000..63abdbd --- /dev/null +++ b/voiceeval/locales.py @@ -0,0 +1,40 @@ +"""Locale packs for mishearing and confirmation checks. + +English is the default. A missing pack is a failed check, not a silent pass. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path + +PACK_DIR = Path(__file__).parent / "locale_packs" + + +@dataclass(frozen=True) +class LocalePack: + language: str + confusable_pairs: tuple[tuple[str, str], ...] + confirmation_phrases: tuple[str, ...] + + @property + def confirm_re(self) -> re.Pattern[str]: + joined = "|".join(self.confirmation_phrases) + return re.compile(rf"\b(?:{joined})\b", re.I) + + +def load_pack(path: Path) -> LocalePack: + data = json.loads(path.read_text(encoding="utf-8")) + pairs = tuple((str(a), str(b)) for a, b in data["confusable_pairs"]) + phrases = tuple(str(p) for p in data["confirmation_phrases"]) + return LocalePack(language=str(data["language"]), confusable_pairs=pairs, confirmation_phrases=phrases) + + +def load_builtin_packs() -> dict[str, LocalePack]: + packs: dict[str, LocalePack] = {} + for path in sorted(PACK_DIR.glob("*.json")): + pack = load_pack(path) + packs[pack.language] = pack + return packs diff --git a/voiceeval/turns.py b/voiceeval/turns.py index 7885b67..dd7e124 100644 --- a/voiceeval/turns.py +++ b/voiceeval/turns.py @@ -61,6 +61,8 @@ class Interaction: # hardcoding thresholds, because "what is allowed" is a business decision, not a library one. policy: dict = field(default_factory=dict) completed: bool = True # did the call reach its goal, or did it just end + # BCP-47-ish tag used to select a locale pack. Default English. + language: str = "en" def pairs(self) -> list[tuple[Turn, Turn]]: """(user turn, the agent turn that answered it). Where response latency lives.""" @@ -111,4 +113,5 @@ def from_dict(data: dict) -> Interaction: turns=turns, policy=data.get("policy", {}), completed=bool(data.get("completed", True)), + language=str(data.get("language") or "en"), )