diff --git a/fixtures/misheard_account.json b/fixtures/misheard_account.json new file mode 100644 index 0000000..8b9be56 --- /dev/null +++ b/fixtures/misheard_account.json @@ -0,0 +1,12 @@ +{ + "id": "account-oh-zero", + "policy": {"max_refund": 50}, + "completed": true, + "turns": [ + {"speaker": "agent", "text": "Can I have the account number?", "start_s": 0.0, "end_s": 1.6}, + {"speaker": "user", "text": "Account number oh four eight double seven.", + "truth": "Account number zero four a seventy seven.", "start_s": 2.0, "end_s": 5.4}, + {"speaker": "agent", "text": "Pulling up oh four eight double seven now.", "start_s": 5.8, "end_s": 8.0, + "actions": [{"name": "lookup_account", "args": {"account": "04877"}, "consequential": true}]} + ] +} diff --git a/tests/test_voiceeval.py b/tests/test_voiceeval.py index b9aa363..4257f3f 100644 --- a/tests/test_voiceeval.py +++ b/tests/test_voiceeval.py @@ -50,6 +50,15 @@ def test_misheard_call_fails_the_suite(): assert score(inter).passed is False +def test_digit_confusions_on_an_account_number_are_high_severity(): + """oh/zero, a/eight, double-seven/seventy-seven on a card-style readback.""" + inter = load(FIXTURES / "misheard_account.json") + findings = analyse(inter) + misheard = [f for f in findings if f.check == "misheard_number"] + assert misheard, "digit-level account confusion was not caught" + assert misheard[0].severity == "high" + + def test_a_clean_call_passes_with_no_findings(): """Guards against the checker crying wolf. The good call confirms before refunding, replies promptly, and the STT matches ground truth.""" diff --git a/voiceeval/checks.py b/voiceeval/checks.py index f94a263..2ddb844 100644 --- a/voiceeval/checks.py +++ b/voiceeval/checks.py @@ -42,6 +42,41 @@ class Finding: (r"\bnineteen\b", r"\bninety\b"), (r"\bthirteen\b", r"\bthirty\b"), (r"\bfourteen\b", r"\bforty\b"), + # Card / account readback: unstressed "oh" and "zero" are the same vowel. + (r"\boh\b", r"\bzero\b"), + # Same one-phoneme collapse as fifteen/fifty. + (r"\ba\b", r"\beight\b"), + # Amounts and quantities: "to"/"too" vs "two". + (r"\bto\b", r"\btwo\b"), + (r"\btoo\b", r"\btwo\b"), + # How people read repeated digits aloud vs the -ty form. + (r"\bdouble\s+one\b", r"\btwenty[\s-]+one\b"), + (r"\bdouble\s+two\b", r"\btwenty[\s-]+two\b"), + (r"\bdouble\s+three\b", r"\bthirty[\s-]+three\b"), + (r"\bdouble\s+four\b", r"\bforty[\s-]+four\b"), + (r"\bdouble\s+five\b", r"\bfifty[\s-]+five\b"), + (r"\bdouble\s+six\b", r"\bsixty[\s-]+six\b"), + (r"\bdouble\s+seven\b", r"\bseventy[\s-]+seven\b"), + (r"\bdouble\s+eight\b", r"\beighty[\s-]+eight\b"), + (r"\bdouble\s+nine\b", r"\bninety[\s-]+nine\b"), + # Narrowband telephony: these letters collapse on a phone line and show + # up in reference codes and postcodes. + (r"\bb\b", r"\bd\b"), + (r"\bb\b", r"\be\b"), + (r"\bb\b", r"\bp\b"), + (r"\bb\b", r"\bt\b"), + (r"\bb\b", r"\bv\b"), + (r"\bb\b", r"\bz\b"), + (r"\bd\b", r"\be\b"), + (r"\bd\b", r"\bt\b"), + (r"\be\b", r"\bp\b"), + (r"\be\b", r"\bt\b"), + (r"\be\b", r"\bv\b"), + (r"\bp\b", r"\bt\b"), + (r"\bp\b", r"\bv\b"), + (r"\bt\b", r"\bv\b"), + (r"\bt\b", r"\bz\b"), + (r"\bv\b", r"\bz\b"), ] _NUMERIC = re.compile(r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|" @@ -70,13 +105,18 @@ def check_misheard(inter: Interaction) -> list[Finding]: heard_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.text)) said_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.truth)) - if heard_nums != said_nums: + pair = _confusable_hit(t.text, t.truth) + if heard_nums != said_nums or pair: + detail = ( + f"STT heard {sorted(heard_nums) or '[]'} but caller said {sorted(said_nums) or '[]'}." + if heard_nums != said_nums + else f"STT swapped a high-cost pair ({pair}): {t.text!r} vs {t.truth!r}." + ) out.append( Finding( "misheard_number", "high", - f"STT heard {sorted(heard_nums) or '[]'} but caller said {sorted(said_nums) or '[]'}. " - "A wrong number the agent acts on is the most expensive failure in voice.", + f"{detail} A wrong number the agent acts on is the most expensive failure in voice.", i, ) ) @@ -224,3 +264,15 @@ def analyse(inter: Interaction) -> list[Finding]: def _norm(s: str) -> str: return re.sub(r"[^a-z0-9 ]", "", s.lower()).strip() + + +def _confusable_hit(text: str, truth: str) -> str | None: + """Return a label if truth and STT differ by a known acoustic pair.""" + for left, right in _CONFUSABLE: + left_in_text = re.search(left, text, re.I) + right_in_truth = re.search(right, truth, re.I) + right_in_text = re.search(right, text, re.I) + left_in_truth = re.search(left, truth, re.I) + if (left_in_text and right_in_truth) or (right_in_text and left_in_truth): + return f"{left} ↔ {right}" + return None