From d8f3ce7524c5ec428c9e9a3499608c16a14dedce Mon Sep 17 00:00:00 2001 From: xin Date: Sat, 24 Oct 2026 15:08:17 +0800 Subject: [PATCH 1/2] VerifyResult: add notice and the cold-start shadow verdict (0.5.0) The API returns a plain-language notice on cold-start calls, and now a 'shadow' verdict with what the shared stock model would have decided. The client dropped both, so a new account's first calls showed only approved=False. Both fields are optional and None when absent, so this works unchanged against servers that don't send them. --- CHANGELOG.md | 12 ++++++++++++ README.md | 13 +++++++++++++ cacheverifier/__init__.py | 6 +++--- cacheverifier/client.py | 36 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_client.py | 28 +++++++++++++++++++++++++++- 6 files changed, 92 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5132175..0e1f656 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.5.0 + +`VerifyResult` gains two optional fields, both None unless the server sends +them: + +- `notice`: the plain-language reason a call wasn't decided by a model (cold + start), which the API already returned but the client dropped. +- `shadow`: on a cold-start call, a `ShadowVerdict` with what the shared stock + model would have decided (`approved`, `score`, `threshold`, `latency_ms`). + The call itself is still rejected, so this is a preview: your first calls + on a new account show what the model thinks before anything is served. + ## 0.4.1 `verify()` and `verify_batch()` now treat a **rate-limited 429** (one with a diff --git a/README.md b/README.md index 95091e2..93e2682 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,19 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86) | `model_version` | `"stock"`, `"v"` (fine-tuned), `"cold_start_fail_closed"` / `"cold_start_auto_pending"`, or `"verify_unavailable"` | | `latency_ms` | server-side inference time | | `degraded` | `True` when the call failed and this result was synthesized client-side | +| `notice` | why no model decided the call (cold start); `None` otherwise | +| `shadow` | cold start only: a `ShadowVerdict` with what the shared stock model would have decided. The call is still rejected, so treat it as a preview | + +A new account starts in cold start: every call returns `approved=False` until a +Health Check shows the shared model is accurate enough on your data. Look at +`result.shadow` to see what it would have decided in the meantime: + +```python +result = cv.verify("when does my flight board", "Boarding starts 40 minutes before departure.") +if result.shadow: + print(result.notice) + print("stock model would approve:", result.shadow.approved, result.shadow.score) +``` ## On your request path diff --git a/cacheverifier/__init__.py b/cacheverifier/__init__.py index 6a2e884..a3edd5b 100644 --- a/cacheverifier/__init__.py +++ b/cacheverifier/__init__.py @@ -16,7 +16,7 @@ Research behind it: https://github.com/imxinchengyou/CacheVerifier """ -from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult +from cacheverifier.client import CacheVerifier, CacheVerifierError, ShadowVerdict, VerifyResult -__version__ = "0.4.1" -__all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"] +__version__ = "0.5.0" +__all__ = ["CacheVerifier", "CacheVerifierError", "ShadowVerdict", "VerifyResult", "__version__"] diff --git a/cacheverifier/client.py b/cacheverifier/client.py index a97d466..890b154 100644 --- a/cacheverifier/client.py +++ b/cacheverifier/client.py @@ -57,6 +57,32 @@ def __init__(self, status_code: int, detail: str, retry_after: str | None = None self.retry_after = retry_after +@dataclass(frozen=True) +class ShadowVerdict: + """What the shared stock model would have decided for a cold-start call. + + Only a preview: during cold start the call itself is always rejected + (`VerifyResult.approved` is False). The stock model hasn't been checked + against your data, so run a Health Check before relying on it. + """ + + approved: bool + score: float + threshold: float + latency_ms: float + model_version: str = "stock" + + @classmethod + def _from_json(cls, d: dict[str, Any]) -> ShadowVerdict: + return cls( + approved=bool(d["approved"]), + score=float(d["score"]), + threshold=float(d["threshold"]), + latency_ms=float(d["latency_ms"]), + model_version=str(d.get("model_version", "stock")), + ) + + @dataclass(frozen=True) class VerifyResult: """One `/v1/verify` decision. @@ -74,6 +100,11 @@ class VerifyResult: 5xx / rate-limited 429) and this result was synthesized by the client rather than returned by the API. `approved` then reflects the client's `fail_open` setting (default `False` -> `approved=False`, i.e. fall through to your LLM). + - `notice`: a plain-language reason when no model decided the call (cold + start); None otherwise. + - `shadow`: on a cold-start call, what the shared stock model would have + decided (a `ShadowVerdict`); None otherwise, and on servers that don't + send it. """ approved: bool @@ -82,15 +113,20 @@ class VerifyResult: model_version: str latency_ms: float degraded: bool = False + notice: str | None = None + shadow: ShadowVerdict | None = None @classmethod def _from_json(cls, d: dict[str, Any]) -> VerifyResult: + shadow = d.get("shadow") return cls( approved=bool(d["approved"]), score=float(d["score"]), threshold=float(d["threshold"]), model_version=str(d["model_version"]), latency_ms=float(d["latency_ms"]), + notice=d.get("notice"), + shadow=ShadowVerdict._from_json(shadow) if shadow else None, ) diff --git a/pyproject.toml b/pyproject.toml index 3f11bb5..33fd64f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "cacheverifier" -version = "0.4.1" +version = "0.5.0" description = "Python client for the hosted CacheVerifier semantic-cache verification API" readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_client.py b/tests/test_client.py index 0d969ca..57ca678 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -6,7 +6,7 @@ import httpx import pytest -from cacheverifier import CacheVerifier, CacheVerifierError, VerifyResult +from cacheverifier import CacheVerifier, CacheVerifierError, ShadowVerdict, VerifyResult def make_client(handler, **kwargs): @@ -51,6 +51,32 @@ def handler(request: httpx.Request) -> httpx.Response: assert seen["path"] == "/v1/verify" assert seen["api_key"] == "cv_test" assert seen["body"] == {"query": "how do I cancel", "candidate_answer": "Go to Settings > Billing."} + assert result.notice is None + assert result.shadow is None + + +def test_verify_parses_cold_start_notice_and_shadow_verdict(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "approved": False, + "score": 0.0, + "latency_ms": 0.0, + "model_version": "cold_start_auto_pending", + "threshold": 0.0, + "decision_rule": "cold_start", + "notice": "No verifier has been checked against your data yet.", + "shadow": {"approved": True, "score": 2.1, "threshold": 0.0, "latency_ms": 18.0, "model_version": "stock"}, + }, + ) + + with make_client(handler) as cv: + result = cv.verify("when does my flight board", "Boarding starts 40 minutes before departure.") + + assert result.approved is False + assert result.notice.startswith("No verifier") + assert result.shadow == ShadowVerdict(approved=True, score=2.1, threshold=0.0, latency_ms=18.0) def test_verify_batch_returns_ordered_results(): From ed069bae4e64908c7b956df10cccf87b60ffc285 Mon Sep 17 00:00:00 2001 From: xin Date: Sat, 24 Oct 2026 15:23:41 +0800 Subject: [PATCH 2/2] LangChain integration: VerifiedCache verifies another cache's hits cacheverifier.integrations.langchain.VerifiedCache wraps any LangChain cache (typically a semantic one such as RedisSemanticCache). On a hit it calls /v1/verify with the user's question and the cached answer; a rejected hit becomes a miss, so the model is called as usual. For chat models the question is the last human message, not the serialized history. Writes and clears pass through. Optional extra: cacheverifier[langchain]. --- CHANGELOG.md | 5 + README.md | 24 ++++ cacheverifier/integrations/langchain.py | 140 ++++++++++++++++++++++++ pyproject.toml | 3 +- tests/test_langchain_integration.py | 90 +++++++++++++++ 5 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 cacheverifier/integrations/langchain.py create mode 100644 tests/test_langchain_integration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e1f656..c8dc353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## 0.5.0 +New LangChain integration: `cacheverifier.integrations.langchain.VerifiedCache` +wraps any LangChain cache (e.g. `RedisSemanticCache`) and verifies each hit +before it is served; a rejected hit becomes a miss. Install with +`pip install "cacheverifier[langchain]"`. + `VerifyResult` gains two optional fields, both None unless the server sends them: diff --git a/README.md b/README.md index 93e2682..47e6181 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,30 @@ evaluator.report_feedback(query, answer, was_correct=False, similarity_score=0.9 See [`examples/gptcache_example.py`](examples/gptcache_example.py). +## LangChain + +`VerifiedCache` wraps any LangChain cache, typically a semantic one, and only +serves a hit after CacheVerifier approves it; a rejected hit is a miss, so +your model is called as usual. For chat models it verifies against the last +user message. + +```bash +pip install "cacheverifier[langchain]" +``` + +```python +from langchain_core.globals import set_llm_cache +from langchain_redis import RedisSemanticCache +from cacheverifier.integrations.langchain import VerifiedCache + +set_llm_cache( + VerifiedCache( + RedisSemanticCache(redis_url="redis://localhost:6379", embeddings=embeddings), + api_key="cv_...", + ) +) +``` + ## Fine-tuning Once you have ~20+ feedback rows (the service found fine-tuning is often a net negative diff --git a/cacheverifier/integrations/langchain.py b/cacheverifier/integrations/langchain.py new file mode 100644 index 0000000..edd60f3 --- /dev/null +++ b/cacheverifier/integrations/langchain.py @@ -0,0 +1,140 @@ +"""LangChain integration: wrap any LangChain semantic cache so a hit is only +served after CacheVerifier approves it. + +A semantic cache (`RedisSemanticCache`, `MongoDBAtlasSemanticCache`, ...) +returns a stored answer whenever a past prompt is similar enough to the new +one, including when the wording differs in a way that changes the answer. +`VerifiedCache` sits in front of it: on a hit it asks `/v1/verify` whether +the cached answer fits the new question, and turns a rejected hit into a +miss, so your model is called as usual. + + from langchain_core.globals import set_llm_cache + from langchain_redis import RedisSemanticCache + from cacheverifier.integrations.langchain import VerifiedCache + + set_llm_cache( + VerifiedCache( + RedisSemanticCache(redis_url="redis://localhost:6379", embeddings=embeddings), + api_key="cv_...", + ) + ) + +Writes, clears and exact-match caches pass straight through. If the verify +call times out or the service is unreachable, the hit is rejected by +default (a verifier outage becomes a normal cache miss); pass +`fail_open=True` to serve it instead. + +`langchain-core` is an optional dependency -- `pip install "cacheverifier[langchain]"`. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +from cacheverifier.client import ( + DEFAULT_BASE_URL, + DEFAULT_TIMEOUT, + DEFAULT_VERIFY_TIMEOUT, + CacheVerifier, +) + +try: + from langchain_core.caches import RETURN_VAL_TYPE, BaseCache +except ImportError as exc: # pragma: no cover - exercised only without the extra + raise ImportError( + 'cacheverifier.integrations.langchain needs langchain-core: pip install "cacheverifier[langchain]"' + ) from exc + + +def _message_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): # multimodal content blocks + return "\n".join( + block if isinstance(block, str) else str(block.get("text", "")) + for block in content + if isinstance(block, str) or (isinstance(block, dict) and block.get("type") == "text") + ) + return "" + + +def user_question(prompt: str) -> str: + """The text to verify against for a LangChain cache key. + + Plain LLMs use the prompt string itself as the key. Chat models use a + JSON dump of the whole message list, system prompt and history + included; the question being asked is the last human message, so that + is what the verifier sees. Anything unrecognized falls back to the raw + prompt.""" + if not prompt.startswith("["): + return prompt + try: + messages = json.loads(prompt) + except ValueError: + return prompt + for message in reversed(messages if isinstance(messages, list) else []): + kwargs = message.get("kwargs", {}) if isinstance(message, dict) else {} + if kwargs.get("type") == "human": + text = _message_text(kwargs.get("content")) + if text: + return text + return prompt + + +class VerifiedCache(BaseCache): + """A LangChain cache that verifies another cache's hits before serving them.""" + + def __init__( + self, + cache: BaseCache, + api_key: str | None = None, + *, + client: CacheVerifier | None = None, + base_url: str = DEFAULT_BASE_URL, + timeout: float = DEFAULT_TIMEOUT, + verify_timeout: float = DEFAULT_VERIFY_TIMEOUT, + fail_open: bool = False, + ) -> None: + if client is None: + if not api_key: + raise ValueError("pass api_key or client") + client = CacheVerifier( + api_key=api_key, + base_url=base_url, + timeout=timeout, + verify_timeout=verify_timeout, + fail_open=fail_open, + ) + self.cache = cache + self.client = client + + def _approved(self, prompt: str, hit: RETURN_VAL_TYPE) -> bool: + answer = "\n".join(generation.text for generation in hit) + return self.client.verify(user_question(prompt), answer).approved + + def lookup(self, prompt: str, llm_string: str) -> RETURN_VAL_TYPE | None: + hit = self.cache.lookup(prompt, llm_string) + if not hit: + return hit + return hit if self._approved(prompt, hit) else None + + async def alookup(self, prompt: str, llm_string: str) -> RETURN_VAL_TYPE | None: + hit = await self.cache.alookup(prompt, llm_string) + if not hit: + return hit + approved = await asyncio.get_running_loop().run_in_executor(None, self._approved, prompt, hit) + return hit if approved else None + + def update(self, prompt: str, llm_string: str, return_val: RETURN_VAL_TYPE) -> None: + self.cache.update(prompt, llm_string, return_val) + + async def aupdate(self, prompt: str, llm_string: str, return_val: RETURN_VAL_TYPE) -> None: + await self.cache.aupdate(prompt, llm_string, return_val) + + def clear(self, **kwargs: Any) -> None: + self.cache.clear(**kwargs) + + async def aclear(self, **kwargs: Any) -> None: + await self.cache.aclear(**kwargs) diff --git a/pyproject.toml b/pyproject.toml index 33fd64f..3e94a1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ Research = "https://github.com/imxinchengyou/CacheVerifier" [project.optional-dependencies] gptcache = ["gptcache>=0.1.30"] +langchain = ["langchain-core>=0.3"] # `cacheverifier healthcheck` -- runs the hosted Health Check Report's # stock-vs-fine-tuned evaluation locally. Deliberately NOT a base # dependency: torch + sentence-transformers dwarf the httpx-only client. @@ -54,7 +55,7 @@ healthcheck = [ "torch>=2.2", "sentence-transformers>=3.0,<4", ] -dev = ["pytest>=7", "mypy>=1.5", "ruff>=0.4"] +dev = ["pytest>=7", "mypy>=1.5", "ruff>=0.4", "langchain-core>=0.3"] [tool.hatch.build.targets.wheel] packages = ["cacheverifier"] diff --git a/tests/test_langchain_integration.py b/tests/test_langchain_integration.py new file mode 100644 index 0000000..7234fc4 --- /dev/null +++ b/tests/test_langchain_integration.py @@ -0,0 +1,90 @@ +"""VerifiedCache against a real langchain-core cache and chat model, with +every verify call served by an httpx.MockTransport.""" + +from __future__ import annotations + +import asyncio +import json + +import httpx +import pytest + +pytest.importorskip("langchain_core") + +from langchain_core.caches import InMemoryCache +from langchain_core.language_models.fake_chat_models import FakeListChatModel +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_core.outputs import Generation + +from cacheverifier import CacheVerifier +from cacheverifier.integrations.langchain import VerifiedCache, user_question + + +def _verified(approve: bool, seen: list | None = None, inner=None) -> VerifiedCache: + def handler(request: httpx.Request) -> httpx.Response: + if seen is not None: + seen.append(json.loads(request.content)) + return httpx.Response( + 200, + json={"approved": approve, "score": 1.0, "latency_ms": 5.0, "model_version": "stock", "threshold": 0.0}, + ) + + client = CacheVerifier(api_key="cv_test", transport=httpx.MockTransport(handler)) + return VerifiedCache(inner or InMemoryCache(), client=client) + + +def test_approved_hit_is_served_and_rejected_hit_becomes_a_miss(): + seen: list = [] + cache = _verified(True, seen) + cache.update("when does my flight board", "llm", [Generation(text="40 minutes before departure.")]) + assert cache.lookup("when does my flight board", "llm")[0].text == "40 minutes before departure." + assert seen == [{"query": "when does my flight board", "candidate_answer": "40 minutes before departure."}] + + rejecting = _verified(False, inner=cache.cache) + assert rejecting.lookup("when does my flight board", "llm") is None + + +def test_a_miss_never_calls_the_verifier(): + seen: list = [] + cache = _verified(True, seen) + assert cache.lookup("never cached", "llm") is None + assert seen == [] + + +def test_chat_model_verifies_against_the_last_human_message(): + seen: list = [] + cache = _verified(True, seen) + model = FakeListChatModel(responses=["Gate B12, boarding at 9:20.", "should not be called"], cache=cache) + messages = [SystemMessage("You are an airline assistant."), HumanMessage("which gate is my flight")] + + assert model.invoke(messages).content == "Gate B12, boarding at 9:20." # miss: model called, then cached + assert model.invoke(messages).content == "Gate B12, boarding at 9:20." # hit: verified, served + assert seen == [{"query": "which gate is my flight", "candidate_answer": "Gate B12, boarding at 9:20."}] + + +def test_rejected_chat_hit_falls_through_to_the_model(): + cache = _verified(False) + model = FakeListChatModel(responses=["first answer", "fresh answer"], cache=cache) + assert model.invoke("what is the baggage allowance").content == "first answer" + assert model.invoke("what is the baggage allowance").content == "fresh answer" + + +def test_async_lookup_verifies_too(): + cache = _verified(False) + asyncio.run(cache.aupdate("q", "llm", [Generation(text="a")])) + assert asyncio.run(cache.alookup("q", "llm")) is None + assert cache.cache.lookup("q", "llm")[0].text == "a" # the inner cache still has it + + +def test_user_question_falls_back_to_the_raw_prompt(): + assert user_question("plain prompt") == "plain prompt" + assert user_question("[not json") == "[not json" + multimodal = json.dumps( + [{"kwargs": {"type": "human", "content": [{"type": "text", "text": "what's in this photo"}, {"type": "image_url"}]}}] + ) + assert user_question(multimodal) == "what's in this photo" + + +def test_needs_a_key_or_a_client(): + with pytest.raises(ValueError): + VerifiedCache(InMemoryCache())