0.5.0: cold-start shadow verdict + LangChain VerifiedCache

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## 0.5.0
+
+New LangChain integration: `cacheverifier.integrations.langchain.VerifiedCache`
+wraps any LangChain cache (e.g. `RedisSemanticCache`) and verifies each hit
+before it is served; a rejected hit becomes a miss. Install with
+`pip install "cacheverifier[langchain]"`.
+
+`VerifyResult` gains two optional fields, both None unless the server sends
+them:
+
+- `notice`: the plain-language reason a call wasn't decided by a model (cold
+  start), which the API already returned but the client dropped.
+- `shadow`: on a cold-start call, a `ShadowVerdict` with what the shared stock
+  model would have decided (`approved`, `score`, `threshold`, `latency_ms`).
+  The call itself is still rejected, so this is a preview: your first calls
+  on a new account show what the model thinks before anything is served.
+
 ## 0.4.1
 
 `verify()` and `verify_batch()` now treat a **rate-limited 429** (one with a

README.md

@@ -62,6 +62,19 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86)
 | `model_version` | `"stock"`, `"v<id>"` (fine-tuned), `"cold_start_fail_closed"` / `"cold_start_auto_pending"`, or `"verify_unavailable"` |
 | `latency_ms` | server-side inference time |
 | `degraded` | `True` when the call failed and this result was synthesized client-side |
+| `notice` | why no model decided the call (cold start); `None` otherwise |
+| `shadow` | cold start only: a `ShadowVerdict` with what the shared stock model would have decided. The call is still rejected, so treat it as a preview |
+
+A new account starts in cold start: every call returns `approved=False` until a
+Health Check shows the shared model is accurate enough on your data. Look at
+`result.shadow` to see what it would have decided in the meantime:
+
+```python
+result = cv.verify("when does my flight board", "Boarding starts 40 minutes before departure.")
+if result.shadow:
+    print(result.notice)
+    print("stock model would approve:", result.shadow.approved, result.shadow.score)
+```
 
 ## On your request path
 
@@ -107,6 +120,30 @@ evaluator.report_feedback(query, answer, was_correct=False, similarity_score=0.9
 
 See [`examples/gptcache_example.py`](examples/gptcache_example.py).
 
+## LangChain
+
+`VerifiedCache` wraps any LangChain cache, typically a semantic one, and only
+serves a hit after CacheVerifier approves it; a rejected hit is a miss, so
+your model is called as usual. For chat models it verifies against the last
+user message.
+
+```bash
+pip install "cacheverifier[langchain]"
+```
+
+```python
+from langchain_core.globals import set_llm_cache
+from langchain_redis import RedisSemanticCache
+from cacheverifier.integrations.langchain import VerifiedCache
+
+set_llm_cache(
+    VerifiedCache(
+        RedisSemanticCache(redis_url="redis://localhost:6379", embeddings=embeddings),
+        api_key="cv_...",
+    )
+)
+```
+
 ## Fine-tuning
 
 Once you have ~20+ feedback rows (the service found fine-tuning is often a net negative

cacheverifier/__init__.py

@@ -16,7 +16,7 @@
 Research behind it: https://github.com/imxinchengyou/CacheVerifier
 """
 
-from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult
+from cacheverifier.client import CacheVerifier, CacheVerifierError, ShadowVerdict, VerifyResult
 
-__version__ = "0.4.1"
-__all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"]
+__version__ = "0.5.0"
+__all__ = ["CacheVerifier", "CacheVerifierError", "ShadowVerdict", "VerifyResult", "__version__"]

cacheverifier/client.py

@@ -57,6 +57,32 @@ def __init__(self, status_code: int, detail: str, retry_after: str | None = None
         self.retry_after = retry_after
 
 
+@dataclass(frozen=True)
+class ShadowVerdict:
+    """What the shared stock model would have decided for a cold-start call.
+
+    Only a preview: during cold start the call itself is always rejected
+    (`VerifyResult.approved` is False). The stock model hasn't been checked
+    against your data, so run a Health Check before relying on it.
+    """
+
+    approved: bool
+    score: float
+    threshold: float
+    latency_ms: float
+    model_version: str = "stock"
+
+    @classmethod
+    def _from_json(cls, d: dict[str, Any]) -> ShadowVerdict:
+        return cls(
+            approved=bool(d["approved"]),
+            score=float(d["score"]),
+            threshold=float(d["threshold"]),
+            latency_ms=float(d["latency_ms"]),
+            model_version=str(d.get("model_version", "stock")),
+        )
+
+
 @dataclass(frozen=True)
 class VerifyResult:
     """One `/v1/verify` decision.
@@ -74,6 +100,11 @@ class VerifyResult:
       5xx / rate-limited 429) and this result was synthesized by the client
       rather than returned by the API. `approved` then reflects the client's `fail_open` setting
       (default `False` -> `approved=False`, i.e. fall through to your LLM).
+    - `notice`: a plain-language reason when no model decided the call (cold
+      start); None otherwise.
+    - `shadow`: on a cold-start call, what the shared stock model would have
+      decided (a `ShadowVerdict`); None otherwise, and on servers that don't
+      send it.
     """
 
     approved: bool
@@ -82,15 +113,20 @@ class VerifyResult:
     model_version: str
     latency_ms: float
     degraded: bool = False
+    notice: str | None = None
+    shadow: ShadowVerdict | None = None
 
     @classmethod
     def _from_json(cls, d: dict[str, Any]) -> VerifyResult:
+        shadow = d.get("shadow")
         return cls(
             approved=bool(d["approved"]),
             score=float(d["score"]),
             threshold=float(d["threshold"]),
             model_version=str(d["model_version"]),
             latency_ms=float(d["latency_ms"]),
+            notice=d.get("notice"),
+            shadow=ShadowVerdict._from_json(shadow) if shadow else None,
         )
 
 

cacheverifier/integrations/langchain.py

@@ -0,0 +1,140 @@
+"""LangChain integration: wrap any LangChain semantic cache so a hit is only
+served after CacheVerifier approves it.
+
+A semantic cache (`RedisSemanticCache`, `MongoDBAtlasSemanticCache`, ...)
+returns a stored answer whenever a past prompt is similar enough to the new
+one, including when the wording differs in a way that changes the answer.
+`VerifiedCache` sits in front of it: on a hit it asks `/v1/verify` whether
+the cached answer fits the new question, and turns a rejected hit into a
+miss, so your model is called as usual.
+
+    from langchain_core.globals import set_llm_cache
+    from langchain_redis import RedisSemanticCache
+    from cacheverifier.integrations.langchain import VerifiedCache
+
+    set_llm_cache(
+        VerifiedCache(
+            RedisSemanticCache(redis_url="redis://localhost:6379", embeddings=embeddings),
+            api_key="cv_...",
+        )
+    )
+
+Writes, clears and exact-match caches pass straight through. If the verify
+call times out or the service is unreachable, the hit is rejected by
+default (a verifier outage becomes a normal cache miss); pass
+`fail_open=True` to serve it instead.
+
+`langchain-core` is an optional dependency -- `pip install "cacheverifier[langchain]"`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Any
+
+from cacheverifier.client import (
+    DEFAULT_BASE_URL,
+    DEFAULT_TIMEOUT,
+    DEFAULT_VERIFY_TIMEOUT,
+    CacheVerifier,
+)
+
+try:
+    from langchain_core.caches import RETURN_VAL_TYPE, BaseCache
+except ImportError as exc:  # pragma: no cover - exercised only without the extra
+    raise ImportError(
+        'cacheverifier.integrations.langchain needs langchain-core: pip install "cacheverifier[langchain]"'
+    ) from exc
+
+
+def _message_text(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):  # multimodal content blocks
+        return "\n".join(
+            block if isinstance(block, str) else str(block.get("text", ""))
+            for block in content
+            if isinstance(block, str) or (isinstance(block, dict) and block.get("type") == "text")
+        )
+    return ""
+
+
+def user_question(prompt: str) -> str:
+    """The text to verify against for a LangChain cache key.
+
+    Plain LLMs use the prompt string itself as the key. Chat models use a
+    JSON dump of the whole message list, system prompt and history
+    included; the question being asked is the last human message, so that
+    is what the verifier sees. Anything unrecognized falls back to the raw
+    prompt."""
+    if not prompt.startswith("["):
+        return prompt
+    try:
+        messages = json.loads(prompt)
+    except ValueError:
+        return prompt
+    for message in reversed(messages if isinstance(messages, list) else []):
+        kwargs = message.get("kwargs", {}) if isinstance(message, dict) else {}
+        if kwargs.get("type") == "human":
+            text = _message_text(kwargs.get("content"))
+            if text:
+                return text
+    return prompt
+
+
+class VerifiedCache(BaseCache):
+    """A LangChain cache that verifies another cache's hits before serving them."""
+
+    def __init__(
+        self,
+        cache: BaseCache,
+        api_key: str | None = None,
+        *,
+        client: CacheVerifier | None = None,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+        verify_timeout: float = DEFAULT_VERIFY_TIMEOUT,
+        fail_open: bool = False,
+    ) -> None:
+        if client is None:
+            if not api_key:
+                raise ValueError("pass api_key or client")
+            client = CacheVerifier(
+                api_key=api_key,
+                base_url=base_url,
+                timeout=timeout,
+                verify_timeout=verify_timeout,
+                fail_open=fail_open,
+            )
+        self.cache = cache
+        self.client = client
+
+    def _approved(self, prompt: str, hit: RETURN_VAL_TYPE) -> bool:
+        answer = "\n".join(generation.text for generation in hit)
+        return self.client.verify(user_question(prompt), answer).approved
+
+    def lookup(self, prompt: str, llm_string: str) -> RETURN_VAL_TYPE | None:
+        hit = self.cache.lookup(prompt, llm_string)
+        if not hit:
+            return hit
+        return hit if self._approved(prompt, hit) else None
+
+    async def alookup(self, prompt: str, llm_string: str) -> RETURN_VAL_TYPE | None:
+        hit = await self.cache.alookup(prompt, llm_string)
+        if not hit:
+            return hit
+        approved = await asyncio.get_running_loop().run_in_executor(None, self._approved, prompt, hit)
+        return hit if approved else None
+
+    def update(self, prompt: str, llm_string: str, return_val: RETURN_VAL_TYPE) -> None:
+        self.cache.update(prompt, llm_string, return_val)
+
+    async def aupdate(self, prompt: str, llm_string: str, return_val: RETURN_VAL_TYPE) -> None:
+        await self.cache.aupdate(prompt, llm_string, return_val)
+
+    def clear(self, **kwargs: Any) -> None:
+        self.cache.clear(**kwargs)
+
+    async def aclear(self, **kwargs: Any) -> None:
+        await self.cache.aclear(**kwargs)

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cacheverifier"
-version = "0.4.1"
+version = "0.5.0"
 description = "Python client for the hosted CacheVerifier semantic-cache verification API"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -40,6 +40,7 @@ Research = "https://github.com/imxinchengyou/CacheVerifier"
 
 [project.optional-dependencies]
 gptcache = ["gptcache>=0.1.30"]
+langchain = ["langchain-core>=0.3"]
 # `cacheverifier healthcheck` -- runs the hosted Health Check Report's
 # stock-vs-fine-tuned evaluation locally. Deliberately NOT a base
 # dependency: torch + sentence-transformers dwarf the httpx-only client.
@@ -54,7 +55,7 @@ healthcheck = [
     "torch>=2.2",
     "sentence-transformers>=3.0,<4",
 ]
-dev = ["pytest>=7", "mypy>=1.5", "ruff>=0.4"]
+dev = ["pytest>=7", "mypy>=1.5", "ruff>=0.4", "langchain-core>=0.3"]
 
 [tool.hatch.build.targets.wheel]
 packages = ["cacheverifier"]

tests/test_client.py

@@ -6,7 +6,7 @@
 import httpx
 import pytest
 
-from cacheverifier import CacheVerifier, CacheVerifierError, VerifyResult
+from cacheverifier import CacheVerifier, CacheVerifierError, ShadowVerdict, VerifyResult
 
 
 def make_client(handler, **kwargs):
@@ -51,6 +51,32 @@ def handler(request: httpx.Request) -> httpx.Response:
     assert seen["path"] == "/v1/verify"
     assert seen["api_key"] == "cv_test"
     assert seen["body"] == {"query": "how do I cancel", "candidate_answer": "Go to Settings > Billing."}
+    assert result.notice is None
+    assert result.shadow is None
+
+
+def test_verify_parses_cold_start_notice_and_shadow_verdict():
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            200,
+            json={
+                "approved": False,
+                "score": 0.0,
+                "latency_ms": 0.0,
+                "model_version": "cold_start_auto_pending",
+                "threshold": 0.0,
+                "decision_rule": "cold_start",
+                "notice": "No verifier has been checked against your data yet.",
+                "shadow": {"approved": True, "score": 2.1, "threshold": 0.0, "latency_ms": 18.0, "model_version": "stock"},
+            },
+        )
+
+    with make_client(handler) as cv:
+        result = cv.verify("when does my flight board", "Boarding starts 40 minutes before departure.")
+
+    assert result.approved is False
+    assert result.notice.startswith("No verifier")
+    assert result.shadow == ShadowVerdict(approved=True, score=2.1, threshold=0.0, latency_ms=18.0)
 
 
 def test_verify_batch_returns_ordered_results():