From 548bda3333240a69406feedfa0d8366bd68edf39 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 09:08:55 +0000 Subject: [PATCH 1/2] fix: entity extraction produced prose names and phantom metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Entity names feed the graph, wiki, query and diff, so these two defects degraded every one of those views at once. On this repo's own README, 15 of 35 entity names were prose rather than concepts, and three of the metrics did not exist. Feature bullets are conventionally "Name -- description", but the splitter only knew ':' and the typographic dashes, missing the ASCII '--' this project actually uses. The whole bullet was therefore kept and merely truncated at 80 characters, which is why graph nodes and wiki headings appeared ellipsised mid-sentence. Splitting now covers the ASCII form, tries multi-character dashes before the bare hyphen so "Auto-PR" is not cut in half, and falls back to the first sentence and then a word-boundary trim when a bullet has no separator at all. The full text is still kept in the entity's description property, so trimming the name loses nothing. The metric pattern let the bare "s", "k" and "M" units match the first letter of any following word, so "12 selectable formats" and "0 sections added" became durations. Units are now ordered longest-first — "30 seconds" was also capturing its unit as a bare "s" — and a trailing lookahead requires the unit to end at a non-letter. Measured on this README: entity names over 60 characters 15 -> 0, and the bogus "12 s" / "0 s" / "5 s" metrics are gone, with real entity count essentially unchanged (35 -> 32), so this is precision gained rather than recall lost. Tests: 574 pass (34 new), including dogfooding assertions that this repo's own README yields no prose-length names and no phantom durations. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01U2mrMSRs3cHBdPgHbRzVDX --- src/opendocs/core/semantic_extractor.py | 60 +++++++- tests/test_extraction_quality.py | 196 ++++++++++++++++++++++++ 2 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 tests/test_extraction_quality.py diff --git a/src/opendocs/core/semantic_extractor.py b/src/opendocs/core/semantic_extractor.py index f533552..4becb40 100644 --- a/src/opendocs/core/semantic_extractor.py +++ b/src/opendocs/core/semantic_extractor.py @@ -206,9 +206,15 @@ # --------------------------------------------------------------------------- _VERSION_RE = re.compile(r"(\d+\.\d+(?:\.\d+)?(?:[+-]\w+)?)") +# Units are ordered longest-first so "30 seconds" captures "seconds" rather +# than matching the bare "s" alternative first. The trailing lookahead is what +# stops a number followed by any s-word from being read as a duration: without +# it, "12 selectable formats" and "0 sections added" both became metrics. _METRIC_RE = re.compile( - r"(\d+(?:\.\d+)?)\s*(%|ms|s|sec|seconds|min|minutes|hz|Hz|MB|GB|TB|KB" - r"|req/s|rps|qps|tps|fps|Mbps|Gbps|rpm|k|K|M)", + r"(\d+(?:\.\d+)?)\s*(" + r"seconds|minutes|req/s|Mbps|Gbps|sec|min|rpm|rps|qps|tps|fps" + r"|ms|Hz|hz|MB|GB|TB|KB|%|s|k|K|M" + r")(?![A-Za-z])", ) _API_PATH_RE = re.compile(r"`?(/api/\S+|/v\d+/\S+)`?") _HTTP_METHOD_RE = re.compile(r"\b(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\b") @@ -216,6 +222,54 @@ _MERMAID_NODE_RE = re.compile(r"(\w+)\[([^\]]+)\]") +#: Separators that divide a feature bullet's *name* from its *description*. +#: Ordered so multi-character dashes are tried before the single hyphen, which +#: would otherwise split hyphenated names like "Auto-PR" in half. +_FEATURE_SEPARATORS = (" -- ", " — ", " – ", ": ", " - ", "--", "—", "–", ":") + +#: A feature name longer than this is prose, not a name. The previous limit of +#: 80 was a truncation point rather than a rejection, so whole sentences became +#: entity names and every downstream view showed them ellipsised. +_MAX_FEATURE_NAME = 60 + + +def _feature_name(text: str) -> str: + """Extract the short name from a feature bullet. + + Feature bullets are conventionally ``Name -- description``. Splitting only + on ``:`` and the typographic dashes missed the ASCII ``--`` form, so the + whole bullet was kept and merely truncated. When no separator is present, + fall back to the first clause and then to a word-count cap, so the result is + always a name rather than a sentence fragment. + """ + candidate = text.strip() + + for separator in _FEATURE_SEPARATORS: + if separator in candidate: + head = candidate.split(separator, 1)[0].strip() + if 3 < len(head) <= _MAX_FEATURE_NAME: + return head + # A head that is still too long means the separator appeared late + # in prose rather than after a name; keep looking. + if head: + candidate = head + break + + # No usable separator — stop at the first sentence. + candidate = re.split(r"(?<=[a-z0-9])\.\s", candidate)[0].strip() + if len(candidate) <= _MAX_FEATURE_NAME: + return candidate + + # Still prose: keep the leading words, cutting on a word boundary. + words = candidate.split() + trimmed: list[str] = [] + for word in words: + if len(" ".join(trimmed + [word])) > _MAX_FEATURE_NAME: + break + trimmed.append(word) + return " ".join(trimmed).rstrip(" ,;:") + + # --------------------------------------------------------------------------- # Extractor # --------------------------------------------------------------------------- @@ -671,7 +725,7 @@ def _extract_from_lists(self, doc: DocumentModel, kg: KnowledgeGraph) -> None: for item in block.items[:15]: # Cap at 15 # Strip bold markers, extract first meaningful phrase clean = re.sub(r"\*\*([^*]+)\*\*", r"\1", item) - name = clean.split(":")[0].split("—")[0].split("–")[0].strip()[:80] + name = _feature_name(clean) if len(name) > 3: eid = self._make_id("feat", name) kg.add_entity( diff --git a/tests/test_extraction_quality.py b/tests/test_extraction_quality.py new file mode 100644 index 0000000..b55c759 --- /dev/null +++ b/tests/test_extraction_quality.py @@ -0,0 +1,196 @@ +"""Extraction-quality regression tests. + +Entity names feed the knowledge graph, the interactive graph, the wiki, the +diff, and the query layer, so a naming defect degrades all of them at once. +These tests pin the two that did: + +- feature bullets were kept whole and merely truncated, because the splitter + did not know about the ASCII ``--`` separator; +- any number followed by an s-word was read as a duration, so "12 selectable + formats" and "0 sections added" became metric entities. +""" + +from __future__ import annotations + +import pytest + +from opendocs.core.knowledge_graph import EntityType +from opendocs.core.parser import ReadmeParser +from opendocs.core.semantic_extractor import _MAX_FEATURE_NAME, _METRIC_RE, SemanticExtractor, _feature_name + + +def extract(markdown: str): + return SemanticExtractor().extract(ReadmeParser().parse(markdown, repo_name="Proj")) + + +def names_of(kg, entity_type: EntityType) -> list[str]: + return [e.name for e in kg.entities if e.entity_type is entity_type] + + +# --------------------------------------------------------------------------- +# Feature names +# --------------------------------------------------------------------------- + + +class TestFeatureName: + @pytest.mark.parametrize( + ("bullet", "expected"), + [ + ("15 Output Formats -- Word, PDF, PPTX, Blog Post and more", "15 Output Formats"), + ("Interactive Graph — Single-file HTML visualization with search", "Interactive Graph"), + ("Community Detection – Label propagation groups entities", "Community Detection"), + ("Smart Table Sorting: 6 strategies for ordering rows", "Smart Table Sorting"), + ("Knowledge Wiki - Wikipedia-style linked Markdown articles", "Knowledge Wiki"), + ], + ) + def test_separator_forms(self, bullet, expected): + """Regression: only ':' and typographic dashes were handled, not '--'.""" + assert _feature_name(bullet) == expected + + def test_hyphenated_names_are_not_split(self): + """A bare '-' must not cut 'Auto-PR' in half.""" + assert _feature_name("File Watcher + Auto-PR -- Monitor repos for changes") == "File Watcher + Auto-PR" + + def test_short_bullet_is_kept_whole(self): + assert _feature_name("Fast and simple") == "Fast and simple" + + def test_prose_without_a_separator_is_trimmed_to_a_name(self): + long_prose = "A feature described in a full sentence with no separator at all that rambles onward" + result = _feature_name(long_prose) + assert len(result) <= _MAX_FEATURE_NAME + assert not result.endswith(" ") + + def test_trimming_falls_on_a_word_boundary(self): + result = _feature_name("word " * 40) + assert not result.endswith("wor") + assert all(w == "word" for w in result.split()) + + def test_stops_at_the_first_sentence(self): + text = "Does one thing. Then it goes on to explain a great many other things at length." + assert _feature_name(text) == "Does one thing" + + def test_never_exceeds_the_cap(self): + for bullet in [ + "x" * 200, + "Name -- " + "y" * 200, + "word " * 100, + "A: " + "z" * 200, + ]: + assert len(_feature_name(bullet)) <= _MAX_FEATURE_NAME + + +class TestFeatureExtraction: + FEATURES_README = """# Proj + +A project with several capabilities worth describing at some length here. + +## Features + +- **15 Output Formats** -- Word, PDF, PPTX, Blog Post, Jira Tickets, Changelog +- **Interactive Knowledge Graph** -- Single-file HTML visualization with search +- **File Watcher + Auto-PR** -- Monitor repos and create pull requests +""" + + def test_feature_names_are_concepts_not_sentences(self): + kg = extract(self.FEATURES_README) + features = names_of(kg, EntityType.FEATURE) + assert "15 Output Formats" in features + assert "Interactive Knowledge Graph" in features + assert "File Watcher + Auto-PR" in features + + def test_no_entity_name_is_prose_length(self): + kg = extract(self.FEATURES_README) + oversized = [e.name for e in kg.entities if len(e.name) > _MAX_FEATURE_NAME] + assert oversized == [] + + def test_description_is_preserved_in_properties(self): + """Trimming the name must not lose the detail.""" + kg = extract(self.FEATURES_README) + feature = next(e for e in kg.entities if e.name == "15 Output Formats") + assert "Word" in feature.properties.get("description", "") + + +# --------------------------------------------------------------------------- +# Metrics +# --------------------------------------------------------------------------- + + +class TestMetricRegex: + @pytest.mark.parametrize( + "text", + [ + "all 12 selectable output formats", + "5 stakeholder views", + "0 sections added", + "10 karat gold", + "3 separate modules", + "7 minor issues", + ], + ) + def test_number_followed_by_a_word_is_not_a_metric(self, text): + """Regression: the bare 's'/'k'/'M' units matched any following word.""" + assert list(_METRIC_RE.finditer(text)) == [] + + @pytest.mark.parametrize( + ("text", "value", "unit"), + [ + ("takes 30 seconds", "30", "seconds"), + ("within 5 min", "5", "min"), + ("latency of 250 ms", "250", "ms"), + ("99.9% uptime", "99.9", "%"), + ("handles 1000 req/s", "1000", "req/s"), + ("cache is 512 MB", "512", "MB"), + ("runs at 60 fps", "60", "fps"), + ("a 10k user base", "10", "k"), + ], + ) + def test_real_metrics_still_match(self, text, value, unit): + match = _METRIC_RE.search(text) + assert match is not None, f"missed metric in {text!r}" + assert (match.group(1), match.group(2)) == (value, unit) + + def test_longest_unit_wins(self): + """'30 seconds' must not capture the unit as a bare 's'.""" + assert _METRIC_RE.search("takes 30 seconds").group(2) == "seconds" + + +class TestMetricExtraction: + def test_counting_prose_produces_no_metrics(self): + kg = extract("# Proj\n\nWe support 12 selectable formats and 5 stakeholder views in total here.\n") + assert names_of(kg, EntityType.METRIC) == [] + + def test_genuine_metrics_are_still_extracted(self): + kg = extract("# Proj\n\nThe service responds in 250 ms and sustains 1000 req/s under load.\n") + metrics = names_of(kg, EntityType.METRIC) + assert any("250" in m for m in metrics) + assert any("1000" in m for m in metrics) + + +# --------------------------------------------------------------------------- +# Dogfooding +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def kg(): + """The project's own README, extracted once for the whole module.""" + from pathlib import Path + + readme = Path(__file__).resolve().parent.parent / "README.md" + return extract(readme.read_text(encoding="utf-8")) + + +class TestAgainstOwnReadme: + def test_no_prose_length_entity_names(self, kg): + oversized = [e.name for e in kg.entities if len(e.name) > _MAX_FEATURE_NAME] + assert oversized == [], f"prose leaked into entity names: {oversized}" + + def test_no_bare_duration_metrics(self, kg): + """The README counts things ("12 formats"); none of those are durations.""" + bogus = [e.name for e in kg.entities if e.entity_type is EntityType.METRIC and e.name.endswith(" s")] + assert bogus == [], f"miscounted durations: {bogus}" + + def test_still_finds_real_entities(self, kg): + """Precision must not have come at the cost of finding nothing.""" + assert len(kg.entities) > 15 + assert names_of(kg, EntityType.FEATURE) From f2d9fd30689022577cb52bd36c3d0a83c0e5c98d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 09:12:36 +0000 Subject: [PATCH 2/2] feat: cache LLM responses so repeat runs are free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--mode llm` re-issued every request on every run, so regenerating docs from an unchanged README paid the full API cost again. Responses are now cached on disk and reused. Caching sits in LLMProvider.chat/chat_json, the single chokepoint all five providers already share, so OpenAI, Anthropic, Google, Ollama and Azure are covered without touching any provider individually. The cache is process-wide, so LLMExtractor, LLMSummarizer and LLMContentEnhancer also benefit from each other's results within one run, not only across runs. The key covers provider, model, temperature, token ceiling, call kind and the exact prompts. Temperature and max_tokens are in there deliberately: replaying an answer produced under different sampling settings would be wrong, so changing either re-asks. This trades novelty for cost and reproducibility. With temperature > 0 the provider would have returned a different answer and you get the earlier one — which is the point for repeated builds, and why OPENDOCS_LLM_CACHE=0 and --no-cache both turn it off. --no-cache now covers artifacts and responses alike, since that is what "don't reuse anything" means. Two details: chat_json caches the raw provider text rather than the parsed object, so entries stay faithful to what was returned and later parsing improvements apply to already-cached responses; and a response that fails to parse is never stored, so a bad reply cannot become permanent. Verified across processes: a second invocation makes zero API calls and returns the identical answer. Tests: 614 pass (40 new). A counting fake provider asserts the property that matters — a cache hit is an API call not made. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01U2mrMSRs3cHBdPgHbRzVDX --- README.md | 18 +++ src/opendocs/cli.py | 27 ++-- src/opendocs/llm/cache.py | 205 +++++++++++++++++++++++++ src/opendocs/llm/providers.py | 73 ++++++++- tests/test_llm_cache.py | 274 ++++++++++++++++++++++++++++++++++ 5 files changed, 586 insertions(+), 11 deletions(-) create mode 100644 src/opendocs/llm/cache.py create mode 100644 tests/test_llm_cache.py diff --git a/README.md b/README.md index 0a44e00..758e17e 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,24 @@ stored artifact byte for byte; the one thing it does not refresh is the generation timestamp, which is what makes identical inputs produce identical output. +**LLM responses are cached too.** In `--mode llm`, re-running over an unchanged +README previously re-paid for every API call. Responses are now stored on disk +keyed by provider, model, temperature, token limit, and the exact prompt, so a +repeat run costs nothing and returns instantly: + +```bash +opendocs generate ./README.md --local --mode llm # first run calls the API +opendocs generate ./README.md --local --mode llm # second run is free + +export OPENDOCS_LLM_CACHE=0 # disable response caching only +opendocs generate ./README.md --local --mode llm --no-cache # disable both +``` + +Note this deliberately trades novelty for cost and reproducibility: with +`temperature > 0` the provider would have returned a *different* answer, and +you get the earlier one. Change the temperature, the model, or the prompt and +it re-asks; use `--no-cache` when you specifically want a fresh generation. + ### What Changed Since the Last Release `opendocs diff` compares two versions of your documentation and reports what diff --git a/src/opendocs/cli.py b/src/opendocs/cli.py index eb17e2f..cd1e263 100644 --- a/src/opendocs/cli.py +++ b/src/opendocs/cli.py @@ -161,7 +161,7 @@ def main(): "no_cache", is_flag=True, default=False, - help="Ignore the build cache and regenerate every format from scratch.", + help="Ignore all caches (generated artifacts and LLM responses) and rebuild from scratch.", ) @click.option( "--cache-dir", @@ -297,8 +297,11 @@ def generate( # Run pipeline — folder path or single file/URL from .core.build_cache import BuildCache + from .llm.cache import LLMCache, reset_shared_cache build_cache = BuildCache(cache_dir, enabled=not no_cache) + # --no-cache means "don't reuse anything", so it covers LLM responses too. + reset_shared_cache(LLMCache(Path(cache_dir) / "llm" if cache_dir else None, enabled=not no_cache)) pipeline = Pipeline(github_token=token) source_path = Path(source) @@ -967,20 +970,26 @@ def cache(do_clear: bool, cache_dir: str | None): opendocs cache --clear # empty it """ from .core.build_cache import BuildCache + from .llm.cache import LLMCache store = BuildCache(cache_dir) + llm_store = LLMCache(Path(cache_dir) / "llm" if cache_dir else None) if do_clear: - removed = store.clear() - console.print(f"[green][OK][/] Cleared {removed} cached artifact(s) from {store.dir}") + artifacts = store.clear() + responses = llm_store.clear() + console.print(f"[green][OK][/] Cleared {artifacts} artifact(s) and {responses} LLM response(s)") return - size = store.size_bytes() - entries = len(list(store.dir.glob("*/*/manifest.json"))) if store.dir.exists() else 0 - console.print(f"\n[bold]Build cache[/] [dim]{store.dir}[/]") - console.print(f" entries: {entries}") - console.print(f" size: {size / 1024 / 1024:.1f} MB" if size else " size: empty") - console.print("\n[dim]Clear it with `opendocs cache --clear`.[/]\n") + def _describe(label: str, path, entries: int, size: int) -> None: + console.print(f"\n[bold]{label}[/] [dim]{path}[/]") + console.print(f" entries: {entries}") + console.print(f" size: {size / 1024 / 1024:.1f} MB" if size else " size: empty") + + build_entries = len(list(store.dir.glob("*/*/manifest.json"))) if store.dir.exists() else 0 + _describe("Build cache", store.dir, build_entries, store.size_bytes()) + _describe("LLM response cache", llm_store.dir, llm_store.entry_count(), llm_store.size_bytes()) + console.print("\n[dim]Clear both with `opendocs cache --clear`.[/]\n") @main.command() diff --git a/src/opendocs/llm/cache.py b/src/opendocs/llm/cache.py new file mode 100644 index 0000000..a6432a3 --- /dev/null +++ b/src/opendocs/llm/cache.py @@ -0,0 +1,205 @@ +"""Response cache for LLM calls. + +``--mode llm`` re-issues every request on every run, so regenerating docs from +an unchanged README pays the full API cost again. This caches responses on +disk, keyed by everything that determines the answer, so a repeat run is free +and instant. + +Caching a non-deterministic API is a deliberate trade: with ``temperature > 0`` +a provider would have returned *a* different answer, and this returns the +earlier one. That is what makes repeated builds reproducible and cheap, and it +is why the temperature is part of the key and why the cache is easy to turn +off — see :func:`llm_cache_enabled`. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +#: Bumped when the on-disk entry layout changes. +CACHE_FORMAT_VERSION = 1 + +_ENV_ENABLED = "OPENDOCS_LLM_CACHE" +_ENV_CACHE_DIR = "OPENDOCS_CACHE_DIR" + +_FALSEY = {"0", "false", "no", "off"} + + +def llm_cache_enabled() -> bool: + """Whether the LLM cache is active. + + On by default; set ``OPENDOCS_LLM_CACHE=0`` to disable it globally. + """ + return os.environ.get(_ENV_ENABLED, "1").strip().lower() not in _FALSEY + + +def llm_cache_dir() -> Path: + """Directory holding cached LLM responses.""" + override = os.environ.get(_ENV_CACHE_DIR) + if override: + return Path(override).expanduser() / "llm" + xdg = os.environ.get("XDG_CACHE_HOME") + base = Path(xdg).expanduser() if xdg else Path.home() / ".cache" + return base / "opendocs" / "llm" + + +@dataclass +class LLMCacheStats: + """Tally of cache activity for a process.""" + + hits: int = 0 + misses: int = 0 + stores: int = 0 + + @property + def total(self) -> int: + return self.hits + self.misses + + def summary(self) -> str: + if not self.total: + return "no LLM calls" + return f"{self.hits}/{self.total} LLM responses cached" + + +class LLMCache: + """Disk cache mapping a request fingerprint to a provider response.""" + + def __init__(self, cache_dir: str | Path | None = None, *, enabled: Optional[bool] = None) -> None: + self.enabled = llm_cache_enabled() if enabled is None else enabled + self.dir = Path(cache_dir) if cache_dir else llm_cache_dir() + self.stats = LLMCacheStats() + + # -- Keying ---------------------------------------------------------- + + @staticmethod + def make_key( + *, + provider: str, + model: str, + system: str, + user: str, + kind: str = "text", + temperature: float = 0.0, + max_tokens: int = 0, + extra: Optional[dict[str, Any]] = None, + ) -> str: + """Fingerprint one request. + + Everything that could change the response belongs here — including the + temperature and token ceiling, so tuning either re-asks the provider + rather than replaying an answer generated under different settings. + """ + payload = { + "v": CACHE_FORMAT_VERSION, + "provider": provider, + "model": model, + "kind": kind, + "temperature": temperature, + "max_tokens": max_tokens, + "system": system, + "user": user, + "extra": extra or {}, + } + blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + # -- Storage --------------------------------------------------------- + + def _path(self, key: str) -> Path: + # Shard by the first two characters to keep directories small. + return self.dir / key[:2] / f"{key}.json" + + def get(self, key: str) -> Optional[str]: + """Return the cached response text, or *None* on a miss.""" + if not self.enabled: + return None + + path = self._path(key) + if not path.exists(): + self.stats.misses += 1 + return None + + try: + entry = json.loads(path.read_text(encoding="utf-8")) + response = entry["response"] + except Exception as exc: + # A damaged entry must behave as a miss, never break a run. + logger.debug("LLM cache read failed for %s: %s", key[:12], exc) + self.stats.misses += 1 + return None + + self.stats.hits += 1 + logger.debug("LLM cache hit %s", key[:12]) + return response + + def put(self, key: str, response: str, *, meta: Optional[dict[str, Any]] = None) -> bool: + """Store *response* under *key*. Returns True when it was written.""" + if not self.enabled: + return False + + path = self._path(key) + try: + path.parent.mkdir(parents=True, exist_ok=True) + # Write via a temp file so a crash cannot leave a half-written + # entry that would later parse as valid but truncated JSON. + tmp = path.with_suffix(".json.tmp") + tmp.write_text( + json.dumps({"response": response, "cached_at": time.time(), "meta": meta or {}}), + encoding="utf-8", + ) + tmp.replace(path) + except Exception as exc: + logger.debug("LLM cache write failed for %s: %s", key[:12], exc) + return False + + self.stats.stores += 1 + return True + + def clear(self) -> int: + """Delete every cached response. Returns how many were removed.""" + if not self.dir.exists(): + return 0 + entries = list(self.dir.glob("*/*.json")) + import shutil + + shutil.rmtree(self.dir, ignore_errors=True) + return len(entries) + + def size_bytes(self) -> int: + if not self.dir.exists(): + return 0 + return sum(p.stat().st_size for p in self.dir.rglob("*.json") if p.is_file()) + + def entry_count(self) -> int: + if not self.dir.exists(): + return 0 + return len(list(self.dir.glob("*/*.json"))) + + +#: Process-wide cache shared by every provider instance, so separate +#: LLMExtractor / LLMSummarizer / LLMContentEnhancer objects benefit from each +#: other's results within a single run as well as across runs. +_SHARED: Optional[LLMCache] = None + + +def shared_cache() -> LLMCache: + """Return the process-wide LLM cache, creating it on first use.""" + global _SHARED + if _SHARED is None: + _SHARED = LLMCache() + return _SHARED + + +def reset_shared_cache(cache: Optional[LLMCache] = None) -> None: + """Replace the process-wide cache (used by tests and by ``--no-cache``).""" + global _SHARED + _SHARED = cache diff --git a/src/opendocs/llm/providers.py b/src/opendocs/llm/providers.py index 762dce1..26ca888 100644 --- a/src/opendocs/llm/providers.py +++ b/src/opendocs/llm/providers.py @@ -100,8 +100,42 @@ def __init__( # ── Public API ──────────────────────────────────────────────────── + # ── Caching ─────────────────────────────────────────────────────── + + def _cache_key(self, system: str, user: str, kind: str) -> str: + from .cache import LLMCache + + return LLMCache.make_key( + provider=type(self).__name__, + model=self.model, + system=system, + user=user, + kind=kind, + temperature=self.temperature, + max_tokens=self.max_tokens, + extra={"base_url": self.base_url or ""}, + ) + def chat(self, system: str, user: str) -> str: - """Send a chat completion with retry. Returns plain text.""" + """Send a chat completion with retry. Returns plain text. + + Responses are cached on disk, so repeating a run over unchanged input + costs nothing. Disable with ``OPENDOCS_LLM_CACHE=0``. + """ + from .cache import shared_cache + + cache = shared_cache() + key = self._cache_key(system, user, "text") + cached = cache.get(key) + if cached is not None: + return cached + + result = self._chat_uncached(system, user) + cache.put(key, result, meta={"model": self.model, "provider": type(self).__name__}) + return result + + def _chat_uncached(self, system: str, user: str) -> str: + """Send a chat completion with retry, bypassing the cache.""" last_exc: Exception | None = None for attempt in range(self.max_retries): try: @@ -124,12 +158,47 @@ def chat_json(self, system: str, user: str) -> dict[str, Any]: Each provider implements its own JSON-mode strategy. Falls back to parsing raw text as JSON. + + The *raw* response is cached rather than the parsed object, so the + stored entry stays a faithful record of what the provider returned and + parsing improvements apply to previously cached responses. """ + from .cache import shared_cache + + cache = shared_cache() + key = self._cache_key(system, user, "json") + cached = cache.get(key) + if cached is not None: + try: + return self._parse_json(cached) + except Exception as exc: + # A cached response we can no longer parse is worthless; fall + # through and ask the provider again. + logger.debug("Cached JSON response failed to parse (%s); re-requesting", exc) + + raw_holder: dict[str, str] = {} + parsed = self._chat_json_uncached(system, user, raw_holder) + if "raw" in raw_holder: + cache.put(key, raw_holder["raw"], meta={"model": self.model, "provider": type(self).__name__}) + return parsed + + def _chat_json_uncached( + self, + system: str, + user: str, + raw_holder: dict[str, str] | None = None, + ) -> dict[str, Any]: + """Send a JSON-mode completion with retry, bypassing the cache.""" last_exc: Exception | None = None for attempt in range(self.max_retries): try: raw = self._call_json(system, user) - return self._parse_json(raw) + parsed = self._parse_json(raw) + # Only expose the raw text once it is known to parse, so an + # unparseable response is never written to the cache. + if raw_holder is not None: + raw_holder["raw"] = raw + return parsed except json.JSONDecodeError as exc: last_exc = exc logger.warning("JSON parse failed (attempt %d): %s", attempt + 1, exc) diff --git a/tests/test_llm_cache.py b/tests/test_llm_cache.py new file mode 100644 index 0000000..2979b48 --- /dev/null +++ b/tests/test_llm_cache.py @@ -0,0 +1,274 @@ +"""Tests for the LLM response cache. + +No real provider is ever contacted: a fake provider counts how often the +network layer *would* have been reached, which is the property that matters — +a cache hit must mean an API call not made, and therefore not paid for. +""" + +from __future__ import annotations + +import json + +import pytest + +from opendocs.llm.cache import ( + LLMCache, + llm_cache_dir, + llm_cache_enabled, + reset_shared_cache, + shared_cache, +) +from opendocs.llm.providers import LLMProvider + + +class CountingProvider(LLMProvider): + """A provider that records how many calls reached the transport layer.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.text_calls = 0 + self.json_calls = 0 + self.json_payload = '{"entities": ["Alpha"]}' + + def _call(self, system: str, user: str) -> str: + self.text_calls += 1 + return f"answer#{self.text_calls} to {user}" + + def _call_json(self, system: str, user: str) -> str: + self.json_calls += 1 + return self.json_payload + + +@pytest.fixture(autouse=True) +def isolated_cache(tmp_path, monkeypatch): + """Give every test its own cache and restore the shared one afterwards.""" + monkeypatch.setenv("OPENDOCS_CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.delenv("OPENDOCS_LLM_CACHE", raising=False) + reset_shared_cache(LLMCache(tmp_path / "cache" / "llm")) + yield + reset_shared_cache(None) + + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + + +class TestConfiguration: + def test_enabled_by_default(self): + assert llm_cache_enabled() is True + + @pytest.mark.parametrize("value", ["0", "false", "no", "off", "OFF", "False"]) + def test_disabled_by_env(self, monkeypatch, value): + monkeypatch.setenv("OPENDOCS_LLM_CACHE", value) + assert llm_cache_enabled() is False + + @pytest.mark.parametrize("value", ["1", "true", "yes", "anything"]) + def test_other_values_leave_it_enabled(self, monkeypatch, value): + monkeypatch.setenv("OPENDOCS_LLM_CACHE", value) + assert llm_cache_enabled() is True + + def test_cache_dir_follows_the_override(self, tmp_path, monkeypatch): + monkeypatch.setenv("OPENDOCS_CACHE_DIR", str(tmp_path / "custom")) + assert llm_cache_dir() == tmp_path / "custom" / "llm" + + def test_cache_dir_falls_back_to_xdg(self, tmp_path, monkeypatch): + monkeypatch.delenv("OPENDOCS_CACHE_DIR", raising=False) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "xdg")) + assert llm_cache_dir() == tmp_path / "xdg" / "opendocs" / "llm" + + +# --------------------------------------------------------------------------- +# Keying +# --------------------------------------------------------------------------- + + +class TestKeying: + def base(self, **over): + args = { + "provider": "OpenAIProvider", + "model": "gpt-4o-mini", + "system": "sys", + "user": "usr", + "kind": "text", + "temperature": 0.2, + "max_tokens": 100, + } + args.update(over) + return LLMCache.make_key(**args) + + def test_identical_requests_share_a_key(self): + assert self.base() == self.base() + + @pytest.mark.parametrize( + "field,value", + [ + ("provider", "AnthropicProvider"), + ("model", "gpt-4o"), + ("system", "different system"), + ("user", "different user"), + ("kind", "json"), + ("temperature", 0.9), + ("max_tokens", 4000), + ], + ) + def test_every_field_changes_the_key(self, field, value): + """A response generated under different settings must not be replayed.""" + assert self.base() != self.base(**{field: value}) + + def test_extra_is_part_of_the_key(self): + assert self.base() != self.base(extra={"base_url": "http://localhost:11434"}) + + +# --------------------------------------------------------------------------- +# Storage +# --------------------------------------------------------------------------- + + +class TestStorage: + @pytest.fixture + def cache(self, tmp_path): + return LLMCache(tmp_path / "store") + + def test_miss_then_hit(self, cache): + assert cache.get("k") is None + cache.put("k", "the answer") + assert cache.get("k") == "the answer" + assert cache.stats.hits == 1 and cache.stats.misses == 1 + + def test_disabled_cache_is_inert(self, tmp_path): + cache = LLMCache(tmp_path / "store", enabled=False) + assert cache.put("k", "v") is False + assert cache.get("k") is None + assert cache.stats.total == 0 + + def test_corrupt_entry_is_a_miss(self, cache): + cache.put("k", "v") + cache._path("k").write_text("{not json", encoding="utf-8") + assert cache.get("k") is None + + def test_entry_missing_response_field_is_a_miss(self, cache): + cache.put("k", "v") + cache._path("k").write_text(json.dumps({"nope": 1}), encoding="utf-8") + assert cache.get("k") is None + + def test_no_partial_files_left_behind(self, cache): + cache.put("k", "v") + assert list(cache.dir.rglob("*.tmp")) == [] + + def test_clear_and_counts(self, cache): + cache.put("a", "1") + cache.put("b", "2") + assert cache.entry_count() == 2 + assert cache.size_bytes() > 0 + assert cache.clear() == 2 + assert cache.entry_count() == 0 + + def test_stats_summary(self, cache): + assert cache.stats.summary() == "no LLM calls" + cache.put("a", "1") + cache.get("a") + cache.get("missing") + assert cache.stats.summary() == "1/2 LLM responses cached" + + +# --------------------------------------------------------------------------- +# Provider integration — the property that saves money +# --------------------------------------------------------------------------- + + +class TestProviderIntegration: + def test_repeat_text_call_hits_no_api(self): + p = CountingProvider(model="m") + first = p.chat("sys", "hello") + for _ in range(4): + assert p.chat("sys", "hello") == first + assert p.text_calls == 1 + + def test_repeat_json_call_hits_no_api(self): + p = CountingProvider(model="m") + first = p.chat_json("sys", "extract") + for _ in range(4): + assert p.chat_json("sys", "extract") == first + assert p.json_calls == 1 + + def test_different_prompt_calls_again(self): + p = CountingProvider(model="m") + p.chat("sys", "one") + p.chat("sys", "two") + assert p.text_calls == 2 + + def test_different_model_calls_again(self): + a = CountingProvider(model="model-a") + b = CountingProvider(model="model-b") + a.chat("sys", "same") + b.chat("sys", "same") + assert a.text_calls == 1 and b.text_calls == 1 + + def test_text_and_json_do_not_collide(self): + """Same prompt, different call kind — must not share an entry.""" + p = CountingProvider(model="m") + p.chat("sys", "same") + p.chat_json("sys", "same") + assert p.text_calls == 1 and p.json_calls == 1 + + def test_separate_provider_instances_share_the_cache(self): + """LLMExtractor and LLMSummarizer are different objects, one run.""" + a = CountingProvider(model="m") + b = CountingProvider(model="m") + a.chat("sys", "shared question") + b.chat("sys", "shared question") + assert a.text_calls == 1 + assert b.text_calls == 0 + + def test_cache_survives_a_new_cache_object(self, tmp_path): + """The on-disk entry is what makes a *later run* free.""" + a = CountingProvider(model="m") + a.chat("sys", "persisted") + + reset_shared_cache(LLMCache(tmp_path / "cache" / "llm")) + b = CountingProvider(model="m") + b.chat("sys", "persisted") + assert b.text_calls == 0 + + def test_disabled_cache_always_calls_the_api(self, tmp_path): + reset_shared_cache(LLMCache(tmp_path / "cache" / "llm", enabled=False)) + p = CountingProvider(model="m") + for _ in range(3): + p.chat("sys", "hello") + assert p.text_calls == 3 + + def test_unparseable_json_is_not_cached(self): + """Caching a broken response would make the failure permanent.""" + p = CountingProvider(model="m") + p.json_payload = "not json at all" + p.max_retries = 1 + with pytest.raises(RuntimeError): + p.chat_json("sys", "extract") + assert shared_cache().entry_count() == 0 + + def test_json_cache_stores_raw_not_parsed(self): + """Storing raw text keeps the entry faithful to what the provider said.""" + p = CountingProvider(model="m") + p.json_payload = '```json\n{"entities": ["Alpha"]}\n```' + assert p.chat_json("sys", "extract") == {"entities": ["Alpha"]} + + entry = next(shared_cache().dir.rglob("*.json")) + stored = json.loads(entry.read_text(encoding="utf-8"))["response"] + assert stored.startswith("```json") + + def test_retries_are_not_defeated_by_the_cache(self): + """A transient failure then success must still end up cached.""" + attempts = {"n": 0} + + class FlakyProvider(CountingProvider): + def _call(self, system, user): + attempts["n"] += 1 + if attempts["n"] == 1: + raise RuntimeError("transient") + return "eventually fine" + + p = FlakyProvider(model="m", max_retries=3) + assert p.chat("sys", "flaky") == "eventually fine" + assert p.chat("sys", "flaky") == "eventually fine" + assert attempts["n"] == 2 # one failure, one success, then cached