diff --git a/README.md b/README.md index 0a44e00..758e17e 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,24 @@ stored artifact byte for byte; the one thing it does not refresh is the generation timestamp, which is what makes identical inputs produce identical output. +**LLM responses are cached too.** In `--mode llm`, re-running over an unchanged +README previously re-paid for every API call. Responses are now stored on disk +keyed by provider, model, temperature, token limit, and the exact prompt, so a +repeat run costs nothing and returns instantly: + +```bash +opendocs generate ./README.md --local --mode llm # first run calls the API +opendocs generate ./README.md --local --mode llm # second run is free + +export OPENDOCS_LLM_CACHE=0 # disable response caching only +opendocs generate ./README.md --local --mode llm --no-cache # disable both +``` + +Note this deliberately trades novelty for cost and reproducibility: with +`temperature > 0` the provider would have returned a *different* answer, and +you get the earlier one. Change the temperature, the model, or the prompt and +it re-asks; use `--no-cache` when you specifically want a fresh generation. + ### What Changed Since the Last Release `opendocs diff` compares two versions of your documentation and reports what diff --git a/src/opendocs/cli.py b/src/opendocs/cli.py index eb17e2f..cd1e263 100644 --- a/src/opendocs/cli.py +++ b/src/opendocs/cli.py @@ -161,7 +161,7 @@ def main(): "no_cache", is_flag=True, default=False, - help="Ignore the build cache and regenerate every format from scratch.", + help="Ignore all caches (generated artifacts and LLM responses) and rebuild from scratch.", ) @click.option( "--cache-dir", @@ -297,8 +297,11 @@ def generate( # Run pipeline — folder path or single file/URL from .core.build_cache import BuildCache + from .llm.cache import LLMCache, reset_shared_cache build_cache = BuildCache(cache_dir, enabled=not no_cache) + # --no-cache means "don't reuse anything", so it covers LLM responses too. + reset_shared_cache(LLMCache(Path(cache_dir) / "llm" if cache_dir else None, enabled=not no_cache)) pipeline = Pipeline(github_token=token) source_path = Path(source) @@ -967,20 +970,26 @@ def cache(do_clear: bool, cache_dir: str | None): opendocs cache --clear # empty it """ from .core.build_cache import BuildCache + from .llm.cache import LLMCache store = BuildCache(cache_dir) + llm_store = LLMCache(Path(cache_dir) / "llm" if cache_dir else None) if do_clear: - removed = store.clear() - console.print(f"[green][OK][/] Cleared {removed} cached artifact(s) from {store.dir}") + artifacts = store.clear() + responses = llm_store.clear() + console.print(f"[green][OK][/] Cleared {artifacts} artifact(s) and {responses} LLM response(s)") return - size = store.size_bytes() - entries = len(list(store.dir.glob("*/*/manifest.json"))) if store.dir.exists() else 0 - console.print(f"\n[bold]Build cache[/] [dim]{store.dir}[/]") - console.print(f" entries: {entries}") - console.print(f" size: {size / 1024 / 1024:.1f} MB" if size else " size: empty") - console.print("\n[dim]Clear it with `opendocs cache --clear`.[/]\n") + def _describe(label: str, path, entries: int, size: int) -> None: + console.print(f"\n[bold]{label}[/] [dim]{path}[/]") + console.print(f" entries: {entries}") + console.print(f" size: {size / 1024 / 1024:.1f} MB" if size else " size: empty") + + build_entries = len(list(store.dir.glob("*/*/manifest.json"))) if store.dir.exists() else 0 + _describe("Build cache", store.dir, build_entries, store.size_bytes()) + _describe("LLM response cache", llm_store.dir, llm_store.entry_count(), llm_store.size_bytes()) + console.print("\n[dim]Clear both with `opendocs cache --clear`.[/]\n") @main.command() diff --git a/src/opendocs/core/semantic_extractor.py b/src/opendocs/core/semantic_extractor.py index f533552..4becb40 100644 --- a/src/opendocs/core/semantic_extractor.py +++ b/src/opendocs/core/semantic_extractor.py @@ -206,9 +206,15 @@ # --------------------------------------------------------------------------- _VERSION_RE = re.compile(r"(\d+\.\d+(?:\.\d+)?(?:[+-]\w+)?)") +# Units are ordered longest-first so "30 seconds" captures "seconds" rather +# than matching the bare "s" alternative first. The trailing lookahead is what +# stops a number followed by any s-word from being read as a duration: without +# it, "12 selectable formats" and "0 sections added" both became metrics. _METRIC_RE = re.compile( - r"(\d+(?:\.\d+)?)\s*(%|ms|s|sec|seconds|min|minutes|hz|Hz|MB|GB|TB|KB" - r"|req/s|rps|qps|tps|fps|Mbps|Gbps|rpm|k|K|M)", + r"(\d+(?:\.\d+)?)\s*(" + r"seconds|minutes|req/s|Mbps|Gbps|sec|min|rpm|rps|qps|tps|fps" + r"|ms|Hz|hz|MB|GB|TB|KB|%|s|k|K|M" + r")(?![A-Za-z])", ) _API_PATH_RE = re.compile(r"`?(/api/\S+|/v\d+/\S+)`?") _HTTP_METHOD_RE = re.compile(r"\b(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\b") @@ -216,6 +222,54 @@ _MERMAID_NODE_RE = re.compile(r"(\w+)\[([^\]]+)\]") +#: Separators that divide a feature bullet's *name* from its *description*. +#: Ordered so multi-character dashes are tried before the single hyphen, which +#: would otherwise split hyphenated names like "Auto-PR" in half. +_FEATURE_SEPARATORS = (" -- ", " — ", " – ", ": ", " - ", "--", "—", "–", ":") + +#: A feature name longer than this is prose, not a name. The previous limit of +#: 80 was a truncation point rather than a rejection, so whole sentences became +#: entity names and every downstream view showed them ellipsised. +_MAX_FEATURE_NAME = 60 + + +def _feature_name(text: str) -> str: + """Extract the short name from a feature bullet. + + Feature bullets are conventionally ``Name -- description``. Splitting only + on ``:`` and the typographic dashes missed the ASCII ``--`` form, so the + whole bullet was kept and merely truncated. When no separator is present, + fall back to the first clause and then to a word-count cap, so the result is + always a name rather than a sentence fragment. + """ + candidate = text.strip() + + for separator in _FEATURE_SEPARATORS: + if separator in candidate: + head = candidate.split(separator, 1)[0].strip() + if 3 < len(head) <= _MAX_FEATURE_NAME: + return head + # A head that is still too long means the separator appeared late + # in prose rather than after a name; keep looking. + if head: + candidate = head + break + + # No usable separator — stop at the first sentence. + candidate = re.split(r"(?<=[a-z0-9])\.\s", candidate)[0].strip() + if len(candidate) <= _MAX_FEATURE_NAME: + return candidate + + # Still prose: keep the leading words, cutting on a word boundary. + words = candidate.split() + trimmed: list[str] = [] + for word in words: + if len(" ".join(trimmed + [word])) > _MAX_FEATURE_NAME: + break + trimmed.append(word) + return " ".join(trimmed).rstrip(" ,;:") + + # --------------------------------------------------------------------------- # Extractor # --------------------------------------------------------------------------- @@ -671,7 +725,7 @@ def _extract_from_lists(self, doc: DocumentModel, kg: KnowledgeGraph) -> None: for item in block.items[:15]: # Cap at 15 # Strip bold markers, extract first meaningful phrase clean = re.sub(r"\*\*([^*]+)\*\*", r"\1", item) - name = clean.split(":")[0].split("—")[0].split("–")[0].strip()[:80] + name = _feature_name(clean) if len(name) > 3: eid = self._make_id("feat", name) kg.add_entity( diff --git a/src/opendocs/llm/cache.py b/src/opendocs/llm/cache.py new file mode 100644 index 0000000..a6432a3 --- /dev/null +++ b/src/opendocs/llm/cache.py @@ -0,0 +1,205 @@ +"""Response cache for LLM calls. + +``--mode llm`` re-issues every request on every run, so regenerating docs from +an unchanged README pays the full API cost again. This caches responses on +disk, keyed by everything that determines the answer, so a repeat run is free +and instant. + +Caching a non-deterministic API is a deliberate trade: with ``temperature > 0`` +a provider would have returned *a* different answer, and this returns the +earlier one. That is what makes repeated builds reproducible and cheap, and it +is why the temperature is part of the key and why the cache is easy to turn +off — see :func:`llm_cache_enabled`. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +#: Bumped when the on-disk entry layout changes. +CACHE_FORMAT_VERSION = 1 + +_ENV_ENABLED = "OPENDOCS_LLM_CACHE" +_ENV_CACHE_DIR = "OPENDOCS_CACHE_DIR" + +_FALSEY = {"0", "false", "no", "off"} + + +def llm_cache_enabled() -> bool: + """Whether the LLM cache is active. + + On by default; set ``OPENDOCS_LLM_CACHE=0`` to disable it globally. + """ + return os.environ.get(_ENV_ENABLED, "1").strip().lower() not in _FALSEY + + +def llm_cache_dir() -> Path: + """Directory holding cached LLM responses.""" + override = os.environ.get(_ENV_CACHE_DIR) + if override: + return Path(override).expanduser() / "llm" + xdg = os.environ.get("XDG_CACHE_HOME") + base = Path(xdg).expanduser() if xdg else Path.home() / ".cache" + return base / "opendocs" / "llm" + + +@dataclass +class LLMCacheStats: + """Tally of cache activity for a process.""" + + hits: int = 0 + misses: int = 0 + stores: int = 0 + + @property + def total(self) -> int: + return self.hits + self.misses + + def summary(self) -> str: + if not self.total: + return "no LLM calls" + return f"{self.hits}/{self.total} LLM responses cached" + + +class LLMCache: + """Disk cache mapping a request fingerprint to a provider response.""" + + def __init__(self, cache_dir: str | Path | None = None, *, enabled: Optional[bool] = None) -> None: + self.enabled = llm_cache_enabled() if enabled is None else enabled + self.dir = Path(cache_dir) if cache_dir else llm_cache_dir() + self.stats = LLMCacheStats() + + # -- Keying ---------------------------------------------------------- + + @staticmethod + def make_key( + *, + provider: str, + model: str, + system: str, + user: str, + kind: str = "text", + temperature: float = 0.0, + max_tokens: int = 0, + extra: Optional[dict[str, Any]] = None, + ) -> str: + """Fingerprint one request. + + Everything that could change the response belongs here — including the + temperature and token ceiling, so tuning either re-asks the provider + rather than replaying an answer generated under different settings. + """ + payload = { + "v": CACHE_FORMAT_VERSION, + "provider": provider, + "model": model, + "kind": kind, + "temperature": temperature, + "max_tokens": max_tokens, + "system": system, + "user": user, + "extra": extra or {}, + } + blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + # -- Storage --------------------------------------------------------- + + def _path(self, key: str) -> Path: + # Shard by the first two characters to keep directories small. + return self.dir / key[:2] / f"{key}.json" + + def get(self, key: str) -> Optional[str]: + """Return the cached response text, or *None* on a miss.""" + if not self.enabled: + return None + + path = self._path(key) + if not path.exists(): + self.stats.misses += 1 + return None + + try: + entry = json.loads(path.read_text(encoding="utf-8")) + response = entry["response"] + except Exception as exc: + # A damaged entry must behave as a miss, never break a run. + logger.debug("LLM cache read failed for %s: %s", key[:12], exc) + self.stats.misses += 1 + return None + + self.stats.hits += 1 + logger.debug("LLM cache hit %s", key[:12]) + return response + + def put(self, key: str, response: str, *, meta: Optional[dict[str, Any]] = None) -> bool: + """Store *response* under *key*. Returns True when it was written.""" + if not self.enabled: + return False + + path = self._path(key) + try: + path.parent.mkdir(parents=True, exist_ok=True) + # Write via a temp file so a crash cannot leave a half-written + # entry that would later parse as valid but truncated JSON. + tmp = path.with_suffix(".json.tmp") + tmp.write_text( + json.dumps({"response": response, "cached_at": time.time(), "meta": meta or {}}), + encoding="utf-8", + ) + tmp.replace(path) + except Exception as exc: + logger.debug("LLM cache write failed for %s: %s", key[:12], exc) + return False + + self.stats.stores += 1 + return True + + def clear(self) -> int: + """Delete every cached response. Returns how many were removed.""" + if not self.dir.exists(): + return 0 + entries = list(self.dir.glob("*/*.json")) + import shutil + + shutil.rmtree(self.dir, ignore_errors=True) + return len(entries) + + def size_bytes(self) -> int: + if not self.dir.exists(): + return 0 + return sum(p.stat().st_size for p in self.dir.rglob("*.json") if p.is_file()) + + def entry_count(self) -> int: + if not self.dir.exists(): + return 0 + return len(list(self.dir.glob("*/*.json"))) + + +#: Process-wide cache shared by every provider instance, so separate +#: LLMExtractor / LLMSummarizer / LLMContentEnhancer objects benefit from each +#: other's results within a single run as well as across runs. +_SHARED: Optional[LLMCache] = None + + +def shared_cache() -> LLMCache: + """Return the process-wide LLM cache, creating it on first use.""" + global _SHARED + if _SHARED is None: + _SHARED = LLMCache() + return _SHARED + + +def reset_shared_cache(cache: Optional[LLMCache] = None) -> None: + """Replace the process-wide cache (used by tests and by ``--no-cache``).""" + global _SHARED + _SHARED = cache diff --git a/src/opendocs/llm/providers.py b/src/opendocs/llm/providers.py index 762dce1..26ca888 100644 --- a/src/opendocs/llm/providers.py +++ b/src/opendocs/llm/providers.py @@ -100,8 +100,42 @@ def __init__( # ── Public API ──────────────────────────────────────────────────── + # ── Caching ─────────────────────────────────────────────────────── + + def _cache_key(self, system: str, user: str, kind: str) -> str: + from .cache import LLMCache + + return LLMCache.make_key( + provider=type(self).__name__, + model=self.model, + system=system, + user=user, + kind=kind, + temperature=self.temperature, + max_tokens=self.max_tokens, + extra={"base_url": self.base_url or ""}, + ) + def chat(self, system: str, user: str) -> str: - """Send a chat completion with retry. Returns plain text.""" + """Send a chat completion with retry. Returns plain text. + + Responses are cached on disk, so repeating a run over unchanged input + costs nothing. Disable with ``OPENDOCS_LLM_CACHE=0``. + """ + from .cache import shared_cache + + cache = shared_cache() + key = self._cache_key(system, user, "text") + cached = cache.get(key) + if cached is not None: + return cached + + result = self._chat_uncached(system, user) + cache.put(key, result, meta={"model": self.model, "provider": type(self).__name__}) + return result + + def _chat_uncached(self, system: str, user: str) -> str: + """Send a chat completion with retry, bypassing the cache.""" last_exc: Exception | None = None for attempt in range(self.max_retries): try: @@ -124,12 +158,47 @@ def chat_json(self, system: str, user: str) -> dict[str, Any]: Each provider implements its own JSON-mode strategy. Falls back to parsing raw text as JSON. + + The *raw* response is cached rather than the parsed object, so the + stored entry stays a faithful record of what the provider returned and + parsing improvements apply to previously cached responses. """ + from .cache import shared_cache + + cache = shared_cache() + key = self._cache_key(system, user, "json") + cached = cache.get(key) + if cached is not None: + try: + return self._parse_json(cached) + except Exception as exc: + # A cached response we can no longer parse is worthless; fall + # through and ask the provider again. + logger.debug("Cached JSON response failed to parse (%s); re-requesting", exc) + + raw_holder: dict[str, str] = {} + parsed = self._chat_json_uncached(system, user, raw_holder) + if "raw" in raw_holder: + cache.put(key, raw_holder["raw"], meta={"model": self.model, "provider": type(self).__name__}) + return parsed + + def _chat_json_uncached( + self, + system: str, + user: str, + raw_holder: dict[str, str] | None = None, + ) -> dict[str, Any]: + """Send a JSON-mode completion with retry, bypassing the cache.""" last_exc: Exception | None = None for attempt in range(self.max_retries): try: raw = self._call_json(system, user) - return self._parse_json(raw) + parsed = self._parse_json(raw) + # Only expose the raw text once it is known to parse, so an + # unparseable response is never written to the cache. + if raw_holder is not None: + raw_holder["raw"] = raw + return parsed except json.JSONDecodeError as exc: last_exc = exc logger.warning("JSON parse failed (attempt %d): %s", attempt + 1, exc) diff --git a/tests/test_extraction_quality.py b/tests/test_extraction_quality.py new file mode 100644 index 0000000..b55c759 --- /dev/null +++ b/tests/test_extraction_quality.py @@ -0,0 +1,196 @@ +"""Extraction-quality regression tests. + +Entity names feed the knowledge graph, the interactive graph, the wiki, the +diff, and the query layer, so a naming defect degrades all of them at once. +These tests pin the two that did: + +- feature bullets were kept whole and merely truncated, because the splitter + did not know about the ASCII ``--`` separator; +- any number followed by an s-word was read as a duration, so "12 selectable + formats" and "0 sections added" became metric entities. +""" + +from __future__ import annotations + +import pytest + +from opendocs.core.knowledge_graph import EntityType +from opendocs.core.parser import ReadmeParser +from opendocs.core.semantic_extractor import _MAX_FEATURE_NAME, _METRIC_RE, SemanticExtractor, _feature_name + + +def extract(markdown: str): + return SemanticExtractor().extract(ReadmeParser().parse(markdown, repo_name="Proj")) + + +def names_of(kg, entity_type: EntityType) -> list[str]: + return [e.name for e in kg.entities if e.entity_type is entity_type] + + +# --------------------------------------------------------------------------- +# Feature names +# --------------------------------------------------------------------------- + + +class TestFeatureName: + @pytest.mark.parametrize( + ("bullet", "expected"), + [ + ("15 Output Formats -- Word, PDF, PPTX, Blog Post and more", "15 Output Formats"), + ("Interactive Graph — Single-file HTML visualization with search", "Interactive Graph"), + ("Community Detection – Label propagation groups entities", "Community Detection"), + ("Smart Table Sorting: 6 strategies for ordering rows", "Smart Table Sorting"), + ("Knowledge Wiki - Wikipedia-style linked Markdown articles", "Knowledge Wiki"), + ], + ) + def test_separator_forms(self, bullet, expected): + """Regression: only ':' and typographic dashes were handled, not '--'.""" + assert _feature_name(bullet) == expected + + def test_hyphenated_names_are_not_split(self): + """A bare '-' must not cut 'Auto-PR' in half.""" + assert _feature_name("File Watcher + Auto-PR -- Monitor repos for changes") == "File Watcher + Auto-PR" + + def test_short_bullet_is_kept_whole(self): + assert _feature_name("Fast and simple") == "Fast and simple" + + def test_prose_without_a_separator_is_trimmed_to_a_name(self): + long_prose = "A feature described in a full sentence with no separator at all that rambles onward" + result = _feature_name(long_prose) + assert len(result) <= _MAX_FEATURE_NAME + assert not result.endswith(" ") + + def test_trimming_falls_on_a_word_boundary(self): + result = _feature_name("word " * 40) + assert not result.endswith("wor") + assert all(w == "word" for w in result.split()) + + def test_stops_at_the_first_sentence(self): + text = "Does one thing. Then it goes on to explain a great many other things at length." + assert _feature_name(text) == "Does one thing" + + def test_never_exceeds_the_cap(self): + for bullet in [ + "x" * 200, + "Name -- " + "y" * 200, + "word " * 100, + "A: " + "z" * 200, + ]: + assert len(_feature_name(bullet)) <= _MAX_FEATURE_NAME + + +class TestFeatureExtraction: + FEATURES_README = """# Proj + +A project with several capabilities worth describing at some length here. + +## Features + +- **15 Output Formats** -- Word, PDF, PPTX, Blog Post, Jira Tickets, Changelog +- **Interactive Knowledge Graph** -- Single-file HTML visualization with search +- **File Watcher + Auto-PR** -- Monitor repos and create pull requests +""" + + def test_feature_names_are_concepts_not_sentences(self): + kg = extract(self.FEATURES_README) + features = names_of(kg, EntityType.FEATURE) + assert "15 Output Formats" in features + assert "Interactive Knowledge Graph" in features + assert "File Watcher + Auto-PR" in features + + def test_no_entity_name_is_prose_length(self): + kg = extract(self.FEATURES_README) + oversized = [e.name for e in kg.entities if len(e.name) > _MAX_FEATURE_NAME] + assert oversized == [] + + def test_description_is_preserved_in_properties(self): + """Trimming the name must not lose the detail.""" + kg = extract(self.FEATURES_README) + feature = next(e for e in kg.entities if e.name == "15 Output Formats") + assert "Word" in feature.properties.get("description", "") + + +# --------------------------------------------------------------------------- +# Metrics +# --------------------------------------------------------------------------- + + +class TestMetricRegex: + @pytest.mark.parametrize( + "text", + [ + "all 12 selectable output formats", + "5 stakeholder views", + "0 sections added", + "10 karat gold", + "3 separate modules", + "7 minor issues", + ], + ) + def test_number_followed_by_a_word_is_not_a_metric(self, text): + """Regression: the bare 's'/'k'/'M' units matched any following word.""" + assert list(_METRIC_RE.finditer(text)) == [] + + @pytest.mark.parametrize( + ("text", "value", "unit"), + [ + ("takes 30 seconds", "30", "seconds"), + ("within 5 min", "5", "min"), + ("latency of 250 ms", "250", "ms"), + ("99.9% uptime", "99.9", "%"), + ("handles 1000 req/s", "1000", "req/s"), + ("cache is 512 MB", "512", "MB"), + ("runs at 60 fps", "60", "fps"), + ("a 10k user base", "10", "k"), + ], + ) + def test_real_metrics_still_match(self, text, value, unit): + match = _METRIC_RE.search(text) + assert match is not None, f"missed metric in {text!r}" + assert (match.group(1), match.group(2)) == (value, unit) + + def test_longest_unit_wins(self): + """'30 seconds' must not capture the unit as a bare 's'.""" + assert _METRIC_RE.search("takes 30 seconds").group(2) == "seconds" + + +class TestMetricExtraction: + def test_counting_prose_produces_no_metrics(self): + kg = extract("# Proj\n\nWe support 12 selectable formats and 5 stakeholder views in total here.\n") + assert names_of(kg, EntityType.METRIC) == [] + + def test_genuine_metrics_are_still_extracted(self): + kg = extract("# Proj\n\nThe service responds in 250 ms and sustains 1000 req/s under load.\n") + metrics = names_of(kg, EntityType.METRIC) + assert any("250" in m for m in metrics) + assert any("1000" in m for m in metrics) + + +# --------------------------------------------------------------------------- +# Dogfooding +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def kg(): + """The project's own README, extracted once for the whole module.""" + from pathlib import Path + + readme = Path(__file__).resolve().parent.parent / "README.md" + return extract(readme.read_text(encoding="utf-8")) + + +class TestAgainstOwnReadme: + def test_no_prose_length_entity_names(self, kg): + oversized = [e.name for e in kg.entities if len(e.name) > _MAX_FEATURE_NAME] + assert oversized == [], f"prose leaked into entity names: {oversized}" + + def test_no_bare_duration_metrics(self, kg): + """The README counts things ("12 formats"); none of those are durations.""" + bogus = [e.name for e in kg.entities if e.entity_type is EntityType.METRIC and e.name.endswith(" s")] + assert bogus == [], f"miscounted durations: {bogus}" + + def test_still_finds_real_entities(self, kg): + """Precision must not have come at the cost of finding nothing.""" + assert len(kg.entities) > 15 + assert names_of(kg, EntityType.FEATURE) diff --git a/tests/test_llm_cache.py b/tests/test_llm_cache.py new file mode 100644 index 0000000..2979b48 --- /dev/null +++ b/tests/test_llm_cache.py @@ -0,0 +1,274 @@ +"""Tests for the LLM response cache. + +No real provider is ever contacted: a fake provider counts how often the +network layer *would* have been reached, which is the property that matters — +a cache hit must mean an API call not made, and therefore not paid for. +""" + +from __future__ import annotations + +import json + +import pytest + +from opendocs.llm.cache import ( + LLMCache, + llm_cache_dir, + llm_cache_enabled, + reset_shared_cache, + shared_cache, +) +from opendocs.llm.providers import LLMProvider + + +class CountingProvider(LLMProvider): + """A provider that records how many calls reached the transport layer.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.text_calls = 0 + self.json_calls = 0 + self.json_payload = '{"entities": ["Alpha"]}' + + def _call(self, system: str, user: str) -> str: + self.text_calls += 1 + return f"answer#{self.text_calls} to {user}" + + def _call_json(self, system: str, user: str) -> str: + self.json_calls += 1 + return self.json_payload + + +@pytest.fixture(autouse=True) +def isolated_cache(tmp_path, monkeypatch): + """Give every test its own cache and restore the shared one afterwards.""" + monkeypatch.setenv("OPENDOCS_CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.delenv("OPENDOCS_LLM_CACHE", raising=False) + reset_shared_cache(LLMCache(tmp_path / "cache" / "llm")) + yield + reset_shared_cache(None) + + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + + +class TestConfiguration: + def test_enabled_by_default(self): + assert llm_cache_enabled() is True + + @pytest.mark.parametrize("value", ["0", "false", "no", "off", "OFF", "False"]) + def test_disabled_by_env(self, monkeypatch, value): + monkeypatch.setenv("OPENDOCS_LLM_CACHE", value) + assert llm_cache_enabled() is False + + @pytest.mark.parametrize("value", ["1", "true", "yes", "anything"]) + def test_other_values_leave_it_enabled(self, monkeypatch, value): + monkeypatch.setenv("OPENDOCS_LLM_CACHE", value) + assert llm_cache_enabled() is True + + def test_cache_dir_follows_the_override(self, tmp_path, monkeypatch): + monkeypatch.setenv("OPENDOCS_CACHE_DIR", str(tmp_path / "custom")) + assert llm_cache_dir() == tmp_path / "custom" / "llm" + + def test_cache_dir_falls_back_to_xdg(self, tmp_path, monkeypatch): + monkeypatch.delenv("OPENDOCS_CACHE_DIR", raising=False) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "xdg")) + assert llm_cache_dir() == tmp_path / "xdg" / "opendocs" / "llm" + + +# --------------------------------------------------------------------------- +# Keying +# --------------------------------------------------------------------------- + + +class TestKeying: + def base(self, **over): + args = { + "provider": "OpenAIProvider", + "model": "gpt-4o-mini", + "system": "sys", + "user": "usr", + "kind": "text", + "temperature": 0.2, + "max_tokens": 100, + } + args.update(over) + return LLMCache.make_key(**args) + + def test_identical_requests_share_a_key(self): + assert self.base() == self.base() + + @pytest.mark.parametrize( + "field,value", + [ + ("provider", "AnthropicProvider"), + ("model", "gpt-4o"), + ("system", "different system"), + ("user", "different user"), + ("kind", "json"), + ("temperature", 0.9), + ("max_tokens", 4000), + ], + ) + def test_every_field_changes_the_key(self, field, value): + """A response generated under different settings must not be replayed.""" + assert self.base() != self.base(**{field: value}) + + def test_extra_is_part_of_the_key(self): + assert self.base() != self.base(extra={"base_url": "http://localhost:11434"}) + + +# --------------------------------------------------------------------------- +# Storage +# --------------------------------------------------------------------------- + + +class TestStorage: + @pytest.fixture + def cache(self, tmp_path): + return LLMCache(tmp_path / "store") + + def test_miss_then_hit(self, cache): + assert cache.get("k") is None + cache.put("k", "the answer") + assert cache.get("k") == "the answer" + assert cache.stats.hits == 1 and cache.stats.misses == 1 + + def test_disabled_cache_is_inert(self, tmp_path): + cache = LLMCache(tmp_path / "store", enabled=False) + assert cache.put("k", "v") is False + assert cache.get("k") is None + assert cache.stats.total == 0 + + def test_corrupt_entry_is_a_miss(self, cache): + cache.put("k", "v") + cache._path("k").write_text("{not json", encoding="utf-8") + assert cache.get("k") is None + + def test_entry_missing_response_field_is_a_miss(self, cache): + cache.put("k", "v") + cache._path("k").write_text(json.dumps({"nope": 1}), encoding="utf-8") + assert cache.get("k") is None + + def test_no_partial_files_left_behind(self, cache): + cache.put("k", "v") + assert list(cache.dir.rglob("*.tmp")) == [] + + def test_clear_and_counts(self, cache): + cache.put("a", "1") + cache.put("b", "2") + assert cache.entry_count() == 2 + assert cache.size_bytes() > 0 + assert cache.clear() == 2 + assert cache.entry_count() == 0 + + def test_stats_summary(self, cache): + assert cache.stats.summary() == "no LLM calls" + cache.put("a", "1") + cache.get("a") + cache.get("missing") + assert cache.stats.summary() == "1/2 LLM responses cached" + + +# --------------------------------------------------------------------------- +# Provider integration — the property that saves money +# --------------------------------------------------------------------------- + + +class TestProviderIntegration: + def test_repeat_text_call_hits_no_api(self): + p = CountingProvider(model="m") + first = p.chat("sys", "hello") + for _ in range(4): + assert p.chat("sys", "hello") == first + assert p.text_calls == 1 + + def test_repeat_json_call_hits_no_api(self): + p = CountingProvider(model="m") + first = p.chat_json("sys", "extract") + for _ in range(4): + assert p.chat_json("sys", "extract") == first + assert p.json_calls == 1 + + def test_different_prompt_calls_again(self): + p = CountingProvider(model="m") + p.chat("sys", "one") + p.chat("sys", "two") + assert p.text_calls == 2 + + def test_different_model_calls_again(self): + a = CountingProvider(model="model-a") + b = CountingProvider(model="model-b") + a.chat("sys", "same") + b.chat("sys", "same") + assert a.text_calls == 1 and b.text_calls == 1 + + def test_text_and_json_do_not_collide(self): + """Same prompt, different call kind — must not share an entry.""" + p = CountingProvider(model="m") + p.chat("sys", "same") + p.chat_json("sys", "same") + assert p.text_calls == 1 and p.json_calls == 1 + + def test_separate_provider_instances_share_the_cache(self): + """LLMExtractor and LLMSummarizer are different objects, one run.""" + a = CountingProvider(model="m") + b = CountingProvider(model="m") + a.chat("sys", "shared question") + b.chat("sys", "shared question") + assert a.text_calls == 1 + assert b.text_calls == 0 + + def test_cache_survives_a_new_cache_object(self, tmp_path): + """The on-disk entry is what makes a *later run* free.""" + a = CountingProvider(model="m") + a.chat("sys", "persisted") + + reset_shared_cache(LLMCache(tmp_path / "cache" / "llm")) + b = CountingProvider(model="m") + b.chat("sys", "persisted") + assert b.text_calls == 0 + + def test_disabled_cache_always_calls_the_api(self, tmp_path): + reset_shared_cache(LLMCache(tmp_path / "cache" / "llm", enabled=False)) + p = CountingProvider(model="m") + for _ in range(3): + p.chat("sys", "hello") + assert p.text_calls == 3 + + def test_unparseable_json_is_not_cached(self): + """Caching a broken response would make the failure permanent.""" + p = CountingProvider(model="m") + p.json_payload = "not json at all" + p.max_retries = 1 + with pytest.raises(RuntimeError): + p.chat_json("sys", "extract") + assert shared_cache().entry_count() == 0 + + def test_json_cache_stores_raw_not_parsed(self): + """Storing raw text keeps the entry faithful to what the provider said.""" + p = CountingProvider(model="m") + p.json_payload = '```json\n{"entities": ["Alpha"]}\n```' + assert p.chat_json("sys", "extract") == {"entities": ["Alpha"]} + + entry = next(shared_cache().dir.rglob("*.json")) + stored = json.loads(entry.read_text(encoding="utf-8"))["response"] + assert stored.startswith("```json") + + def test_retries_are_not_defeated_by_the_cache(self): + """A transient failure then success must still end up cached.""" + attempts = {"n": 0} + + class FlakyProvider(CountingProvider): + def _call(self, system, user): + attempts["n"] += 1 + if attempts["n"] == 1: + raise RuntimeError("transient") + return "eventually fine" + + p = FlakyProvider(model="m", max_retries=3) + assert p.chat("sys", "flaky") == "eventually fine" + assert p.chat("sys", "flaky") == "eventually fine" + assert attempts["n"] == 2 # one failure, one success, then cached