Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,24 @@ stored artifact byte for byte; the one thing it does not refresh is the
generation timestamp, which is what makes identical inputs produce identical
output.

**LLM responses are cached too.** In `--mode llm`, re-running over an unchanged
README previously re-paid for every API call. Responses are now stored on disk
keyed by provider, model, temperature, token limit, and the exact prompt, so a
repeat run costs nothing and returns instantly:

```bash
opendocs generate ./README.md --local --mode llm # first run calls the API
opendocs generate ./README.md --local --mode llm # second run is free

export OPENDOCS_LLM_CACHE=0 # disable response caching only
opendocs generate ./README.md --local --mode llm --no-cache # disable both
```

Note this deliberately trades novelty for cost and reproducibility: with
`temperature > 0` the provider would have returned a *different* answer, and
you get the earlier one. Change the temperature, the model, or the prompt and
it re-asks; use `--no-cache` when you specifically want a fresh generation.

### What Changed Since the Last Release

`opendocs diff` compares two versions of your documentation and reports what
Expand Down
27 changes: 18 additions & 9 deletions src/opendocs/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ def main():
"no_cache",
is_flag=True,
default=False,
help="Ignore the build cache and regenerate every format from scratch.",
help="Ignore all caches (generated artifacts and LLM responses) and rebuild from scratch.",
)
@click.option(
"--cache-dir",
Expand Down Expand Up @@ -297,8 +297,11 @@ def generate(

# Run pipeline — folder path or single file/URL
from .core.build_cache import BuildCache
from .llm.cache import LLMCache, reset_shared_cache

build_cache = BuildCache(cache_dir, enabled=not no_cache)
# --no-cache means "don't reuse anything", so it covers LLM responses too.
reset_shared_cache(LLMCache(Path(cache_dir) / "llm" if cache_dir else None, enabled=not no_cache))

pipeline = Pipeline(github_token=token)
source_path = Path(source)
Expand Down Expand Up @@ -967,20 +970,26 @@ def cache(do_clear: bool, cache_dir: str | None):
opendocs cache --clear # empty it
"""
from .core.build_cache import BuildCache
from .llm.cache import LLMCache

store = BuildCache(cache_dir)
llm_store = LLMCache(Path(cache_dir) / "llm" if cache_dir else None)

if do_clear:
removed = store.clear()
console.print(f"[green][OK][/] Cleared {removed} cached artifact(s) from {store.dir}")
artifacts = store.clear()
responses = llm_store.clear()
console.print(f"[green][OK][/] Cleared {artifacts} artifact(s) and {responses} LLM response(s)")
return

size = store.size_bytes()
entries = len(list(store.dir.glob("*/*/manifest.json"))) if store.dir.exists() else 0
console.print(f"\n[bold]Build cache[/] [dim]{store.dir}[/]")
console.print(f" entries: {entries}")
console.print(f" size: {size / 1024 / 1024:.1f} MB" if size else " size: empty")
console.print("\n[dim]Clear it with `opendocs cache --clear`.[/]\n")
def _describe(label: str, path, entries: int, size: int) -> None:
console.print(f"\n[bold]{label}[/] [dim]{path}[/]")
console.print(f" entries: {entries}")
console.print(f" size: {size / 1024 / 1024:.1f} MB" if size else " size: empty")

build_entries = len(list(store.dir.glob("*/*/manifest.json"))) if store.dir.exists() else 0
_describe("Build cache", store.dir, build_entries, store.size_bytes())
_describe("LLM response cache", llm_store.dir, llm_store.entry_count(), llm_store.size_bytes())
console.print("\n[dim]Clear both with `opendocs cache --clear`.[/]\n")


@main.command()
Expand Down
60 changes: 57 additions & 3 deletions src/opendocs/core/semantic_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,16 +206,70 @@
# ---------------------------------------------------------------------------

_VERSION_RE = re.compile(r"(\d+\.\d+(?:\.\d+)?(?:[+-]\w+)?)")
# Units are ordered longest-first so "30 seconds" captures "seconds" rather
# than matching the bare "s" alternative first. The trailing lookahead is what
# stops a number followed by any s-word from being read as a duration: without
# it, "12 selectable formats" and "0 sections added" both became metrics.
_METRIC_RE = re.compile(
r"(\d+(?:\.\d+)?)\s*(%|ms|s|sec|seconds|min|minutes|hz|Hz|MB|GB|TB|KB"
r"|req/s|rps|qps|tps|fps|Mbps|Gbps|rpm|k|K|M)",
r"(\d+(?:\.\d+)?)\s*("
r"seconds|minutes|req/s|Mbps|Gbps|sec|min|rpm|rps|qps|tps|fps"
r"|ms|Hz|hz|MB|GB|TB|KB|%|s|k|K|M"
r")(?![A-Za-z])",
)
_API_PATH_RE = re.compile(r"`?(/api/\S+|/v\d+/\S+)`?")
_HTTP_METHOD_RE = re.compile(r"\b(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\b")
_MERMAID_EDGE_RE = re.compile(r"(\w[\w\s]*?)\s*-->?\|?([^|]*?)\|?\s*(\w[\w\s]*?)$", re.MULTILINE)
_MERMAID_NODE_RE = re.compile(r"(\w+)\[([^\]]+)\]")


#: Separators that divide a feature bullet's *name* from its *description*.
#: Ordered so multi-character dashes are tried before the single hyphen, which
#: would otherwise split hyphenated names like "Auto-PR" in half.
_FEATURE_SEPARATORS = (" -- ", " — ", " – ", ": ", " - ", "--", "—", "–", ":")

#: A feature name longer than this is prose, not a name. The previous limit of
#: 80 was a truncation point rather than a rejection, so whole sentences became
#: entity names and every downstream view showed them ellipsised.
_MAX_FEATURE_NAME = 60


def _feature_name(text: str) -> str:
"""Extract the short name from a feature bullet.

Feature bullets are conventionally ``Name -- description``. Splitting only
on ``:`` and the typographic dashes missed the ASCII ``--`` form, so the
whole bullet was kept and merely truncated. When no separator is present,
fall back to the first clause and then to a word-count cap, so the result is
always a name rather than a sentence fragment.
"""
candidate = text.strip()

for separator in _FEATURE_SEPARATORS:
if separator in candidate:
head = candidate.split(separator, 1)[0].strip()
if 3 < len(head) <= _MAX_FEATURE_NAME:
return head
# A head that is still too long means the separator appeared late
# in prose rather than after a name; keep looking.
if head:
candidate = head
break

# No usable separator — stop at the first sentence.
candidate = re.split(r"(?<=[a-z0-9])\.\s", candidate)[0].strip()
if len(candidate) <= _MAX_FEATURE_NAME:
return candidate

# Still prose: keep the leading words, cutting on a word boundary.
words = candidate.split()
trimmed: list[str] = []
for word in words:
if len(" ".join(trimmed + [word])) > _MAX_FEATURE_NAME:
break
trimmed.append(word)
return " ".join(trimmed).rstrip(" ,;:")


# ---------------------------------------------------------------------------
# Extractor
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -671,7 +725,7 @@ def _extract_from_lists(self, doc: DocumentModel, kg: KnowledgeGraph) -> None:
for item in block.items[:15]: # Cap at 15
# Strip bold markers, extract first meaningful phrase
clean = re.sub(r"\*\*([^*]+)\*\*", r"\1", item)
name = clean.split(":")[0].split("—")[0].split("–")[0].strip()[:80]
name = _feature_name(clean)
if len(name) > 3:
eid = self._make_id("feat", name)
kg.add_entity(
Expand Down
205 changes: 205 additions & 0 deletions src/opendocs/llm/cache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
"""Response cache for LLM calls.

``--mode llm`` re-issues every request on every run, so regenerating docs from
an unchanged README pays the full API cost again. This caches responses on
disk, keyed by everything that determines the answer, so a repeat run is free
and instant.

Caching a non-deterministic API is a deliberate trade: with ``temperature > 0``
a provider would have returned *a* different answer, and this returns the
earlier one. That is what makes repeated builds reproducible and cheap, and it
is why the temperature is part of the key and why the cache is easy to turn
off — see :func:`llm_cache_enabled`.
"""

from __future__ import annotations

import hashlib
import json
import logging
import os
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional

logger = logging.getLogger(__name__)

#: Bumped when the on-disk entry layout changes.
CACHE_FORMAT_VERSION = 1

_ENV_ENABLED = "OPENDOCS_LLM_CACHE"
_ENV_CACHE_DIR = "OPENDOCS_CACHE_DIR"

_FALSEY = {"0", "false", "no", "off"}


def llm_cache_enabled() -> bool:
"""Whether the LLM cache is active.

On by default; set ``OPENDOCS_LLM_CACHE=0`` to disable it globally.
"""
return os.environ.get(_ENV_ENABLED, "1").strip().lower() not in _FALSEY


def llm_cache_dir() -> Path:
"""Directory holding cached LLM responses."""
override = os.environ.get(_ENV_CACHE_DIR)
if override:
return Path(override).expanduser() / "llm"
xdg = os.environ.get("XDG_CACHE_HOME")
base = Path(xdg).expanduser() if xdg else Path.home() / ".cache"
return base / "opendocs" / "llm"


@dataclass
class LLMCacheStats:
"""Tally of cache activity for a process."""

hits: int = 0
misses: int = 0
stores: int = 0

@property
def total(self) -> int:
return self.hits + self.misses

def summary(self) -> str:
if not self.total:
return "no LLM calls"
return f"{self.hits}/{self.total} LLM responses cached"


class LLMCache:
"""Disk cache mapping a request fingerprint to a provider response."""

def __init__(self, cache_dir: str | Path | None = None, *, enabled: Optional[bool] = None) -> None:
self.enabled = llm_cache_enabled() if enabled is None else enabled
self.dir = Path(cache_dir) if cache_dir else llm_cache_dir()
self.stats = LLMCacheStats()

# -- Keying ----------------------------------------------------------

@staticmethod
def make_key(
*,
provider: str,
model: str,
system: str,
user: str,
kind: str = "text",
temperature: float = 0.0,
max_tokens: int = 0,
extra: Optional[dict[str, Any]] = None,
) -> str:
"""Fingerprint one request.

Everything that could change the response belongs here — including the
temperature and token ceiling, so tuning either re-asks the provider
rather than replaying an answer generated under different settings.
"""
payload = {
"v": CACHE_FORMAT_VERSION,
"provider": provider,
"model": model,
"kind": kind,
"temperature": temperature,
"max_tokens": max_tokens,
"system": system,
"user": user,
"extra": extra or {},
}
blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
return hashlib.sha256(blob.encode("utf-8")).hexdigest()

# -- Storage ---------------------------------------------------------

def _path(self, key: str) -> Path:
# Shard by the first two characters to keep directories small.
return self.dir / key[:2] / f"{key}.json"

def get(self, key: str) -> Optional[str]:
"""Return the cached response text, or *None* on a miss."""
if not self.enabled:
return None

path = self._path(key)
if not path.exists():
self.stats.misses += 1
return None

try:
entry = json.loads(path.read_text(encoding="utf-8"))
response = entry["response"]
except Exception as exc:
# A damaged entry must behave as a miss, never break a run.
logger.debug("LLM cache read failed for %s: %s", key[:12], exc)
self.stats.misses += 1
return None

self.stats.hits += 1
logger.debug("LLM cache hit %s", key[:12])
return response

def put(self, key: str, response: str, *, meta: Optional[dict[str, Any]] = None) -> bool:
"""Store *response* under *key*. Returns True when it was written."""
if not self.enabled:
return False

path = self._path(key)
try:
path.parent.mkdir(parents=True, exist_ok=True)
# Write via a temp file so a crash cannot leave a half-written
# entry that would later parse as valid but truncated JSON.
tmp = path.with_suffix(".json.tmp")
tmp.write_text(
json.dumps({"response": response, "cached_at": time.time(), "meta": meta or {}}),
encoding="utf-8",
)
tmp.replace(path)
except Exception as exc:
logger.debug("LLM cache write failed for %s: %s", key[:12], exc)
return False

self.stats.stores += 1
return True

def clear(self) -> int:
"""Delete every cached response. Returns how many were removed."""
if not self.dir.exists():
return 0
entries = list(self.dir.glob("*/*.json"))
import shutil

shutil.rmtree(self.dir, ignore_errors=True)
return len(entries)

def size_bytes(self) -> int:
if not self.dir.exists():
return 0
return sum(p.stat().st_size for p in self.dir.rglob("*.json") if p.is_file())

def entry_count(self) -> int:
if not self.dir.exists():
return 0
return len(list(self.dir.glob("*/*.json")))


#: Process-wide cache shared by every provider instance, so separate
#: LLMExtractor / LLMSummarizer / LLMContentEnhancer objects benefit from each
#: other's results within a single run as well as across runs.
_SHARED: Optional[LLMCache] = None


def shared_cache() -> LLMCache:
"""Return the process-wide LLM cache, creating it on first use."""
global _SHARED
if _SHARED is None:
_SHARED = LLMCache()
return _SHARED


def reset_shared_cache(cache: Optional[LLMCache] = None) -> None:
"""Replace the process-wide cache (used by tests and by ``--no-cache``)."""
global _SHARED
_SHARED = cache
Loading
Loading