From 5bead0bf5b4304629471fa91ad084de1110edea4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 07:49:00 +0000 Subject: [PATCH 1/6] fix: correctness, injection, packaging, and test-isolation defects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes found while reviewing the repository. Each fix has a regression test that fails against the previous code. Correctness: - cli: `opendocs inspect ` crashed with UnboundLocalError because `name` was only assigned on the Markdown branch. - cli: `mindmap` was present in FORMAT_MAP and in the `--format all` set but missing from the click Choice, so `-f mindmap` was rejected. The choice list and the all-formats list are now derived from FORMAT_MAP, which is what let them drift apart across three commands. - cli/pipeline: `--no-outputs` was parsed but never forwarded, so notebook cell outputs were always included. - latex: `_escape` never escaped the backslash, so any README containing one produced a .tex that would not compile (`50% \& x` became `50\% \\& x`, where `\\` is a line break). - watcher: `_create_docs_pr` stashed the user's uncommitted work but returned without popping it on three failure paths, including the common failed-push case. It also popped unconditionally, discarding a stash the user had made themselves when there was nothing of ours to stash. - watcher: the PR base branch was hardcoded to `main`, breaking any repo that uses another default. It is now resolved from the remote. - knowledge_graph: label propagation shuffled with the global unseeded RNG, so community IDs in graph.json and the wiki changed between runs. The shuffle is now seeded (overridable via `seed=`). Injection: - interactive_graph: entity names reach the page unsanitised. They are untrusted — mermaid node labels are lifted verbatim from README content, and in LLM mode names come from model output. `json.dumps` does not escape `/`, so a name containing `` closed the script element early, and four panels then wrote names straight to `innerHTML`. Serialisation now escapes ` Claude-Session: https://claude.ai/code/session_01U2mrMSRs3cHBdPgHbRzVDX --- .github/workflows/generate-docs.yml | 10 +- .github/workflows/test.yml | 5 +- README.md | 8 +- pyproject.toml | 6 + src/opendocs/cli.py | 97 ++--------- src/opendocs/core/knowledge_graph.py | 14 +- src/opendocs/core/watcher.py | 102 ++++++++--- src/opendocs/generators/interactive_graph.py | 56 ++++-- src/opendocs/generators/latex_generator.py | 8 +- src/opendocs/generators/mermaid_renderer.py | 42 ++++- src/opendocs/pipeline.py | 5 + tests/test_cli.py | 172 +++++++++++++++++++ tests/test_interactive_graph_security.py | 113 ++++++++++++ tests/test_latex_escaping.py | 100 +++++++++++ tests/test_watcher_git.py | 132 ++++++++++++++ 15 files changed, 737 insertions(+), 133 deletions(-) create mode 100644 tests/test_cli.py create mode 100644 tests/test_interactive_graph_security.py create mode 100644 tests/test_latex_escaping.py create mode 100644 tests/test_watcher_git.py diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index fd3d64a..a31eddc 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -34,9 +34,15 @@ jobs: run: pip install opendocs - name: Generate documentation + # Inputs are passed through env rather than interpolated into the + # script body: `${{ }}` substitution happens before the shell runs, so + # a crafted input would otherwise execute as shell code. + env: + INPUT_README: ${{ github.event.inputs.readme_path || './README.md' }} + INPUT_FORMATS: ${{ github.event.inputs.formats || 'word,pdf,pptx' }} run: | - README="${{ github.event.inputs.readme_path || './README.md' }}" - FORMATS="${{ github.event.inputs.formats || 'word,pdf,pptx' }}" + README="$INPUT_README" + FORMATS="$INPUT_FORMATS" # Convert comma-separated to individual --format flags for FMT in $(echo "$FORMATS" | tr ',' '\n'); do diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4784f9f..738ba1d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,5 +42,8 @@ jobs: - name: Run tests run: pytest tests/ -v --tb=short env: - # Prevent any accidental network calls during tests + # Disable diagram rendering so the suite makes no network calls and + # never triggers an `npx @mermaid-js/mermaid-cli` download (which + # pulls Chromium and dominated the run time). + OPENDOCS_MERMAID_BACKEND: none PYTHONDONTWRITEBYTECODE: "1" diff --git a/README.md b/README.md index 6d6b658..1987703 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ OpenDocs (by [ioteverythin](https://www.ioteverythin.com/)) takes a GitHub repos | Knowledge Wiki | Markdown folder | Available | | LLM Summaries | Stakeholder views | Available | -### What's New in v0.5.0 +### What's New in v0.9.0 - **Interactive Knowledge Graph** -- Explorable HTML graph (vis.js) with search, filtering, god-node analysis, community clusters, provenance labels, and surprising connections - **Graph JSON Export** -- Persistent `graph.json` with nodes, edges, communities, provenance, god nodes, surprising connections, and suggested questions. Query weeks later without re-processing @@ -64,7 +64,7 @@ OpenDocs (by [ioteverythin](https://www.ioteverythin.com/)) takes a GitHub repos ### 1. Pipeline (Deterministic + LLM) -The core pipeline parses Markdown/Notebooks and generates all 11 output formats: +The core pipeline parses Markdown/Notebooks and generates all 12 selectable output formats: - **Basic mode** -- Pure Markdown AST parsing, no LLM required. Fast, free, predictable. - **LLM mode** -- Uses any supported LLM provider to extract entities, build knowledge graphs, and generate executive summaries + stakeholder views (CTO, Investor, Developer). @@ -315,8 +315,8 @@ pipeline.run( ## Features -- **15 Output Formats** -- Word, PDF, PPTX, Blog Post, Jira Tickets, Changelog, LaTeX Paper, One-Pager PDF, Social Cards, FAQ, Architecture Diagrams, Interactive Graph, Graph JSON, Knowledge Wiki -- **Interactive Knowledge Graph** -- Self-contained HTML visualization (vis.js) with search, legend, community clusters, god nodes, surprising connections, provenance bar, and suggested questions +- **15 Output Formats** -- Word, PDF, PPTX, Blog Post, Jira Tickets, Changelog, LaTeX Paper, One-Pager PDF, Social Cards, FAQ, Architecture Diagrams, Mindmap, Interactive Graph, Graph JSON, Knowledge Wiki +- **Interactive Knowledge Graph** -- Single-file HTML visualization with search, legend, community clusters, god nodes, surprising connections, provenance bar, and suggested questions (loads vis.js from a CDN, so first render needs network access) - **Graph JSON Export** -- Persistent queryable `graph.json` with nodes, edges, communities, provenance labels, god nodes, surprising connections, and suggested questions - **Knowledge Wiki** -- Wikipedia-style inter-linked Markdown articles (one per community) with navigable index and full entity catalog - **Community Detection** -- Label propagation algorithm groups entities into clusters by edge density (zero external dependencies) diff --git a/pyproject.toml b/pyproject.toml index f10ca7d..a29a885 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,6 +117,12 @@ Issues = "https://github.com/ioteverythin/OpenDocs/issues" where = ["src"] exclude = ["opendocs.agents", "opendocs.agents.*"] +# The docagent templates are loaded at runtime by +# opendocs.docagent.templates.get_template(); without this they are dropped +# from the wheel and every lookup raises FileNotFoundError once installed. +[tool.setuptools.package-data] +"opendocs.docagent.templates" = ["*.md"] + [tool.ruff] line-length = 120 target-version = "py310" diff --git a/src/opendocs/cli.py b/src/opendocs/cli.py index a4e3159..dcd30a0 100644 --- a/src/opendocs/cli.py +++ b/src/opendocs/cli.py @@ -40,6 +40,13 @@ "all": OutputFormat.ALL, } +# Derived from FORMAT_MAP so the CLI choices can never drift out of sync with +# the formats the pipeline actually knows how to build. +FORMAT_CHOICES = list(FORMAT_MAP) + +# Every concrete format produced by `--format all`. +ALL_FORMATS = [fmt for key, fmt in FORMAT_MAP.items() if key != "all"] + @click.group() @click.version_option(version="0.9.0", prog_name="opendocs") @@ -54,23 +61,7 @@ def main(): "-f", "--format", "fmt", - type=click.Choice( - [ - "word", - "pdf", - "pptx", - "blog", - "jira", - "changelog", - "latex", - "onepager", - "social", - "faq", - "architecture", - "all", - ], - case_sensitive=False, - ), + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), default="all", help="Output format (default: all).", ) @@ -277,20 +268,7 @@ def generate( # Resolve formats chosen = FORMAT_MAP[fmt.lower()] if chosen == OutputFormat.ALL: - formats = [ - OutputFormat.WORD, - OutputFormat.PDF, - OutputFormat.PPTX, - OutputFormat.BLOG, - OutputFormat.JIRA, - OutputFormat.CHANGELOG, - OutputFormat.LATEX, - OutputFormat.ONEPAGER, - OutputFormat.SOCIAL, - OutputFormat.FAQ, - OutputFormat.ARCHITECTURE, - OutputFormat.MINDMAP, - ] + formats = list(ALL_FORMATS) else: formats = [chosen] @@ -329,6 +307,7 @@ def generate( sort_tables=sort_tables, provider=llm_provider, template_vars=tvars, + include_outputs=include_outputs, ) # ---- AI Reader files summary ------------------------------------ @@ -450,8 +429,9 @@ def inspect(source: str, local: bool, token: str | None): from .core.parser import ReadmeParser if is_notebook(source): + name = Path(source).stem parser = NotebookParser() - doc = parser.parse(source, repo_name=Path(source).stem) + doc = parser.parse(source, repo_name=name) else: fetcher = ReadmeFetcher(github_token=token) if local: @@ -504,7 +484,7 @@ def inspect(source: str, local: bool, token: str | None): "--branch", "branch_name", default="docs/auto-update", - help="Base branch name for auto-PR (default: docs/auto-update).", + help="Branch-name prefix for auto-PR; a UTC timestamp is appended (default: docs/auto-update).", ) @click.option( "--patterns", @@ -515,23 +495,7 @@ def inspect(source: str, local: bool, token: str | None): "-f", "--format", "fmt", - type=click.Choice( - [ - "word", - "pdf", - "pptx", - "blog", - "jira", - "changelog", - "latex", - "onepager", - "social", - "faq", - "architecture", - "all", - ], - case_sensitive=False, - ), + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), default="all", help="Output format (default: all).", ) @@ -643,23 +607,7 @@ def _add_section_tree(parent, section): "-f", "--format", "fmt", - type=click.Choice( - [ - "word", - "pdf", - "pptx", - "blog", - "jira", - "changelog", - "latex", - "onepager", - "social", - "faq", - "architecture", - "all", - ], - case_sensitive=False, - ), + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), default="all", help="Output format (default: all).", ) @@ -783,20 +731,7 @@ def codebase( # Resolve formats chosen = FORMAT_MAP[fmt.lower()] if chosen == OutputFormat.ALL: - formats = [ - OutputFormat.WORD, - OutputFormat.PDF, - OutputFormat.PPTX, - OutputFormat.BLOG, - OutputFormat.JIRA, - OutputFormat.CHANGELOG, - OutputFormat.LATEX, - OutputFormat.ONEPAGER, - OutputFormat.SOCIAL, - OutputFormat.FAQ, - OutputFormat.ARCHITECTURE, - OutputFormat.MINDMAP, - ] + formats = list(ALL_FORMATS) else: formats = [chosen] diff --git a/src/opendocs/core/knowledge_graph.py b/src/opendocs/core/knowledge_graph.py index f06ab42..a79d0c0 100644 --- a/src/opendocs/core/knowledge_graph.py +++ b/src/opendocs/core/knowledge_graph.py @@ -350,7 +350,7 @@ def surprising_connections(self, top_n: int = 5) -> list[tuple[float, Relation]] # -- Community detection --------------------------------------------- - def detect_communities(self, max_iterations: int = 50) -> dict[str, int]: + def detect_communities(self, max_iterations: int = 50, *, seed: int = 0) -> dict[str, int]: """Detect communities using label propagation. A lightweight, dependency-free community detection algorithm. @@ -358,6 +358,15 @@ def detect_communities(self, max_iterations: int = 50) -> dict[str, int]: the most common community among its neighbours. Converges when no labels change. + Parameters + ---------- + max_iterations + Upper bound on propagation rounds. + seed + Seed for the shuffle that randomises visit order. Fixed by + default so the exported ``graph.json`` and wiki are reproducible + across runs; pass a different value to sample another partition. + Returns ------- dict[str, int] @@ -376,10 +385,11 @@ def detect_communities(self, max_iterations: int = 50) -> dict[str, int]: # Initialise: each node in its own community labels: dict[str, int] = {e.id: i for i, e in enumerate(self.entities)} ids = [e.id for e in self.entities] + rng = random.Random(seed) for _ in range(max_iterations): changed = False - random.shuffle(ids) # randomised order for convergence + rng.shuffle(ids) # randomised order for convergence for nid in ids: nbrs = adj.get(nid, []) if not nbrs: diff --git a/src/opendocs/core/watcher.py b/src/opendocs/core/watcher.py index 663ffab..9ce2fe1 100644 --- a/src/opendocs/core/watcher.py +++ b/src/opendocs/core/watcher.py @@ -159,6 +159,49 @@ def _git_run(repo_dir: Path, *args: str) -> subprocess.CompletedProcess: ) +def _default_branch(repo_dir: Path) -> str | None: + """Resolve the remote's default branch (e.g. ``main`` or ``master``). + + Returns *None* when it cannot be determined, in which case callers should + let the tool they are invoking apply its own default. + """ + result = _git_run(repo_dir, "symbolic-ref", "refs/remotes/origin/HEAD") + if result.returncode == 0: + ref = result.stdout.strip() + if ref: + return ref.rsplit("/", 1)[-1] + + # Fall back to asking the remote directly. + result = _git_run(repo_dir, "remote", "show", "origin") + if result.returncode == 0: + for line in result.stdout.splitlines(): + line = line.strip() + if line.startswith("HEAD branch:"): + branch = line.split(":", 1)[1].strip() + if branch and branch != "(unknown)": + return branch + return None + + +def _restore(repo_dir: Path, stashed: bool, *, checkout_back: bool = True) -> None: + """Return the repo to its pre-run state. + + Always call this before leaving :func:`_create_docs_pr`, on success and on + every failure path alike — otherwise a failed push would silently leave the + user's uncommitted work sitting in the stash. + """ + if checkout_back: + _git_run(repo_dir, "checkout", "-") + if stashed: + result = _git_run(repo_dir, "stash", "pop") + if result.returncode != 0: + log.error( + "Could not restore stashed changes (%s). Your work is safe in " + "`git stash list` — run `git stash pop` to recover it.", + result.stderr.strip(), + ) + + def _git_branch_exists(repo_dir: Path, branch: str) -> bool: """Check if a git branch exists locally.""" result = _git_run(repo_dir, "branch", "--list", branch) @@ -183,14 +226,19 @@ def _create_docs_pr( branch = f"{branch_name}-{timestamp}" message = commit_message or f"docs: auto-update generated documentation ({timestamp})" + stashed = False try: - # Ensure we're on a clean state - _git_run(repo_dir, "stash", "--include-untracked") + # Ensure we're on a clean state. git reports "No local changes to save" + # (exit 0) when there was nothing to stash — only pop later if we + # actually created a stash entry, otherwise we would pop someone else's. + stash_result = _git_run(repo_dir, "stash", "--include-untracked") + stashed = stash_result.returncode == 0 and "No local changes" not in stash_result.stdout # Create and switch to new branch result = _git_run(repo_dir, "checkout", "-b", branch) if result.returncode != 0: log.error("Failed to create branch %s: %s", branch, result.stderr) + _restore(repo_dir, stashed, checkout_back=False) return False # Add output files @@ -201,37 +249,43 @@ def _create_docs_pr( result = _git_run(repo_dir, "commit", "-m", message) if result.returncode != 0: log.warning("Nothing to commit: %s", result.stderr) - _git_run(repo_dir, "checkout", "-") + _restore(repo_dir, stashed) return False # Push result = _git_run(repo_dir, "push", "-u", "origin", branch) if result.returncode != 0: log.error("Failed to push branch: %s", result.stderr) - _git_run(repo_dir, "checkout", "-") + _restore(repo_dir, stashed) return False console.print(f"[green]✓[/] Pushed branch [bold]{branch}[/] to origin") # Try to create a PR using gh CLI try: + pr_cmd = [ + "gh", + "pr", + "create", + "--title", + f"📄 Auto-update documentation ({timestamp})", + "--body", + ( + "This PR was automatically generated by **OpenDocs** watcher.\n\n" + f"**Branch:** `{branch}`\n" + f"**Generated at:** {timestamp}\n\n" + "Review the updated documentation and merge when ready." + ), + ] + # Target the repo's real default branch. Hardcoding "main" breaks + # every repo that still uses "master" (or anything else); when it + # cannot be resolved, let `gh` pick the default itself. + base = _default_branch(repo_dir) + if base: + pr_cmd += ["--base", base] + pr_result = subprocess.run( - [ - "gh", - "pr", - "create", - "--title", - f"📄 Auto-update documentation ({timestamp})", - "--body", - ( - "This PR was automatically generated by **OpenDocs** watcher.\n\n" - f"**Branch:** `{branch}`\n" - f"**Generated at:** {timestamp}\n\n" - "Review the updated documentation and merge when ready." - ), - "--base", - "main", - ], + pr_cmd, cwd=str(repo_dir), capture_output=True, text=True, @@ -250,18 +304,16 @@ def _create_docs_pr( ) # Switch back to previous branch - _git_run(repo_dir, "checkout", "-") - _git_run(repo_dir, "stash", "pop") + _restore(repo_dir, stashed) return True except Exception as exc: log.error("Auto-PR failed: %s", exc) # Try to recover - _git_run(repo_dir, "checkout", "-") try: - _git_run(repo_dir, "stash", "pop") + _restore(repo_dir, stashed) except Exception: - pass + log.exception("Failed to restore working tree; run `git stash pop` manually") return False diff --git a/src/opendocs/generators/interactive_graph.py b/src/opendocs/generators/interactive_graph.py index e5808b2..ff0071c 100644 --- a/src/opendocs/generators/interactive_graph.py +++ b/src/opendocs/generators/interactive_graph.py @@ -1,8 +1,9 @@ """Generate an interactive HTML knowledge-graph visualisation. -Produces a self-contained HTML file powered by vis-network (CDN) that -lets users pan, zoom, click nodes, search, and filter by entity type. -No local JavaScript toolchain is required. +Produces a single HTML file powered by vis-network that lets users pan, +zoom, click nodes, search, and filter by entity type. No local JavaScript +toolchain is required, but vis-network is loaded from a CDN, so viewing the +page needs network access. """ from __future__ import annotations @@ -39,6 +40,23 @@ _DEFAULT_COLOUR = "#94a3b8" +def _script_json(data) -> str: + """Serialise *data* as JSON that is safe to embed inside a ```` would terminate the script element + early and allow arbitrary markup to be injected into the page. + """ + payload = json.dumps(data) + return ( + payload.replace(" str: "group": e.entity_type.value, } ) - return json.dumps(nodes) + return _script_json(nodes) def _build_edges_json(kg: KnowledgeGraph) -> str: @@ -146,7 +164,7 @@ def _build_edges_json(kg: KnowledgeGraph) -> str: "font": {"size": 9, "color": "#94a3b8", "strokeWidth": 0}, } ) - return json.dumps(edges) + return _script_json(edges) def _god_nodes_json(kg: KnowledgeGraph, top_n: int = 5) -> str: @@ -158,7 +176,7 @@ def _god_nodes_json(kg: KnowledgeGraph, top_n: int = 5) -> str: ent = kg.get_entity(eid) if ent: result.append({"name": ent.name, "type": ent.entity_type.value, "degree": deg}) - return json.dumps(result) + return _script_json(result) def _surprising_connections_json(kg: KnowledgeGraph, top_n: int = 5) -> str: @@ -190,7 +208,7 @@ def _surprising_connections_json(kg: KnowledgeGraph, top_n: int = 5) -> str: "score": round(score, 2), } ) - return json.dumps(result) + return _script_json(result) def _legend_items(kg: KnowledgeGraph) -> str: @@ -212,12 +230,12 @@ def _legend_items(kg: KnowledgeGraph) -> str: def _communities_json(kg: KnowledgeGraph) -> str: """Build community summaries as JSON for the sidebar.""" summaries = kg.community_summary() - return json.dumps(summaries) + return _script_json(summaries) def _questions_json(kg: KnowledgeGraph) -> str: """Build suggested questions as JSON.""" - return json.dumps(kg.suggested_questions(top_n=5)) + return _script_json(kg.suggested_questions(top_n=5)) def _provenance_stats(kg: KnowledgeGraph) -> tuple[int, int, int]: @@ -339,12 +357,22 @@ def _build_html(kg: KnowledgeGraph, project_name: str) -> str: layout: {{improvedLayout: true}} }}); +// Entity names originate from README content, so escape before any innerHTML use. +function esc(v) {{ + return String(v) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +}} + // God nodes panel var gp = document.getElementById('god-nodes'); godNodes.forEach(function(g) {{ var d = document.createElement('div'); d.className = 'panel-card'; - d.innerHTML = '

' + g.name + '

' + g.type.replace(/_/g,' ') + ' Degree: ' + g.degree + ''; + d.innerHTML = '

' + esc(g.name) + '

' + esc(g.type.replace(/_/g,' ')) + ' Degree: ' + esc(g.degree) + ''; gp.appendChild(d); }}); @@ -353,7 +381,7 @@ def _build_html(kg: KnowledgeGraph, project_name: str) -> str: surprises.forEach(function(s) {{ var d = document.createElement('div'); d.className = 'panel-card'; - d.innerHTML = '

' + s.source + ' \\u2194 ' + s.target + '

' + s.relation.replace(/_/g,' ') + ' score ' + s.score + '

'; + d.innerHTML = '

' + esc(s.source) + ' \\u2194 ' + esc(s.target) + '

' + esc(s.relation.replace(/_/g,' ')) + ' score ' + esc(s.score) + '

'; sp.appendChild(d); }}); @@ -362,9 +390,9 @@ def _build_html(kg: KnowledgeGraph, project_name: str) -> str: communities.forEach(function(c) {{ var d = document.createElement('div'); d.className = 'panel-card'; - var members = c.members.slice(0,3).join(', '); + var members = c.members.slice(0,3).map(esc).join(', '); if (c.members.length > 3) members += ' (+' + (c.members.length - 3) + ')'; - d.innerHTML = '

Cluster ' + c.id + ' ' + c.size + ' nodes

' + c.dominant_type + ' | ' + c.internal_edges + ' edges | ' + members + '

'; + d.innerHTML = '

Cluster ' + esc(c.id) + ' ' + esc(c.size) + ' nodes

' + esc(c.dominant_type) + ' | ' + esc(c.internal_edges) + ' edges | ' + members + '

'; cp.appendChild(d); }}); @@ -373,7 +401,7 @@ def _build_html(kg: KnowledgeGraph, project_name: str) -> str: questions.forEach(function(q) {{ var d = document.createElement('div'); d.className = 'question-card'; - d.innerHTML = '' + q + ''; + d.innerHTML = '' + esc(q) + ''; qp.appendChild(d); }}); diff --git a/src/opendocs/generators/latex_generator.py b/src/opendocs/generators/latex_generator.py index fd5facf..e0ba887 100644 --- a/src/opendocs/generators/latex_generator.py +++ b/src/opendocs/generators/latex_generator.py @@ -358,7 +358,13 @@ def _build_conclusion(self, doc: DocumentModel, name: str) -> str: @staticmethod def _escape(text: str) -> str: """Escape special LaTeX characters.""" + # Backslash must go first: every replacement below introduces one, so + # escaping it later would corrupt them all. It maps to a placeholder + # rather than \textbackslash{} directly, because that expansion itself + # contains braces which the "{"/"}" rules would then double-escape. + _BACKSLASH = "\x00opendocs-backslash\x00" replacements = { + "\\": _BACKSLASH, "&": r"\&", "%": r"\%", "$": r"\$", @@ -371,7 +377,7 @@ def _escape(text: str) -> str: } for char, replacement in replacements.items(): text = text.replace(char, replacement) - return text + return text.replace(_BACKSLASH, r"\textbackslash{}") @staticmethod def _latex_lang(lang: str) -> str: diff --git a/src/opendocs/generators/mermaid_renderer.py b/src/opendocs/generators/mermaid_renderer.py index 1a574cb..c303e4d 100644 --- a/src/opendocs/generators/mermaid_renderer.py +++ b/src/opendocs/generators/mermaid_renderer.py @@ -19,8 +19,10 @@ from __future__ import annotations import base64 +import functools import hashlib import logging +import os import shutil import subprocess import tempfile @@ -74,8 +76,21 @@ def _diagram_hash(code: str) -> str: return hashlib.sha256(code.encode("utf-8")).hexdigest()[:12] +#: Environment override for the rendering backend. Accepts the same values as +#: the ``backend`` argument plus ``"none"``, which disables diagram rendering +#: entirely. Useful for offline builds and for test suites that must not make +#: network calls or trigger an ``npx`` package download. +_BACKEND_ENV_VAR = "OPENDOCS_MERMAID_BACKEND" + + +@functools.lru_cache(maxsize=1) def _mmdc_available() -> bool: - """Check whether ``npx mmdc`` is available on the system.""" + """Check whether ``npx mmdc`` is available on the system. + + The result is cached for the life of the process: probing spawns ``npx``, + which on a cold machine downloads the mermaid-cli package (and Chromium + with it). Without caching, every ``MermaidRenderer`` would pay that cost. + """ if shutil.which("mmdc"): return True # Try npx @@ -107,6 +122,9 @@ class MermaidRenderer: backend ``"auto"`` tries *mmdc* first, falls back to *mermaid.ink*. ``"mmdc"`` forces the CLI. ``"ink"`` forces the HTTP API. + ``"none"`` disables rendering (no subprocess, no network). + The ``OPENDOCS_MERMAID_BACKEND`` environment variable overrides + this argument, so offline builds and CI can opt out globally. theme Mermaid theme passed to both backends (``default``, ``dark``, ``forest``, ``neutral``). @@ -115,7 +133,7 @@ class MermaidRenderer: def __init__( self, cache_dir: Path | None = None, - backend: Literal["auto", "mmdc", "ink"] = "auto", + backend: Literal["auto", "mmdc", "ink", "none"] = "auto", theme: str = "default", ) -> None: if cache_dir is None: @@ -128,7 +146,17 @@ def __init__( self.theme = theme - # Resolve backend + # Resolve backend. The environment only overrides "auto" — that value + # means "pick for me", so it is the caller delegating the choice. An + # explicit backend= is a deliberate decision and is always honoured. + if backend == "auto": + env_backend = os.environ.get(_BACKEND_ENV_VAR, "").strip().lower() + if env_backend in {"auto", "mmdc", "ink", "none"}: + backend = env_backend # type: ignore[assignment] + elif env_backend: + logger.warning("Ignoring unknown %s=%r", _BACKEND_ENV_VAR, env_backend) + + self._disabled = backend == "none" if backend == "auto": self._use_mmdc = _mmdc_available() elif backend == "mmdc": @@ -147,6 +175,10 @@ def render(self, code: str, *, label: str = "") -> Path | None: Results are cached by content hash so duplicate diagrams are rendered only once. """ + if self._disabled: + logger.debug("Mermaid rendering disabled (%s=none)", _BACKEND_ENV_VAR) + return None + h = _diagram_hash(code) cached = self.cache_dir / f"mermaid_{h}.png" if cached.exists() and cached.stat().st_size > 0: @@ -198,6 +230,10 @@ def download_image(self, url: str) -> Path | None: Returns the local path on success, or *None* on failure. Skips badge/shield images and SVGs that cannot be embedded. """ + if self._disabled: + logger.debug("Image download disabled (%s=none)", _BACKEND_ENV_VAR) + return None + # Skip badge images — they are small decorative shields url_lower = url.lower() if any(pat in url_lower for pat in self._BADGE_PATTERNS): diff --git a/src/opendocs/pipeline.py b/src/opendocs/pipeline.py index 2d57cfc..a719720 100644 --- a/src/opendocs/pipeline.py +++ b/src/opendocs/pipeline.py @@ -88,6 +88,7 @@ def run( provider: str = "openai", template_vars: TemplateVars | None = None, config_path: str | None = None, + include_outputs: bool = True, ) -> PipelineResult: """Run the full pipeline. @@ -123,6 +124,9 @@ def run( Pre-built ``TemplateVars`` instance with report variables. config_path Path to a YAML/JSON config file with template variables. + include_outputs + When *source* is a Jupyter Notebook, include executed cell + outputs (text, images, tables) in the parsed document. """ output_path = Path(output_dir).resolve() output_path.mkdir(parents=True, exist_ok=True) @@ -192,6 +196,7 @@ def run( repo_name=name, repo_url=repo_url, source_path=source, + include_outputs=include_outputs, ) else: console.print("[bold blue]Parsing Markdown...[/]") diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..e5e894d --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,172 @@ +"""Tests for the ``opendocs`` command-line interface. + +``cli.py`` previously had no test coverage, which is how several +user-facing defects (a crash in ``inspect``, an unselectable format, and a +flag that was accepted but never forwarded) went unnoticed. +""" + +from __future__ import annotations + +import json + +import pytest +from click.testing import CliRunner + +from opendocs.cli import ALL_FORMATS, FORMAT_CHOICES, FORMAT_MAP, main +from opendocs.core.models import OutputFormat + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def runner() -> CliRunner: + return CliRunner() + + +@pytest.fixture +def notebook_with_output(tmp_path): + """A minimal notebook whose single code cell has a recognisable output.""" + nb = { + "cells": [ + {"cell_type": "markdown", "source": ["# Sample Notebook\n"]}, + { + "cell_type": "code", + "execution_count": 1, + "source": ["print('hi')"], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": ["UNIQUE_OUTPUT_MARKER\n"], + } + ], + }, + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5, + } + path = tmp_path / "sample.ipynb" + path.write_text(json.dumps(nb), encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- +# Format choices +# --------------------------------------------------------------------------- + + +class TestFormatChoices: + def test_every_mapped_format_is_selectable(self, runner): + """Regression: 'mindmap' was in FORMAT_MAP but missing from --format.""" + result = runner.invoke(main, ["generate", "--help"]) + assert result.exit_code == 0 + for key in FORMAT_MAP: + assert key in result.output, f"--format is missing the '{key}' choice" + + def test_mindmap_is_accepted(self, runner, tmp_path): + """Regression: `-f mindmap` used to be rejected by click.""" + readme = tmp_path / "README.md" + readme.write_text("# Title\n\nSome text.\n", encoding="utf-8") + result = runner.invoke( + main, + ["generate", str(readme), "--local", "-f", "mindmap", "-o", str(tmp_path / "out")], + ) + assert "is not one of" not in result.output + + @pytest.mark.parametrize("command", ["generate", "watch", "codebase"]) + def test_all_commands_share_the_same_choices(self, runner, command): + """The three --format options must not drift apart again.""" + result = runner.invoke(main, [command, "--help"]) + assert result.exit_code == 0 + for key in FORMAT_CHOICES: + assert key in result.output, f"{command} --format is missing '{key}'" + + def test_all_formats_excludes_the_all_sentinel(self): + assert OutputFormat.ALL not in ALL_FORMATS + assert OutputFormat.MINDMAP in ALL_FORMATS + assert len(ALL_FORMATS) == len(FORMAT_MAP) - 1 + + +# --------------------------------------------------------------------------- +# inspect +# --------------------------------------------------------------------------- + + +class TestInspect: + def test_inspect_notebook_does_not_crash(self, runner, notebook_with_output): + """Regression: `name` was unbound on the notebook branch.""" + result = runner.invoke(main, ["inspect", str(notebook_with_output)]) + assert result.exit_code == 0, result.output + assert result.exception is None + + def test_inspect_notebook_reports_the_file_stem(self, runner, notebook_with_output): + result = runner.invoke(main, ["inspect", str(notebook_with_output)]) + assert "sample" in result.output + + def test_inspect_markdown_still_works(self, runner, tmp_path): + readme = tmp_path / "README.md" + readme.write_text("# Hello\n\nBody text.\n\n## Sub\n", encoding="utf-8") + result = runner.invoke(main, ["inspect", str(readme), "--local"]) + assert result.exit_code == 0, result.output + assert "Sections" in result.output + + +# --------------------------------------------------------------------------- +# Notebook output inclusion +# --------------------------------------------------------------------------- + + +class TestNotebookOutputFlag: + def _blog_text(self, tmp_path, name): + return "".join(p.read_text(encoding="utf-8") for p in (tmp_path / name).glob("*.md")) + + def test_outputs_included_by_default(self, runner, notebook_with_output, tmp_path): + result = runner.invoke( + main, + ["generate", str(notebook_with_output), "--local", "-f", "blog", "-o", str(tmp_path / "inc")], + ) + assert result.exit_code == 0, result.output + assert "UNIQUE_OUTPUT_MARKER" in self._blog_text(tmp_path, "inc") + + def test_no_outputs_flag_is_honoured(self, runner, notebook_with_output, tmp_path): + """Regression: --no-outputs was parsed but never reached the parser.""" + result = runner.invoke( + main, + [ + "generate", + str(notebook_with_output), + "--local", + "-f", + "blog", + "-o", + str(tmp_path / "exc"), + "--no-outputs", + ], + ) + assert result.exit_code == 0, result.output + assert "UNIQUE_OUTPUT_MARKER" not in self._blog_text(tmp_path, "exc") + + +# --------------------------------------------------------------------------- +# Smoke tests for the remaining commands +# --------------------------------------------------------------------------- + + +class TestSmoke: + def test_version(self, runner): + result = runner.invoke(main, ["--version"]) + assert result.exit_code == 0 + assert "0.9.0" in result.output + + def test_themes_lists_themes(self, runner): + result = runner.invoke(main, ["themes"]) + assert result.exit_code == 0 + assert "corporate" in result.output + + @pytest.mark.parametrize("command", ["generate", "watch", "codebase", "inspect", "themes"]) + def test_help_is_available(self, runner, command): + result = runner.invoke(main, [command, "--help"]) + assert result.exit_code == 0 diff --git a/tests/test_interactive_graph_security.py b/tests/test_interactive_graph_security.py new file mode 100644 index 0000000..13dd8cc --- /dev/null +++ b/tests/test_interactive_graph_security.py @@ -0,0 +1,113 @@ +"""Injection regression tests for the interactive HTML knowledge graph. + +Entity names are not trusted input: mermaid node labels are lifted verbatim +out of README content, and in LLM mode names come straight from model output. +Pointing OpenDocs at an untrusted repository must not produce an HTML file +that executes attacker-controlled markup when opened. +""" + +from __future__ import annotations + +import pytest + +from opendocs.core.knowledge_graph import ( + Entity, + EntityType, + KnowledgeGraph, + Relation, + RelationType, +) +from opendocs.core.models import DocumentMetadata, DocumentModel +from opendocs.core.parser import ReadmeParser +from opendocs.core.semantic_extractor import SemanticExtractor +from opendocs.generators.interactive_graph import _script_json, generate_interactive_graph + +SCRIPT_BREAKOUT = "" +IMG_PAYLOAD = "" + + +# --------------------------------------------------------------------------- +# The serialisation helper +# --------------------------------------------------------------------------- + + +class TestScriptJson: + def test_closing_tag_is_escaped(self): + assert " KnowledgeGraph: + kg = KnowledgeGraph() + kg.add_entity(Entity(id="a", name=SCRIPT_BREAKOUT, entity_type=EntityType.PROJECT, source_section="S")) + kg.add_entity(Entity(id="b", name=IMG_PAYLOAD, entity_type=EntityType.TECHNOLOGY, source_section="S")) + kg.add_relation(Relation(source_id="a", target_id="b", relation_type=RelationType.USES)) + return kg + + +class TestGeneratedPage: + @pytest.fixture + def page(self, tmp_path): + doc = DocumentModel(metadata=DocumentMetadata(repo_name="poc")) + result = generate_interactive_graph(doc, _hostile_kg(), tmp_path) + assert result.success, result.error + return result.output_path.read_text(encoding="utf-8") + + def test_script_element_is_not_terminated_early(self, page): + assert SCRIPT_BREAKOUT not in page + + def test_script_tag_count_is_balanced(self, page): + """Exactly two script elements: the vis-network CDN tag and the app tag.""" + assert page.count("") == 2 + + def test_payload_survives_only_as_inert_json_data(self, page): + # The name is still present (we escape, not drop) but in escaped form. + assert "<\\/script>" in page + + def test_innerhtml_sinks_are_escaped(self, page): + """Every panel that writes untrusted names must route through esc().""" + for sink in ["esc(g.name)", "esc(s.source)", "esc(s.target)", "esc(q)"]: + assert sink in page, f"unescaped innerHTML sink: {sink}" + assert "c.members.slice(0,3).map(esc)" in page + + +# --------------------------------------------------------------------------- +# End-to-end: the payload reaches the graph through ordinary basic-mode parsing +# --------------------------------------------------------------------------- + + +class TestEndToEnd: + def test_mermaid_labels_cannot_break_out(self, tmp_path): + """Mermaid node labels become entity names verbatim, with no LLM involved.""" + readme = f"# Demo\n\n```mermaid\ngraph LR\n A[{SCRIPT_BREAKOUT}] --> B[{IMG_PAYLOAD}]\n```\n" + doc = ReadmeParser().parse(readme, repo_name="demo") + kg = SemanticExtractor().extract(doc) + + # Guard the premise: the payload really does become an entity name. + assert any(SCRIPT_BREAKOUT in e.name for e in kg.entities) + + result = generate_interactive_graph(doc, kg, tmp_path) + assert result.success, result.error + page = result.output_path.read_text(encoding="utf-8") + + assert SCRIPT_BREAKOUT not in page + assert page.count("") == 2 diff --git a/tests/test_latex_escaping.py b/tests/test_latex_escaping.py new file mode 100644 index 0000000..8d68ec2 --- /dev/null +++ b/tests/test_latex_escaping.py @@ -0,0 +1,100 @@ +"""LaTeX escaping regression tests. + +``_escape`` did not handle the backslash, so any README containing one +produced a ``.tex`` file that would not compile: the replacements below all +introduce backslashes, and escaping ``\\`` afterwards (or not at all) leaves +stray control sequences behind. +""" + +from __future__ import annotations + +import pytest + +from opendocs.generators.latex_generator import LatexGenerator + +escape = LatexGenerator._escape + + +class TestBackslash: + def test_lone_backslash_becomes_textbackslash(self): + assert escape("\\") == r"\textbackslash{}" + + def test_windows_path(self): + assert escape(r"C:\path_to\file") == r"C:\textbackslash{}path\_to\textbackslash{}file" + + def test_latex_command_is_neutralised(self): + """A command in the source must not survive as a live command.""" + result = escape(r"\textbf{hi}") + assert result == r"\textbackslash{}textbf\{hi\}" + + def test_backslash_before_special_char_does_not_double_escape(self): + """Regression: this used to yield '\\\\&', where '\\\\' is a line break.""" + result = escape(r"50% \& rising") + assert r"\\" not in result + assert result == r"50\% \textbackslash{}\& rising" + + +class TestSpecialCharacters: + @pytest.mark.parametrize( + ("raw", "expected"), + [ + ("&", r"\&"), + ("%", r"\%"), + ("$", r"\$"), + ("#", r"\#"), + ("_", r"\_"), + ("{", r"\{"), + ("}", r"\}"), + ("~", r"\textasciitilde{}"), + ("^", r"\textasciicircum{}"), + ], + ) + def test_each_special_char(self, raw, expected): + assert escape(raw) == expected + + def test_plain_text_is_untouched(self): + assert escape("plain text, with punctuation!") == "plain text, with punctuation!" + + def test_combined(self): + assert escape("a_b & c%d") == r"a\_b \& c\%d" + + +class TestNoUnescapedControlSequences: + """Whatever the input, the output must not contain a stray command.""" + + @pytest.mark.parametrize( + "raw", + [ + r"C:\Users\test", + r"\alpha \beta", + r"100% \& more_stuff", + r"\\", + r"~/.config/opendocs", + r"a^b_c{d}e", + "\\textbackslash", + ], + ) + def test_only_known_commands_survive(self, raw): + allowed = { + r"\&", + r"\%", + r"\$", + r"\#", + r"\_", + r"\{", + r"\}", + r"\textasciitilde", + r"\textasciicircum", + r"\textbackslash", + } + result = escape(raw) + # Walk every backslash and confirm it starts one of the allowed escapes. + i = 0 + while i < len(result): + if result[i] == "\\": + assert any(result.startswith(cmd, i) for cmd in allowed), ( + f"unescaped control sequence at {i} in {result!r} (input {raw!r})" + ) + i += max(len(c) for c in allowed if result.startswith(c, i)) + else: + i += 1 diff --git a/tests/test_watcher_git.py b/tests/test_watcher_git.py new file mode 100644 index 0000000..3549cdf --- /dev/null +++ b/tests/test_watcher_git.py @@ -0,0 +1,132 @@ +"""Tests for the watcher's auto-PR git handling. + +``_create_docs_pr`` stashes the user's uncommitted work before switching +branches. Every exit path must put that work back — a failed push is the +common case (no remote, no credentials) and must not silently leave the +user's changes sitting in the stash. +""" + +from __future__ import annotations + +import subprocess + +import pytest + +from opendocs.core import watcher +from opendocs.core.watcher import _create_docs_pr + +pytestmark = pytest.mark.skipif( + subprocess.run(["git", "--version"], capture_output=True).returncode != 0, + reason="git is not available", +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _git(repo, *args): + return subprocess.run(["git", *args], cwd=str(repo), capture_output=True, text=True) + + +@pytest.fixture +def repo(tmp_path): + """A git repo with one commit, on branch 'main', with no remote.""" + r = tmp_path / "repo" + r.mkdir() + _git(r, "init", "-b", "main") + _git(r, "config", "user.email", "test@example.com") + _git(r, "config", "user.name", "Test") + _git(r, "config", "commit.gpgsign", "false") + (r / "README.md").write_text("# Original\n", encoding="utf-8") + _git(r, "add", ".") + _git(r, "commit", "-m", "initial") + return r + + +def _stash_depth(repo) -> int: + out = _git(repo, "stash", "list").stdout.strip() + return len(out.splitlines()) if out else 0 + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestAutoPrRestoresWorkingTree: + def test_uncommitted_work_survives_a_failed_push(self, repo): + """Regression: the push failure path returned without popping the stash.""" + (repo / "README.md").write_text("# Work in progress\n", encoding="utf-8") + output_dir = repo / "output" + output_dir.mkdir() + (output_dir / "doc.md").write_text("generated\n", encoding="utf-8") + + # No remote is configured, so the push fails. + ok = _create_docs_pr(repo, output_dir, branch_name="docs/auto-update") + + assert ok is False + assert _stash_depth(repo) == 0, "user's changes were left in the stash" + assert (repo / "README.md").read_text(encoding="utf-8") == "# Work in progress\n" + + def test_returns_to_the_original_branch(self, repo): + (repo / "README.md").write_text("# Edited\n", encoding="utf-8") + output_dir = repo / "output" + output_dir.mkdir() + (output_dir / "doc.md").write_text("generated\n", encoding="utf-8") + + _create_docs_pr(repo, output_dir, branch_name="docs/auto-update") + + current = _git(repo, "rev-parse", "--abbrev-ref", "HEAD").stdout.strip() + assert current == "main" + + def test_nothing_to_commit_still_restores(self, repo): + """With no generated output there is nothing to commit — still restore.""" + (repo / "README.md").write_text("# Edited\n", encoding="utf-8") + output_dir = repo / "output" + output_dir.mkdir() + + _create_docs_pr(repo, output_dir, branch_name="docs/auto-update") + + assert _stash_depth(repo) == 0 + assert (repo / "README.md").read_text(encoding="utf-8") == "# Edited\n" + + def test_clean_tree_does_not_pop_an_unrelated_stash(self, repo): + """With nothing to stash we must not pop a stash the user created.""" + (repo / "README.md").write_text("# User's own stash\n", encoding="utf-8") + _git(repo, "stash", "--include-untracked") + assert _stash_depth(repo) == 1 + + output_dir = repo / "output" + output_dir.mkdir() + (output_dir / "doc.md").write_text("generated\n", encoding="utf-8") + + _create_docs_pr(repo, output_dir, branch_name="docs/auto-update") + + assert _stash_depth(repo) == 1, "popped a stash we did not create" + + def test_restores_when_an_exception_escapes(self, repo, monkeypatch): + (repo / "README.md").write_text("# Work in progress\n", encoding="utf-8") + output_dir = repo / "output" + output_dir.mkdir() + (output_dir / "doc.md").write_text("generated\n", encoding="utf-8") + + real_git_run = watcher._git_run + calls = {"n": 0} + + def exploding_git_run(repo_dir, *args): + # Let the stash through, then fail on the next git call. + calls["n"] += 1 + if calls["n"] == 2: + raise RuntimeError("boom") + return real_git_run(repo_dir, *args) + + monkeypatch.setattr(watcher, "_git_run", exploding_git_run) + + ok = _create_docs_pr(repo, output_dir, branch_name="docs/auto-update") + + assert ok is False + monkeypatch.undo() + assert _stash_depth(repo) == 0 + assert (repo / "README.md").read_text(encoding="utf-8") == "# Work in progress\n" From 9a9ed738b6e032e4d1d3f80edcde637ed1675cc1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 07:56:28 +0000 Subject: [PATCH 2/6] feat: offline-capable interactive knowledge graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generated graph loaded vis-network from unpkg.com, so the page did not render at all without network access — despite the README calling it "self-contained". Adds `--embed-graph-assets`, which inlines the library into the HTML. The bundle is resolved cheapest-first: an `OPENDOCS_VIS_NETWORK_JS` path (for air-gapped builds that cannot download anything), then a local cache, then a one-time download that is written to the cache. If none succeed it falls back to the CDN tag rather than emitting a page that cannot work. Inlined library source is escaped so a ` ~716 KB. Also documents OPENDOCS_MERMAID_BACKEND, OPENDOCS_VIS_NETWORK_JS and OPENDOCS_CACHE_DIR in a new offline/air-gapped section of the README, and makes generate_interactive_graph create its output directory. Tests: 364 pass (15 new). None of them touch the network. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01U2mrMSRs3cHBdPgHbRzVDX --- README.md | 32 +++- src/opendocs/cli.py | 9 + src/opendocs/generators/graph_assets.py | 124 ++++++++++++ src/opendocs/generators/interactive_graph.py | 40 +++- src/opendocs/pipeline.py | 9 +- tests/test_graph_assets.py | 187 +++++++++++++++++++ 6 files changed, 394 insertions(+), 7 deletions(-) create mode 100644 src/opendocs/generators/graph_assets.py create mode 100644 tests/test_graph_assets.py diff --git a/README.md b/README.md index 1987703..76a838b 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,36 @@ opendocs generate ./README.md --local --mode llm --provider anthropic opendocs themes ``` +### Offline / Air-Gapped Use + +The interactive knowledge graph normally loads vis.js from a CDN. To produce a +page that renders with no network access at all: + +```bash +opendocs generate ./README.md --local --embed-graph-assets +``` + +The library is downloaded once and cached (`~/.cache/opendocs`), so subsequent +runs need no network. On a fully air-gapped machine, supply it yourself: + +```bash +export OPENDOCS_VIS_NETWORK_JS=/path/to/vis-network.min.js +opendocs generate ./README.md --local --embed-graph-assets +``` + +Diagram rendering can be disabled the same way, which also makes builds fully +offline and much faster: + +```bash +export OPENDOCS_MERMAID_BACKEND=none # auto | mmdc | ink | none +``` + +| Variable | Purpose | +|----------|---------| +| `OPENDOCS_MERMAID_BACKEND` | Diagram backend: `auto` (default), `mmdc`, `ink`, or `none` to disable | +| `OPENDOCS_VIS_NETWORK_JS` | Path to a local vis-network bundle for `--embed-graph-assets` | +| `OPENDOCS_CACHE_DIR` | Override the asset cache location (default `~/.cache/opendocs`) | + ### Jupyter Notebook Ingestion Generate polished reports from research notebooks and data-science projects: @@ -316,7 +346,7 @@ pipeline.run( ## Features - **15 Output Formats** -- Word, PDF, PPTX, Blog Post, Jira Tickets, Changelog, LaTeX Paper, One-Pager PDF, Social Cards, FAQ, Architecture Diagrams, Mindmap, Interactive Graph, Graph JSON, Knowledge Wiki -- **Interactive Knowledge Graph** -- Single-file HTML visualization with search, legend, community clusters, god nodes, surprising connections, provenance bar, and suggested questions (loads vis.js from a CDN, so first render needs network access) +- **Interactive Knowledge Graph** -- Single-file HTML visualization with search, legend, community clusters, god nodes, surprising connections, provenance bar, and suggested questions. Loads vis.js from a CDN by default; pass `--embed-graph-assets` to inline it and get a page that renders fully offline - **Graph JSON Export** -- Persistent queryable `graph.json` with nodes, edges, communities, provenance labels, god nodes, surprising connections, and suggested questions - **Knowledge Wiki** -- Wikipedia-style inter-linked Markdown articles (one per community) with navigable index and full entity catalog - **Community Detection** -- Label propagation algorithm groups entities into clusters by edge density (zero external dependencies) diff --git a/src/opendocs/cli.py b/src/opendocs/cli.py index dcd30a0..eb065bf 100644 --- a/src/opendocs/cli.py +++ b/src/opendocs/cli.py @@ -150,6 +150,12 @@ def main(): default=True, help="Include cell outputs when parsing Jupyter Notebooks (default: yes).", ) +@click.option( + "--embed-graph-assets", + is_flag=True, + default=False, + help="Inline vis-network into the interactive graph so it renders offline (~700 KB larger).", +) @click.option( "--folder-recursive/--no-folder-recursive", "folder_recursive", @@ -227,6 +233,7 @@ def generate( department: str | None, confidentiality: str | None, include_outputs: bool, + embed_graph_assets: bool, folder_recursive: bool, folder_title: str | None, notion_page_id: str | None, @@ -292,6 +299,7 @@ def generate( sort_tables=sort_tables, provider=llm_provider, template_vars=tvars, + embed_graph_assets=embed_graph_assets, ) else: result = pipeline.run( @@ -308,6 +316,7 @@ def generate( provider=llm_provider, template_vars=tvars, include_outputs=include_outputs, + embed_graph_assets=embed_graph_assets, ) # ---- AI Reader files summary ------------------------------------ diff --git a/src/opendocs/generators/graph_assets.py b/src/opendocs/generators/graph_assets.py new file mode 100644 index 0000000..a027084 --- /dev/null +++ b/src/opendocs/generators/graph_assets.py @@ -0,0 +1,124 @@ +"""Resolution of the vis-network bundle used by the interactive graph. + +By default the generated page loads vis-network from a CDN, which keeps the +HTML small but means it will not render on a machine without network access. +``resolve_vis_network`` instead returns the library source so it can be +inlined, producing a page that is genuinely self-contained. + +The source is looked up in three places, cheapest first: + +1. ``OPENDOCS_VIS_NETWORK_JS`` — a path to a local copy. Lets air-gapped + builds work with no download at all. +2. The on-disk cache (``~/.cache/opendocs`` by default). +3. The CDN, which is then written to the cache for subsequent runs. + +Only step 3 touches the network, and only once per machine. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path + +import httpx + +logger = logging.getLogger(__name__) + +#: Pinned so a cached copy stays valid and the output is reproducible. +VIS_NETWORK_VERSION = "9.1.9" +VIS_NETWORK_URL = f"https://unpkg.com/vis-network@{VIS_NETWORK_VERSION}/standalone/umd/vis-network.min.js" + +_ENV_LOCAL_PATH = "OPENDOCS_VIS_NETWORK_JS" +_ENV_CACHE_DIR = "OPENDOCS_CACHE_DIR" + +_DOWNLOAD_TIMEOUT = 30.0 +#: The real bundle is ~700 KB; anything far below that is a truncated download +#: or an error page rather than the library. +_MIN_PLAUSIBLE_SIZE = 100_000 + + +def cache_dir() -> Path: + """Return the directory used to cache downloaded front-end assets.""" + override = os.environ.get(_ENV_CACHE_DIR) + if override: + return Path(override).expanduser() + xdg = os.environ.get("XDG_CACHE_HOME") + base = Path(xdg).expanduser() if xdg else Path.home() / ".cache" + return base / "opendocs" + + +def _cached_bundle() -> Path: + return cache_dir() / f"vis-network-{VIS_NETWORK_VERSION}.min.js" + + +def escape_for_inline_script(source: str) -> str: + """Make JavaScript safe to place inside an inline ``' + + source = resolve_vis_network() + if source is None: + logger.warning("Could not embed vis-network; falling back to the CDN. The page will need network access.") + return f'' + + return f"" + + +def _build_html(kg: KnowledgeGraph, project_name: str, *, embed_assets: bool = False) -> str: + """Assemble the full HTML page.""" nodes_json = _build_nodes_json(kg) edges_json = _build_edges_json(kg) god_json = _god_nodes_json(kg) @@ -258,6 +287,7 @@ def _build_html(kg: KnowledgeGraph, project_name: str) -> str: prov_ext, prov_inf, prov_amb = _provenance_stats(kg) num_communities = max(kg.communities.values(), default=-1) + 1 if kg.communities else 0 title = html.escape(project_name) + vis_network_tag = _vis_network_tag(embed_assets) return f""" @@ -265,7 +295,7 @@ def _build_html(kg: KnowledgeGraph, project_name: str) -> str: {title} Knowledge Graph - +{vis_network_tag}