diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index fd3d64a..a31eddc 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -34,9 +34,15 @@ jobs: run: pip install opendocs - name: Generate documentation + # Inputs are passed through env rather than interpolated into the + # script body: `${{ }}` substitution happens before the shell runs, so + # a crafted input would otherwise execute as shell code. + env: + INPUT_README: ${{ github.event.inputs.readme_path || './README.md' }} + INPUT_FORMATS: ${{ github.event.inputs.formats || 'word,pdf,pptx' }} run: | - README="${{ github.event.inputs.readme_path || './README.md' }}" - FORMATS="${{ github.event.inputs.formats || 'word,pdf,pptx' }}" + README="$INPUT_README" + FORMATS="$INPUT_FORMATS" # Convert comma-separated to individual --format flags for FMT in $(echo "$FORMATS" | tr ',' '\n'); do diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index b691ba3..c6b9352 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -7,12 +7,16 @@ on: - "src/**/*.py" - "tests/**/*.py" - "pyproject.toml" + - "README.md" + - ".github/workflows/quality.yml" pull_request: branches: [main] paths: - "src/**/*.py" - "tests/**/*.py" - "pyproject.toml" + - "README.md" + - ".github/workflows/quality.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -41,3 +45,25 @@ jobs: - name: Format check (ruff format) run: ruff format --check src/ tests/ + + docs: + name: Documentation lint + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install package + run: pip install -e . + + # Dogfooding: OpenDocs gates its own README with the linter it ships. + # Link checking is left off so a third-party outage cannot fail the build. + - name: Lint README + run: opendocs lint ./README.md --local diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4784f9f..738ba1d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,5 +42,8 @@ jobs: - name: Run tests run: pytest tests/ -v --tb=short env: - # Prevent any accidental network calls during tests + # Disable diagram rendering so the suite makes no network calls and + # never triggers an `npx @mermaid-js/mermaid-cli` download (which + # pulls Chromium and dominated the run time). + OPENDOCS_MERMAID_BACKEND: none PYTHONDONTWRITEBYTECODE: "1" diff --git a/README.md b/README.md index 6d6b658..0a44e00 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ OpenDocs (by [ioteverythin](https://www.ioteverythin.com/)) takes a GitHub repos | Knowledge Wiki | Markdown folder | Available | | LLM Summaries | Stakeholder views | Available | -### What's New in v0.5.0 +### What's New in v0.9.0 - **Interactive Knowledge Graph** -- Explorable HTML graph (vis.js) with search, filtering, god-node analysis, community clusters, provenance labels, and surprising connections - **Graph JSON Export** -- Persistent `graph.json` with nodes, edges, communities, provenance, god nodes, surprising connections, and suggested questions. Query weeks later without re-processing @@ -64,7 +64,7 @@ OpenDocs (by [ioteverythin](https://www.ioteverythin.com/)) takes a GitHub repos ### 1. Pipeline (Deterministic + LLM) -The core pipeline parses Markdown/Notebooks and generates all 11 output formats: +The core pipeline parses Markdown/Notebooks and generates all 12 selectable output formats: - **Basic mode** -- Pure Markdown AST parsing, no LLM required. Fast, free, predictable. - **LLM mode** -- Uses any supported LLM provider to extract entities, build knowledge graphs, and generate executive summaries + stakeholder views (CTO, Investor, Developer). @@ -129,6 +129,166 @@ opendocs generate ./README.md --local --mode llm --provider anthropic opendocs themes ``` +### Incremental Builds + +Generated artifacts are cached by content, so re-running the pipeline only +rebuilds what actually changed. This matters most for `opendocs watch`, which +otherwise regenerates every format on every save. + +```bash +opendocs generate ./README.md --local # cached automatically +opendocs generate ./README.md --local --no-cache # force a full rebuild +opendocs generate ./README.md --local --cache-dir /tmp/od-cache + +opendocs cache # show location, entry count, and size +opendocs cache --clear # empty it +``` + +The cache key covers the document content, the knowledge graph, the theme, the +template variables, the output format, and the installed opendocs version, so +changing any of them rebuilds rather than reusing. A cache hit reproduces the +stored artifact byte for byte; the one thing it does not refresh is the +generation timestamp, which is what makes identical inputs produce identical +output. + +### What Changed Since the Last Release + +`opendocs diff` compares two versions of your documentation and reports what +moved — new and removed sections, concepts, and relationships — then renders +it as release notes. Deterministic and offline. + +```bash +# Two files +opendocs diff old/README.md new/README.md + +# Two git revisions of the same file +opendocs diff v0.8.0 HEAD --git . --path README.md + +# Two previously exported graphs +opendocs diff v1_graph.json v2_graph.json + +# Draft release notes +opendocs diff v0.8.0 HEAD --git . --format markdown -o RELEASE_NOTES.md + +# Machine-readable, including which formats are worth regenerating +opendocs diff old.md new.md --format json + +# CI gate: fail if documentation drifted +opendocs diff committed.md generated.md --fail-on-change +``` + +The summary also reports which output formats are worth regenerating, so a +docs pipeline can rebuild only what the change actually affects. + +### Linting Documentation in CI + +`opendocs lint` checks documentation quality and exits non-zero when it +regresses, so a pull request can fail on a broken README the same way it fails +on a broken test: + +```bash +opendocs lint ./README.md --local # errors fail the build +opendocs lint ./README.md --local --fail-on warning # be stricter +opendocs lint ./README.md --local --fail-on never # report only +opendocs lint ./README.md --local --json # machine-readable +opendocs lint https://github.com/owner/repo --check-links +``` + +All rules run offline; `--check-links` is the only one that uses the network. + +| Rule | Severity | Catches | +|------|----------|---------| +| `no-title` | error | No level-1 heading to use as a title | +| `no-description` | error | No prose at all, only headings and code | +| `placeholder` | error | Unreplaced template text (`your-project-name`, `CHANGEME`) | +| `dead-link` | error | 4xx links (`--check-links`) | +| `missing-installation` / `missing-usage` / `missing-license` | warning | Conventional section absent | +| `thin-content` | warning | Fewer than 50 words of prose | +| `todo-marker` | warning | `TODO` / `FIXME` / `TBD` left in published docs | +| `ragged-table` | warning | Table rows that do not match the header, so it silently renders as plain text | +| `image-no-alt` | warning | Images without alt text | +| `heading-jump` | info | Heading levels skipping (H2 to H4) | +| `duplicate-heading` | info | The same heading repeated at one level | +| `unlabelled-code` | info | Fenced code with no language annotation | + +Use it in a workflow: + +```yaml +- name: Lint documentation + run: | + pip install opendocs + opendocs lint ./README.md --local +``` + +Exit codes: `0` clean, `1` findings at or above `--fail-on`, `2` the source +could not be read. + +### Querying a Graph Later + +`opendocs generate` writes a `graph.json` alongside the documents. `opendocs +query` answers questions about it without re-processing the source — no LLM, +no API key, no network: + +```bash +# Overview: top entities and suggested questions +opendocs query output/myproject_graph.json + +# Structural queries +opendocs query graph.json --stats +opendocs query graph.json --list-types +opendocs query graph.json --search redis +opendocs query graph.json --entity "PostgreSQL" +opendocs query graph.json --dependents "PostgreSQL" # what would be affected +opendocs query graph.json --dependencies "API Gateway" # what it relies on +opendocs query graph.json --neighbors "Redis" +opendocs query graph.json --path "API Gateway" "S3" # how two things connect +opendocs query graph.json --type database +opendocs query graph.json --community 2 +opendocs query graph.json --provenance AMBIGUOUS # review low-confidence finds +opendocs query graph.json --god-nodes + +# Plain-English questions, answered from graph structure +opendocs query graph.json "what depends on Redis?" +opendocs query graph.json "how are the API and S3 connected?" +opendocs query graph.json "which databases are used?" +``` + +Add `--json` to any query for machine-readable output, and `--limit N` to +control how many rows come back. + +Exit codes make it scriptable: `0` success, `1` the query matched nothing, +`2` the graph file could not be read. + +### Offline / Air-Gapped Use + +The interactive knowledge graph normally loads vis.js from a CDN. To produce a +page that renders with no network access at all: + +```bash +opendocs generate ./README.md --local --embed-graph-assets +``` + +The library is downloaded once and cached (`~/.cache/opendocs`), so subsequent +runs need no network. On a fully air-gapped machine, supply it yourself: + +```bash +export OPENDOCS_VIS_NETWORK_JS=/path/to/vis-network.min.js +opendocs generate ./README.md --local --embed-graph-assets +``` + +Diagram rendering can be disabled the same way, which also makes builds fully +offline and much faster: + +```bash +export OPENDOCS_MERMAID_BACKEND=none # auto | mmdc | ink | none +``` + +| Variable | Purpose | +|----------|---------| +| `OPENDOCS_MERMAID_BACKEND` | Diagram backend: `auto` (default), `mmdc`, `ink`, or `none` to disable | +| `OPENDOCS_VIS_NETWORK_JS` | Path to a local vis-network bundle for `--embed-graph-assets` | +| `OPENDOCS_CACHE_DIR` | Override the asset cache location (default `~/.cache/opendocs`) | + ### Jupyter Notebook Ingestion Generate polished reports from research notebooks and data-science projects: @@ -315,8 +475,8 @@ pipeline.run( ## Features -- **15 Output Formats** -- Word, PDF, PPTX, Blog Post, Jira Tickets, Changelog, LaTeX Paper, One-Pager PDF, Social Cards, FAQ, Architecture Diagrams, Interactive Graph, Graph JSON, Knowledge Wiki -- **Interactive Knowledge Graph** -- Self-contained HTML visualization (vis.js) with search, legend, community clusters, god nodes, surprising connections, provenance bar, and suggested questions +- **15 Output Formats** -- Word, PDF, PPTX, Blog Post, Jira Tickets, Changelog, LaTeX Paper, One-Pager PDF, Social Cards, FAQ, Architecture Diagrams, Mindmap, Interactive Graph, Graph JSON, Knowledge Wiki +- **Interactive Knowledge Graph** -- Single-file HTML visualization with search, legend, community clusters, god nodes, surprising connections, provenance bar, and suggested questions. Loads vis.js from a CDN by default; pass `--embed-graph-assets` to inline it and get a page that renders fully offline - **Graph JSON Export** -- Persistent queryable `graph.json` with nodes, edges, communities, provenance labels, god nodes, surprising connections, and suggested questions - **Knowledge Wiki** -- Wikipedia-style inter-linked Markdown articles (one per community) with navigable index and full entity catalog - **Community Detection** -- Label propagation algorithm groups entities into clusters by edge density (zero external dependencies) diff --git a/pyproject.toml b/pyproject.toml index f10ca7d..a29a885 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,6 +117,12 @@ Issues = "https://github.com/ioteverythin/OpenDocs/issues" where = ["src"] exclude = ["opendocs.agents", "opendocs.agents.*"] +# The docagent templates are loaded at runtime by +# opendocs.docagent.templates.get_template(); without this they are dropped +# from the wheel and every lookup raises FileNotFoundError once installed. +[tool.setuptools.package-data] +"opendocs.docagent.templates" = ["*.md"] + [tool.ruff] line-length = 120 target-version = "py310" diff --git a/src/opendocs/agents/__init__.py b/src/opendocs/agents/__init__.py index 66caced..2f1c9cc 100644 --- a/src/opendocs/agents/__init__.py +++ b/src/opendocs/agents/__init__.py @@ -1,6 +1,27 @@ """Agentic documentation layer for opendocs. -This package implements the Planner → Executor → Critic architecture +.. warning:: + + **Status: architectural scaffolding, not a working feature.** This package + is excluded from the built wheel (see ``pyproject.toml``) and is not + reachable from the CLI. Several key methods are stubs that return empty or + pass-through results: + + - ``diff.DiffAgent._compute_diff`` returns an empty ``DiffSummary`` + regardless of the refs it is given, so the whole diff chain reports no + changes even when a repository has them. + - ``diff.RegenerationAgent._regenerate`` returns the requested format list + without regenerating anything. + - ``diff.ImpactAgent._compute_impact`` is implemented, but indexes entities + by a ``source_file`` / ``file_path`` / ``path`` property that nothing in + the extractor ever sets, so its file-to-entity map is always empty. + + Do not expose these through user-facing commands until those are + implemented — they would silently succeed while doing nothing. For working + change detection, see :mod:`opendocs.core.doc_diff`, which powers + ``opendocs diff``. + +This package sketches the Planner → Executor → Critic architecture that differentiates opendocs from static doc generators like Mintlify, GitBook AI, DocuWriter, and repo-diagram tools. diff --git a/src/opendocs/cli.py b/src/opendocs/cli.py index a4e3159..eb17e2f 100644 --- a/src/opendocs/cli.py +++ b/src/opendocs/cli.py @@ -40,6 +40,13 @@ "all": OutputFormat.ALL, } +# Derived from FORMAT_MAP so the CLI choices can never drift out of sync with +# the formats the pipeline actually knows how to build. +FORMAT_CHOICES = list(FORMAT_MAP) + +# Every concrete format produced by `--format all`. +ALL_FORMATS = [fmt for key, fmt in FORMAT_MAP.items() if key != "all"] + @click.group() @click.version_option(version="0.9.0", prog_name="opendocs") @@ -54,23 +61,7 @@ def main(): "-f", "--format", "fmt", - type=click.Choice( - [ - "word", - "pdf", - "pptx", - "blog", - "jira", - "changelog", - "latex", - "onepager", - "social", - "faq", - "architecture", - "all", - ], - case_sensitive=False, - ), + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), default="all", help="Output format (default: all).", ) @@ -159,6 +150,26 @@ def main(): default=True, help="Include cell outputs when parsing Jupyter Notebooks (default: yes).", ) +@click.option( + "--embed-graph-assets", + is_flag=True, + default=False, + help="Inline vis-network into the interactive graph so it renders offline (~700 KB larger).", +) +@click.option( + "--no-cache", + "no_cache", + is_flag=True, + default=False, + help="Ignore the build cache and regenerate every format from scratch.", +) +@click.option( + "--cache-dir", + "cache_dir", + type=click.Path(), + default=None, + help="Where to keep cached build artifacts (default: ~/.cache/opendocs/build).", +) @click.option( "--folder-recursive/--no-folder-recursive", "folder_recursive", @@ -236,6 +247,9 @@ def generate( department: str | None, confidentiality: str | None, include_outputs: bool, + embed_graph_assets: bool, + no_cache: bool, + cache_dir: str | None, folder_recursive: bool, folder_title: str | None, notion_page_id: str | None, @@ -277,24 +291,15 @@ def generate( # Resolve formats chosen = FORMAT_MAP[fmt.lower()] if chosen == OutputFormat.ALL: - formats = [ - OutputFormat.WORD, - OutputFormat.PDF, - OutputFormat.PPTX, - OutputFormat.BLOG, - OutputFormat.JIRA, - OutputFormat.CHANGELOG, - OutputFormat.LATEX, - OutputFormat.ONEPAGER, - OutputFormat.SOCIAL, - OutputFormat.FAQ, - OutputFormat.ARCHITECTURE, - OutputFormat.MINDMAP, - ] + formats = list(ALL_FORMATS) else: formats = [chosen] # Run pipeline — folder path or single file/URL + from .core.build_cache import BuildCache + + build_cache = BuildCache(cache_dir, enabled=not no_cache) + pipeline = Pipeline(github_token=token) source_path = Path(source) @@ -314,6 +319,8 @@ def generate( sort_tables=sort_tables, provider=llm_provider, template_vars=tvars, + embed_graph_assets=embed_graph_assets, + cache=build_cache, ) else: result = pipeline.run( @@ -329,6 +336,9 @@ def generate( sort_tables=sort_tables, provider=llm_provider, template_vars=tvars, + include_outputs=include_outputs, + embed_graph_assets=embed_graph_assets, + cache=build_cache, ) # ---- AI Reader files summary ------------------------------------ @@ -450,8 +460,9 @@ def inspect(source: str, local: bool, token: str | None): from .core.parser import ReadmeParser if is_notebook(source): + name = Path(source).stem parser = NotebookParser() - doc = parser.parse(source, repo_name=Path(source).stem) + doc = parser.parse(source, repo_name=name) else: fetcher = ReadmeFetcher(github_token=token) if local: @@ -504,7 +515,7 @@ def inspect(source: str, local: bool, token: str | None): "--branch", "branch_name", default="docs/auto-update", - help="Base branch name for auto-PR (default: docs/auto-update).", + help="Branch-name prefix for auto-PR; a UTC timestamp is appended (default: docs/auto-update).", ) @click.option( "--patterns", @@ -515,23 +526,7 @@ def inspect(source: str, local: bool, token: str | None): "-f", "--format", "fmt", - type=click.Choice( - [ - "word", - "pdf", - "pptx", - "blog", - "jira", - "changelog", - "latex", - "onepager", - "social", - "faq", - "architecture", - "all", - ], - case_sensitive=False, - ), + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), default="all", help="Output format (default: all).", ) @@ -643,23 +638,7 @@ def _add_section_tree(parent, section): "-f", "--format", "fmt", - type=click.Choice( - [ - "word", - "pdf", - "pptx", - "blog", - "jira", - "changelog", - "latex", - "onepager", - "social", - "faq", - "architecture", - "all", - ], - case_sensitive=False, - ), + type=click.Choice(FORMAT_CHOICES, case_sensitive=False), default="all", help="Output format (default: all).", ) @@ -783,20 +762,7 @@ def codebase( # Resolve formats chosen = FORMAT_MAP[fmt.lower()] if chosen == OutputFormat.ALL: - formats = [ - OutputFormat.WORD, - OutputFormat.PDF, - OutputFormat.PPTX, - OutputFormat.BLOG, - OutputFormat.JIRA, - OutputFormat.CHANGELOG, - OutputFormat.LATEX, - OutputFormat.ONEPAGER, - OutputFormat.SOCIAL, - OutputFormat.FAQ, - OutputFormat.ARCHITECTURE, - OutputFormat.MINDMAP, - ] + formats = list(ALL_FORMATS) else: formats = [chosen] @@ -989,5 +955,558 @@ def finetune( raise SystemExit(1) +@main.command() +@click.option("--clear", "do_clear", is_flag=True, default=False, help="Delete every cached artifact.") +@click.option("--cache-dir", "cache_dir", type=click.Path(), default=None, help="Cache location to inspect.") +def cache(do_clear: bool, cache_dir: str | None): + """Inspect or clear the incremental build cache. + + \b + Examples: + opendocs cache # show location and size + opendocs cache --clear # empty it + """ + from .core.build_cache import BuildCache + + store = BuildCache(cache_dir) + + if do_clear: + removed = store.clear() + console.print(f"[green][OK][/] Cleared {removed} cached artifact(s) from {store.dir}") + return + + size = store.size_bytes() + entries = len(list(store.dir.glob("*/*/manifest.json"))) if store.dir.exists() else 0 + console.print(f"\n[bold]Build cache[/] [dim]{store.dir}[/]") + console.print(f" entries: {entries}") + console.print(f" size: {size / 1024 / 1024:.1f} MB" if size else " size: empty") + console.print("\n[dim]Clear it with `opendocs cache --clear`.[/]\n") + + +@main.command() +@click.argument("old", metavar="OLD") +@click.argument("new", metavar="NEW") +@click.option( + "--git", + "git_repo", + type=click.Path(exists=True, file_okay=False), + default=None, + help="Treat OLD and NEW as git revisions of --path inside this repository.", +) +@click.option( + "--path", + "git_file", + default="README.md", + show_default=True, + help="File to compare when --git is used.", +) +@click.option( + "--format", + "out_format", + type=click.Choice(["markdown", "json", "summary"], case_sensitive=False), + default="summary", + show_default=True, + help="Output style: a short summary, full release notes, or JSON.", +) +@click.option("-o", "--output", "output_file", type=click.Path(), default=None, help="Write the result to a file.") +@click.option("--title", default="Documentation Changes", help="Heading for generated release notes.") +@click.option( + "--fail-on-change", + is_flag=True, + default=False, + help="Exit non-zero when anything changed (for CI drift detection).", +) +def diff( + old: str, + new: str, + git_repo: str | None, + git_file: str, + out_format: str, + output_file: str | None, + title: str, + fail_on_change: bool, +): + """Report what changed between two versions of documentation. + + OLD and NEW are Markdown files, notebooks, or exported graph.json files. + With --git they are treated as git revisions of --path instead. + + \b + Examples: + opendocs diff old/README.md new/README.md + opendocs diff v1_graph.json v2_graph.json + opendocs diff v0.8.0 HEAD --git . --path README.md + opendocs diff v0.8.0 HEAD --git . --format markdown -o RELEASE_NOTES.md + opendocs diff a.md b.md --fail-on-change # CI drift gate + """ + import json as _json + + from .core.doc_diff import ( + diff_snapshots, + impacted_formats, + render_release_notes, + snapshot_from_git, + snapshot_from_path, + ) + + try: + if git_repo: + old_snap = snapshot_from_git(git_repo, old, git_file) + new_snap = snapshot_from_git(git_repo, new, git_file) + else: + old_snap = snapshot_from_path(old) + new_snap = snapshot_from_path(new) + except Exception as exc: + console.print(f"[bold red]Could not load sources:[/] {exc}") + raise SystemExit(2) from exc + + delta = diff_snapshots(old_snap, new_snap) + + if out_format.lower() == "json": + payload = {**delta.to_dict(), "impacted_formats": impacted_formats(delta)} + rendered = _json.dumps(payload, indent=2) + if output_file: + Path(output_file).write_text(rendered, encoding="utf-8") + console.print(f"[green][OK][/] Wrote {output_file}") + else: + console.print_json(rendered) + elif out_format.lower() == "markdown": + rendered = render_release_notes(delta, title=title) + if output_file: + Path(output_file).write_text(rendered, encoding="utf-8") + console.print(f"[green][OK][/] Wrote {output_file}") + else: + console.print(rendered) + else: + _print_diff_summary(delta, impacted_formats(delta)) + if output_file: + Path(output_file).write_text(render_release_notes(delta, title=title), encoding="utf-8") + console.print(f"[green][OK][/] Wrote {output_file}") + + if fail_on_change and not delta.is_empty: + raise SystemExit(1) + + +def _print_diff_summary(delta, formats: list[str]) -> None: + """Render a delta as a compact terminal summary.""" + console.print(f"\n[bold]{delta.old_label}[/] [dim]->[/] [bold]{delta.new_label}[/]") + + if delta.is_empty: + console.print("[green]No documentation changes detected.[/]\n") + return + + counts = delta.counts() + rows = [ + ("Sections added", counts["sections_added"], "green"), + ("Sections removed", counts["sections_removed"], "red"), + ("Concepts added", counts["entities_added"], "green"), + ("Concepts removed", counts["entities_removed"], "red"), + ("Concepts reclassified", counts["entities_retyped"], "yellow"), + ("Relations added", counts["relations_added"], "green"), + ("Relations removed", counts["relations_removed"], "red"), + ] + console.print() + for label, count, colour in rows: + if count: + console.print(f" [{colour}]{count:>4}[/] {label}") + + for section in [s for s in delta.sections if s.change == "added"][:10]: + console.print(f" [green]+[/] section: {section.title}") + for section in [s for s in delta.sections if s.change == "removed"][:10]: + console.print(f" [red]-[/] section: {section.title}") + + if formats: + console.print(f"\n[dim]Worth regenerating:[/] {', '.join(formats)}") + console.print() + + +@main.command() +@click.argument("source", metavar="SOURCE") +@click.option("--local", is_flag=True, default=False, help="Treat SOURCE as a local file path.") +@click.option("--token", envvar="GITHUB_TOKEN", default=None, help="GitHub token for remote sources.") +@click.option( + "--fail-on", + type=click.Choice(["error", "warning", "info", "never"], case_sensitive=False), + default="error", + show_default=True, + help="Lowest severity that should make the command exit non-zero.", +) +@click.option("--check-links", is_flag=True, default=False, help="Also request every external link (uses network).") +@click.option("--include-badges", is_flag=True, default=False, help="Include badge/shield URLs in link checking.") +@click.option("--link-timeout", type=float, default=10.0, show_default=True, help="Per-link timeout in seconds.") +@click.option("--json", "as_json", is_flag=True, default=False, help="Emit findings as JSON.") +def lint( + source: str, + local: bool, + token: str | None, + fail_on: str, + check_links: bool, + include_badges: bool, + link_timeout: float, + as_json: bool, +): + """Check documentation quality and exit non-zero when it regresses. + + SOURCE is a GitHub URL, a local Markdown file, or a Jupyter Notebook. + All rules run offline unless --check-links is passed. + + \b + Examples: + opendocs lint ./README.md --local + opendocs lint ./README.md --local --fail-on warning + opendocs lint https://github.com/owner/repo --check-links + opendocs lint ./README.md --local --json + """ + import json as _json + + # -- Load and parse the source ------------------------------------- + from .core.fetcher import ReadmeFetcher, is_github_url + from .core.linter import Severity, lint_document + from .core.notebook_parser import NotebookParser, is_notebook + from .core.parser import ReadmeParser + + try: + if is_notebook(source): + doc = NotebookParser().parse(source, repo_name=Path(source).stem) + else: + fetcher = ReadmeFetcher(github_token=token) + if local or not is_github_url(source): + content, name = fetcher._fetch_local(source) + else: + content, name = fetcher.fetch(source) + doc = ReadmeParser().parse(content, repo_name=name) + except Exception as exc: + console.print(f"[bold red]Could not read {source}:[/] {exc}") + raise SystemExit(2) from exc + + report = lint_document( + doc, + check_links_too=check_links, + link_timeout=link_timeout, + include_badges=include_badges, + ) + + if as_json: + console.print_json(_json.dumps(report.to_dict())) + else: + _print_lint_report(report, source) + + if fail_on.lower() == "never": + return + raise SystemExit(report.exit_code(fail_on=Severity(fail_on.lower()))) + + +def _print_lint_report(report, source: str) -> None: + """Render a lint report as grouped, colour-coded output.""" + styles = {"error": "bold red", "warning": "yellow", "info": "dim cyan"} + labels = {"error": "ERROR", "warning": "WARN ", "info": "INFO "} + + console.print(f"\n[bold]Linting[/] {source}") + + if not report.findings: + console.print(f"[green]No issues found[/] [dim]({report.checked} checks)[/]\n") + return + + for finding in report.findings: + sev = finding.severity.value + console.print(f" [{styles[sev]}]{labels[sev]}[/] [bold]{finding.rule}[/] {finding.message}") + if finding.context: + console.print(f" [dim]{finding.context}[/]") + + counts = report.counts() + summary = ", ".join(f"{n} {name}{'s' if n != 1 else ''}" for name, n in counts.items() if n) + console.print(f"\n[bold]{summary or 'no issues'}[/] [dim]({report.checked} checks)[/]\n") + + +@main.command() +@click.argument("graph_path", type=click.Path(exists=True), metavar="GRAPH_JSON") +@click.argument("question", required=False, default=None) +@click.option("--search", "search_term", default=None, help="Substring search over entity names.") +@click.option("--entity", default=None, help="Show one entity with its incoming and outgoing relations.") +@click.option("--neighbors", "neighbors_of", default=None, help="Entities directly connected to this one.") +@click.option("--dependents", default=None, help="Entities that point at this one (what would be affected).") +@click.option("--dependencies", default=None, help="Entities this one points at (what it relies on).") +@click.option("--path", "path_ends", nargs=2, default=None, help="Shortest path between two entities.") +@click.option("--type", "entity_type", default=None, help="List all entities of a type (e.g. database).") +@click.option("--community", "community_id", type=int, default=None, help="List members of a community.") +@click.option("--provenance", default=None, help="Filter by EXTRACTED, INFERRED, or AMBIGUOUS.") +@click.option("--god-nodes", "show_god_nodes", is_flag=True, help="Highest-degree hub entities.") +@click.option("--list-types", "list_types", is_flag=True, help="Show entity and relation type counts.") +@click.option("--questions", "show_questions", is_flag=True, help="Show the stored suggested questions.") +@click.option("--stats", "show_stats", is_flag=True, help="Show graph statistics.") +@click.option("--json", "as_json", is_flag=True, help="Emit machine-readable JSON instead of tables.") +@click.option("--limit", type=int, default=25, show_default=True, help="Maximum rows to display.") +def query( + graph_path: str, + question: str | None, + search_term: str | None, + entity: str | None, + neighbors_of: str | None, + dependents: str | None, + dependencies: str | None, + path_ends: tuple[str, str] | None, + entity_type: str | None, + community_id: int | None, + provenance: str | None, + show_god_nodes: bool, + list_types: bool, + show_questions: bool, + show_stats: bool, + as_json: bool, + limit: int, +): + """Query an exported graph.json without re-processing the source. + + GRAPH_JSON is a graph file produced by a previous `opendocs generate` run. + Everything here works offline — no LLM, no API key, no network. + + \b + Examples: + opendocs query graph.json --stats + opendocs query graph.json --search redis + opendocs query graph.json --entity "PostgreSQL" + opendocs query graph.json --dependents "PostgreSQL" + opendocs query graph.json --path "API" "S3" + opendocs query graph.json --type database + opendocs query graph.json "what depends on Redis?" + """ + from .core.graph_query import GraphQuery, GraphQueryError, nodes_to_dicts + + try: + graph = GraphQuery.load(graph_path) + except GraphQueryError as exc: + console.print(f"[bold red]Error:[/] {exc}") + raise SystemExit(2) from exc + + try: + _run_query( + graph, + question=question, + search_term=search_term, + entity=entity, + neighbors_of=neighbors_of, + dependents=dependents, + dependencies=dependencies, + path_ends=path_ends, + entity_type=entity_type, + community_id=community_id, + provenance=provenance, + show_god_nodes=show_god_nodes, + list_types=list_types, + show_questions=show_questions, + show_stats=show_stats, + as_json=as_json, + limit=limit, + nodes_to_dicts=nodes_to_dicts, + ) + except GraphQueryError as exc: + console.print(f"[bold red]Error:[/] {exc}") + raise SystemExit(1) from exc + + +def _node_table(title: str, nodes, limit: int): + """Render entities as a Rich table.""" + from rich.table import Table as RichTable + + table = RichTable(title=title, show_lines=False, title_justify="left") + table.add_column("Name", style="bold cyan") + table.add_column("Type", style="magenta") + table.add_column("Degree", justify="right") + table.add_column("Community", justify="right") + table.add_column("Provenance", style="dim") + for n in nodes[:limit]: + table.add_row(n.name, n.type.replace("_", " "), str(n.degree), str(n.community), n.provenance) + return table + + +def _run_query(graph, **opts): + """Dispatch a single query mode and print the result.""" + import json as _json + + as_json = opts["as_json"] + limit = opts["limit"] + nodes_to_dicts = opts["nodes_to_dicts"] + + def emit_nodes(title, nodes): + if as_json: + console.print_json(_json.dumps({"query": title, "results": nodes_to_dicts(nodes[:limit])})) + else: + if not nodes: + console.print(f"[yellow]No results for:[/] {title}") + return + console.print(_node_table(title, nodes, limit)) + if len(nodes) > limit: + console.print(f"[dim]... {len(nodes) - limit} more (raise --limit to see them)[/dim]") + + # -- Metadata modes ------------------------------------------------- + if opts["show_stats"]: + stats = {"project": graph.project_name, "generated_at": graph.generated_at, **graph.stats} + if as_json: + console.print_json(_json.dumps(stats)) + else: + from rich.table import Table as RichTable + + table = RichTable(title=f"{graph.project_name} — graph statistics", title_justify="left") + table.add_column("Metric", style="bold") + table.add_column("Value", justify="right") + for key, value in stats.items(): + table.add_row(str(key).replace("_", " "), str(value)) + console.print(table) + return + + if opts["list_types"]: + payload = {"entity_types": graph.entity_types(), "relation_types": graph.relation_types()} + if as_json: + console.print_json(_json.dumps(payload)) + else: + from rich.table import Table as RichTable + + table = RichTable(title="Entity types", title_justify="left") + table.add_column("Type", style="bold cyan") + table.add_column("Count", justify="right") + for key, count in payload["entity_types"].items(): + table.add_row(key.replace("_", " "), str(count)) + console.print(table) + + table2 = RichTable(title="Relation types", title_justify="left") + table2.add_column("Relation", style="bold magenta") + table2.add_column("Count", justify="right") + for key, count in payload["relation_types"].items(): + table2.add_row(key.replace("_", " "), str(count)) + console.print(table2) + return + + if opts["show_questions"]: + questions = graph.suggested_questions() + if as_json: + console.print_json(_json.dumps({"suggested_questions": questions})) + else: + console.print("[bold]Questions this graph can answer:[/]") + for i, q in enumerate(questions, 1): + console.print(f" [cyan]{i}.[/] {q}") + return + + # -- Entity modes --------------------------------------------------- + if opts["show_god_nodes"]: + emit_nodes("God nodes (highest degree)", graph.god_nodes(top_n=limit)) + return + + if opts["search_term"]: + emit_nodes(f"Search: {opts['search_term']!r}", graph.search(opts["search_term"], limit=limit)) + return + + if opts["entity_type"]: + emit_nodes(f"Type: {opts['entity_type']}", graph.of_type(opts["entity_type"])) + return + + if opts["provenance"]: + emit_nodes(f"Provenance: {opts['provenance'].upper()}", graph.by_provenance(opts["provenance"])) + return + + if opts["community_id"] is not None: + emit_nodes(f"Community {opts['community_id']}", graph.community_members(opts["community_id"])) + return + + if opts["neighbors_of"]: + emit_nodes(f"Neighbors of {opts['neighbors_of']!r}", graph.neighbors(opts["neighbors_of"])) + return + + if opts["dependents"]: + emit_nodes(f"Depends on {opts['dependents']!r}", graph.dependents_of(opts["dependents"])) + return + + if opts["dependencies"]: + emit_nodes(f"{opts['dependencies']!r} relies on", graph.dependencies_of(opts["dependencies"])) + return + + if opts["entity"]: + node = graph.resolve(opts["entity"]) + outgoing = graph.outgoing(node.name) + incoming = graph.incoming(node.name) + if as_json: + console.print_json( + _json.dumps( + { + "entity": nodes_to_dicts([node])[0], + "outgoing": [{"relation": e.relation, "target": t.name} for e, t in outgoing], + "incoming": [{"relation": e.relation, "source": s.name} for e, s in incoming], + } + ) + ) + else: + console.print(f"\n[bold cyan]{node.name}[/] [dim]({node.type.replace('_', ' ')})[/]") + console.print( + f"[dim]degree {node.degree} | community {node.community} | " + f"{node.provenance} | confidence {node.confidence:.0%}[/]" + ) + if node.source_section: + console.print(f"[dim]found in section: {node.source_section}[/]") + if outgoing: + console.print("\n[bold]Points at:[/]") + for e, t in outgoing[:limit]: + console.print(f" [green]--{e.relation.replace('_', ' ')}-->[/] {t.name}") + if incoming: + console.print("\n[bold]Pointed at by:[/]") + for e, s in incoming[:limit]: + console.print(f" {s.name} [green]--{e.relation.replace('_', ' ')}-->[/]") + if not outgoing and not incoming: + console.print("\n[yellow]No relations recorded for this entity.[/]") + return + + if opts["path_ends"]: + start, end = opts["path_ends"] + hops = graph.path(start, end) + if as_json: + console.print_json( + _json.dumps( + { + "start": start, + "end": end, + "hops": [{"from": h.frm.name, "relation": h.edge.relation, "to": h.to.name} for h in hops], + } + ) + ) + elif not hops: + console.print(f"[yellow]No path found between {start!r} and {end!r}.[/]") + else: + console.print(f"\n[bold]Path ({len(hops)} hop(s)):[/]") + console.print(f" [cyan]{hops[0].frm.name}[/]") + for h in hops: + arrow = "<--" if h.reversed_ else "-->" + console.print(f" [green]{arrow} {h.edge.relation.replace('_', ' ')} {arrow}[/] [cyan]{h.to.name}[/]") + return + + # -- Natural-language fallback -------------------------------------- + if opts["question"]: + answer = graph.answer(opts["question"]) + if as_json: + console.print_json(_json.dumps(answer.to_dict())) + return + if answer.hops: + console.print(f"\n[dim]{answer.summary}[/]") + console.print(f" [cyan]{answer.hops[0].frm.name}[/]") + for h in answer.hops: + arrow = "<--" if h.reversed_ else "-->" + console.print(f" [green]{arrow} {h.edge.relation.replace('_', ' ')} {arrow}[/] [cyan]{h.to.name}[/]") + elif answer.nodes: + # The summary is the table title; printing it separately duplicates it. + console.print(_node_table(answer.summary, answer.nodes, limit)) + else: + console.print(f"\n[dim]{answer.summary}[/]") + console.print("[yellow]No matching entities. Try --search or --list-types.[/]") + return + + # -- Nothing selected: show an overview ------------------------------ + console.print(f"\n[bold]{graph.project_name}[/] [dim]graph exported {graph.generated_at or 'unknown'}[/]") + console.print(f"[dim]{len(graph.nodes)} entities, {len(graph.edges)} relations[/]\n") + console.print(_node_table("Top entities", graph.god_nodes(top_n=limit), limit)) + questions = graph.suggested_questions() + if questions: + console.print("\n[bold]Try asking:[/]") + for q in questions[:3]: + console.print(f" [dim]-[/] {q}") + console.print("\n[dim]Run `opendocs query --help` for all query modes.[/]") + + if __name__ == "__main__": main() diff --git a/src/opendocs/core/build_cache.py b/src/opendocs/core/build_cache.py new file mode 100644 index 0000000..207bfcc --- /dev/null +++ b/src/opendocs/core/build_cache.py @@ -0,0 +1,280 @@ +"""Content-addressed cache for generated documentation artifacts. + +``opendocs watch`` re-runs the full pipeline whenever a watched file changes, +regenerating every format even when a given output would be byte-for-byte +identical. This cache skips that work by fingerprinting everything an output +depends on and reusing the stored artifact when the fingerprint matches. + +Correctness rests entirely on the fingerprint covering every input. If an +input is missed, a stale artifact is served silently, which is worse than no +cache at all — so :func:`fingerprint` folds in the document content, the +knowledge graph, the theme, the template variables, the output format, and the +running opendocs version, and any unknown extra is included rather than +ignored. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +#: Bumped when the cache layout itself changes, invalidating every entry. +CACHE_FORMAT_VERSION = 1 + +_ENV_CACHE_DIR = "OPENDOCS_CACHE_DIR" + + +def default_cache_dir() -> Path: + """Return the directory used for cached build artifacts.""" + import os + + override = os.environ.get(_ENV_CACHE_DIR) + if override: + return Path(override).expanduser() / "build" + xdg = os.environ.get("XDG_CACHE_HOME") + base = Path(xdg).expanduser() if xdg else Path.home() / ".cache" + return base / "opendocs" / "build" + + +def _opendocs_version() -> str: + """Version of the running package, so a code change invalidates the cache.""" + try: + from .. import __version__ # type: ignore[attr-defined] + + return str(__version__) + except Exception: + pass + try: + from importlib.metadata import version + + return version("opendocs") + except Exception: + return "unknown" + + +def _stable(value: Any) -> Any: + """Convert *value* into something JSON can serialise deterministically.""" + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, dict): + return {str(k): _stable(v) for k, v in sorted(value.items(), key=lambda kv: str(kv[0]))} + if isinstance(value, (list, tuple)): + return [_stable(v) for v in value] + if isinstance(value, (set, frozenset)): + return sorted(_stable(v) for v in value) + if hasattr(value, "model_dump"): # pydantic + try: + return _stable(value.model_dump()) + except Exception: + pass + if hasattr(value, "__dict__"): + return _stable(vars(value)) + return repr(value) + + +#: Metadata that changes on every parse without changing the content. +#: ``generated_at`` is a fresh timestamp each run, so including it would make +#: the fingerprint unique every time — the cache would never hit and would grow +#: a new entry per build. The trade-off is that a cache hit keeps the +#: timestamp from when the artifact was first built, which is the usual and +#: desirable behaviour for a build cache: identical inputs, identical output. +_VOLATILE_METADATA = ("generated_at",) + + +def document_fingerprint(doc: Any) -> str: + """Fingerprint a DocumentModel by its content, ignoring volatile metadata.""" + try: + payload_obj = doc.model_dump(mode="json") + except Exception: + try: + payload_obj = _stable(doc) + except Exception: + return hashlib.sha256(repr(doc).encode("utf-8")).hexdigest() + + metadata = payload_obj.get("metadata") if isinstance(payload_obj, dict) else None + if isinstance(metadata, dict): + for field_name in _VOLATILE_METADATA: + metadata.pop(field_name, None) + + payload = json.dumps(payload_obj, sort_keys=True, default=str) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def graph_fingerprint(kg: Any) -> str: + """Fingerprint a KnowledgeGraph. + + Included separately from the document because LLM mode enriches the graph + with content the document alone does not determine. + """ + if kg is None: + return "no-kg" + try: + payload = kg.model_dump_json() + except Exception: + payload = json.dumps(_stable(kg), sort_keys=True, default=str) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def fingerprint( + *, + doc: Any, + kg: Any, + output_format: str, + theme_name: str, + template_vars: Any = None, + extra: Optional[dict[str, Any]] = None, +) -> str: + """Compute the cache key for one generated artifact.""" + key = { + "cache_format": CACHE_FORMAT_VERSION, + "opendocs": _opendocs_version(), + "format": output_format, + "theme": theme_name, + "doc": document_fingerprint(doc), + "kg": graph_fingerprint(kg), + "template_vars": _stable(template_vars), + "extra": _stable(extra or {}), + } + blob = json.dumps(key, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +@dataclass +class CacheStats: + """Tally of cache activity for one pipeline run.""" + + hits: int = 0 + misses: int = 0 + stores: int = 0 + + @property + def total(self) -> int: + return self.hits + self.misses + + def summary(self) -> str: + if not self.total: + return "cache not used" + return f"{self.hits}/{self.total} cached" + + +class BuildCache: + """Stores and retrieves generated artifacts by content fingerprint. + + Each entry is a directory named after the fingerprint containing the + artifact plus a small manifest. Artifacts may be single files or whole + directories (the architecture generator produces a directory). + """ + + def __init__(self, cache_dir: str | Path | None = None, *, enabled: bool = True) -> None: + self.enabled = enabled + self.dir = Path(cache_dir) if cache_dir else default_cache_dir() + self.stats = CacheStats() + + # -- Internal -------------------------------------------------------- + + def _entry(self, key: str) -> Path: + # Shard by the first two characters to avoid one enormous directory. + return self.dir / key[:2] / key + + def _manifest(self, key: str) -> Path: + return self._entry(key) / "manifest.json" + + # -- Public API ------------------------------------------------------ + + def restore_into(self, key: str, output_dir: Path) -> Optional[Path]: + """Restore a cached artifact into *output_dir*. + + The artifact's filename comes from the manifest, so callers do not need + to know what a generator would have named its output — which is what + makes this usable without touching every generator. + + Returns the restored path on a hit, or *None* on a miss. + """ + if not self.enabled: + return None + + manifest_path = self._manifest(key) + if not manifest_path.exists(): + self.stats.misses += 1 + return None + + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + name = manifest["name"] + stored = self._entry(key) / name + if not stored.exists(): + self.stats.misses += 1 + return None + + destination = Path(output_dir) / name + destination.parent.mkdir(parents=True, exist_ok=True) + if stored.is_dir(): + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(stored, destination) + else: + shutil.copy2(stored, destination) + except Exception as exc: + # A damaged entry must degrade to a miss, never break the build. + logger.debug("Cache read failed for %s: %s", key, exc) + self.stats.misses += 1 + return None + + self.stats.hits += 1 + logger.debug("Cache hit %s -> %s", key[:12], destination) + return destination + + def put(self, key: str, artifact: Path) -> bool: + """Store *artifact* under *key*. Returns True when it was stored.""" + if not self.enabled: + return False + + artifact = Path(artifact) + if not artifact.exists(): + return False + + entry = self._entry(key) + try: + if entry.exists(): + shutil.rmtree(entry) + entry.mkdir(parents=True, exist_ok=True) + + target = entry / artifact.name + if artifact.is_dir(): + shutil.copytree(artifact, target) + else: + shutil.copy2(artifact, target) + + self._manifest(key).write_text( + json.dumps({"name": artifact.name, "is_dir": artifact.is_dir()}), + encoding="utf-8", + ) + except Exception as exc: + logger.debug("Cache write failed for %s: %s", key, exc) + # Leave no half-written entry behind. + shutil.rmtree(entry, ignore_errors=True) + return False + + self.stats.stores += 1 + return True + + def clear(self) -> int: + """Delete every cached entry. Returns how many were removed.""" + if not self.dir.exists(): + return 0 + removed = sum(1 for _ in self.dir.glob("*/*/manifest.json")) + shutil.rmtree(self.dir, ignore_errors=True) + return removed + + def size_bytes(self) -> int: + """Total size of the cache on disk.""" + if not self.dir.exists(): + return 0 + return sum(p.stat().st_size for p in self.dir.rglob("*") if p.is_file()) diff --git a/src/opendocs/core/doc_diff.py b/src/opendocs/core/doc_diff.py new file mode 100644 index 0000000..db25043 --- /dev/null +++ b/src/opendocs/core/doc_diff.py @@ -0,0 +1,401 @@ +"""Compare two versions of a document and report what changed. + +Answers "what changed in the docs since the last release?" by diffing both +the prose structure (sections) and the extracted knowledge graph (entities and +relations) between two versions of a source, then rendering the delta as +release notes. + +Sources can be Markdown/Notebook files, previously exported ``graph.json`` +files, or two git revisions of the same path. Everything is deterministic and +offline. +""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +from .knowledge_graph import KnowledgeGraph +from .models import DocumentModel, HeadingBlock + +# --------------------------------------------------------------------------- +# Delta models +# --------------------------------------------------------------------------- + + +@dataclass +class EntityChange: + """One entity that appeared, disappeared, or was reclassified.""" + + name: str + entity_type: str + change: str # added | removed | retyped + detail: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "type": self.entity_type, + "change": self.change, + "detail": self.detail, + } + + +@dataclass +class RelationChange: + """One relation that appeared or disappeared.""" + + source: str + target: str + relation: str + change: str # added | removed + + def to_dict(self) -> dict[str, Any]: + return { + "source": self.source, + "target": self.target, + "relation": self.relation, + "change": self.change, + } + + +@dataclass +class SectionChange: + """One documentation section that appeared or disappeared.""" + + title: str + level: int + change: str # added | removed + + def to_dict(self) -> dict[str, Any]: + return {"title": self.title, "level": self.level, "change": self.change} + + +@dataclass +class DocDelta: + """Everything that changed between two versions.""" + + old_label: str = "old" + new_label: str = "new" + entities: list[EntityChange] = field(default_factory=list) + relations: list[RelationChange] = field(default_factory=list) + sections: list[SectionChange] = field(default_factory=list) + old_counts: dict[str, int] = field(default_factory=dict) + new_counts: dict[str, int] = field(default_factory=dict) + + @property + def is_empty(self) -> bool: + return not (self.entities or self.relations or self.sections) + + def added_entities(self) -> list[EntityChange]: + return [e for e in self.entities if e.change == "added"] + + def removed_entities(self) -> list[EntityChange]: + return [e for e in self.entities if e.change == "removed"] + + def retyped_entities(self) -> list[EntityChange]: + return [e for e in self.entities if e.change == "retyped"] + + def counts(self) -> dict[str, int]: + return { + "entities_added": len(self.added_entities()), + "entities_removed": len(self.removed_entities()), + "entities_retyped": len(self.retyped_entities()), + "relations_added": sum(1 for r in self.relations if r.change == "added"), + "relations_removed": sum(1 for r in self.relations if r.change == "removed"), + "sections_added": sum(1 for s in self.sections if s.change == "added"), + "sections_removed": sum(1 for s in self.sections if s.change == "removed"), + } + + def to_dict(self) -> dict[str, Any]: + return { + "old": self.old_label, + "new": self.new_label, + "counts": self.counts(), + "old_totals": self.old_counts, + "new_totals": self.new_counts, + "entities": [e.to_dict() for e in self.entities], + "relations": [r.to_dict() for r in self.relations], + "sections": [s.to_dict() for s in self.sections], + } + + +# --------------------------------------------------------------------------- +# Loading a comparable snapshot +# --------------------------------------------------------------------------- + + +@dataclass +class Snapshot: + """One side of a comparison: a parsed document plus its knowledge graph.""" + + label: str + entities: dict[str, str] = field(default_factory=dict) # name -> type + relations: set[tuple[str, str, str]] = field(default_factory=set) + sections: dict[str, int] = field(default_factory=dict) # title -> level + + @property + def totals(self) -> dict[str, int]: + return { + "entities": len(self.entities), + "relations": len(self.relations), + "sections": len(self.sections), + } + + @classmethod + def from_document(cls, doc: DocumentModel, kg: KnowledgeGraph, label: str) -> Snapshot: + entities = {e.name: e.entity_type.value for e in kg.entities} + by_id = {e.id: e.name for e in kg.entities} + relations = { + (by_id[r.source_id], r.relation_type.value, by_id[r.target_id]) + for r in kg.relations + if r.source_id in by_id and r.target_id in by_id + } + sections = {b.text.strip(): b.level for b in doc.all_blocks if isinstance(b, HeadingBlock) and b.text.strip()} + return cls(label=label, entities=entities, relations=relations, sections=sections) + + @classmethod + def from_graph_json(cls, payload: dict[str, Any], label: str) -> Snapshot: + """Build a snapshot from an exported graph.json. + + Section titles are not stored in the export, so a graph-to-graph + comparison reports entity and relation changes only. + """ + nodes = payload.get("nodes", []) or [] + entities = { + str(n.get("name", "")): str(n.get("type", "")) for n in nodes if isinstance(n, dict) and n.get("name") + } + by_id = {str(n.get("id", "")): str(n.get("name", "")) for n in nodes if isinstance(n, dict)} + relations: set[tuple[str, str, str]] = set() + for e in payload.get("edges", []) or []: + if not isinstance(e, dict): + continue + src = by_id.get(str(e.get("source", ""))) + tgt = by_id.get(str(e.get("target", ""))) + if src and tgt: + relations.add((src, str(e.get("relation", "")), tgt)) + return cls(label=label, entities=entities, relations=relations, sections={}) + + +def _parse_source(content: str, *, is_notebook_source: bool, name: str) -> tuple[DocumentModel, KnowledgeGraph]: + from .notebook_parser import NotebookParser + from .parser import ReadmeParser + from .semantic_extractor import SemanticExtractor + + if is_notebook_source: + doc = NotebookParser().parse_content(content, repo_name=name) + else: + doc = ReadmeParser().parse(content, repo_name=name) + return doc, SemanticExtractor().extract(doc) + + +#: Identity given to both sides of a comparison. The extractor turns the +#: repo name into a PROJECT entity that everything else links to, so feeding it +#: a per-version label (a path or a git ref) would make that entity — and every +#: relation touching it — appear added on one side and removed on the other. +#: The display label is kept separate from this parsed identity. +_STABLE_PROJECT_NAME = "project" + + +def snapshot_from_path( + path: str | Path, + *, + label: Optional[str] = None, + repo_name: str = _STABLE_PROJECT_NAME, +) -> Snapshot: + """Build a snapshot from a Markdown file, notebook, or exported graph.json. + + *label* names this side in the report; *repo_name* is the identity handed + to the parser and must match on both sides of a comparison. + """ + p = Path(path) + text = p.read_text(encoding="utf-8") + display = label or p.name + + if p.suffix.lower() == ".json": + payload = json.loads(text) + if not isinstance(payload, dict) or "nodes" not in payload: + raise ValueError(f"{p} is not an opendocs graph.json (no 'nodes' array)") + return Snapshot.from_graph_json(payload, display) + + doc, kg = _parse_source(text, is_notebook_source=p.suffix.lower() == ".ipynb", name=repo_name) + return Snapshot.from_document(doc, kg, display) + + +def snapshot_from_git( + repo_dir: str | Path, + ref: str, + file_path: str, + *, + repo_name: str = _STABLE_PROJECT_NAME, +) -> Snapshot: + """Build a snapshot from a file as it existed at a git revision.""" + result = subprocess.run( + ["git", "show", f"{ref}:{file_path}"], + cwd=str(repo_dir), + capture_output=True, + text=True, + timeout=60, + ) + if result.returncode != 0: + raise ValueError(f"Could not read {file_path} at {ref}: {result.stderr.strip()}") + + doc, kg = _parse_source( + result.stdout, + is_notebook_source=file_path.lower().endswith(".ipynb"), + name=repo_name, + ) + return Snapshot.from_document(doc, kg, f"{ref}:{file_path}") + + +# --------------------------------------------------------------------------- +# Diffing +# --------------------------------------------------------------------------- + + +def diff_snapshots(old: Snapshot, new: Snapshot) -> DocDelta: + """Compare two snapshots and return the delta.""" + delta = DocDelta( + old_label=old.label, + new_label=new.label, + old_counts=old.totals, + new_counts=new.totals, + ) + + old_names = set(old.entities) + new_names = set(new.entities) + + for name in sorted(new_names - old_names): + delta.entities.append(EntityChange(name=name, entity_type=new.entities[name], change="added")) + for name in sorted(old_names - new_names): + delta.entities.append(EntityChange(name=name, entity_type=old.entities[name], change="removed")) + for name in sorted(old_names & new_names): + if old.entities[name] != new.entities[name]: + delta.entities.append( + EntityChange( + name=name, + entity_type=new.entities[name], + change="retyped", + detail=f"{old.entities[name]} -> {new.entities[name]}", + ) + ) + + for src, rel, tgt in sorted(new.relations - old.relations): + delta.relations.append(RelationChange(source=src, target=tgt, relation=rel, change="added")) + for src, rel, tgt in sorted(old.relations - new.relations): + delta.relations.append(RelationChange(source=src, target=tgt, relation=rel, change="removed")) + + old_sections = set(old.sections) + new_sections = set(new.sections) + for title in sorted(new_sections - old_sections): + delta.sections.append(SectionChange(title=title, level=new.sections[title], change="added")) + for title in sorted(old_sections - new_sections): + delta.sections.append(SectionChange(title=title, level=old.sections[title], change="removed")) + + return delta + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + + +def render_release_notes(delta: DocDelta, *, title: str = "Documentation Changes") -> str: + """Render a delta as Markdown release notes.""" + lines: list[str] = [f"# {title}", ""] + lines.append(f"Comparing `{delta.old_label}` -> `{delta.new_label}`.") + lines.append("") + + if delta.is_empty: + lines.append("No documentation changes detected.") + lines.append("") + return "\n".join(lines) + + counts = delta.counts() + summary_bits = [ + f"{counts['sections_added']} section(s) added" if counts["sections_added"] else "", + f"{counts['sections_removed']} section(s) removed" if counts["sections_removed"] else "", + f"{counts['entities_added']} concept(s) added" if counts["entities_added"] else "", + f"{counts['entities_removed']} concept(s) removed" if counts["entities_removed"] else "", + ] + summary = ", ".join(b for b in summary_bits if b) + if summary: + lines.append(f"**Summary:** {summary}.") + lines.append("") + + added_sections = [s for s in delta.sections if s.change == "added"] + if added_sections: + lines.append("## New sections") + lines.append("") + for s in added_sections: + lines.append(f"- {s.title}") + lines.append("") + + removed_sections = [s for s in delta.sections if s.change == "removed"] + if removed_sections: + lines.append("## Removed sections") + lines.append("") + for s in removed_sections: + lines.append(f"- {s.title}") + lines.append("") + + added = delta.added_entities() + if added: + lines.append("## New concepts") + lines.append("") + by_type: dict[str, list[str]] = {} + for e in added: + by_type.setdefault(e.entity_type.replace("_", " ").title(), []).append(e.name) + for type_name in sorted(by_type): + lines.append(f"**{type_name}**") + for name in sorted(by_type[type_name]): + lines.append(f"- {name}") + lines.append("") + + removed = delta.removed_entities() + if removed: + lines.append("## Concepts no longer mentioned") + lines.append("") + for e in removed: + lines.append(f"- {e.name} _({e.entity_type.replace('_', ' ')})_") + lines.append("") + + retyped = delta.retyped_entities() + if retyped: + lines.append("## Reclassified concepts") + lines.append("") + for e in retyped: + lines.append(f"- {e.name}: {e.detail}") + lines.append("") + + added_rel = [r for r in delta.relations if r.change == "added"] + if added_rel: + lines.append("## New relationships") + lines.append("") + for r in added_rel[:40]: + lines.append(f"- {r.source} --{r.relation.replace('_', ' ')}--> {r.target}") + if len(added_rel) > 40: + lines.append(f"- _...and {len(added_rel) - 40} more_") + lines.append("") + + return "\n".join(lines) + + +def impacted_formats(delta: DocDelta) -> list[str]: + """Which output formats are worth regenerating for this delta. + + Deliberately conservative: any structural change invalidates the + document-shaped outputs, while graph-shaped outputs only need rebuilding + when the graph itself moved. + """ + formats: set[str] = set() + if delta.sections or delta.entities: + formats.update({"word", "pdf", "pptx", "blog", "latex", "onepager", "faq"}) + if delta.entities or delta.relations: + formats.update({"architecture", "mindmap", "graph", "wiki"}) + if delta.sections: + formats.add("changelog") + return sorted(formats) diff --git a/src/opendocs/core/graph_query.py b/src/opendocs/core/graph_query.py new file mode 100644 index 0000000..5f47e38 --- /dev/null +++ b/src/opendocs/core/graph_query.py @@ -0,0 +1,517 @@ +"""Query a previously exported ``graph.json`` without re-processing the source. + +The graph export already carries everything needed to answer structural +questions about a project — entities, typed relations, communities, degrees +and provenance. This module loads that file and answers those questions +offline, so a graph produced weeks ago stays useful with no LLM, no API key +and no network access. + + from opendocs.core.graph_query import GraphQuery + + q = GraphQuery.load("output/myproject_graph.json") + q.dependents_of("Redis") # what points at Redis + q.path("API Gateway", "S3") # how two concepts connect +""" + +from __future__ import annotations + +import json +from collections import deque +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Optional + + +class GraphQueryError(RuntimeError): + """Raised when a graph file cannot be loaded or a query cannot be answered.""" + + +@dataclass(frozen=True) +class Node: + """One entity from the exported graph.""" + + id: str + name: str + type: str + confidence: float = 1.0 + provenance: str = "EXTRACTED" + extraction_method: str = "deterministic" + community: int = -1 + degree: int = 0 + source_section: str = "" + properties: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Node: + return cls( + id=str(data.get("id", "")), + name=str(data.get("name", "")), + type=str(data.get("type", "")), + confidence=float(data.get("confidence", 1.0)), + provenance=str(data.get("provenance", "EXTRACTED")), + extraction_method=str(data.get("extraction_method", "deterministic")), + community=int(data.get("community", -1)), + degree=int(data.get("degree", 0)), + source_section=str(data.get("source_section", "")), + properties=data.get("properties") or {}, + ) + + +@dataclass(frozen=True) +class Edge: + """One typed relation from the exported graph.""" + + source: str + target: str + relation: str + confidence: float = 1.0 + provenance: str = "EXTRACTED" + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Edge: + return cls( + source=str(data.get("source", "")), + target=str(data.get("target", "")), + relation=str(data.get("relation", "")), + confidence=float(data.get("confidence", 1.0)), + provenance=str(data.get("provenance", "EXTRACTED")), + ) + + +@dataclass(frozen=True) +class Hop: + """A single step along a path between two entities.""" + + frm: Node + edge: Edge + to: Node + #: True when the edge is stored target -> source and we walked it backwards. + reversed_: bool = False + + +class GraphQuery: + """Read-only queries over an exported ``graph.json``.""" + + def __init__(self, payload: dict[str, Any]) -> None: + if not isinstance(payload, dict): + raise GraphQueryError("Graph file must contain a JSON object") + + raw_nodes = payload.get("nodes") + if not isinstance(raw_nodes, list): + raise GraphQueryError("Graph file has no 'nodes' array — is this an opendocs graph.json?") + + self.payload = payload + self.nodes: list[Node] = [Node.from_dict(n) for n in raw_nodes if isinstance(n, dict)] + self.edges: list[Edge] = [Edge.from_dict(e) for e in payload.get("edges", []) if isinstance(e, dict)] + + self._by_id: dict[str, Node] = {n.id: n for n in self.nodes} + # Names are not guaranteed unique; keep every match. + self._by_name: dict[str, list[Node]] = {} + for n in self.nodes: + self._by_name.setdefault(n.name.lower(), []).append(n) + + self._out: dict[str, list[Edge]] = {} + self._in: dict[str, list[Edge]] = {} + for e in self.edges: + self._out.setdefault(e.source, []).append(e) + self._in.setdefault(e.target, []).append(e) + + # -- Loading --------------------------------------------------------- + + @classmethod + def load(cls, path: str | Path) -> GraphQuery: + """Load a graph from *path*.""" + p = Path(path) + try: + raw = p.read_text(encoding="utf-8") + except OSError as exc: + raise GraphQueryError(f"Could not read {p}: {exc}") from exc + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise GraphQueryError(f"{p} is not valid JSON: {exc}") from exc + return cls(payload) + + # -- Metadata -------------------------------------------------------- + + @property + def project_name(self) -> str: + return str(self.payload.get("project", {}).get("name", "unknown")) + + @property + def generated_at(self) -> str: + return str(self.payload.get("generated_at", "")) + + @property + def stats(self) -> dict[str, Any]: + return dict(self.payload.get("stats", {})) + + def entity_types(self) -> dict[str, int]: + """Return ``{type: count}``, most common first.""" + counts: dict[str, int] = {} + for n in self.nodes: + counts[n.type] = counts.get(n.type, 0) + 1 + return dict(sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))) + + def relation_types(self) -> dict[str, int]: + counts: dict[str, int] = {} + for e in self.edges: + counts[e.relation] = counts.get(e.relation, 0) + 1 + return dict(sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))) + + # -- Lookup ---------------------------------------------------------- + + def get(self, id_or_name: str) -> Optional[Node]: + """Resolve an entity by exact id, then exact name (case-insensitive).""" + if id_or_name in self._by_id: + return self._by_id[id_or_name] + matches = self._by_name.get(id_or_name.lower()) + return matches[0] if matches else None + + def resolve(self, term: str) -> Node: + """Resolve *term* to a node, falling back to search. + + Raises ``GraphQueryError`` with near-miss suggestions when the term + does not identify anything, which is far more useful at a CLI than an + empty result. + """ + node = self.get(term) + if node is not None: + return node + + candidates = self.search(term) + if len(candidates) == 1: + return candidates[0] + if candidates: + names = ", ".join(repr(c.name) for c in candidates[:8]) + raise GraphQueryError(f"{term!r} is ambiguous — did you mean one of: {names}?") + raise GraphQueryError(f"No entity matching {term!r}. Try `--search` or `--list-types`.") + + def search(self, term: str, *, limit: int = 25) -> list[Node]: + """Case-insensitive substring search over entity names. + + Exact matches sort first, then prefix matches, then the rest — each + group by descending degree so hubs surface above leaves. + """ + needle = term.strip().lower() + if not needle: + return [] + + def rank(n: Node) -> tuple[int, int, str]: + lowered = n.name.lower() + if lowered == needle: + tier = 0 + elif lowered.startswith(needle): + tier = 1 + else: + tier = 2 + return (tier, -n.degree, lowered) + + hits = [n for n in self.nodes if needle in n.name.lower()] + return sorted(hits, key=rank)[:limit] + + def of_type(self, entity_type: str) -> list[Node]: + """All entities of a given type, highest degree first.""" + wanted = entity_type.strip().lower().replace(" ", "_") + found = [n for n in self.nodes if n.type.lower() == wanted] + return sorted(found, key=lambda n: (-n.degree, n.name.lower())) + + # -- Relations ------------------------------------------------------- + + def _node_or_none(self, node_id: str) -> Optional[Node]: + return self._by_id.get(node_id) + + def outgoing(self, term: str) -> list[tuple[Edge, Node]]: + """Edges leaving the entity, paired with the entity they point at.""" + node = self.resolve(term) + pairs = [] + for e in self._out.get(node.id, []): + target = self._node_or_none(e.target) + if target is not None: + pairs.append((e, target)) + return sorted(pairs, key=lambda p: (p[0].relation, p[1].name.lower())) + + def incoming(self, term: str) -> list[tuple[Edge, Node]]: + """Edges arriving at the entity, paired with their origin.""" + node = self.resolve(term) + pairs = [] + for e in self._in.get(node.id, []): + source = self._node_or_none(e.source) + if source is not None: + pairs.append((e, source)) + return sorted(pairs, key=lambda p: (p[0].relation, p[1].name.lower())) + + def neighbors(self, term: str) -> list[Node]: + """Every entity directly connected, in either direction.""" + seen: dict[str, Node] = {} + for _edge, other in self.outgoing(term) + self.incoming(term): + seen.setdefault(other.id, other) + return sorted(seen.values(), key=lambda n: (-n.degree, n.name.lower())) + + def dependents_of(self, term: str) -> list[Node]: + """Entities that point at this one — "what would break if it changed".""" + return sorted( + {s.id: s for _e, s in self.incoming(term)}.values(), + key=lambda n: (-n.degree, n.name.lower()), + ) + + def dependencies_of(self, term: str) -> list[Node]: + """Entities this one points at — "what it relies on".""" + return sorted( + {t.id: t for _e, t in self.outgoing(term)}.values(), + key=lambda n: (-n.degree, n.name.lower()), + ) + + # -- Paths ----------------------------------------------------------- + + def path(self, start: str, end: str, *, max_depth: int = 8) -> list[Hop]: + """Shortest undirected path between two entities. + + Relations are treated as undirected here: the question "how are these + two things connected?" rarely cares which way an edge was recorded. + Returns an empty list when no path exists within *max_depth*. + """ + src = self.resolve(start) + dst = self.resolve(end) + if src.id == dst.id: + return [] + + # BFS, recording how each node was reached. + came_from: dict[str, tuple[str, Edge, bool]] = {} + queue: deque[tuple[str, int]] = deque([(src.id, 0)]) + visited = {src.id} + + while queue: + current, depth = queue.popleft() + if depth >= max_depth: + continue + for edge in self._out.get(current, []): + if edge.target not in visited: + visited.add(edge.target) + came_from[edge.target] = (current, edge, False) + queue.append((edge.target, depth + 1)) + for edge in self._in.get(current, []): + if edge.source not in visited: + visited.add(edge.source) + came_from[edge.source] = (current, edge, True) + queue.append((edge.source, depth + 1)) + if dst.id in visited: + break + + if dst.id not in came_from: + return [] + + # Walk back to the start, then reverse. + hops: list[Hop] = [] + cursor = dst.id + while cursor != src.id: + prev_id, edge, was_reversed = came_from[cursor] + frm = self._by_id.get(prev_id) + to = self._by_id.get(cursor) + if frm is None or to is None: # pragma: no cover - defensive + break + hops.append(Hop(frm=frm, edge=edge, to=to, reversed_=was_reversed)) + cursor = prev_id + hops.reverse() + return hops + + # -- Structural views ------------------------------------------------ + + def god_nodes(self, top_n: int = 5) -> list[Node]: + """Highest-degree entities, recomputed from the edges we actually hold.""" + return sorted(self.nodes, key=lambda n: (-n.degree, n.name.lower()))[:top_n] + + def communities(self) -> list[dict[str, Any]]: + raw = self.payload.get("communities", []) + return [c for c in raw if isinstance(c, dict)] + + def community_members(self, community_id: int) -> list[Node]: + found = [n for n in self.nodes if n.community == community_id] + return sorted(found, key=lambda n: (-n.degree, n.name.lower())) + + def surprising_connections(self) -> list[dict[str, Any]]: + raw = self.payload.get("surprising_connections", []) + return [s for s in raw if isinstance(s, dict)] + + def suggested_questions(self) -> list[str]: + raw = self.payload.get("suggested_questions", []) + return [str(q) for q in raw] + + def by_provenance(self, provenance: str) -> list[Node]: + """Entities carrying a given provenance label (EXTRACTED/INFERRED/AMBIGUOUS).""" + wanted = provenance.strip().upper() + found = [n for n in self.nodes if n.provenance.upper() == wanted] + return sorted(found, key=lambda n: (-n.degree, n.name.lower())) + + # -- Natural-language routing --------------------------------------- + + def answer(self, question: str) -> QueryAnswer: + """Answer a natural-language question using structural heuristics. + + This deliberately avoids an LLM: it recognises a handful of question + shapes and maps them onto the structural queries above, so it works + offline and gives reproducible answers. Unrecognised questions fall + back to a name search over the terms in the question. + """ + text = question.strip() + lowered = text.lower() + + def _subject(*markers: str) -> str | None: + for marker in markers: + if marker in lowered: + tail = lowered.split(marker, 1)[1] + return tail.strip(" ?.,'\"") + return None + + # "what depends on X" / "what uses X" -> incoming edges + subject = _subject("depends on", "depend on", "uses ", "use ", "relies on") + if subject: + try: + node = self.resolve(subject) + except GraphQueryError: + pass + else: + return QueryAnswer( + kind="dependents", + question=text, + subject=node, + nodes=self.dependents_of(node.name), + summary=f"Entities pointing at {node.name}", + ) + + # "what does X need / require / connect to" -> outgoing edges + subject = _subject("what does ", "what do ") + if subject: + for verb in (" need", " require", " connect to", " use", " depend on"): + if verb in f" {subject}": + subject = subject.split(verb.strip(), 1)[0].strip() + break + try: + node = self.resolve(subject) + except GraphQueryError: + pass + else: + return QueryAnswer( + kind="dependencies", + question=text, + subject=node, + nodes=self.dependencies_of(node.name), + summary=f"Entities {node.name} points at", + ) + + # "how are X and Y connected" -> path + if " and " in lowered and any(w in lowered for w in ("connect", "related", "relate", "link")): + head = lowered + for prefix in ("how are ", "how is ", "how do ", "are "): + if head.startswith(prefix): + head = head[len(prefix) :] + break + for suffix in ("connected", "related", "relate", "linked", "connect"): + head = head.split(suffix)[0] + parts = [p.strip(" ?.,'\"") for p in head.split(" and ", 1)] + if len(parts) == 2 and all(parts): + try: + hops = self.path(parts[0], parts[1]) + except GraphQueryError: + pass + else: + a = self.resolve(parts[0]) + b = self.resolve(parts[1]) + return QueryAnswer( + kind="path", + question=text, + subject=a, + hops=hops, + summary=( + f"{a.name} -> {b.name} in {len(hops)} hop(s)" + if hops + else f"No path found between {a.name} and {b.name}" + ), + ) + + # "what are the most important / central things" -> god nodes + if any(w in lowered for w in ("most important", "most central", "hub", "god node", "key concept")): + return QueryAnswer( + kind="god_nodes", + question=text, + nodes=self.god_nodes(top_n=10), + summary="Highest-degree entities", + ) + + # Type-scoped: "which databases", "what technologies" + for type_name in self.entity_types(): + label = type_name.replace("_", " ") + if label in lowered or f"{label}s" in lowered: + return QueryAnswer( + kind="by_type", + question=text, + nodes=self.of_type(type_name), + summary=f"Entities of type {label}", + ) + + # Fall back to searching the longest word in the question. + words = sorted( + (w.strip(" ?.,'\"") for w in text.split()), + key=len, + reverse=True, + ) + for word in words: + if len(word) < 3: + continue + hits = self.search(word) + if hits: + return QueryAnswer( + kind="search", + question=text, + nodes=hits, + summary=f"Entities matching {word!r}", + ) + + return QueryAnswer(kind="empty", question=text, summary="No matching entities") + + +@dataclass +class QueryAnswer: + """The result of :meth:`GraphQuery.answer`.""" + + kind: str + question: str + summary: str = "" + subject: Optional[Node] = None + nodes: list[Node] = field(default_factory=list) + hops: list[Hop] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "kind": self.kind, + "question": self.question, + "summary": self.summary, + "subject": _node_dict(self.subject) if self.subject else None, + "nodes": [_node_dict(n) for n in self.nodes], + "hops": [ + { + "from": _node_dict(h.frm), + "relation": h.edge.relation, + "reversed": h.reversed_, + "to": _node_dict(h.to), + } + for h in self.hops + ], + } + + +def _node_dict(node: Node) -> dict[str, Any]: + return { + "id": node.id, + "name": node.name, + "type": node.type, + "degree": node.degree, + "community": node.community, + "provenance": node.provenance, + "confidence": node.confidence, + } + + +def nodes_to_dicts(nodes: Iterable[Node]) -> list[dict[str, Any]]: + """Public helper for serialising query results.""" + return [_node_dict(n) for n in nodes] diff --git a/src/opendocs/core/knowledge_graph.py b/src/opendocs/core/knowledge_graph.py index f06ab42..a79d0c0 100644 --- a/src/opendocs/core/knowledge_graph.py +++ b/src/opendocs/core/knowledge_graph.py @@ -350,7 +350,7 @@ def surprising_connections(self, top_n: int = 5) -> list[tuple[float, Relation]] # -- Community detection --------------------------------------------- - def detect_communities(self, max_iterations: int = 50) -> dict[str, int]: + def detect_communities(self, max_iterations: int = 50, *, seed: int = 0) -> dict[str, int]: """Detect communities using label propagation. A lightweight, dependency-free community detection algorithm. @@ -358,6 +358,15 @@ def detect_communities(self, max_iterations: int = 50) -> dict[str, int]: the most common community among its neighbours. Converges when no labels change. + Parameters + ---------- + max_iterations + Upper bound on propagation rounds. + seed + Seed for the shuffle that randomises visit order. Fixed by + default so the exported ``graph.json`` and wiki are reproducible + across runs; pass a different value to sample another partition. + Returns ------- dict[str, int] @@ -376,10 +385,11 @@ def detect_communities(self, max_iterations: int = 50) -> dict[str, int]: # Initialise: each node in its own community labels: dict[str, int] = {e.id: i for i, e in enumerate(self.entities)} ids = [e.id for e in self.entities] + rng = random.Random(seed) for _ in range(max_iterations): changed = False - random.shuffle(ids) # randomised order for convergence + rng.shuffle(ids) # randomised order for convergence for nid in ids: nbrs = adj.get(nid, []) if not nbrs: diff --git a/src/opendocs/core/linter.py b/src/opendocs/core/linter.py new file mode 100644 index 0000000..8a199d5 --- /dev/null +++ b/src/opendocs/core/linter.py @@ -0,0 +1,497 @@ +"""A documentation quality gate. + +``opendocs lint`` checks a README (or any Markdown/Notebook source) against a +set of rules and reports findings with severities, so CI can fail a build when +documentation regresses. + +Every rule works on the parsed :class:`~opendocs.core.models.DocumentModel`, +which means the linter sees exactly what the generators see. Link checking is +opt-in because it is the only rule that touches the network. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Callable, Iterable, Optional + +from .models import ( + CodeBlock, + DocumentModel, + HeadingBlock, + ImageBlock, + ListBlock, + ParagraphBlock, + TableBlock, +) + + +class Severity(str, Enum): + """How seriously to take a finding.""" + + ERROR = "error" + WARNING = "warning" + INFO = "info" + + @property + def rank(self) -> int: + return {"error": 3, "warning": 2, "info": 1}[self.value] + + +@dataclass(frozen=True) +class Finding: + """A single lint result.""" + + rule: str + severity: Severity + message: str + context: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "rule": self.rule, + "severity": self.severity.value, + "message": self.message, + "context": self.context, + } + + +@dataclass +class LintReport: + """The result of a lint run.""" + + findings: list[Finding] = field(default_factory=list) + checked: int = 0 + + @property + def errors(self) -> list[Finding]: + return [f for f in self.findings if f.severity is Severity.ERROR] + + @property + def warnings(self) -> list[Finding]: + return [f for f in self.findings if f.severity is Severity.WARNING] + + @property + def infos(self) -> list[Finding]: + return [f for f in self.findings if f.severity is Severity.INFO] + + def counts(self) -> dict[str, int]: + return { + "error": len(self.errors), + "warning": len(self.warnings), + "info": len(self.infos), + } + + def exit_code(self, *, fail_on: Severity = Severity.ERROR) -> int: + """Return 0 when nothing at or above *fail_on* was found, else 1.""" + return 1 if any(f.severity.rank >= fail_on.rank for f in self.findings) else 0 + + def to_dict(self) -> dict[str, Any]: + return { + "checks_run": self.checked, + "counts": self.counts(), + "findings": [f.to_dict() for f in self.findings], + } + + +# --------------------------------------------------------------------------- +# Rule helpers +# --------------------------------------------------------------------------- + +#: Sections a README is generally expected to have. Matched loosely, since +#: projects name them in many ways ("Setup" vs "Installation"). +_EXPECTED_SECTIONS: dict[str, tuple[str, ...]] = { + "installation": ("install", "setup", "getting started", "quick start", "quickstart"), + "usage": ("usage", "example", "how to use", "quick start", "quickstart", "api"), + "license": ("license", "licence", "licensing"), +} + +_URL_RE = re.compile(r"https?://[^\s<>\)\]\"']+") +_TODO_RE = re.compile(r"\b(TODO|FIXME|XXX|TBD|WIP|COMING SOON)\b", re.IGNORECASE) +_PLACEHOLDER_RE = re.compile( + r"(lorem ipsum|your-project-name|yourusername| str: + """Concatenate every piece of prose in the document.""" + parts: list[str] = [] + for block in doc.all_blocks: + if isinstance(block, (ParagraphBlock, HeadingBlock)): + parts.append(getattr(block, "text", "")) + elif isinstance(block, ListBlock): + parts.extend(block.items) + elif isinstance(block, TableBlock): + for row in block.rows: + parts.extend(row) + return "\n".join(parts) + + +_FENCE_RE = re.compile(r"^\s*```.*?^\s*```", re.MULTILINE | re.DOTALL) +_INLINE_CODE_RE = re.compile(r"`[^`\n]+`") + + +def _prose_only(doc: DocumentModel) -> str: + """Return document text with all code removed. + + An unfinished marker only means something in running prose. A README that + *documents* markers — "`TODO` and `FIXME` are reported" — writes them as + code, and flagging that is a false positive, so both fenced blocks and + inline spans are stripped. + """ + source = doc.raw_markdown + if not source: + return _all_text(doc) + return _INLINE_CODE_RE.sub(" ", _FENCE_RE.sub(" ", source)) + + +def _prose_and_blocks(doc: DocumentModel) -> str: + """Return document text with only inline code spans removed. + + Placeholders are treated the other way round from markers: one left inside + a fenced block is *worse* than one in prose, because that is the command a + reader copies and runs. Only inline spans are stripped, since those are + how documentation refers to a placeholder rather than shipping one. + """ + source = doc.raw_markdown + if not source: + return _all_text(doc) + return _INLINE_CODE_RE.sub(" ", source) + + +def _headings(doc: DocumentModel) -> list[HeadingBlock]: + return [b for b in doc.all_blocks if isinstance(b, HeadingBlock)] + + +def collect_links(doc: DocumentModel) -> list[str]: + """Return every external URL referenced by the document, deduplicated.""" + seen: dict[str, None] = {} + + def add(url: str) -> None: + url = url.rstrip(".,;:") + if url.startswith("http"): + seen.setdefault(url, None) + + for block in doc.all_blocks: + if isinstance(block, ImageBlock): + add(block.src) + spans = getattr(block, "spans", None) or [] + for span in spans: + if span.url: + add(span.url) + for item_spans in getattr(block, "rich_items", None) or []: + for span in item_spans: + if span.url: + add(span.url) + for row in getattr(block, "rich_rows", None) or []: + for cell in row: + for span in cell: + if span.url: + add(span.url) + + # Bare URLs in prose that the parser did not turn into links. + for match in _URL_RE.finditer(doc.raw_markdown or _all_text(doc)): + add(match.group(0)) + + return list(seen) + + +def is_badge(url: str) -> bool: + return any(host in url.lower() for host in _BADGE_HOSTS) + + +# --------------------------------------------------------------------------- +# Rules +# --------------------------------------------------------------------------- + +Rule = Callable[[DocumentModel], Iterable[Finding]] + + +def rule_has_title(doc: DocumentModel) -> Iterable[Finding]: + """The document should open with a level-1 heading.""" + headings = _headings(doc) + if not headings: + yield Finding("no-title", Severity.ERROR, "Document has no headings at all") + return + if not any(h.level == 1 for h in headings): + yield Finding( + "no-title", + Severity.ERROR, + "Document has no level-1 heading to use as a title", + context=f"first heading is H{headings[0].level}: {headings[0].text!r}", + ) + + +def rule_has_description(doc: DocumentModel) -> Iterable[Finding]: + """There should be prose, not just headings and code.""" + paragraphs = [b for b in doc.all_blocks if isinstance(b, ParagraphBlock) and b.text.strip()] + if not paragraphs: + yield Finding("no-description", Severity.ERROR, "Document contains no prose paragraphs") + elif len(_all_text(doc).split()) < 50: + yield Finding( + "thin-content", + Severity.WARNING, + f"Document is very short ({len(_all_text(doc).split())} words)", + ) + + +def rule_expected_sections(doc: DocumentModel) -> Iterable[Finding]: + """Warn when a conventional README section is missing.""" + titles = " ".join(h.text.lower() for h in _headings(doc)) + for name, aliases in _EXPECTED_SECTIONS.items(): + if not any(alias in titles for alias in aliases): + yield Finding( + f"missing-{name}", + Severity.WARNING, + f"No '{name}' section found", + context=f"looked for any of: {', '.join(aliases)}", + ) + + +def rule_heading_hierarchy(doc: DocumentModel) -> Iterable[Finding]: + """Heading levels should not jump (H2 -> H4).""" + previous = 0 + for heading in _headings(doc): + if previous and heading.level > previous + 1: + yield Finding( + "heading-jump", + Severity.INFO, + f"Heading level jumps from H{previous} to H{heading.level}", + context=heading.text[:80], + ) + previous = heading.level + + +def rule_duplicate_headings(doc: DocumentModel) -> Iterable[Finding]: + """Repeated headings at the same level make navigation ambiguous.""" + seen: dict[tuple[int, str], int] = {} + for heading in _headings(doc): + key = (heading.level, heading.text.strip().lower()) + seen[key] = seen.get(key, 0) + 1 + for (level, text), count in seen.items(): + if count > 1 and text: + yield Finding( + "duplicate-heading", + Severity.INFO, + f"Heading appears {count} times at H{level}", + context=text[:80], + ) + + +def rule_unlabelled_code(doc: DocumentModel) -> Iterable[Finding]: + """Fenced code without a language loses syntax highlighting everywhere.""" + unlabelled = [b for b in doc.all_blocks if isinstance(b, CodeBlock) and not b.language.strip()] + if unlabelled: + yield Finding( + "unlabelled-code", + Severity.INFO, + f"{len(unlabelled)} code block(s) have no language annotation", + context=unlabelled[0].code.strip().splitlines()[0][:80] if unlabelled[0].code.strip() else "", + ) + + +def rule_images_have_alt_text(doc: DocumentModel) -> Iterable[Finding]: + """Images without alt text are inaccessible.""" + for block in doc.all_blocks: + if isinstance(block, ImageBlock) and not block.alt.strip() and not is_badge(block.src): + yield Finding( + "image-no-alt", + Severity.WARNING, + "Image has no alt text", + context=block.src[:100], + ) + + +def rule_todo_markers(doc: DocumentModel) -> Iterable[Finding]: + """Unfinished markers left in published documentation.""" + prose = _prose_only(doc) + for match in _TODO_RE.finditer(prose): + yield Finding( + "todo-marker", + Severity.WARNING, + f"Unfinished marker in documentation: {match.group(0)}", + context=_snippet(prose, match.start()), + ) + + +def rule_placeholders(doc: DocumentModel) -> Iterable[Finding]: + """Template placeholders that were never filled in.""" + text = _prose_and_blocks(doc) + for match in _PLACEHOLDER_RE.finditer(text): + yield Finding( + "placeholder", + Severity.ERROR, + f"Unreplaced template placeholder: {match.group(0)!r}", + context=_snippet(text, match.start()), + ) + + +def rule_table_shape(doc: DocumentModel) -> Iterable[Finding]: + """Find pipe tables whose rows do not match their header width. + + A ragged table is worse than it looks: the Markdown parser rejects the + whole block, so it silently renders as plain text in every output format + rather than as a table. Because the parser drops it, this rule has to + work from the raw source rather than the parsed blocks. + """ + source = doc.raw_markdown + if not source: + return + + def cells(line: str) -> list[str]: + stripped = line.strip() + if stripped.startswith("|"): + stripped = stripped[1:] + if stripped.endswith("|"): + stripped = stripped[:-1] + return [c.strip() for c in stripped.split("|")] + + lines = source.splitlines() + in_fence = False + block: list[tuple[int, str]] = [] + + def flush(rows: list[tuple[int, str]]) -> Iterable[Finding]: + # A table needs a header, a separator, and at least one body row. + if len(rows) < 3: + return + separator = rows[1][1] + if not re.fullmatch(r"\|?[\s:\-|]+\|?", separator.strip()) or "-" not in separator: + return + width = len(cells(rows[0][1])) + for lineno, raw in rows[2:]: + found = len(cells(raw)) + if found != width: + yield Finding( + "ragged-table", + Severity.WARNING, + f"Table row has {found} cells but the header has {width}; the table will not render as a table", + context=f"line {lineno}: {raw.strip()[:70]}", + ) + return # one finding per table is enough + + for i, line in enumerate(lines, 1): + if line.lstrip().startswith("```"): + in_fence = not in_fence + continue + if in_fence: + continue + if "|" in line and line.strip(): + block.append((i, line)) + else: + yield from flush(block) + block = [] + yield from flush(block) + + +def _snippet(text: str, position: int, width: int = 60) -> str: + start = max(0, position - width // 2) + return text[start : start + width].replace("\n", " ").strip() + + +#: Rules that never touch the network. +OFFLINE_RULES: tuple[Rule, ...] = ( + rule_has_title, + rule_has_description, + rule_expected_sections, + rule_heading_hierarchy, + rule_duplicate_headings, + rule_unlabelled_code, + rule_images_have_alt_text, + rule_todo_markers, + rule_placeholders, + rule_table_shape, +) + + +# --------------------------------------------------------------------------- +# Link checking (opt-in, network) +# --------------------------------------------------------------------------- + + +def check_links( + doc: DocumentModel, + *, + timeout: float = 10.0, + include_badges: bool = False, +) -> list[Finding]: + """Request every external link and report the ones that fail. + + This is the only rule that uses the network, so it is never part of + :data:`OFFLINE_RULES` and must be requested explicitly. + """ + import httpx + + findings: list[Finding] = [] + urls = [u for u in collect_links(doc) if include_badges or not is_badge(u)] + if not urls: + return findings + + with httpx.Client(timeout=timeout, follow_redirects=True) as client: + for url in urls: + try: + # HEAD first; some hosts reject it, so fall back to GET. + response = client.head(url) + if response.status_code >= 400: + response = client.get(url) + except Exception as exc: + findings.append( + Finding("dead-link", Severity.ERROR, f"Link could not be reached: {type(exc).__name__}", url) + ) + continue + + if response.status_code >= 500: + findings.append(Finding("dead-link", Severity.WARNING, f"Server error {response.status_code}", url)) + elif response.status_code >= 400: + findings.append(Finding("dead-link", Severity.ERROR, f"HTTP {response.status_code}", url)) + return findings + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def lint_document( + doc: DocumentModel, + *, + rules: Optional[Iterable[Rule]] = None, + check_links_too: bool = False, + link_timeout: float = 10.0, + include_badges: bool = False, +) -> LintReport: + """Run the lint rules over *doc* and return a report.""" + active = tuple(rules) if rules is not None else OFFLINE_RULES + report = LintReport(checked=len(active) + (1 if check_links_too else 0)) + + for rule in active: + try: + report.findings.extend(rule(doc)) + except Exception as exc: # a broken rule must not fail the whole run + report.findings.append( + Finding( + getattr(rule, "__name__", "rule"), + Severity.INFO, + f"Rule raised {type(exc).__name__}: {exc}", + ) + ) + + if check_links_too: + report.findings.extend(check_links(doc, timeout=link_timeout, include_badges=include_badges)) + + # Most severe first, then by rule for stable output. + report.findings.sort(key=lambda f: (-f.severity.rank, f.rule, f.context)) + return report diff --git a/src/opendocs/core/watcher.py b/src/opendocs/core/watcher.py index 663ffab..6852215 100644 --- a/src/opendocs/core/watcher.py +++ b/src/opendocs/core/watcher.py @@ -159,6 +159,49 @@ def _git_run(repo_dir: Path, *args: str) -> subprocess.CompletedProcess: ) +def _default_branch(repo_dir: Path) -> str | None: + """Resolve the remote's default branch (e.g. ``main`` or ``master``). + + Returns *None* when it cannot be determined, in which case callers should + let the tool they are invoking apply its own default. + """ + result = _git_run(repo_dir, "symbolic-ref", "refs/remotes/origin/HEAD") + if result.returncode == 0: + ref = result.stdout.strip() + if ref: + return ref.rsplit("/", 1)[-1] + + # Fall back to asking the remote directly. + result = _git_run(repo_dir, "remote", "show", "origin") + if result.returncode == 0: + for line in result.stdout.splitlines(): + line = line.strip() + if line.startswith("HEAD branch:"): + branch = line.split(":", 1)[1].strip() + if branch and branch != "(unknown)": + return branch + return None + + +def _restore(repo_dir: Path, stashed: bool, *, checkout_back: bool = True) -> None: + """Return the repo to its pre-run state. + + Always call this before leaving :func:`_create_docs_pr`, on success and on + every failure path alike — otherwise a failed push would silently leave the + user's uncommitted work sitting in the stash. + """ + if checkout_back: + _git_run(repo_dir, "checkout", "-") + if stashed: + result = _git_run(repo_dir, "stash", "pop") + if result.returncode != 0: + log.error( + "Could not restore stashed changes (%s). Your work is safe in " + "`git stash list` — run `git stash pop` to recover it.", + result.stderr.strip(), + ) + + def _git_branch_exists(repo_dir: Path, branch: str) -> bool: """Check if a git branch exists locally.""" result = _git_run(repo_dir, "branch", "--list", branch) @@ -183,14 +226,19 @@ def _create_docs_pr( branch = f"{branch_name}-{timestamp}" message = commit_message or f"docs: auto-update generated documentation ({timestamp})" + stashed = False try: - # Ensure we're on a clean state - _git_run(repo_dir, "stash", "--include-untracked") + # Ensure we're on a clean state. git reports "No local changes to save" + # (exit 0) when there was nothing to stash — only pop later if we + # actually created a stash entry, otherwise we would pop someone else's. + stash_result = _git_run(repo_dir, "stash", "--include-untracked") + stashed = stash_result.returncode == 0 and "No local changes" not in stash_result.stdout # Create and switch to new branch result = _git_run(repo_dir, "checkout", "-b", branch) if result.returncode != 0: log.error("Failed to create branch %s: %s", branch, result.stderr) + _restore(repo_dir, stashed, checkout_back=False) return False # Add output files @@ -201,37 +249,43 @@ def _create_docs_pr( result = _git_run(repo_dir, "commit", "-m", message) if result.returncode != 0: log.warning("Nothing to commit: %s", result.stderr) - _git_run(repo_dir, "checkout", "-") + _restore(repo_dir, stashed) return False # Push result = _git_run(repo_dir, "push", "-u", "origin", branch) if result.returncode != 0: log.error("Failed to push branch: %s", result.stderr) - _git_run(repo_dir, "checkout", "-") + _restore(repo_dir, stashed) return False console.print(f"[green]✓[/] Pushed branch [bold]{branch}[/] to origin") # Try to create a PR using gh CLI try: + pr_cmd = [ + "gh", + "pr", + "create", + "--title", + f"📄 Auto-update documentation ({timestamp})", + "--body", + ( + "This PR was automatically generated by **OpenDocs** watcher.\n\n" + f"**Branch:** `{branch}`\n" + f"**Generated at:** {timestamp}\n\n" + "Review the updated documentation and merge when ready." + ), + ] + # Target the repo's real default branch. Hardcoding "main" breaks + # every repo that still uses "master" (or anything else); when it + # cannot be resolved, let `gh` pick the default itself. + base = _default_branch(repo_dir) + if base: + pr_cmd += ["--base", base] + pr_result = subprocess.run( - [ - "gh", - "pr", - "create", - "--title", - f"📄 Auto-update documentation ({timestamp})", - "--body", - ( - "This PR was automatically generated by **OpenDocs** watcher.\n\n" - f"**Branch:** `{branch}`\n" - f"**Generated at:** {timestamp}\n\n" - "Review the updated documentation and merge when ready." - ), - "--base", - "main", - ], + pr_cmd, cwd=str(repo_dir), capture_output=True, text=True, @@ -250,18 +304,16 @@ def _create_docs_pr( ) # Switch back to previous branch - _git_run(repo_dir, "checkout", "-") - _git_run(repo_dir, "stash", "pop") + _restore(repo_dir, stashed) return True except Exception as exc: log.error("Auto-PR failed: %s", exc) # Try to recover - _git_run(repo_dir, "checkout", "-") try: - _git_run(repo_dir, "stash", "pop") + _restore(repo_dir, stashed) except Exception: - pass + log.exception("Failed to restore working tree; run `git stash pop` manually") return False @@ -282,6 +334,7 @@ def _regenerate( model: str = "gpt-4o-mini", provider: str = "openai", config_path: str | None = None, + use_cache: bool = True, ) -> bool: """Regenerate documentation for changed files. @@ -309,6 +362,11 @@ def _regenerate( if formats: out_formats = [format_map[f] for f in formats if f in format_map] + # The watcher is the main reason the build cache exists: it re-runs the + # pipeline on every change, and most formats are usually unaffected. + from ..core.build_cache import BuildCache + + build_cache = BuildCache(enabled=use_cache) pipeline = Pipeline() success = True @@ -327,6 +385,7 @@ def _regenerate( api_key=api_key, model=model, provider=provider, + cache=build_cache, ) if not any(r.success for r in result.results): diff --git a/src/opendocs/generators/graph_assets.py b/src/opendocs/generators/graph_assets.py new file mode 100644 index 0000000..a027084 --- /dev/null +++ b/src/opendocs/generators/graph_assets.py @@ -0,0 +1,124 @@ +"""Resolution of the vis-network bundle used by the interactive graph. + +By default the generated page loads vis-network from a CDN, which keeps the +HTML small but means it will not render on a machine without network access. +``resolve_vis_network`` instead returns the library source so it can be +inlined, producing a page that is genuinely self-contained. + +The source is looked up in three places, cheapest first: + +1. ``OPENDOCS_VIS_NETWORK_JS`` — a path to a local copy. Lets air-gapped + builds work with no download at all. +2. The on-disk cache (``~/.cache/opendocs`` by default). +3. The CDN, which is then written to the cache for subsequent runs. + +Only step 3 touches the network, and only once per machine. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path + +import httpx + +logger = logging.getLogger(__name__) + +#: Pinned so a cached copy stays valid and the output is reproducible. +VIS_NETWORK_VERSION = "9.1.9" +VIS_NETWORK_URL = f"https://unpkg.com/vis-network@{VIS_NETWORK_VERSION}/standalone/umd/vis-network.min.js" + +_ENV_LOCAL_PATH = "OPENDOCS_VIS_NETWORK_JS" +_ENV_CACHE_DIR = "OPENDOCS_CACHE_DIR" + +_DOWNLOAD_TIMEOUT = 30.0 +#: The real bundle is ~700 KB; anything far below that is a truncated download +#: or an error page rather than the library. +_MIN_PLAUSIBLE_SIZE = 100_000 + + +def cache_dir() -> Path: + """Return the directory used to cache downloaded front-end assets.""" + override = os.environ.get(_ENV_CACHE_DIR) + if override: + return Path(override).expanduser() + xdg = os.environ.get("XDG_CACHE_HOME") + base = Path(xdg).expanduser() if xdg else Path.home() / ".cache" + return base / "opendocs" + + +def _cached_bundle() -> Path: + return cache_dir() / f"vis-network-{VIS_NETWORK_VERSION}.min.js" + + +def escape_for_inline_script(source: str) -> str: + """Make JavaScript safe to place inside an inline ```` would terminate the script element + early and allow arbitrary markup to be injected into the page. + """ + payload = json.dumps(data) + return ( + payload.replace(" GenerationResult: """Build an interactive HTML graph and write it to *output_dir*. @@ -57,6 +81,10 @@ def generate_interactive_graph( Populated knowledge graph. output_dir Directory where ``graph.html`` will be written. + embed_assets + Inline vis-network into the page instead of linking the CDN, so the + result renders without network access. Adds roughly 700 KB to the + file; the library is cached after the first fetch. Returns ------- @@ -65,10 +93,11 @@ def generate_interactive_graph( name = doc.metadata.repo_name or "project" safe = "".join(c if c.isalnum() or c in "-_ " else "_" for c in name) safe = safe.strip().replace(" ", "_")[:80] or "project" - output_path = output_dir / f"{safe}_graph.html" + output_path = Path(output_dir) / f"{safe}_graph.html" try: - html_content = _build_html(kg, name) + output_path.parent.mkdir(parents=True, exist_ok=True) + html_content = _build_html(kg, name, embed_assets=embed_assets) output_path.write_text(html_content, encoding="utf-8") return GenerationResult( format=OutputFormat.ARCHITECTURE, @@ -122,7 +151,7 @@ def _build_nodes_json(kg: KnowledgeGraph) -> str: "group": e.entity_type.value, } ) - return json.dumps(nodes) + return _script_json(nodes) def _build_edges_json(kg: KnowledgeGraph) -> str: @@ -146,7 +175,7 @@ def _build_edges_json(kg: KnowledgeGraph) -> str: "font": {"size": 9, "color": "#94a3b8", "strokeWidth": 0}, } ) - return json.dumps(edges) + return _script_json(edges) def _god_nodes_json(kg: KnowledgeGraph, top_n: int = 5) -> str: @@ -158,7 +187,7 @@ def _god_nodes_json(kg: KnowledgeGraph, top_n: int = 5) -> str: ent = kg.get_entity(eid) if ent: result.append({"name": ent.name, "type": ent.entity_type.value, "degree": deg}) - return json.dumps(result) + return _script_json(result) def _surprising_connections_json(kg: KnowledgeGraph, top_n: int = 5) -> str: @@ -190,7 +219,7 @@ def _surprising_connections_json(kg: KnowledgeGraph, top_n: int = 5) -> str: "score": round(score, 2), } ) - return json.dumps(result) + return _script_json(result) def _legend_items(kg: KnowledgeGraph) -> str: @@ -212,12 +241,12 @@ def _legend_items(kg: KnowledgeGraph) -> str: def _communities_json(kg: KnowledgeGraph) -> str: """Build community summaries as JSON for the sidebar.""" summaries = kg.community_summary() - return json.dumps(summaries) + return _script_json(summaries) def _questions_json(kg: KnowledgeGraph) -> str: """Build suggested questions as JSON.""" - return json.dumps(kg.suggested_questions(top_n=5)) + return _script_json(kg.suggested_questions(top_n=5)) def _provenance_stats(kg: KnowledgeGraph) -> tuple[int, int, int]: @@ -228,8 +257,26 @@ def _provenance_stats(kg: KnowledgeGraph) -> tuple[int, int, int]: return ext, inf, amb -def _build_html(kg: KnowledgeGraph, project_name: str) -> str: - """Assemble the full self-contained HTML page.""" +def _vis_network_tag(embed_assets: bool) -> str: + """Return the ``' + + source = resolve_vis_network() + if source is None: + logger.warning("Could not embed vis-network; falling back to the CDN. The page will need network access.") + return f'' + + return f"" + + +def _build_html(kg: KnowledgeGraph, project_name: str, *, embed_assets: bool = False) -> str: + """Assemble the full HTML page.""" nodes_json = _build_nodes_json(kg) edges_json = _build_edges_json(kg) god_json = _god_nodes_json(kg) @@ -240,6 +287,7 @@ def _build_html(kg: KnowledgeGraph, project_name: str) -> str: prov_ext, prov_inf, prov_amb = _provenance_stats(kg) num_communities = max(kg.communities.values(), default=-1) + 1 if kg.communities else 0 title = html.escape(project_name) + vis_network_tag = _vis_network_tag(embed_assets) return f""" @@ -247,7 +295,7 @@ def _build_html(kg: KnowledgeGraph, project_name: str) -> str: {title} Knowledge Graph - +{vis_network_tag}