From a4ec0d8f0c2a7b55a3f6f55871ff36401fc5ee4d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 15:58:59 +0000 Subject: [PATCH] feat: add `opendocs coverage` to report what the docs do not cover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `opendocs lint` judges the quality of what documentation says. This reports what it misses, by comparing the real API surface — which code_analyzer already derives — against what the prose actually mentions. Four dimensions are scored, each picked because it has an objective answer: docstrings on public symbols, environment variables the code reads, CLI flags it defines, and detected technologies. Any judgement of whether prose is *good* is deliberately excluded; that is not objectively measurable, and a score nobody can verify is worse than no score. Two decisions determine whether the number can be trusted: Test files are excluded. Counting them made the metric actively misleading — on this repo, test classes were most of the "undocumented" classes, reporting 61.4% where the shipped code was at 98.9%. CodebaseModel has a test_files field but the analyser never populates it, so classification is by path convention, with --include-tests to opt back in. Private helpers are excluded for the same reason, since they carry no promise to a reader. The overall figure is weighted by item count rather than averaging the per-dimension percentages, so one missed flag out of two cannot outweigh hundreds of documented symbols. Dimensions with nothing to measure are left out entirely rather than banking a vacuous 100%. Dogfooded: run against this repo it found one undocumented function, two undocumented classes, and three undocumented environment variables (AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_API_VERSION, OPENDOCS_MODEL_CACHE), all genuine. Fixing them takes docstrings and env-vars to 100% and the overall figure from 86.3% to 89.2%. The remaining gaps — 30 undocumented CLI flags and 10 undocumented technologies — are real and left for a follow-up. Scriptable: --fail-under for CI, --json, --show-missing, and exit codes 0 pass, 1 below threshold, 2 analysis failed. Tests: 671 pass (57 new), weighted mostly toward what must *not* be counted. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01U2mrMSRs3cHBdPgHbRzVDX --- README.md | 41 ++++ src/opendocs/cli.py | 117 ++++++++++ src/opendocs/core/coverage.py | 344 ++++++++++++++++++++++++++++ src/opendocs/core/linter.py | 1 + src/opendocs/generators/styles.py | 4 + tests/test_coverage.py | 367 ++++++++++++++++++++++++++++++ 6 files changed, 874 insertions(+) create mode 100644 src/opendocs/core/coverage.py create mode 100644 tests/test_coverage.py diff --git a/README.md b/README.md index 758e17e..8aaad38 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,43 @@ opendocs diff committed.md generated.md --fail-on-change The summary also reports which output formats are worth regenerating, so a docs pipeline can rebuild only what the change actually affects. +### What Isn't Documented + +`opendocs lint` checks the quality of what your docs *say*. `opendocs coverage` +reports what they *miss*, by comparing the real API surface against what the +documentation actually mentions: + +```bash +opendocs coverage . # report +opendocs coverage . --show-missing # list every gap +opendocs coverage . --fail-under 80 # CI gate +opendocs coverage . --json # machine-readable +opendocs coverage . --docs README.md --docs docs/guide.md +``` + +``` +Dimension Covered Total Coverage +docstrings:functions 87 88 98.9% +docstrings:classes 185 187 98.9% +env-vars 4 7 57.1% +cli-flags 41 76 53.9% +tech-stack 4 14 28.6% + Overall: 86.3% +``` + +Four dimensions are scored, each chosen because it has an objective answer: +docstrings on public symbols, environment variables the code reads, CLI flags +it defines, and detected technologies. Test files and private helpers are +excluded by default (`--include-tests`, `--include-private` to include them), +and the overall figure is weighted by item count so a single missed flag does +not outweigh hundreds of documented symbols. + +It deliberately does not judge whether prose is *good* — that is not +objectively measurable, and a score nobody can verify is worse than none. + +Everything runs offline with no LLM. Exit codes: `0` pass, `1` below +`--fail-under`, `2` analysis failed. + ### Linting Documentation in CI `opendocs lint` checks documentation quality and exits non-zero when it @@ -306,6 +343,10 @@ export OPENDOCS_MERMAID_BACKEND=none # auto | mmdc | ink | none | `OPENDOCS_MERMAID_BACKEND` | Diagram backend: `auto` (default), `mmdc`, `ink`, or `none` to disable | | `OPENDOCS_VIS_NETWORK_JS` | Path to a local vis-network bundle for `--embed-graph-assets` | | `OPENDOCS_CACHE_DIR` | Override the asset cache location (default `~/.cache/opendocs`) | +| `OPENDOCS_LLM_CACHE` | Set to `0` to disable LLM response caching | +| `OPENDOCS_MODEL_CACHE` | Where local SLM weights are cached (`--provider slm`) | +| `AZURE_OPENAI_ENDPOINT` | Azure endpoint URL (`--provider azure`) | +| `AZURE_OPENAI_API_VERSION` | Azure API version (`--provider azure`) | ### Jupyter Notebook Ingestion diff --git a/src/opendocs/cli.py b/src/opendocs/cli.py index cd1e263..dd26aca 100644 --- a/src/opendocs/cli.py +++ b/src/opendocs/cli.py @@ -958,6 +958,123 @@ def finetune( raise SystemExit(1) +@main.command() +@click.argument("codebase_dir", type=click.Path(exists=True, file_okay=False), default=".") +@click.option( + "--docs", + "doc_paths", + type=click.Path(exists=True), + multiple=True, + default=None, + help="Documentation file(s) to check against (default: README.md in the project root).", +) +@click.option( + "--fail-under", + type=float, + default=None, + help="Exit non-zero if overall coverage is below this percentage.", +) +@click.option("--include-private", is_flag=True, default=False, help="Also score underscore-prefixed symbols.") +@click.option("--include-tests", is_flag=True, default=False, help="Also score test files (excluded by default).") +@click.option("--show-missing", is_flag=True, default=False, help="List every undocumented item, not just a sample.") +@click.option("--json", "as_json", is_flag=True, default=False, help="Emit the report as JSON.") +@click.option("--limit", type=int, default=10, show_default=True, help="Sampled missing items to show per dimension.") +def coverage( + codebase_dir: str, + doc_paths: tuple[str, ...], + fail_under: float | None, + include_private: bool, + include_tests: bool, + show_missing: bool, + as_json: bool, + limit: int, +): + """Report how much of the codebase the documentation actually covers. + + Compares the real API surface against what the docs mention: docstrings on + public symbols, environment variables the code reads, CLI flags it defines, + and detected technologies. Entirely offline and deterministic. + + \b + Examples: + opendocs coverage . + opendocs coverage . --docs README.md --docs docs/guide.md + opendocs coverage . --show-missing + opendocs coverage . --fail-under 80 # CI gate + opendocs coverage . --json + """ + import json as _json + + from .core.coverage import analyse_coverage + + try: + report = analyse_coverage( + codebase_dir, + list(doc_paths) or None, + include_private=include_private, + include_tests=include_tests, + ) + except Exception as exc: + console.print(f"[bold red]Coverage analysis failed:[/] {exc}") + raise SystemExit(2) from exc + + if as_json: + console.print_json(_json.dumps(report.to_dict())) + else: + _print_coverage(report, show_missing=show_missing, limit=limit) + + if fail_under is not None and report.overall < fail_under: + if not as_json: + console.print(f"[bold red]FAIL[/] coverage {report.overall}% is below --fail-under {fail_under}%\n") + raise SystemExit(1) + + +def _print_coverage(report, *, show_missing: bool, limit: int) -> None: + """Render a coverage report as a table plus optional detail.""" + from rich.table import Table as RichTable + + def _colour(pct: float) -> str: + return "green" if pct >= 80 else "yellow" if pct >= 50 else "red" + + console.print(f"\n[bold]Documentation coverage[/] [dim]{report.project_name}[/]") + if report.docs_analysed: + console.print(f"[dim]checked against: {', '.join(report.docs_analysed)}[/]") + else: + console.print("[yellow]No documentation files found — nothing to check against.[/]") + + table = RichTable(show_lines=False) + table.add_column("Dimension", style="bold") + table.add_column("Covered", justify="right") + table.add_column("Total", justify="right") + table.add_column("Coverage", justify="right") + + for dim in report.dimensions: + if not dim.applicable: + table.add_row(dim.name, "-", "0", "[dim]n/a[/]") + continue + table.add_row( + dim.name, + str(dim.covered), + str(dim.total), + f"[{_colour(dim.percent)}]{dim.percent}%[/]", + ) + console.print(table) + + overall = report.overall + console.print(f" [bold]Overall: [{_colour(overall)}]{overall}%[/][/]\n") + + for dim in report.applicable_dimensions: + if not dim.missing: + continue + shown = dim.missing if show_missing else dim.missing[:limit] + console.print(f"[bold]Undocumented — {dim.name}[/] [dim]({len(dim.missing)})[/]") + for item in shown: + console.print(f" [red]-[/] {item}") + if len(dim.missing) > len(shown): + console.print(f" [dim]... {len(dim.missing) - len(shown)} more (use --show-missing)[/]") + console.print() + + @main.command() @click.option("--clear", "do_clear", is_flag=True, default=False, help="Delete every cached artifact.") @click.option("--cache-dir", "cache_dir", type=click.Path(), default=None, help="Cache location to inspect.") diff --git a/src/opendocs/core/coverage.py b/src/opendocs/core/coverage.py new file mode 100644 index 0000000..30dd858 --- /dev/null +++ b/src/opendocs/core/coverage.py @@ -0,0 +1,344 @@ +"""Measure how much of a codebase its documentation actually covers. + +Most documentation tooling reports on what the docs *say*. This reports what +they *miss*, by comparing two things the project already knows how to derive: +the real surface area of the code (via :mod:`opendocs.core.code_analyzer`) and +what the prose actually mentions (via the parser and semantic extractor). + +Four dimensions are scored, chosen because each has an objective answer: + +- **docstrings** — public functions and classes carrying a docstring; +- **environment variables** — every ``os.environ`` / ``getenv`` key the code + reads, checked against the documentation; +- **CLI options** — every flag the code defines, checked the same way; +- **tech stack** — detected technologies the documentation names. + +Deliberately excluded is any judgement of whether prose is *good*. That is not +objectively measurable, and a score nobody can verify is worse than no score. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Optional + +# --------------------------------------------------------------------------- +# Extraction patterns +# --------------------------------------------------------------------------- + +#: os.environ["X"], os.environ.get("X"), os.getenv("X"), environ["X"] +_ENV_RE = re.compile(r"""(?:os\.)?(?:environ(?:\.get)?\s*[\[(]\s*|getenv\s*\(\s*)["']([A-Z][A-Z0-9_]{2,})["']""") + +#: click.option("--flag"), add_argument("--flag"), @option("--flag") +_CLI_FLAG_RE = re.compile(r"""(?:option|add_argument|argument)\s*\(\s*["'](--[a-z][a-z0-9-]*)["']""") + +#: Environment names that are conventions of the wider ecosystem rather than +#: something this project is expected to document. +_STANDARD_ENV = frozenset( + { + "PATH", + "HOME", + "USER", + "PWD", + "TMPDIR", + "TEMP", + "TMP", + "LANG", + "LC_ALL", + "SHELL", + "TERM", + "PYTHONPATH", + "PYTHONDONTWRITEBYTECODE", + "VIRTUAL_ENV", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "CI", + "DEBUG", + } +) + +#: Flags every CLI provides; not evidence of missing documentation. +_STANDARD_FLAGS = frozenset({"--help", "--version"}) + +#: Directory names whose contents are tests. +_TEST_DIRS = frozenset({"tests", "test", "testing", "spec", "specs"}) + + +def is_test_path(path: str) -> bool: + """Whether *path* looks like test code rather than shipped code. + + Test helpers and test classes are not part of a project's documented + surface, so counting them makes the score meaningless — on this project + they accounted for most of the "undocumented" classes. ``CodebaseModel`` + has a ``test_files`` field but the analyser never populates it, so the + classification is done here by path convention instead. + """ + normalised = path.replace("\\", "/") + parts = normalised.split("/") + if any(part in _TEST_DIRS for part in parts[:-1]): + return True + name = parts[-1] + return name.startswith("test_") or name.endswith("_test.py") or name == "conftest.py" + + +# --------------------------------------------------------------------------- +# Result models +# --------------------------------------------------------------------------- + + +@dataclass +class Dimension: + """One scored aspect of coverage.""" + + name: str + total: int = 0 + covered: int = 0 + missing: list[str] = field(default_factory=list) + + @property + def percent(self) -> float: + """Percentage covered; a dimension with nothing to cover scores 100.""" + if self.total == 0: + return 100.0 + return round(self.covered / self.total * 100, 1) + + @property + def applicable(self) -> bool: + return self.total > 0 + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "total": self.total, + "covered": self.covered, + "percent": self.percent, + "missing": self.missing, + } + + +@dataclass +class CoverageReport: + """The full result of a coverage run.""" + + project_name: str = "" + dimensions: list[Dimension] = field(default_factory=list) + files_analysed: int = 0 + docs_analysed: list[str] = field(default_factory=list) + + def dimension(self, name: str) -> Optional[Dimension]: + for d in self.dimensions: + if d.name == name: + return d + return None + + @property + def applicable_dimensions(self) -> list[Dimension]: + """Dimensions with something to measure. + + A project with no environment variables should not be scored on + documenting them — including it would dilute the overall figure with a + vacuous 100%. + """ + return [d for d in self.dimensions if d.applicable] + + @property + def overall(self) -> float: + """Weighted overall percentage across applicable dimensions. + + Weighted by item count rather than averaging the per-dimension + percentages, so one undocumented flag out of two does not outweigh + three hundred documented functions. + """ + applicable = self.applicable_dimensions + if not applicable: + return 100.0 + total = sum(d.total for d in applicable) + covered = sum(d.covered for d in applicable) + if total == 0: + return 100.0 + return round(covered / total * 100, 1) + + def to_dict(self) -> dict[str, Any]: + return { + "project": self.project_name, + "overall_percent": self.overall, + "files_analysed": self.files_analysed, + "docs_analysed": self.docs_analysed, + "dimensions": [d.to_dict() for d in self.dimensions], + } + + +# --------------------------------------------------------------------------- +# Extraction from source +# --------------------------------------------------------------------------- + + +def extract_env_vars(sources: Iterable[str]) -> set[str]: + """Return every environment variable name the given sources read.""" + found: set[str] = set() + for source in sources: + for match in _ENV_RE.finditer(source): + name = match.group(1) + if name not in _STANDARD_ENV: + found.add(name) + return found + + +def extract_cli_flags(sources: Iterable[str]) -> set[str]: + """Return every long-form CLI flag the given sources define.""" + found: set[str] = set() + for source in sources: + for match in _CLI_FLAG_RE.finditer(source): + flag = match.group(1) + # click declares paired boolean flags as "--x/--no-x"; both halves + # are captured separately by the pattern, which is what we want. + if flag not in _STANDARD_FLAGS: + found.add(flag) + return found + + +# --------------------------------------------------------------------------- +# The analysis +# --------------------------------------------------------------------------- + + +def _read_sources(root: Path, *, include_tests: bool = False, limit_bytes: int = 500_000) -> list[str]: + """Read Python sources under *root*, skipping vendored and (by default) test trees.""" + from .code_analyzer import _IGNORE_DIRS, _iter_source_files + + sources: list[str] = [] + for path in _iter_source_files(root): + if path.suffix != ".py": + continue + if any(part in _IGNORE_DIRS for part in path.parts): + continue + if not include_tests: + try: + relative = str(path.relative_to(root)) + except ValueError: + relative = path.name + if is_test_path(relative): + continue + try: + if path.stat().st_size > limit_bytes: + continue + sources.append(path.read_text(encoding="utf-8", errors="replace")) + except OSError: + continue + return sources + + +def _docs_text(doc_paths: Iterable[Path]) -> str: + """Concatenate documentation sources into one searchable blob.""" + parts: list[str] = [] + for path in doc_paths: + try: + parts.append(path.read_text(encoding="utf-8", errors="replace")) + except OSError: + continue + return "\n".join(parts) + + +def _mentions(haystack: str, needle: str) -> bool: + """Whether *needle* appears in the documentation as a distinct token.""" + return re.search(rf"(? CoverageReport: + """Compare a codebase against its documentation. + + Parameters + ---------- + codebase_dir + Root of the project to analyse. + doc_paths + Documentation files to check against. Defaults to ``README.md`` in + the project root when present. + include_private + Score underscore-prefixed functions and classes too. Off by default, + since a private helper carries no promise to the reader. + include_tests + Score test files too. Off by default — test code is not part of the + surface a project documents. + """ + from .code_analyzer import CodebaseAnalyzer + + root = Path(codebase_dir).resolve() + model = CodebaseAnalyzer().analyze(root) + + if doc_paths is None: + default_readme = root / "README.md" + resolved_docs = [default_readme] if default_readme.exists() else [] + else: + resolved_docs = [Path(p) for p in doc_paths] + + docs = _docs_text(resolved_docs) + report = CoverageReport( + project_name=model.project_name, + files_analysed=model.total_files, + docs_analysed=[str(p) for p in resolved_docs], + ) + + # -- Docstrings ---------------------------------------------------- + def _is_public(name: str) -> bool: + return include_private or not name.startswith("_") + + fn_dim = Dimension(name="docstrings:functions") + cls_dim = Dimension(name="docstrings:classes") + for file_analysis in model.files: + if not include_tests and is_test_path(file_analysis.path): + continue + for fn in file_analysis.functions: + if not _is_public(fn.name): + continue + fn_dim.total += 1 + if fn.docstring.strip(): + fn_dim.covered += 1 + else: + fn_dim.missing.append(f"{file_analysis.path}::{fn.name}") + for cls in file_analysis.classes: + if not _is_public(cls.name): + continue + cls_dim.total += 1 + if cls.docstring.strip(): + cls_dim.covered += 1 + else: + cls_dim.missing.append(f"{file_analysis.path}::{cls.name}") + + # -- Environment variables and CLI flags --------------------------- + sources = _read_sources(root, include_tests=include_tests) + env_dim = Dimension(name="env-vars") + for name in sorted(extract_env_vars(sources)): + env_dim.total += 1 + if _mentions(docs, name): + env_dim.covered += 1 + else: + env_dim.missing.append(name) + + flag_dim = Dimension(name="cli-flags") + for flag in sorted(extract_cli_flags(sources)): + flag_dim.total += 1 + if _mentions(docs, flag): + flag_dim.covered += 1 + else: + flag_dim.missing.append(flag) + + # -- Tech stack ---------------------------------------------------- + tech_dim = Dimension(name="tech-stack") + for item in model.tech_stack: + tech_dim.total += 1 + if _mentions(docs, item.name): + tech_dim.covered += 1 + else: + tech_dim.missing.append(item.name) + + report.dimensions = [fn_dim, cls_dim, env_dim, flag_dim, tech_dim] + return report diff --git a/src/opendocs/core/linter.py b/src/opendocs/core/linter.py index 8a199d5..302e972 100644 --- a/src/opendocs/core/linter.py +++ b/src/opendocs/core/linter.py @@ -211,6 +211,7 @@ def add(url: str) -> None: def is_badge(url: str) -> bool: + """Whether *url* points at a decorative badge rather than real content.""" return any(host in url.lower() for host in _BADGE_HOSTS) diff --git a/src/opendocs/generators/styles.py b/src/opendocs/generators/styles.py index 0d06d01..39c005c 100644 --- a/src/opendocs/generators/styles.py +++ b/src/opendocs/generators/styles.py @@ -18,6 +18,8 @@ class Colors: + """Theme colour palette, mutated in place by :func:`apply_theme`.""" + # Brand PRIMARY = (0, 102, 204) # Core blue PRIMARY_DARK = (0, 71, 153) # Darker blue for depth @@ -66,6 +68,8 @@ class Colors: class Fonts: + """Theme font families, mutated in place by :func:`apply_theme`.""" + HEADING = "Calibri" BODY = "Calibri" CODE = "Courier" diff --git a/tests/test_coverage.py b/tests/test_coverage.py new file mode 100644 index 0000000..4eda025 --- /dev/null +++ b/tests/test_coverage.py @@ -0,0 +1,367 @@ +"""Tests for documentation coverage analysis. + +The value of a coverage number is entirely in whether it can be trusted, so +most of these tests are about what must *not* be counted: test code, private +helpers, and standard environment variables. A score inflated or deflated by +noise is worse than no score. +""" + +from __future__ import annotations + +import json +import textwrap + +import pytest +from click.testing import CliRunner + +from opendocs.cli import main +from opendocs.core.coverage import ( + CoverageReport, + Dimension, + analyse_coverage, + extract_cli_flags, + extract_env_vars, + is_test_path, +) + + +@pytest.fixture +def runner(): + return CliRunner() + + +@pytest.fixture +def project(tmp_path): + """A small project with known documented / undocumented surface.""" + src = tmp_path / "src" / "demo" + src.mkdir(parents=True) + (src / "__init__.py").write_text("", encoding="utf-8") + (src / "core.py").write_text( + textwrap.dedent( + ''' + """Module docstring.""" + import os + import click + + + def documented(a, b): + """This one has a docstring.""" + return a + b + + + def undocumented(a): + return a + + + def _private_helper(): + return None + + + class Documented: + """A documented class.""" + + + class Undocumented: + pass + + + def read_config(): + """Read settings from the environment.""" + token = os.environ["DEMO_TOKEN"] + region = os.getenv("DEMO_REGION") + home = os.environ.get("HOME") + return token, region, home + + + @click.option("--documented-flag", help="x") + @click.option("--secret-flag", help="y") + def cli(): + """Entry point.""" + ''' + ).strip(), + encoding="utf-8", + ) + + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "test_core.py").write_text( + textwrap.dedent( + """ + import os + + + def helper_without_docstring(): + return os.environ["TEST_ONLY_VAR"] + + + class TestSomething: + def test_it(self): + assert True + """ + ).strip(), + encoding="utf-8", + ) + + (tmp_path / "README.md").write_text( + textwrap.dedent( + """ + # Demo + + A demo project used for testing coverage analysis end to end. + + Set `DEMO_TOKEN` before running. Use `--documented-flag` to enable + the documented behaviour. + """ + ).strip(), + encoding="utf-8", + ) + return tmp_path + + +# --------------------------------------------------------------------------- +# Test-path classification — the fix that makes the score meaningful +# --------------------------------------------------------------------------- + + +class TestIsTestPath: + @pytest.mark.parametrize( + "path", + [ + "tests/test_core.py", + "tests/helpers.py", + "src/pkg/tests/thing.py", + "test/foo.py", + "spec/bar.py", + "src/pkg/test_module.py", + "src/pkg/module_test.py", + "conftest.py", + "tests/conftest.py", + ], + ) + def test_identifies_test_code(self, path): + assert is_test_path(path) is True + + @pytest.mark.parametrize( + "path", + [ + "src/opendocs/cli.py", + "src/pkg/latest.py", + "src/pkg/contest.py", + "src/protest/module.py", + "README.md", + ], + ) + def test_leaves_real_code_alone(self, path): + assert is_test_path(path) is False + + def test_handles_windows_separators(self): + assert is_test_path("tests\\test_core.py") is True + + +# --------------------------------------------------------------------------- +# Extraction +# --------------------------------------------------------------------------- + + +class TestEnvExtraction: + @pytest.mark.parametrize( + "snippet", + [ + 'os.environ["MY_VAR"]', + "os.environ['MY_VAR']", + 'os.environ.get("MY_VAR")', + 'os.getenv("MY_VAR")', + 'environ["MY_VAR"]', + 'os.environ.get( "MY_VAR", "default")', + ], + ) + def test_recognises_access_forms(self, snippet): + assert "MY_VAR" in extract_env_vars([snippet]) + + def test_skips_standard_names(self): + found = extract_env_vars(['os.environ["PATH"]', 'os.getenv("HOME")', 'os.environ["XDG_CACHE_HOME"]']) + assert found == set() + + def test_ignores_lowercase_locals(self): + assert extract_env_vars(['config["my_var"]']) == set() + + def test_deduplicates_across_sources(self): + assert extract_env_vars(['os.getenv("A_VAR")', 'os.environ["A_VAR"]']) == {"A_VAR"} + + +class TestFlagExtraction: + @pytest.mark.parametrize( + "snippet", + [ + '@click.option("--my-flag")', + "click.option('--my-flag', is_flag=True)", + 'parser.add_argument("--my-flag")', + ], + ) + def test_recognises_declaration_forms(self, snippet): + assert "--my-flag" in extract_cli_flags([snippet]) + + def test_skips_universal_flags(self): + assert extract_cli_flags(['@click.option("--help")', 'option("--version")']) == set() + + def test_ignores_usage_in_prose_strings(self): + """A flag mentioned in help text is not a declaration.""" + assert extract_cli_flags(['help="pass --other-flag to enable"']) == set() + + +# --------------------------------------------------------------------------- +# Scoring +# --------------------------------------------------------------------------- + + +class TestDimension: + def test_percent(self): + assert Dimension(name="d", total=4, covered=3).percent == 75.0 + + def test_empty_dimension_scores_full(self): + assert Dimension(name="d").percent == 100.0 + + def test_empty_dimension_is_not_applicable(self): + assert Dimension(name="d").applicable is False + + +class TestReportScoring: + def test_overall_is_weighted_by_item_count(self): + """One missed flag out of two must not outweigh 99 documented symbols.""" + report = CoverageReport( + dimensions=[ + Dimension(name="big", total=100, covered=99), + Dimension(name="small", total=2, covered=1), + ] + ) + # Naive averaging would give (99 + 50) / 2 = 74.5 + assert report.overall == pytest.approx(98.0, abs=0.1) + + def test_inapplicable_dimensions_are_excluded(self): + """A project with no env vars should not bank a vacuous 100%.""" + report = CoverageReport( + dimensions=[ + Dimension(name="real", total=10, covered=5), + Dimension(name="absent", total=0, covered=0), + ] + ) + assert report.overall == 50.0 + + def test_all_empty_scores_full(self): + assert CoverageReport(dimensions=[Dimension(name="a")]).overall == 100.0 + + def test_serialises(self): + report = CoverageReport(project_name="x", dimensions=[Dimension(name="a", total=2, covered=1)]) + json.dumps(report.to_dict()) + assert report.to_dict()["overall_percent"] == 50.0 + + +# --------------------------------------------------------------------------- +# End to end +# --------------------------------------------------------------------------- + + +class TestAnalyseCoverage: + @pytest.fixture + def report(self, project): + return analyse_coverage(project) + + def test_finds_the_readme_by_default(self, report): + assert any("README.md" in d for d in report.docs_analysed) + + def test_scores_undocumented_function(self, report): + missing = report.dimension("docstrings:functions").missing + assert any("undocumented" in m for m in missing) + assert not any("::documented" in m for m in missing) + + def test_private_helpers_are_excluded(self, report): + missing = report.dimension("docstrings:functions").missing + assert not any("_private_helper" in m for m in missing) + + def test_test_files_are_excluded(self, report): + """Regression: test classes dominated the score before this.""" + for name in ("docstrings:functions", "docstrings:classes"): + missing = report.dimension(name).missing + assert not any("test_core" in m for m in missing), missing + + def test_test_only_env_vars_are_excluded(self, report): + assert "TEST_ONLY_VAR" not in report.dimension("env-vars").missing + + def test_documented_env_var_counts_as_covered(self, report): + assert "DEMO_TOKEN" not in report.dimension("env-vars").missing + + def test_undocumented_env_var_is_reported(self, report): + assert "DEMO_REGION" in report.dimension("env-vars").missing + + def test_standard_env_var_is_not_required(self, report): + assert "HOME" not in report.dimension("env-vars").missing + + def test_documented_flag_counts_as_covered(self, report): + assert "--documented-flag" not in report.dimension("cli-flags").missing + + def test_undocumented_flag_is_reported(self, report): + assert "--secret-flag" in report.dimension("cli-flags").missing + + def test_include_tests_changes_the_result(self, project): + without = analyse_coverage(project) + with_tests = analyse_coverage(project, include_tests=True) + assert with_tests.dimension("docstrings:classes").total > without.dimension("docstrings:classes").total + + def test_include_private_changes_the_result(self, project): + without = analyse_coverage(project) + with_private = analyse_coverage(project, include_private=True) + assert with_private.dimension("docstrings:functions").total > without.dimension("docstrings:functions").total + + def test_explicit_doc_paths(self, project, tmp_path): + other = tmp_path / "GUIDE.md" + other.write_text("Mentions DEMO_REGION and --secret-flag here.", encoding="utf-8") + report = analyse_coverage(project, [other]) + assert "DEMO_REGION" not in report.dimension("env-vars").missing + assert "--secret-flag" not in report.dimension("cli-flags").missing + + def test_missing_docs_does_not_crash(self, tmp_path): + (tmp_path / "mod.py").write_text("def f():\n return 1\n", encoding="utf-8") + report = analyse_coverage(tmp_path) + assert report.docs_analysed == [] + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +class TestCoverageCommand: + def test_reports(self, runner, project): + result = runner.invoke(main, ["coverage", str(project)]) + assert result.exit_code == 0, result.output + assert "Documentation coverage" in result.output + + def test_json_output(self, runner, project): + result = runner.invoke(main, ["coverage", str(project), "--json"]) + assert result.exit_code == 0 + payload = json.loads(result.output) + assert "overall_percent" in payload + assert {d["name"] for d in payload["dimensions"]} >= {"env-vars", "cli-flags"} + + def test_fail_under_gate_passes(self, runner, project): + result = runner.invoke(main, ["coverage", str(project), "--fail-under", "1"]) + assert result.exit_code == 0 + + def test_fail_under_gate_fails(self, runner, project): + result = runner.invoke(main, ["coverage", str(project), "--fail-under", "100"]) + assert result.exit_code == 1 + assert "below" in result.output + + def test_show_missing_lists_everything(self, runner, project): + brief = runner.invoke(main, ["coverage", str(project), "--limit", "1"]) + full = runner.invoke(main, ["coverage", str(project), "--show-missing"]) + assert len(full.output) >= len(brief.output) + + def test_nonexistent_directory_is_rejected(self, runner, tmp_path): + result = runner.invoke(main, ["coverage", str(tmp_path / "absent")]) + assert result.exit_code != 0 + + def test_help(self, runner): + result = runner.invoke(main, ["coverage", "--help"]) + assert result.exit_code == 0 + assert "--fail-under" in result.output