diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab85062..dbec7b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: run: | pip install --no-deps semantica pip install numpy pandas scipy networkx pypdf python-multipart \ - fastapi "uvicorn[standard]" pytest httpx + fastapi "uvicorn[standard]" pytest httpx matplotlib - name: Configure git for snapshot tests run: | git config --global user.email ci@graphier.local diff --git a/CHANGELOG.md b/CHANGELOG.md index 60dc95c..2b8f4b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Library API: `import graphier; v = graphier.open("~/notes")` — entities, + relations, Datalog queries, conflicts, timeline, `to_networkx()`, and + matplotlib plotting (`plot_graph()`, `plot_timeline()`) via the + `graphier[viz]` extra. Same determinism, same sentence-level evidence. + - Lean PyPI install: `pip install graphier` no longer drags in multi-gigabyte ML dependencies; `graphier setup` fetches the extraction engine with `--no-deps`, and a friendly message explains the one diff --git a/README.md b/README.md index 87d304a..56b298c 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,28 @@ be rebuilt from the files at any time. Extraction is deterministic (no LLM, no network) — the pattern extractors run on an offset-preserving masked copy of each note so spans map exactly onto what you typed. +## Use it as a library + +`graphier` is also importable, matplotlib-style — same graph, same +provenance, in a script or notebook, no web app required: + +```python +import graphier + +v = graphier.open("~/notes") +v.entities("PERSON") # plain dicts, evidence included +v.relations("founded by") # each edge carries its sentence +v.query("?- empire_builder(P, B)") # Datalog over your own rules +v.to_networkx() # hand the graph to any tool + +v.plot_graph() # needs: pip install graphier[viz] +v.plot_timeline() +``` + +![plot_graph output](docs/plot-graph.png) + +![plot_timeline output](docs/plot-timeline.png) + ## Ask your AI about your notes' knowledge Graphier ships an [MCP](https://modelcontextprotocol.io/) server, so diff --git a/backend/graphier/__init__.py b/backend/graphier/__init__.py index 26a77f2..7096cba 100644 --- a/backend/graphier/__init__.py +++ b/backend/graphier/__init__.py @@ -1,3 +1,12 @@ -"""Graphier backend: a Markdown vault with a Semantica-powered knowledge graph.""" +"""Graphier: a knowledge workspace where the graph builds itself. + +Library use: + + import graphier + v = graphier.open("~/notes") + v.plot_graph(); v.plot_timeline() +""" __version__ = "0.1.0" + +from .api import VaultSession, open # noqa: F401,A004 diff --git a/backend/graphier/api.py b/backend/graphier/api.py new file mode 100644 index 0000000..b0a3b1c --- /dev/null +++ b/backend/graphier/api.py @@ -0,0 +1,307 @@ +"""Graphier as a library: the vault's knowledge in a script or notebook. + + import graphier + + v = graphier.open("~/notes") + v.entities("PERSON") # rows, plain dicts + v.query("?- empire_builder(P, B)") + v.to_networkx() # hand the graph to any tool you like + + v.plot_graph() # matplotlib: the knowledge graph + v.plot_timeline() # matplotlib: entity lifelines over time + +Plotting needs matplotlib (`pip install graphier[viz]`); everything else +is dependency-free beyond Graphier itself. Same determinism, same +provenance: every row that asserts something carries its evidence. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +os.environ.setdefault("SEMANTICA_DISABLE_PROGRESS", "1") + +from .enrichment import Enricher, enrich # noqa: E402 +from .extraction import ExtractionService # noqa: E402 +from .graph import build_graph, entity_page # noqa: E402 +from .search import search as _search # noqa: E402 +from .timeline import build_timeline # noqa: E402 +from .vault import Vault # noqa: E402 + +# Mirrors the app's light-theme palette (frontend/src/index.css). +LABEL_COLORS = { + "PERSON": "#b3541e", + "ORG": "#6e40c9", + "GPE": "#2e7d32", + "DATE": "#1565c0", + "CONCEPT": "#8a7500", + "NOTE": "#1f6f61", +} +_CUSTOM_PALETTE = ["#c2185b", "#00838f", "#7cb342", "#5e35b1", "#ef6c00", "#7d6608"] + + +def label_color(label: str) -> str: + if label in LABEL_COLORS: + return LABEL_COLORS[label] + return _CUSTOM_PALETTE[sum(ord(c) for c in label) % len(_CUSTOM_PALETTE)] + + +class VaultSession: + """A read view over one vault. Cheap to keep around: extraction is + cached by content hash, so repeated calls only re-read changed notes.""" + + def __init__(self, vault_dir: str | Path): + self.vault = Vault(Path(vault_dir).expanduser()) + self._extractor = ExtractionService() + + # ---- data access ---- + + def graph(self) -> dict[str, Any]: + return build_graph(self.vault, self._extractor) + + def entities(self, label: str | None = None) -> list[dict[str, Any]]: + return [ + n for n in self.graph()["nodes"] + if n["label"] != "NOTE" and (label is None or n["label"] == label.upper()) + ] + + def relations(self, predicate: str | None = None) -> list[dict[str, Any]]: + graph = self.graph() + display = {n["id"]: n["text"] for n in graph["nodes"]} + wanted = predicate.lower().replace(" ", "_") if predicate else None + return [ + {**e, "subject": display.get(e["source"]), "object": display.get(e["target"])} + for e in graph["edges"] + if wanted is None or e["predicate"] == wanted + ] + + def entity(self, name: str) -> dict[str, Any] | None: + graph = self.graph() + wanted = name.strip().lower() + node = next((n for n in graph["nodes"] if n["text"].strip().lower() == wanted), None) + return entity_page(graph, node["id"]) if node else None + + def search(self, query: str) -> dict[str, Any]: + notes = {m.id: self.vault.read(m.id) for m in self.vault.list_notes()} + graph = self.graph() + return _search(query, notes, graph["note_titles"], graph) + + def query(self, pattern: str) -> list[dict[str, str]]: + pattern = pattern.strip() + if pattern.startswith("?-"): + pattern = pattern[2:].strip() + return Enricher(self.graph()).datalog_query(pattern) + + def conflicts(self) -> list[dict[str, Any]]: + return enrich(self.graph())["conflicts"] + + def inferred(self) -> list[dict[str, Any]]: + return enrich(self.graph())["inferred"] + + def timeline(self) -> list[dict[str, Any]]: + return build_timeline(self.vault, self._extractor) + + def to_networkx( + self, + include_notes: bool = False, + exclude_labels: tuple[str, ...] = ("UNKNOWN",), + ): + """The knowledge graph as a networkx.DiGraph, ready for any tool. + + UNKNOWN-type nodes (low-confidence extraction noise) are excluded + by default; pass exclude_labels=() to keep everything. + """ + import networkx as nx + + graph = self.graph() + g = nx.DiGraph() + for node in graph["nodes"]: + if not include_notes and node["label"] == "NOTE": + continue + if node["label"] in exclude_labels: + continue + g.add_node(node["text"], label=node["label"], mentions=node["count"]) + for edge in graph["edges"]: + display = {n["id"]: n["text"] for n in graph["nodes"]} + src, dst = display.get(edge["source"]), display.get(edge["target"]) + if g.has_node(src) and g.has_node(dst): + g.add_edge(src, dst, predicate=edge["predicate"], origin=edge["origin"], + evidence=edge["evidence"]) + return g + + # ---- plotting (matplotlib, optional) ---- + + @staticmethod + def _plt(): + try: + import matplotlib.pyplot as plt + except ImportError as exc: + raise ImportError( + "plotting needs matplotlib — install with: pip install graphier[viz]" + ) from exc + return plt + + def plot_graph( + self, + ax=None, + edge_labels: bool = True, + seed: int = 7, + exclude_labels: tuple[str, ...] = ("UNKNOWN", "DATE"), + include_related: bool = False, + ): + """Draw the knowledge graph. Returns the matplotlib Axes. + + Defaults are opinionated for legibility: UNKNOWN and DATE nodes and + generic co-occurrence ("related_to") edges are excluded from the + drawing — the data keeps them; pass exclude_labels=("UNKNOWN",) / + include_related=True to draw everything. + """ + plt = self._plt() + import matplotlib.patheffects as pe + import networkx as nx + + g = self.to_networkx(exclude_labels=exclude_labels) + if not include_related: + g.remove_edges_from( + [(u, v) for u, v, d in g.edges(data=True) if d["predicate"] == "related_to"] + ) + + if ax is None: + _, ax = plt.subplots(figsize=(11, 8)) + fig = ax.figure + fig.set_facecolor("#fbfaf8") + ax.set_facecolor("#fbfaf8") + + try: + pos = nx.kamada_kawai_layout(g.to_undirected()) + except Exception: # disconnected corner cases fall back to spring + pos = nx.spring_layout(g, seed=seed, k=3.0 / max(1, len(g)) ** 0.5, iterations=200) + + ink, ink_soft, line = "#232725", "#6b716e", "#c9c5bd" + halo = [pe.withStroke(linewidth=3, foreground="#fbfaf8")] + + nx.draw_networkx_edges( + g, pos, ax=ax, edge_color=line, width=1.3, alpha=0.9, + arrows=True, arrowsize=13, arrowstyle="-|>", + connectionstyle="arc3,rad=0.08", node_size=520, + ) + colors = [label_color(g.nodes[n]["label"]) for n in g] + sizes = [260 + 90 * g.degree(n) for n in g] + nx.draw_networkx_nodes( + g, pos, ax=ax, node_color=colors, node_size=sizes, + edgecolors="white", linewidths=2.0, + ) + + # Labels sit below their node with a halo, never on top of it, and + # get greedily nudged apart when two would collide. + placed: list[tuple[float, float]] = [] + xs = [p[0] for p in pos.values()] + ys = [p[1] for p in pos.values()] + span_x = (max(xs) - min(xs)) or 1.0 + span_y = (max(ys) - min(ys)) or 1.0 + for node in sorted(g, key=lambda n: -g.degree(n)): + x, y = pos[node] + ly = y - 0.045 * span_y - (sizes[list(g).index(node)] ** 0.5) * 0.0016 * span_y + for px, py in placed: + if abs(x - px) < 0.14 * span_x and abs(ly - py) < 0.045 * span_y: + ly = py - 0.05 * span_y + placed.append((x, ly)) + ax.text( + x, ly, node, ha="center", va="top", fontsize=9.5, color=ink, + path_effects=halo, zorder=6, + ) + + if edge_labels: + labels = { + (u, v): d["predicate"].replace("_", " ") + for u, v, d in g.edges(data=True) + if d["predicate"] != "related_to" + } + texts = nx.draw_networkx_edge_labels( + g, pos, ax=ax, edge_labels=labels, font_size=7.5, + font_color=ink_soft, rotate=False, + bbox={"boxstyle": "round,pad=0.15", "fc": "#fbfaf8", "ec": "none", "alpha": 0.9}, + ) + for t in texts.values(): + t.set_fontstyle("italic") + + present = sorted({g.nodes[n]["label"] for n in g}) + handles = [ + plt.Line2D([], [], marker="o", linestyle="", color=label_color(l), + markeredgecolor="white", markeredgewidth=1.2, + label=l.title(), markersize=9) + for l in present + ] + ax.legend(handles=handles, loc="upper left", frameon=False, fontsize=8.5, + labelcolor=ink_soft, handletextpad=0.4, borderaxespad=0.2) + ax.set_axis_off() + ax.margins(0.10) + ax.set_title("Knowledge graph", fontsize=13, color=ink, loc="left", + fontweight="bold", pad=12) + return ax + + def plot_timeline(self, ax=None, max_rows: int = 8): + """Draw entity lifelines over time. Returns the matplotlib Axes.""" + plt = self._plt() + import matplotlib.patheffects as pe + + events = self.timeline() + if not events: + raise ValueError("no dated events in this vault yet") + + def t_of(e): + y, m, d = e["sort_key"] + return y + (m - 1) / 12 if m else y + 0.5 + + rows: dict[tuple[str, str], list[float]] = {} + for e in events: + for ent in e["entities"]: + rows.setdefault((ent["text"], ent["label"]), []).append(t_of(e)) + top = sorted(rows.items(), key=lambda kv: -len(kv[1]))[:max_rows] + top.sort(key=lambda kv: kv[1][0]) + + ink, ink_soft, grid = "#232725", "#6b716e", "#e7e3db" + if ax is None: + _, ax = plt.subplots(figsize=(10, 0.62 * len(top) + 1.4)) + fig = ax.figure + fig.set_facecolor("#fbfaf8") + ax.set_facecolor("#fbfaf8") + + all_t = [t for _, ts in top for t in ts] + pad = max(1.5, (max(all_t) - min(all_t)) * 0.04) + x_hi = max(all_t) + pad + + halo = [pe.withStroke(linewidth=3, foreground="#fbfaf8")] + for i, ((text, label), ts) in enumerate(top): + color = label_color(label) + if len(ts) > 1: + ax.hlines(i, min(ts), max(ts), color=color, alpha=0.30, + linewidth=2.5, capstyle="round") + ax.scatter(ts, [i] * len(ts), color=color, s=58, zorder=3, + edgecolors="#fbfaf8", linewidths=1.4) + years = sorted({int(t) for t in ts}) + span = str(years[0]) if len(years) == 1 else f"{years[0]}–{years[-1]}" + ax.annotate(span, (max(ts), i), xytext=(9, 0), + textcoords="offset points", va="center", fontsize=7.5, + color=ink_soft, path_effects=halo) + + ax.set_yticks(range(len(top))) + ax.set_yticklabels([text for (text, _), _ in top], fontsize=9.5, color=ink) + ax.invert_yaxis() + ax.set_xlim(min(all_t) - pad, x_hi + pad) + ax.grid(axis="x", color=grid, linewidth=0.9) + ax.set_axisbelow(True) + ax.tick_params(axis="x", colors=ink_soft, labelsize=8.5, length=0) + ax.tick_params(axis="y", length=0) + for spine in ax.spines.values(): + spine.set_visible(False) + ax.set_title("Entity lifelines", fontsize=13, color=ink, loc="left", + fontweight="bold", pad=12) + return ax + + +def open(vault_dir: str | Path) -> VaultSession: # noqa: A001 - deliberate, matplotlib-style + """Open a vault for reading: `v = graphier.open("~/notes")`.""" + return VaultSession(vault_dir) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 88f631f..1dc90cb 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -37,6 +37,8 @@ Changelog = "https://github.com/ioteverythin/Graphier/blob/main/CHANGELOG.md" dev = ["pytest>=8.0", "httpx>=0.27"] # Full Semantica including its ML extractors (large download): ml = ["semantica>=0.6.0"] +# Plotting from the library API (plot_graph / plot_timeline): +viz = ["matplotlib>=3.7"] [project.scripts] graphier = "graphier.cli:main" diff --git a/backend/tests/test_api_library.py b/backend/tests/test_api_library.py new file mode 100644 index 0000000..a514acd --- /dev/null +++ b/backend/tests/test_api_library.py @@ -0,0 +1,62 @@ +import pytest + +import graphier +from graphier.demo import seed_demo +from graphier.vault import Vault + + +@pytest.fixture() +def session(tmp_path): + seed_demo(Vault(str(tmp_path))) + return graphier.open(tmp_path) + + +def test_data_access(session): + people = session.entities("PERSON") + assert any(e["text"] == "Ada Lovelace" for e in people) + + founded = session.relations("founded by") + assert founded and founded[0]["evidence"][0]["sentence"] + + rows = session.query("?- empire_builder(P, B)") + assert {"P": "Ada Lovelace", "B": "Widget Inc"} in rows + + page = session.entity("Acme Corp") + assert page and any("founded Acme Corp" in m["sentence"] for m in page["mentions"]) + + assert session.conflicts() and session.timeline() + + +def test_to_networkx(session): + g = session.to_networkx() + assert g.has_node("Ada Lovelace") + data = g.get_edge_data("Acme Corp", "Ada Lovelace") + assert data and data["predicate"] == "founded_by" and data["evidence"] + + +def test_plots_return_axes(session): + mpl = pytest.importorskip("matplotlib") + mpl.use("Agg") + + ax = session.plot_graph() + assert ax.has_data() or len(ax.collections) > 0 + + ax2 = session.plot_timeline() + assert len(ax2.collections) > 0 # lifeline scatter marks + labels = [t.get_text() for t in ax2.get_yticklabels()] + assert any("Ada Lovelace" in l for l in labels) + + +def test_plot_without_matplotlib_message(session, monkeypatch): + import builtins + + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name.startswith("matplotlib"): + raise ImportError("nope") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + with pytest.raises(ImportError, match=r"graphier\[viz\]"): + session.plot_graph() diff --git a/docs/plot-graph.png b/docs/plot-graph.png new file mode 100644 index 0000000..2498392 Binary files /dev/null and b/docs/plot-graph.png differ diff --git a/docs/plot-timeline.png b/docs/plot-timeline.png new file mode 100644 index 0000000..c0165c3 Binary files /dev/null and b/docs/plot-timeline.png differ