Repository navigation
Expand file tree
/
Copy pathhost.py
More file actions
225 lines (186 loc) · 8.48 KB
/
Copy pathhost.py
File metadata and controls
225 lines (186 loc) · 8.48 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
"""Out-of-band host glue — embed MOP at a delivery chokepoint.
A host (a Hermes plugin, a patchbay filter, any Python send-path) calls
``gate(text, host=...)`` at the point its agent's outgoing text is finalized.
MOP resolves the rule set, evaluates the text, audits the verdict, and returns
a :class:`GateResult` saying what to deliver.
Two modes — the "gears" the caller shifts between:
* ``mode="log"`` (default, gears **disengaged**): evaluate + audit, but never
alter delivery. ``GateResult.deliver`` is always the original text. This is
the safe shadow launch — you get real verdicts in the audit log without any
behaviour change the user can see.
* ``mode="enforce"`` (gears **engaged**): ``deliver`` reflects the verdict —
the rewrite on Rewritten, a redaction notice on Rejected.
Every audit record is tagged with ``host`` so logs from multiple surfaces stay
attributable and dedupable. Because a relay (patchbay) wraps the agent it runs
(Pi), gating in BOTH double-logs the same message — install ONE gate per
delivery path, at the outermost boundary. See docs/integration.md.
The callback surface a host registers must be **synchronous** (Hermes'
``invoke_hook`` calls ``cb(**kwargs)`` without awaiting), so ``gate`` is sync.
It only spins a worker thread for the LLM call when an ``llm`` rule is actually
active; with no active rules it never touches litellm or an event loop.
"""
from __future__ import annotations
import asyncio
import os
import threading
from dataclasses import dataclass
from pathlib import Path
from .cli import _DETERMINISTIC, _deterministic_hits, check
from .discovery import resolve_rules
from .types import Accepted, Evaluator, Rejected, Rewritten, Verdict
REDACTION_NOTICE = "[message withheld by MOP]"
@dataclass
class GateResult:
"""Outcome of gating one outgoing message."""
deliver: str # text the host should send
verdict: Verdict # the raw MOP verdict (for logging / branching)
changed: bool # did enforce alter the original text?
mode: str # "log" | "enforce"
def replacement(self) -> str | None:
"""The string a transform-hook should return, or None to leave as-is.
In ``log`` mode this is always None (passthrough). In ``enforce`` mode
it is the delivered text when it differs from the original, else None.
"""
return self.deliver if self.changed else None
def _run_coro(coro):
"""Run *coro* to completion whether or not a loop is already running here.
Hermes calls hook callbacks synchronously from inside its async loop, so
``asyncio.run`` would raise "loop already running". When that's the case we
execute the coroutine in a short-lived worker thread with its own loop.
"""
try:
asyncio.get_running_loop()
except RuntimeError:
return asyncio.run(coro)
box: dict = {}
def runner() -> None:
try:
box["result"] = asyncio.run(coro)
except BaseException as exc: # re-raised on the caller thread below
box["error"] = exc
worker = threading.Thread(target=runner, name="mop-gate")
worker.start()
worker.join()
if "error" in box:
raise box["error"]
return box["result"]
def _evaluate(
text: str,
*,
rules_dir: Path | None,
use_builtins: bool,
model: str | None,
allow_rewrite: bool,
evaluator: Evaluator | None,
):
"""Return (verdict, rules). Skips litellm entirely when no LLM call is needed.
``allow_rewrite`` False (log mode) is judge-only: a deterministic-only rule
set rejects without any LLM call. The evaluator (litellm) is built lazily —
only when an ``llm`` rule is active, or a deterministic hit needs a rewrite
attempt (``allow_rewrite`` True). A caller-supplied ``evaluator`` overrides
the built one (injection for hosts and tests).
"""
rules = resolve_rules(rules_dir=rules_dir, use_builtins=use_builtins)
det = _deterministic_hits(text, rules)
has_llm = any(r.detector == "llm" for r in rules)
needs_llm = has_llm or (bool(det) and allow_rewrite)
if not det and not has_llm:
return Accepted(), rules
if evaluator is None and needs_llm:
from .evaluators import build_evaluator
evaluator = build_evaluator(rules=rules, model=model)
verdict = _run_coro(
check(text, rules, evaluator or _unused_evaluator, allow_rewrite=allow_rewrite)
)
return verdict, rules
async def _unused_evaluator(text, hints, justification): # pragma: no cover
"""Placeholder passed when check() provably won't call the evaluator."""
raise AssertionError("evaluator called unexpectedly")
def _delivery(text: str, verdict: Verdict, rules, *, mode: str) -> tuple[str, bool]:
"""Map a verdict to (text_to_deliver, changed) under the active mode.
Enforce contract (ADR-0002; docs/mop-enforce-decision.md, Issue 1 / 1A):
MOP never delivers text on which an active *deterministic* rule still fires.
``cli.py`` re-checks the deterministic rules against the rewrite and folds
any still-firing names into ``Rewritten.unresolved``; so a partial rewrite
whose ``unresolved`` names a regex/script/length rule is redacted, not
delivered. ``llm``-rule residuals are judged, best-effort, and deliver.
"""
if mode == "log":
return text, False
if isinstance(verdict, Rewritten):
det_names = {r.name for r in rules if r.detector in _DETERMINISTIC}
if det_names & set(verdict.unresolved):
return REDACTION_NOTICE, True
return verdict.rewritten, verdict.rewritten.strip() != text.strip()
if isinstance(verdict, Rejected):
return REDACTION_NOTICE, True
return text, False
def gate(
text: str,
*,
host: str,
mode: str = "log",
rules_dir: Path | str | None = None,
use_builtins: bool = False,
model: str | None = None,
audit_dir: Path | str | None = None,
evaluator: Evaluator | None = None,
) -> GateResult:
"""Evaluate *text*, audit the verdict, and decide what to deliver.
``host`` tags the audit record. ``mode`` selects log (passthrough, judge-only)
vs. enforce (apply rewrites, redact rejections). ``rules_dir`` /
``use_builtins`` / ``model`` feed rule resolution and the evaluator;
``audit_dir`` (falsy = no audit) is where the JSONL flight recorder is
written; ``evaluator`` overrides the built-in litellm evaluator.
"""
if mode not in ("log", "enforce"):
raise ValueError(f"mode must be 'log' or 'enforce', got {mode!r}")
rd = Path(rules_dir) if rules_dir else None
verdict, rules = _evaluate(
text,
rules_dir=rd,
use_builtins=use_builtins,
model=model,
allow_rewrite=(mode == "enforce"),
evaluator=evaluator,
)
if audit_dir:
from .audit import JsonlAuditor
JsonlAuditor(audit_dir).record(
original=text,
verdict=verdict,
rule_names=[r.name for r in rules],
attempt=0,
host=host,
)
deliver, changed = _delivery(text, verdict, rules, mode=mode)
return GateResult(deliver=deliver, verdict=verdict, changed=changed, mode=mode)
def gate_from_env(text: str, *, host: str) -> GateResult:
"""`gate()` with config read from the standard MOP_* env vars.
The one env surface every in-process host shim uses (Hermes plugin,
patchbay filter):
MOP_MODE=log|enforce default log
MOP_RULES_DIR=<dir> a .mop-style rules dir (unset = no local rules)
MOP_AUDIT_LOG=<dir> JSONL flight recorder (unset = no audit)
MOP_BUILTINS=1 opt in to packaged built-in rules
MOP_EVALUATOR_MODEL=<m> litellm model/tier (only used once llm rules active)
"""
return gate(
text,
host=host,
mode=os.environ.get("MOP_MODE", "log"),
rules_dir=os.environ.get("MOP_RULES_DIR") or None,
use_builtins=os.environ.get("MOP_BUILTINS", "") not in ("", "0", "false"),
model=os.environ.get("MOP_EVALUATOR_MODEL") or None,
audit_dir=os.environ.get("MOP_AUDIT_LOG") or None,
)
def filter_text(text: str, *, host: str) -> str:
"""Text-in / text-out filter for send-path hosts (e.g. patchbay `_send_response`).
Returns the string to deliver: the original in log mode, the rewrite or a
redaction notice in enforce mode. Config from the MOP_* env vars. Fail-open —
on any error the original text is returned unchanged, never withheld.
"""
try:
return gate_from_env(text, host=host).deliver
except Exception:
return text