The recall bench labels a run by whether the prompt carried known, records it, and tables the two groups (#588)

scripts/bench/README.md

@@ -72,6 +72,24 @@ node scripts/bench/judge_open.mjs --kb <id> --sample 200     # 开放陈述有
 服务端带 `UTOPIA_EXTRACT_KNOWN_IN_PROMPT=false` 起，照常跑 `recall.mjs`，与不带时的几轮比。
 这个开关只动提示词，落库时按名字认回前面分块的实体照旧。
 
+```
+# 服务端照常起：每轮记一份
+node scripts/bench/recall.mjs --kb <id> --known shown --out runs/588
+# 服务端带 UTOPIA_EXTRACT_KNOWN_IN_PROMPT=false 重起：同样几轮
+node scripts/bench/recall.mjs --kb <id> --known empty --out runs/588
+# 按 known 分组排表
+node scripts/bench/recall.mjs --table runs/588
+```
+
+`--known` 只是标签，台子查不到服务端的开关，两边对不上那一轮就作废。每轮除了总分还记每篇的
+分块、事实、实体、**跨块认回**（一个实体出现在这篇两个以上分块的事实里）、**排给裁决的对**（这一轮进了
+`resolution_reviews` 的疑似同一对，#877；提示词里没有 known 时同一个东西换个写法再列一遍，落在这里而不是静默合并。
+不分召回通道：一对裁过之后 reason 被裁决改写，`name_vector|` 留不下来。全库总数另按待裁 / 合并 / 分开和现在的
+reason 前缀记；只算这一轮排队之后排的对，`--score` 没有排队时刻，算的是库里现有的——每轮开头删实体时它们随之删掉）、丢弃账（`extraction_drops`
+按原因）和排队到最后一块抽完的秒数。`--table` 给每组的均值 ± 标准差，再给事实集合（去重的主/谓/宾）
+的重合度：组内两两比是运行间的抖动，跨组两两比是 known 的影响——跨组落在组内的范围里，空 known 就没把
+输出挪出方差。
+
 抽取写的是开放图谱（0044 决定 2，没有带本体的第二条路）：陈述按原文短语落成 `layer='open'` 的
 事实行，限定挂在 `statement_qualifiers`，时间词进 `time_mentions`；打分口径照旧（谓词取
 `proposed_predicate`，限定也拼进那一行），所以下表里带本体那些轮次的分数仍然可比。`judge_open.mjs` 是

scripts/bench/recall.mjs

@@ -16,6 +16,12 @@
 //   node scripts/bench/recall.mjs --kb <id>            # 已有库（本体向量已就绪）
 //   node scripts/bench/recall.mjs --kb <id> --score    # 只打分，不重抽
 //   node scripts/bench/recall.mjs --kb <id> --reprocess # 改了解析器：连分块一起重来
+//   node scripts/bench/recall.mjs --kb <id> --known empty --out runs/588   # 记一轮，写成 JSON
+//   node scripts/bench/recall.mjs --table runs/588                        # 把记下的几轮排成表
+//
+// `--known shown|empty` 只是**标签**：提示词里带不带前面分块认下的实体，由服务端的
+// `UTOPIA_EXTRACT_KNOWN_IN_PROMPT` 决定（#588），起服务时定死，台子改不了也查不到。
+// 标签写进记录，`--table` 按它分组；与服务端对不上，那一轮的数就是错的。
 //
 // 环境变量：BENCH_BASE / BENCH_EMAIL / BENCH_PASSWORD / BENCH_PSQL（同 run.mjs）。
 //
@@ -40,6 +46,18 @@ const args = Object.fromEntries(
     return acc;
   }, []),
 );
+if (args.table) {
+  table(args.table);
+  process.exit(0);
+}
+if (args.known !== undefined && !["shown", "empty"].includes(args.known)) {
+  console.error("--known 只认 shown 或 empty（与服务端的 UTOPIA_EXTRACT_KNOWN_IN_PROMPT 对上）");
+  process.exit(1);
+}
+if (args.out && !args.known) {
+  console.error("--out 要带 --known shown|empty：不标清这一轮提示词里有没有 known，记下来的数没法分组");
+  process.exit(1);
+}
 const KB = args.kb;
 if (!KB) {
   console.error("要一个 --kb <id>：建一个装了本体包的库，等它的本体向量补齐，再把 id 给这里。");
@@ -170,10 +188,179 @@ function score() {
   // **52 个条目、九成命中率，一个标准差 ≈ 2.7 条。** 单轮差一两条不是证据，
   // 同一份代码重跑一遍再说；结构性的改善看 drops 计数与某一类是否整类进来了
   console.log("\n（52 条真值，1σ ≈ 2.7 条：单轮小幅波动不作数）");
+
+  // 去重后的 (主, 谓, 宾)：两轮之间比的是「抽出来的是不是同一批」，不只是个数
+  const sigs = {};
+  for (const [doc, facts] of byDoc) sigs[doc] = [...new Set(facts.map((f) => `${norm(f.subj)}\u001f${norm(f.pred)}\u001f${norm(f.obj)}`))].sort();
+  return {
+    pass,
+    total: truth.length,
+    perDoc: Object.fromEntries([...perDoc].map(([k, v]) => [k, v.pass])),
+    misses: misses.map((m) => m.id),
+    sigs,
+  };
+}
+
+// 每篇文档的形状：分块、事实、实体、跨块认回的实体、丢弃账（extraction_drops），
+// 以及排队到最后一块抽完的秒数（给了 queuedAt 才有）。
+// 「跨块认回」= 一个实体出现在这篇文档**两个以上分块**的事实里：前面的块认下、后面的块接着用，
+// known 管的就是这一件（名字认回也算在内——那半步不受开关管，差出来的是提示词的份）
+// 「排给裁决的对」= 这一轮进了 resolution_reviews 的疑似同一对，不分是哪条召回通道提的：
+// 提示词里没有 known，后面的块更容易换个写法再列一遍同一个东西，这种不会静默合并（#877），
+// 而是在这里多排一对——召回分不动、这个数涨，就是空 known 的代价。一对算进两边实体出现的每篇。
+// **不按 `name_vector|` 筛**：一对一旦裁了（治理、升级给人、随合并作废），reason 就被裁决改写
+// （`governed|…` 等），原来是哪条通道提的不留痕；按它筛，一轮跑完几乎总是 0
+function stats(queuedAt) {
+  const rows = psql(`
+    WITH live AS (
+      SELECT c.id, c.document_id, c.extracted_at FROM chunks c JOIN documents d ON d.id = c.document_id
+       WHERE d.kb_id = '${KB}' AND c.superseded_at IS NULL),
+    ev AS (
+      SELECT l.document_id, fe.chunk_id, f.id AS fact_id, f.subject_id, f.object_id
+        FROM fact_evidence fe JOIN live l ON l.id = fe.chunk_id JOIN facts f ON f.id = fe.fact_id
+       WHERE f.kb_id = '${KB}' AND f.invalidated_at IS NULL),
+    ents AS (
+      SELECT document_id, e, count(DISTINCT chunk_id) AS n FROM (
+        SELECT document_id, chunk_id, subject_id AS e FROM ev
+        UNION ALL SELECT document_id, chunk_id, object_id FROM ev WHERE object_id IS NOT NULL) x
+       GROUP BY 1, 2)
+    SELECT concat_ws(chr(31), d.filename,
+      (SELECT count(*) FROM live WHERE document_id = d.id),
+      (SELECT count(*) FROM live WHERE document_id = d.id AND extracted_at IS NOT NULL),
+      (SELECT count(DISTINCT fact_id) FROM ev WHERE document_id = d.id),
+      (SELECT count(*) FROM ents WHERE document_id = d.id),
+      (SELECT count(*) FROM ents WHERE document_id = d.id AND n > 1),
+      (SELECT count(*) FROM resolution_reviews r
+        WHERE r.kb_id = '${KB}'${queuedAt ? ` AND r.created_at >= '${queuedAt}'::timestamptz` : ""}
+          AND EXISTS (SELECT 1 FROM ents WHERE document_id = d.id AND e IN (r.left_id, r.right_id))),
+      coalesce((SELECT string_agg(reason || '=' || s, ' ' ORDER BY reason) FROM
+        (SELECT reason, sum(count) AS s FROM extraction_drops WHERE document_id = d.id GROUP BY reason) r), ''),
+      ${queuedAt ? `coalesce((SELECT round(extract(epoch FROM max(extracted_at) - '${queuedAt}'::timestamptz))::text FROM live WHERE document_id = d.id), '')` : "''"})
+    FROM documents d WHERE d.kb_id = '${KB}' AND d.filename IN (${DOCS.map((x) => `'${x}.html'`).join(",")})
+    ORDER BY d.filename`)
+    .split("\n")
+    .filter(Boolean);
+  const out = {};
+  for (const l of rows) {
+    const [file, chunks, extracted, facts, entities, carried, pairs, drops, secs] = l.split("\u001f");
+    out[file.replace(/\.(html|md|pdf)$/i, "")] = {
+      chunks: +chunks,
+      extracted: +extracted,
+      facts: +facts,
+      entities: +entities,
+      carried: +carried,
+      pairs: +pairs,
+      drops: Object.fromEntries(drops.split(" ").filter(Boolean).map((kv) => [kv.slice(0, kv.lastIndexOf("=")), +kv.slice(kv.lastIndexOf("=") + 1)])),
+      seconds: secs === "" ? null : +secs,
+    };
+  }
+  console.log("\n每篇：块（已抽）/ 事实 / 实体 / 跨块认回 / 排给裁决的对 / 秒");
+  for (const [doc, v] of Object.entries(out)) {
+    console.log(`  ${doc}  ${v.chunks}(${v.extracted}) / ${v.facts} / ${v.entities} / ${v.carried} / ${v.pairs} / ${v.seconds ?? "-"}`);
+    const d = Object.entries(v.drops);
+    if (d.length) console.log(`    丢弃 ${d.map(([k, n]) => `${k}=${n}`).join(" ")}`);
+  }
+  return out;
+}
+
+// 全库排给裁决的对，按裁决到哪一步分开：一对可能跨两篇，按篇加起来会重，总数看这里。
+// 另列现在的 reason 前缀（还待裁的仍是 `name_vector|` / `contains|` 这些通道名，裁过的是裁决写的）。
+// 轮内裁决器可能已经判了一部分（merged / kept），所以三种状态都算——排过这一对就是代价。
+// 只算这一轮排的（created_at 不早于排队那一刻）：每轮开头删实体会连带删掉这些对（外键 CASCADE），
+// 但重抽本身不清它们，不靠那条连带也不会把上一轮的对算进来。--score 没有排队时刻，算的是库里现有的
+function pairs(queuedAt) {
+  const scope = `kb_id = '${KB}' ${queuedAt ? `AND created_at >= '${queuedAt}'::timestamptz` : ""}`;
+  const [total, pending, merged, kept] = psql(`
+    SELECT concat_ws(chr(31), count(*), count(*) FILTER (WHERE status = 'pending'),
+           count(*) FILTER (WHERE status = 'merged'), count(*) FILTER (WHERE status = 'kept'))
+      FROM resolution_reviews WHERE ${scope}`)
+    .split("\u001f")
+    .map(Number);
+  const byReason = Object.fromEntries(
+    psql(`SELECT k || '=' || n FROM (SELECT split_part(coalesce(reason, '-'), '|', 1) AS k, count(*) AS n
+           FROM resolution_reviews WHERE ${scope} GROUP BY 1) x ORDER BY k`)
+      .split("\n")
+      .filter(Boolean)
+      .map((kv) => [kv.slice(0, kv.lastIndexOf("=")), +kv.slice(kv.lastIndexOf("=") + 1)]),
+  );
+  console.log(`\n排给裁决的对 ${total}：待裁 ${pending} / 合并 ${merged} / 分开 ${kept}`);
+  console.log(`  现在的 reason：${Object.entries(byReason).map(([k, n]) => `${k}=${n}`).join(" ") || "-"}`);
+  return { total, pending, merged, kept, byReason };
+}
+
+function record(result, perDoc, extra) {
+  if (!args.out) return;
+  fs.mkdirSync(args.out, { recursive: true });
+  let sha = "";
+  try {
+    sha = execFileSync("git", ["rev-parse", "--short", "HEAD"], { cwd: HERE, encoding: "utf8" }).trim();
+  } catch {}
+  const at = new Date().toISOString();
+  const file = path.join(args.out, `recall-${at.replace(/[:.]/g, "-")}-${args.known}.json`);
+  fs.writeFileSync(
+    file,
+    JSON.stringify({ at, sha, kb: KB, known: args.known, label: args.label ?? null, ...extra, score: result, docs: perDoc }, null, 1),
+  );
+  console.log(`\n记下 ${file}`);
+}
+
+// 把 --out 记下的几轮按 known 分组排成表：每组的均值 ± 标准差，
+// 以及事实集合的重合度——组内两两比（运行间方差本身）与跨组两两比（known 的影响）。
+// 跨组的重合度落在组内的范围里，就是说空 known 挪动的没超出模型自己的抖动
+function table(dir) {
+  const runs = fs
+    .readdirSync(dir)
+    .filter((f) => f.endsWith(".json"))
+    .map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), "utf8")));
+  if (!runs.length) {
+    console.error(`${dir} 里没有记录`);
+    process.exit(1);
+  }
+  const groups = ["shown", "empty"].map((k) => [k, runs.filter((r) => r.known === k)]).filter(([, rs]) => rs.length);
+  const ms = (xs) => {
+    const v = xs.filter((x) => x !== null && x !== undefined);
+    if (!v.length) return "-";
+    const m = v.reduce((a, b) => a + b, 0) / v.length;
+    const sd = v.length > 1 ? Math.sqrt(v.reduce((a, b) => a + (b - m) ** 2, 0) / (v.length - 1)) : 0;
+    return `${m.toFixed(1)} ± ${sd.toFixed(1)}`;
+  };
+  const docs = [...new Set(runs.flatMap((r) => Object.keys(r.docs)))].sort();
+  const cols = groups.map(([k, rs]) => `known=${k} (n=${rs.length})`);
+  const row = (name, f) => `| ${name} | ${groups.map(([, rs]) => ms(rs.map(f))).join(" | ")} |`;
+  const lines = [`| | ${cols.join(" | ")} |`, `| --- |${cols.map(() => " --- |").join("")}`];
+  lines.push(row("总分（/52）", (r) => r.score.pass));
+  lines.push(row("排给裁决的对（全库）", (r) => r.pairs?.total));
+  lines.push(row("其中合并的", (r) => r.pairs?.merged));
+  for (const doc of docs) {
+    const d = (r) => r.docs[doc] ?? {};
+    lines.push(row(`${doc} 事实`, (r) => d(r).facts));
+    lines.push(row(`${doc} 实体`, (r) => d(r).entities));
+    lines.push(row(`${doc} 跨块认回`, (r) => d(r).carried));
+    lines.push(row(`${doc} 排给裁决的对`, (r) => d(r).pairs));
+    lines.push(row(`${doc} 丢弃`, (r) => Object.values(d(r).drops ?? {}).reduce((a, b) => a + b, 0)));
+    lines.push(row(`${doc} 秒`, (r) => d(r).seconds));
+  }
+  console.log(lines.join("\n"));
+
+  // 事实集合的 Jaccard：每篇文档各算，再平均
+  const jac = (a, b) => {
+    const xs = docs.map((doc) => {
+      const A = new Set(a.score.sigs?.[doc] ?? []), B = new Set(b.score.sigs?.[doc] ?? []);
+      const inter = [...A].filter((x) => B.has(x)).length;
+      const uni = new Set([...A, ...B]).size;
+      return uni ? inter / uni : 1;
+    });
+    return xs.reduce((s, x) => s + x, 0) / xs.length;
+  };
+  const pairs = (xs, ys) => (ys ? xs.flatMap((a) => ys.map((b) => jac(a, b))) : xs.flatMap((a, i) => xs.slice(i + 1).map((b) => jac(a, b))));
+  const fmt = (v) => (v.length ? `${ms(v.map((x) => x * 100))}%（${v.length} 对）` : "-");
+  console.log("\n事实集合重合度（Jaccard，按文档平均）：");
+  for (const [k, rs] of groups) console.log(`  组内 known=${k}：${fmt(pairs(rs))}`);
+  if (groups.length === 2) console.log(`  跨组：${fmt(pairs(groups[0][1], groups[1][1]))}`);
 }
 
 if (args.score) {
-  score();
+  record(score(), stats(null), { pairs: pairs(null) });
   process.exit(0);
 }
 
@@ -217,6 +404,9 @@ console.log(`本体 ${num(`SELECT count(*) FROM relation_types WHERE kb_id='${KB
 
 const docs = (await api("GET", `/api/v1/kbs/${KB}/documents?limit=200`)).docs.filter((d) => want.includes(d.filename));
 const endpoint = args.reprocess ? "reprocess" : "extract";
+// 墙钟从这一刻算到每篇最后一块抽完：取库的时钟，与 extracted_at 同一个钟
+const queuedAt = psql("SELECT now()::text");
+if (args.known) console.log(`这一轮标 known=${args.known}：服务端的 UTOPIA_EXTRACT_KNOWN_IN_PROMPT 得对得上`);
 for (const d of docs) {
   await api("POST", `/api/v1/documents/${d.id}/${endpoint}`, {});
   console.log(`${stamp()} 排队 ${endpoint} ${d.filename}`);
@@ -241,4 +431,4 @@ for (;;) {
   await sleep(30000);
 }
 
-score();
+record(score(), stats(queuedAt), { endpoint, pairs: pairs(queuedAt) });