diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 1f9713386..004854875 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -72,6 +72,24 @@ node scripts/bench/judge_open.mjs --kb --sample 200 # 开放陈述有 服务端带 `UTOPIA_EXTRACT_KNOWN_IN_PROMPT=false` 起,照常跑 `recall.mjs`,与不带时的几轮比。 这个开关只动提示词,落库时按名字认回前面分块的实体照旧。 +``` +# 服务端照常起:每轮记一份 +node scripts/bench/recall.mjs --kb --known shown --out runs/588 +# 服务端带 UTOPIA_EXTRACT_KNOWN_IN_PROMPT=false 重起:同样几轮 +node scripts/bench/recall.mjs --kb --known empty --out runs/588 +# 按 known 分组排表 +node scripts/bench/recall.mjs --table runs/588 +``` + +`--known` 只是标签,台子查不到服务端的开关,两边对不上那一轮就作废。每轮除了总分还记每篇的 +分块、事实、实体、**跨块认回**(一个实体出现在这篇两个以上分块的事实里)、**排给裁决的对**(这一轮进了 +`resolution_reviews` 的疑似同一对,#877;提示词里没有 known 时同一个东西换个写法再列一遍,落在这里而不是静默合并。 +不分召回通道:一对裁过之后 reason 被裁决改写,`name_vector|` 留不下来。全库总数另按待裁 / 合并 / 分开和现在的 +reason 前缀记;只算这一轮排队之后排的对,`--score` 没有排队时刻,算的是库里现有的——每轮开头删实体时它们随之删掉)、丢弃账(`extraction_drops` +按原因)和排队到最后一块抽完的秒数。`--table` 给每组的均值 ± 标准差,再给事实集合(去重的主/谓/宾) +的重合度:组内两两比是运行间的抖动,跨组两两比是 known 的影响——跨组落在组内的范围里,空 known 就没把 +输出挪出方差。 + 抽取写的是开放图谱(0044 决定 2,没有带本体的第二条路):陈述按原文短语落成 `layer='open'` 的 事实行,限定挂在 `statement_qualifiers`,时间词进 `time_mentions`;打分口径照旧(谓词取 `proposed_predicate`,限定也拼进那一行),所以下表里带本体那些轮次的分数仍然可比。`judge_open.mjs` 是 diff --git a/scripts/bench/recall.mjs b/scripts/bench/recall.mjs index cc9f85b9f..bcc12f005 100644 --- a/scripts/bench/recall.mjs +++ b/scripts/bench/recall.mjs @@ -16,6 +16,12 @@ // node scripts/bench/recall.mjs --kb # 已有库(本体向量已就绪) // node scripts/bench/recall.mjs --kb --score # 只打分,不重抽 // node scripts/bench/recall.mjs --kb --reprocess # 改了解析器:连分块一起重来 +// node scripts/bench/recall.mjs --kb --known empty --out runs/588 # 记一轮,写成 JSON +// node scripts/bench/recall.mjs --table runs/588 # 把记下的几轮排成表 +// +// `--known shown|empty` 只是**标签**:提示词里带不带前面分块认下的实体,由服务端的 +// `UTOPIA_EXTRACT_KNOWN_IN_PROMPT` 决定(#588),起服务时定死,台子改不了也查不到。 +// 标签写进记录,`--table` 按它分组;与服务端对不上,那一轮的数就是错的。 // // 环境变量:BENCH_BASE / BENCH_EMAIL / BENCH_PASSWORD / BENCH_PSQL(同 run.mjs)。 // @@ -40,6 +46,18 @@ const args = Object.fromEntries( return acc; }, []), ); +if (args.table) { + table(args.table); + process.exit(0); +} +if (args.known !== undefined && !["shown", "empty"].includes(args.known)) { + console.error("--known 只认 shown 或 empty(与服务端的 UTOPIA_EXTRACT_KNOWN_IN_PROMPT 对上)"); + process.exit(1); +} +if (args.out && !args.known) { + console.error("--out 要带 --known shown|empty:不标清这一轮提示词里有没有 known,记下来的数没法分组"); + process.exit(1); +} const KB = args.kb; if (!KB) { console.error("要一个 --kb :建一个装了本体包的库,等它的本体向量补齐,再把 id 给这里。"); @@ -170,10 +188,179 @@ function score() { // **52 个条目、九成命中率,一个标准差 ≈ 2.7 条。** 单轮差一两条不是证据, // 同一份代码重跑一遍再说;结构性的改善看 drops 计数与某一类是否整类进来了 console.log("\n(52 条真值,1σ ≈ 2.7 条:单轮小幅波动不作数)"); + + // 去重后的 (主, 谓, 宾):两轮之间比的是「抽出来的是不是同一批」,不只是个数 + const sigs = {}; + for (const [doc, facts] of byDoc) sigs[doc] = [...new Set(facts.map((f) => `${norm(f.subj)}\u001f${norm(f.pred)}\u001f${norm(f.obj)}`))].sort(); + return { + pass, + total: truth.length, + perDoc: Object.fromEntries([...perDoc].map(([k, v]) => [k, v.pass])), + misses: misses.map((m) => m.id), + sigs, + }; +} + +// 每篇文档的形状:分块、事实、实体、跨块认回的实体、丢弃账(extraction_drops), +// 以及排队到最后一块抽完的秒数(给了 queuedAt 才有)。 +// 「跨块认回」= 一个实体出现在这篇文档**两个以上分块**的事实里:前面的块认下、后面的块接着用, +// known 管的就是这一件(名字认回也算在内——那半步不受开关管,差出来的是提示词的份) +// 「排给裁决的对」= 这一轮进了 resolution_reviews 的疑似同一对,不分是哪条召回通道提的: +// 提示词里没有 known,后面的块更容易换个写法再列一遍同一个东西,这种不会静默合并(#877), +// 而是在这里多排一对——召回分不动、这个数涨,就是空 known 的代价。一对算进两边实体出现的每篇。 +// **不按 `name_vector|` 筛**:一对一旦裁了(治理、升级给人、随合并作废),reason 就被裁决改写 +// (`governed|…` 等),原来是哪条通道提的不留痕;按它筛,一轮跑完几乎总是 0 +function stats(queuedAt) { + const rows = psql(` + WITH live AS ( + SELECT c.id, c.document_id, c.extracted_at FROM chunks c JOIN documents d ON d.id = c.document_id + WHERE d.kb_id = '${KB}' AND c.superseded_at IS NULL), + ev AS ( + SELECT l.document_id, fe.chunk_id, f.id AS fact_id, f.subject_id, f.object_id + FROM fact_evidence fe JOIN live l ON l.id = fe.chunk_id JOIN facts f ON f.id = fe.fact_id + WHERE f.kb_id = '${KB}' AND f.invalidated_at IS NULL), + ents AS ( + SELECT document_id, e, count(DISTINCT chunk_id) AS n FROM ( + SELECT document_id, chunk_id, subject_id AS e FROM ev + UNION ALL SELECT document_id, chunk_id, object_id FROM ev WHERE object_id IS NOT NULL) x + GROUP BY 1, 2) + SELECT concat_ws(chr(31), d.filename, + (SELECT count(*) FROM live WHERE document_id = d.id), + (SELECT count(*) FROM live WHERE document_id = d.id AND extracted_at IS NOT NULL), + (SELECT count(DISTINCT fact_id) FROM ev WHERE document_id = d.id), + (SELECT count(*) FROM ents WHERE document_id = d.id), + (SELECT count(*) FROM ents WHERE document_id = d.id AND n > 1), + (SELECT count(*) FROM resolution_reviews r + WHERE r.kb_id = '${KB}'${queuedAt ? ` AND r.created_at >= '${queuedAt}'::timestamptz` : ""} + AND EXISTS (SELECT 1 FROM ents WHERE document_id = d.id AND e IN (r.left_id, r.right_id))), + coalesce((SELECT string_agg(reason || '=' || s, ' ' ORDER BY reason) FROM + (SELECT reason, sum(count) AS s FROM extraction_drops WHERE document_id = d.id GROUP BY reason) r), ''), + ${queuedAt ? `coalesce((SELECT round(extract(epoch FROM max(extracted_at) - '${queuedAt}'::timestamptz))::text FROM live WHERE document_id = d.id), '')` : "''"}) + FROM documents d WHERE d.kb_id = '${KB}' AND d.filename IN (${DOCS.map((x) => `'${x}.html'`).join(",")}) + ORDER BY d.filename`) + .split("\n") + .filter(Boolean); + const out = {}; + for (const l of rows) { + const [file, chunks, extracted, facts, entities, carried, pairs, drops, secs] = l.split("\u001f"); + out[file.replace(/\.(html|md|pdf)$/i, "")] = { + chunks: +chunks, + extracted: +extracted, + facts: +facts, + entities: +entities, + carried: +carried, + pairs: +pairs, + drops: Object.fromEntries(drops.split(" ").filter(Boolean).map((kv) => [kv.slice(0, kv.lastIndexOf("=")), +kv.slice(kv.lastIndexOf("=") + 1)])), + seconds: secs === "" ? null : +secs, + }; + } + console.log("\n每篇:块(已抽)/ 事实 / 实体 / 跨块认回 / 排给裁决的对 / 秒"); + for (const [doc, v] of Object.entries(out)) { + console.log(` ${doc} ${v.chunks}(${v.extracted}) / ${v.facts} / ${v.entities} / ${v.carried} / ${v.pairs} / ${v.seconds ?? "-"}`); + const d = Object.entries(v.drops); + if (d.length) console.log(` 丢弃 ${d.map(([k, n]) => `${k}=${n}`).join(" ")}`); + } + return out; +} + +// 全库排给裁决的对,按裁决到哪一步分开:一对可能跨两篇,按篇加起来会重,总数看这里。 +// 另列现在的 reason 前缀(还待裁的仍是 `name_vector|` / `contains|` 这些通道名,裁过的是裁决写的)。 +// 轮内裁决器可能已经判了一部分(merged / kept),所以三种状态都算——排过这一对就是代价。 +// 只算这一轮排的(created_at 不早于排队那一刻):每轮开头删实体会连带删掉这些对(外键 CASCADE), +// 但重抽本身不清它们,不靠那条连带也不会把上一轮的对算进来。--score 没有排队时刻,算的是库里现有的 +function pairs(queuedAt) { + const scope = `kb_id = '${KB}' ${queuedAt ? `AND created_at >= '${queuedAt}'::timestamptz` : ""}`; + const [total, pending, merged, kept] = psql(` + SELECT concat_ws(chr(31), count(*), count(*) FILTER (WHERE status = 'pending'), + count(*) FILTER (WHERE status = 'merged'), count(*) FILTER (WHERE status = 'kept')) + FROM resolution_reviews WHERE ${scope}`) + .split("\u001f") + .map(Number); + const byReason = Object.fromEntries( + psql(`SELECT k || '=' || n FROM (SELECT split_part(coalesce(reason, '-'), '|', 1) AS k, count(*) AS n + FROM resolution_reviews WHERE ${scope} GROUP BY 1) x ORDER BY k`) + .split("\n") + .filter(Boolean) + .map((kv) => [kv.slice(0, kv.lastIndexOf("=")), +kv.slice(kv.lastIndexOf("=") + 1)]), + ); + console.log(`\n排给裁决的对 ${total}:待裁 ${pending} / 合并 ${merged} / 分开 ${kept}`); + console.log(` 现在的 reason:${Object.entries(byReason).map(([k, n]) => `${k}=${n}`).join(" ") || "-"}`); + return { total, pending, merged, kept, byReason }; +} + +function record(result, perDoc, extra) { + if (!args.out) return; + fs.mkdirSync(args.out, { recursive: true }); + let sha = ""; + try { + sha = execFileSync("git", ["rev-parse", "--short", "HEAD"], { cwd: HERE, encoding: "utf8" }).trim(); + } catch {} + const at = new Date().toISOString(); + const file = path.join(args.out, `recall-${at.replace(/[:.]/g, "-")}-${args.known}.json`); + fs.writeFileSync( + file, + JSON.stringify({ at, sha, kb: KB, known: args.known, label: args.label ?? null, ...extra, score: result, docs: perDoc }, null, 1), + ); + console.log(`\n记下 ${file}`); +} + +// 把 --out 记下的几轮按 known 分组排成表:每组的均值 ± 标准差, +// 以及事实集合的重合度——组内两两比(运行间方差本身)与跨组两两比(known 的影响)。 +// 跨组的重合度落在组内的范围里,就是说空 known 挪动的没超出模型自己的抖动 +function table(dir) { + const runs = fs + .readdirSync(dir) + .filter((f) => f.endsWith(".json")) + .map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"))); + if (!runs.length) { + console.error(`${dir} 里没有记录`); + process.exit(1); + } + const groups = ["shown", "empty"].map((k) => [k, runs.filter((r) => r.known === k)]).filter(([, rs]) => rs.length); + const ms = (xs) => { + const v = xs.filter((x) => x !== null && x !== undefined); + if (!v.length) return "-"; + const m = v.reduce((a, b) => a + b, 0) / v.length; + const sd = v.length > 1 ? Math.sqrt(v.reduce((a, b) => a + (b - m) ** 2, 0) / (v.length - 1)) : 0; + return `${m.toFixed(1)} ± ${sd.toFixed(1)}`; + }; + const docs = [...new Set(runs.flatMap((r) => Object.keys(r.docs)))].sort(); + const cols = groups.map(([k, rs]) => `known=${k} (n=${rs.length})`); + const row = (name, f) => `| ${name} | ${groups.map(([, rs]) => ms(rs.map(f))).join(" | ")} |`; + const lines = [`| | ${cols.join(" | ")} |`, `| --- |${cols.map(() => " --- |").join("")}`]; + lines.push(row("总分(/52)", (r) => r.score.pass)); + lines.push(row("排给裁决的对(全库)", (r) => r.pairs?.total)); + lines.push(row("其中合并的", (r) => r.pairs?.merged)); + for (const doc of docs) { + const d = (r) => r.docs[doc] ?? {}; + lines.push(row(`${doc} 事实`, (r) => d(r).facts)); + lines.push(row(`${doc} 实体`, (r) => d(r).entities)); + lines.push(row(`${doc} 跨块认回`, (r) => d(r).carried)); + lines.push(row(`${doc} 排给裁决的对`, (r) => d(r).pairs)); + lines.push(row(`${doc} 丢弃`, (r) => Object.values(d(r).drops ?? {}).reduce((a, b) => a + b, 0))); + lines.push(row(`${doc} 秒`, (r) => d(r).seconds)); + } + console.log(lines.join("\n")); + + // 事实集合的 Jaccard:每篇文档各算,再平均 + const jac = (a, b) => { + const xs = docs.map((doc) => { + const A = new Set(a.score.sigs?.[doc] ?? []), B = new Set(b.score.sigs?.[doc] ?? []); + const inter = [...A].filter((x) => B.has(x)).length; + const uni = new Set([...A, ...B]).size; + return uni ? inter / uni : 1; + }); + return xs.reduce((s, x) => s + x, 0) / xs.length; + }; + const pairs = (xs, ys) => (ys ? xs.flatMap((a) => ys.map((b) => jac(a, b))) : xs.flatMap((a, i) => xs.slice(i + 1).map((b) => jac(a, b)))); + const fmt = (v) => (v.length ? `${ms(v.map((x) => x * 100))}%(${v.length} 对)` : "-"); + console.log("\n事实集合重合度(Jaccard,按文档平均):"); + for (const [k, rs] of groups) console.log(` 组内 known=${k}:${fmt(pairs(rs))}`); + if (groups.length === 2) console.log(` 跨组:${fmt(pairs(groups[0][1], groups[1][1]))}`); } if (args.score) { - score(); + record(score(), stats(null), { pairs: pairs(null) }); process.exit(0); } @@ -217,6 +404,9 @@ console.log(`本体 ${num(`SELECT count(*) FROM relation_types WHERE kb_id='${KB const docs = (await api("GET", `/api/v1/kbs/${KB}/documents?limit=200`)).docs.filter((d) => want.includes(d.filename)); const endpoint = args.reprocess ? "reprocess" : "extract"; +// 墙钟从这一刻算到每篇最后一块抽完:取库的时钟,与 extracted_at 同一个钟 +const queuedAt = psql("SELECT now()::text"); +if (args.known) console.log(`这一轮标 known=${args.known}:服务端的 UTOPIA_EXTRACT_KNOWN_IN_PROMPT 得对得上`); for (const d of docs) { await api("POST", `/api/v1/documents/${d.id}/${endpoint}`, {}); console.log(`${stamp()} 排队 ${endpoint} ${d.filename}`); @@ -241,4 +431,4 @@ for (;;) { await sleep(30000); } -score(); +record(score(), stats(queuedAt), { endpoint, pairs: pairs(queuedAt) });