From 0703e3fc4dcf0fbc954bfc7de1195c9d365cd55f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 16 Sep 2026 19:00:05 -0400 Subject: [PATCH] history: Store compact per-item AIC totals Consumers need durable issue and pull-request AIC attribution, not a copy of metadata already available from GitHub. Keep detailed records limited to identity and usage while preserving full activity aggregates and auditable run evidence. Assisted-by: AI Signed-off-by: Colin Walters --- scripts/README.md | 6 ++- scripts/org-history.js | 41 +++++++++++++----- tests/org-history.test.js | 91 +++++++++++++++++---------------------- 3 files changed, 76 insertions(+), 62 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index ab55446..eea064a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -18,7 +18,11 @@ node scripts/org-history.js --previous-iso-week Schema v5 retains the v4 period field names and changes detailed `items`: it contains only items with exact evidence linking them to a relevant workflow run with known AIC. -Each detailed item has sorted, unique `aicRunIds`. Repository aggregates still describe +Each record contains only `repository`, `number`, `type`, `aic`, and sorted, unique +`aicRunIds`; generic GitHub metadata is intentionally omitted. `aic` is the normalized +sum for its linked runs, including zero. A run linked exactly to multiple items contributes +its full AIC to each association, so item AIC values are not additive; use +`coverage.aic.total` for the organization-wide total. Repository aggregates still describe the full collected activity cohort, not just these AIC-linked detailed items; the linked run evidence is serialized for audit. diff --git a/scripts/org-history.js b/scripts/org-history.js index 4d7906c..d051559 100644 --- a/scripts/org-history.js +++ b/scripts/org-history.js @@ -11,6 +11,7 @@ const path = require('path'); const SCHEMA_VERSION = 5; const AGENT_LABELS = ['agent/code', 'agent/fixme', 'agent/lgtm']; +const DETAILED_ITEM_FIELDS = ['repository', 'number', 'type', 'aic', 'aicRunIds']; const AGGREGATE_USAGE_FILE = 'agent_usage.json'; const TOKEN_USAGE_FILE = 'agent/token_usage.jsonl'; @@ -326,7 +327,10 @@ function aicLinkedItems(items, runs, commentAic, org) { ids.add(runId); links.set(item, ids); }; - for (const evidence of commentAic) link(evidence.repository, evidence.itemNumber, evidence.runId); + for (const evidence of commentAic) { + const item = itemsByKey.get(`${evidence.repository}/${evidence.itemNumber}`); + if (item && evidence.itemUrl === item.url) link(evidence.repository, evidence.itemNumber, evidence.runId); + } for (const run of runById.values()) { for (const pullRequest of normalizedPullRequests(run, org, run.repository)) { const candidate = itemsByKey.get(`${run.repository}/${pullRequest.number}`); @@ -335,26 +339,43 @@ function aicLinkedItems(items, runs, commentAic, org) { } } } - return [...links.entries()].map(([item, runIds]) => ({ ...item, aicRunIds: [...runIds].sort((a, b) => a - b) })) + return [...links.entries()].map(([item, runIds]) => { + const aicRunIds = [...runIds].sort((a, b) => a - b); + return { + repository: item.repository, + number: item.number, + type: item.type, + aic: normalizeAic(aicRunIds.reduce((total, runId) => total + runById.get(runId).aic, 0)), + aicRunIds, + }; + }) .sort((a, b) => a.repository.localeCompare(b.repository) || a.number - b.number); } function validateDetailedItems(items, workflowRuns, commentAic, org) { - const runById = new Map(workflowRuns.map((run) => [run.id, run])); + const identities = new Set(); for (const item of items) { - if (!Array.isArray(item.aicRunIds) || !item.aicRunIds.length) throw new Error(`Detailed item ${item.repository}#${item.number} has no AIC runs`); - const itemPath = item.type === 'pull_request' ? 'pull' : item.type === 'issue' ? 'issues' : null; - const canonicalItemUrl = itemPath && `https://github.com/${org}/${item.repository}/${itemPath}/${item.number}`; - if (item.url !== canonicalItemUrl) throw new Error(`Detailed item ${item.repository}#${item.number} has a mismatched canonical URL`); + if (!item || typeof item !== 'object' || Array.isArray(item) || Object.keys(item).length !== DETAILED_ITEM_FIELDS.length || Object.keys(item).some((field) => !DETAILED_ITEM_FIELDS.includes(field))) throw new Error('Detailed item has unexpected fields'); + if (typeof item.repository !== 'string' || !/^[A-Za-z0-9_.-]+$/.test(item.repository) || !Number.isSafeInteger(item.number) || item.number <= 0 || !['issue', 'pull_request'].includes(item.type)) throw new Error('Detailed item has an invalid identity'); + const identity = `${item.repository}/${item.number}`; + if (identities.has(identity)) throw new Error(`Detailed item ${item.repository}#${item.number} is duplicated`); + identities.add(identity); + if (!finiteNonnegative(item.aic)) throw new Error(`Detailed item ${item.repository}#${item.number} has invalid AIC`); + if (!Array.isArray(item.aicRunIds) || !item.aicRunIds.length || item.aicRunIds.some((runId) => !Number.isSafeInteger(runId) || runId <= 0)) throw new Error(`Detailed item ${item.repository}#${item.number} has no valid AIC runs`); const sortedUnique = [...new Set(item.aicRunIds)].sort((a, b) => a - b); if (JSON.stringify(item.aicRunIds) !== JSON.stringify(sortedUnique)) throw new Error(`Detailed item ${item.repository}#${item.number} has unsorted or duplicate AIC runs`); + const itemPath = item.type === 'pull_request' ? 'pull' : 'issues'; + const canonicalItemUrl = `https://github.com/${org}/${item.repository}/${itemPath}/${item.number}`; + let total = 0; for (const runId of item.aicRunIds) { - const run = runById.get(runId); + const run = workflowRuns.find((candidate) => candidate && candidate.id === runId && candidate.repository === item.repository); if (!run || run.repository !== item.repository || !finiteNonnegative(run.aic)) throw new Error(`Detailed item ${item.repository}#${item.number} references unknown-AIC run ${runId}`); - const commentEvidence = (commentAic || []).some((record) => record.repository === item.repository && record.runId === runId && record.itemNumber === item.number && record.itemUrl === item.url); - const pullRequestEvidence = item.type === 'pull_request' && (run.pullRequests || []).some((reference) => reference.number === item.number && reference.url === item.url); + total += run.aic; + const commentEvidence = (commentAic || []).some((record) => record.repository === item.repository && record.runId === runId && record.itemNumber === item.number && record.itemUrl === canonicalItemUrl); + const pullRequestEvidence = item.type === 'pull_request' && (run.pullRequests || []).some((reference) => reference.number === item.number && reference.url === canonicalItemUrl); if (!commentEvidence && !pullRequestEvidence) throw new Error(`Detailed item ${item.repository}#${item.number} lacks exact evidence for run ${runId}`); } + if (item.aic !== normalizeAic(total)) throw new Error(`Detailed item ${item.repository}#${item.number} has an incorrect AIC total`); } } diff --git a/tests/org-history.test.js b/tests/org-history.test.js index bba6d58..14c538a 100644 --- a/tests/org-history.test.js +++ b/tests/org-history.test.js @@ -53,76 +53,65 @@ test('supports deterministic as-of week selection from the CLI', () => { assert.throws(() => childProcess.execFileSync(process.execPath, [script, '--previous-iso-week', '--as-of', 'not-a-date'], { encoding: 'utf8' })); }); -test('links detailed items only through exact known-AIC comment or PR evidence', () => { +test('projects compact AIC-linked items from exact known-AIC evidence', () => { const items = [ - { repository: 'repo', number: 1, type: 'issue', url: 'https://github.com/org/repo/issues/1' }, - { repository: 'repo', number: 2, type: 'pull_request', url: 'https://github.com/org/repo/pull/2' }, + { repository: 'repo', number: 1, type: 'issue', url: 'https://github.com/org/repo/issues/1', updatedAt: 'metadata' }, + { repository: 'repo', number: 2, type: 'pull_request', url: 'https://github.com/org/repo/pull/2', title: 'metadata' }, { repository: 'repo', number: 3, type: 'issue', url: 'https://github.com/org/repo/issues/3' }, ]; const run = (id, aic, pullRequests = []) => ({ id, repository: 'repo', kind: 'drafter', aic, pull_requests: pullRequests }); const pull = (number, repository = 'repo') => ({ url: `https://api.github.com/repos/org/${repository}/pulls/${number}` }); + const comment = (runId, itemNumber) => ({ repository: 'repo', runId, itemNumber, itemUrl: items.find((item) => item.number === itemNumber).url }); for (const scenario of [ - { name: 'comment', runs: [run(10, 1)], comments: [{ repository: 'repo', runId: 10, itemNumber: 1 }], expected: [[1, [10]]] }, - { name: 'pull request', runs: [run(11, 2, [pull(2)])], comments: [], expected: [[2, [11]]] }, - { name: 'unrelated evidence', runs: [run(12, 3, [pull(3, 'other')]), run(13, null, [pull(2)])], comments: [{ repository: 'repo', runId: 12, itemNumber: 99 }], expected: [] }, - { name: 'deduplicates links', runs: [run(14, 4, [pull(2), pull(2)])], comments: [{ repository: 'repo', runId: 14, itemNumber: 2 }, { repository: 'repo', runId: 14, itemNumber: 2 }], expected: [[2, [14]]] }, - { name: 'keeps multiple comment targets', runs: [run(16, 5)], comments: [{ repository: 'repo', runId: 16, itemNumber: 1 }, { repository: 'repo', runId: 16, itemNumber: 3 }], expected: [[1, [16]], [3, [16]]] }, - { name: 'retains zero AIC', runs: [run(15, 0)], comments: [{ repository: 'repo', runId: 15, itemNumber: 1 }], expected: [[1, [15]]] }, - ]) { - const actual = aicLinkedItems(items, scenario.runs, scenario.comments, 'org').map((item) => [item.number, item.aicRunIds]); - assert.deepEqual(actual, scenario.expected, scenario.name); - } + { name: 'comment', runs: [run(10, 1)], comments: [comment(10, 1)], expected: [{ repository: 'repo', number: 1, type: 'issue', aic: 1, aicRunIds: [10] }] }, + { name: 'pull request', runs: [run(11, 2, [pull(2)])], comments: [], expected: [{ repository: 'repo', number: 2, type: 'pull_request', aic: 2, aicRunIds: [11] }] }, + { name: 'unrelated or cross-repository evidence', runs: [run(12, 3, [pull(3, 'other')]), run(13, null, [pull(2)])], comments: [{ ...comment(12, 1), itemNumber: 99 }], expected: [] }, + { name: 'sums multiple runs', runs: [run(10, 1.25), run(14, 2.5)], comments: [comment(10, 1), comment(14, 1)], expected: [{ repository: 'repo', number: 1, type: 'issue', aic: 3.75, aicRunIds: [10, 14] }] }, + { name: 'deduplicates evidence', runs: [run(14, 4, [pull(2), pull(2)])], comments: [comment(14, 2), comment(14, 2)], expected: [{ repository: 'repo', number: 2, type: 'pull_request', aic: 4, aicRunIds: [14] }] }, + { name: 'attributes one run fully to multiple items', runs: [run(16, 5)], comments: [comment(16, 1), comment(16, 3)], expected: [{ repository: 'repo', number: 1, type: 'issue', aic: 5, aicRunIds: [16] }, { repository: 'repo', number: 3, type: 'issue', aic: 5, aicRunIds: [16] }] }, + { name: 'retains zero AIC', runs: [run(15, 0)], comments: [comment(15, 1)], expected: [{ repository: 'repo', number: 1, type: 'issue', aic: 0, aicRunIds: [15] }] }, + ]) assert.deepEqual(aicLinkedItems(items, scenario.runs, scenario.comments, 'org'), scenario.expected, scenario.name); }); test('parses comment AIC with transparent same-repository item identity', () => { - const comment = parseCommentAic({ - id: 1, - issue_url: 'https://api.github.com/repos/org/repo/issues/7', html_url: 'https://github.com/org/repo/pull/7#issuecomment-9', - body: '> Generated by [workflow](https://github.com/org/repo/actions/runs/42) · 0 AIC', - }, 'org', 'repo'); - assert.deepEqual({ runId: comment.runId, itemNumber: comment.itemNumber, itemUrl: comment.itemUrl, aic: comment.aic }, { - runId: 42, itemNumber: 7, itemUrl: 'https://github.com/org/repo/pull/7', aic: 0, - }); + const comment = parseCommentAic({ id: 1, issue_url: 'https://api.github.com/repos/org/repo/issues/7', html_url: 'https://github.com/org/repo/pull/7#issuecomment-9', body: '> Generated by [workflow](https://github.com/org/repo/actions/runs/42) · 0 AIC' }, 'org', 'repo'); + assert.deepEqual({ runId: comment.runId, itemNumber: comment.itemNumber, itemUrl: comment.itemUrl, aic: comment.aic }, { runId: 42, itemNumber: 7, itemUrl: 'https://github.com/org/repo/pull/7', aic: 0 }); assert.equal(parseCommentAic({ id: 1, issue_url: 'https://api.github.com/repos/other/repo/issues/7', html_url: 'https://github.com/other/repo/issues/7#issuecomment-9', body: '> Generated by [workflow](https://github.com/org/repo/actions/runs/42) · 1 AIC' }, 'org', 'repo'), null); assert.equal(parseCommentAic({ id: 1, issue_url: 'https://api.github.com/repos/org/repo/issues/7', html_url: 'https://github.com/org/repo/issues/7#issuecomment-9', body: '> Generated by [workflow](https://github.com/other/repo/actions/runs/42) · 1 AIC' }, 'org', 'repo'), null); assert.equal(parseCommentAic({ id: 1, issue_url: 'https://api.github.com/repos/org/repo/issues/7', html_url: 'https://github.com/org/repo/issues/7#issuecomment-9', body: '> Generated by [workflow](https://github.com/org/repo/actions/runs/42) · 1 AIC\n' }, 'org', 'repo'), null); }); test('retains consistent comment evidence for every exact item target', () => { - const records = deduplicateCommentAic([ - { runId: 20, commentId: 1, commentUrl: 'one', agentAic: 1, detectionAic: null, aic: 1, repository: 'repo', itemNumber: 1 }, - { runId: 20, commentId: 2, commentUrl: 'two', agentAic: 1, detectionAic: null, aic: 1, repository: 'repo', itemNumber: 2 }, - ]).records; - const items = [{ repository: 'repo', number: 1, type: 'issue' }, { repository: 'repo', number: 2, type: 'issue' }]; - assert.deepEqual(aicLinkedItems(items, [{ id: 20, repository: 'repo', kind: 'drafter', aic: 1 }], records, 'org').map((item) => [item.number, item.aicRunIds]), [[1, [20]], [2, [20]]]); + const records = deduplicateCommentAic([{ runId: 20, commentId: 1, commentUrl: 'one', agentAic: 1, detectionAic: null, aic: 1, repository: 'repo', itemNumber: 1 }, { runId: 20, commentId: 2, commentUrl: 'two', agentAic: 1, detectionAic: null, aic: 1, repository: 'repo', itemNumber: 2 }]).records; + const items = [{ repository: 'repo', number: 1, type: 'issue', url: 'https://github.com/org/repo/issues/1' }, { repository: 'repo', number: 2, type: 'issue', url: 'https://github.com/org/repo/issues/2' }]; + for (const record of records) record.itemUrl = `https://github.com/org/repo/issues/${record.itemNumber}`; + assert.deepEqual(aicLinkedItems(items, [{ id: 20, repository: 'repo', kind: 'drafter', aic: 1 }], records, 'org'), [{ repository: 'repo', number: 1, type: 'issue', aic: 1, aicRunIds: [20] }, { repository: 'repo', number: 2, type: 'issue', aic: 1, aicRunIds: [20] }]); }); test('normalizes only same-repository workflow PR references', () => { - assert.deepEqual(normalizedPullRequests({ pull_requests: [ - { url: 'https://api.github.com/repos/org/repo/pulls/2' }, - { url: 'https://api.github.com/repos/org/repo/pulls/1' }, - { url: 'https://api.github.com/repos/org/repo/pulls/2' }, - { url: 'https://api.github.com/repos/org/other/pulls/3' }, - ] }, 'org', 'repo'), [ - { number: 1, url: 'https://github.com/org/repo/pull/1' }, - { number: 2, url: 'https://github.com/org/repo/pull/2' }, - ]); + assert.deepEqual(normalizedPullRequests({ pull_requests: [{ url: 'https://api.github.com/repos/org/repo/pulls/2' }, { url: 'https://api.github.com/repos/org/repo/pulls/1' }, { url: 'https://api.github.com/repos/org/repo/pulls/2' }, { url: 'https://api.github.com/repos/org/other/pulls/3' }] }, 'org', 'repo'), [{ number: 1, url: 'https://github.com/org/repo/pull/1' }, { number: 2, url: 'https://github.com/org/repo/pull/2' }]); }); -test('validates detailed item links have exact serialized evidence', () => { - const issue = { repository: 'repo', number: 1, type: 'issue', url: 'https://github.com/org/repo/issues/1', aicRunIds: [1] }; - const pullRequest = { repository: 'repo', number: 2, type: 'pull_request', url: 'https://github.com/org/repo/pull/2', aicRunIds: [2] }; - const commentAic = [{ repository: 'repo', runId: 1, itemNumber: 1, itemUrl: issue.url }]; - const workflowRuns = [ - { id: 1, repository: 'repo', aic: 0, pullRequests: [] }, - { id: 2, repository: 'repo', aic: 1, pullRequests: [{ number: 2, url: pullRequest.url }] }, - ]; +test('validates compact detailed item records and exact serialized evidence', () => { + const issue = { repository: 'repo', number: 1, type: 'issue', aic: 0, aicRunIds: [1] }; + const pullRequest = { repository: 'repo', number: 2, type: 'pull_request', aic: 1, aicRunIds: [2] }; + const issueUrl = 'https://github.com/org/repo/issues/1'; + const pullRequestUrl = 'https://github.com/org/repo/pull/2'; + const commentAic = [{ repository: 'repo', runId: 1, itemNumber: 1, itemUrl: issueUrl }]; + const workflowRuns = [{ id: 1, repository: 'repo', aic: 0, pullRequests: [] }, { id: 2, repository: 'repo', aic: 1, pullRequests: [{ number: 2, url: pullRequestUrl }] }]; assert.doesNotThrow(() => validateDetailedItems([issue, pullRequest], workflowRuns, commentAic, 'org')); - assert.throws(() => validateDetailedItems([issue], [{ id: 1, repository: 'repo', aic: 0, pullRequests: [] }], [], 'org')); - assert.throws(() => validateDetailedItems([pullRequest], [{ id: 2, repository: 'repo', aic: 1, pullRequests: [{ number: 2, url: issue.url }] }], [], 'org')); - assert.throws(() => validateDetailedItems([{ ...issue, type: 'pull_request' }], workflowRuns, commentAic, 'org')); - assert.throws(() => validateDetailedItems([{ ...issue, url: 'https://github.com/other/repo/issues/1' }], workflowRuns, [{ ...commentAic[0], itemUrl: 'https://github.com/other/repo/issues/1' }], 'org')); - assert.throws(() => validateDetailedItems([{ ...issue, aicRunIds: [1, 1] }], workflowRuns, commentAic, 'org')); - assert.throws(() => validateDetailedItems([{ ...issue, aicRunIds: [2] }], [{ id: 2, repository: 'repo', aic: null, pullRequests: [] }], commentAic, 'org')); - assert.throws(() => validateDetailedItems([{ ...issue, aicRunIds: [3] }], [{ id: 3, repository: 'other', aic: 1, pullRequests: [] }], commentAic, 'org')); + for (const [name, items, runs, comments] of [ + ['missing evidence', [issue], [{ id: 1, repository: 'repo', aic: 0, pullRequests: [] }], []], + ['wrong PR evidence', [pullRequest], [{ id: 2, repository: 'repo', aic: 1, pullRequests: [{ number: 2, url: issueUrl }] }], []], + ['wrong type', [{ ...issue, type: 'pull_request' }], workflowRuns, commentAic], + ['cross-repository comment URL', [issue], workflowRuns, [{ ...commentAic[0], itemUrl: 'https://github.com/other/repo/issues/1' }]], + ['duplicate run ID', [{ ...issue, aicRunIds: [1, 1] }], workflowRuns, commentAic], + ['unknown AIC', [{ ...issue, aicRunIds: [2] }], [{ id: 2, repository: 'repo', aic: null, pullRequests: [] }], commentAic], + ['cross-repository run', [{ ...issue, aicRunIds: [3] }], [{ id: 3, repository: 'other', aic: 1, pullRequests: [] }], commentAic], + ['wrong total', [{ ...pullRequest, aic: 2 }], workflowRuns, commentAic], + ['duplicate item identity', [issue, { ...issue, type: 'pull_request' }], workflowRuns, commentAic], + ['unexpected metadata', [{ ...issue, updatedAt: '2026-09-01T00:00:00Z' }], workflowRuns, commentAic], + ['non-finite AIC', [{ ...issue, aic: Number.NaN }], workflowRuns, commentAic], + ['malformed identity', [{ ...issue, repository: 'bad/repo' }], workflowRuns, commentAic], + ]) assert.throws(() => validateDetailedItems(items, runs, comments, 'org'), name); });