Skip to content

history: Store compact per-item AIC totals #101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cgwalters merged 1 commit into from
Sep 16, 2026
Merged

history: Store compact per-item AIC totals #101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion scripts/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@ node scripts/org-history.js --previous-iso-week

Schema v5 retains the v4 period field names and changes detailed `items`: it contains
only items with exact evidence linking them to a relevant workflow run with known AIC.
Each detailed item has sorted, unique `aicRunIds`. Repository aggregates still describe
Each record contains only `repository`, `number`, `type`, `aic`, and sorted, unique
`aicRunIds`; generic GitHub metadata is intentionally omitted. `aic` is the normalized
sum for its linked runs, including zero. A run linked exactly to multiple items contributes
its full AIC to each association, so item AIC values are not additive; use
`coverage.aic.total` for the organization-wide total. Repository aggregates still describe
the full collected activity cohort, not just these AIC-linked detailed items; the linked
run evidence is serialized for audit.

Expand Down
41 changes: 31 additions & 10 deletions scripts/org-history.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const path = require('path');

const SCHEMA_VERSION = 5;
const AGENT_LABELS = ['agent/code', 'agent/fixme', 'agent/lgtm'];
const DETAILED_ITEM_FIELDS = ['repository', 'number', 'type', 'aic', 'aicRunIds'];
const AGGREGATE_USAGE_FILE = 'agent_usage.json';
const TOKEN_USAGE_FILE = 'agent/token_usage.jsonl';

Expand Down Expand Up @@ -326,7 +327,10 @@ function aicLinkedItems(items, runs, commentAic, org) {
ids.add(runId);
links.set(item, ids);
};
for (const evidence of commentAic) link(evidence.repository, evidence.itemNumber, evidence.runId);
for (const evidence of commentAic) {
const item = itemsByKey.get(`${evidence.repository}/${evidence.itemNumber}`);
if (item && evidence.itemUrl === item.url) link(evidence.repository, evidence.itemNumber, evidence.runId);
}
for (const run of runById.values()) {
for (const pullRequest of normalizedPullRequests(run, org, run.repository)) {
const candidate = itemsByKey.get(`${run.repository}/${pullRequest.number}`);
Expand All @@ -335,26 +339,43 @@ function aicLinkedItems(items, runs, commentAic, org) {
}
}
}
return [...links.entries()].map(([item, runIds]) => ({ ...item, aicRunIds: [...runIds].sort((a, b) => a - b) }))
return [...links.entries()].map(([item, runIds]) => {
const aicRunIds = [...runIds].sort((a, b) => a - b);
return {
repository: item.repository,
number: item.number,
type: item.type,
aic: normalizeAic(aicRunIds.reduce((total, runId) => total + runById.get(runId).aic, 0)),
aicRunIds,
};
})
.sort((a, b) => a.repository.localeCompare(b.repository) || a.number - b.number);
}

function validateDetailedItems(items, workflowRuns, commentAic, org) {
const runById = new Map(workflowRuns.map((run) => [run.id, run]));
const identities = new Set();
for (const item of items) {
if (!Array.isArray(item.aicRunIds) || !item.aicRunIds.length) throw new Error(`Detailed item ${item.repository}#${item.number} has no AIC runs`);
const itemPath = item.type === 'pull_request' ? 'pull' : item.type === 'issue' ? 'issues' : null;
const canonicalItemUrl = itemPath && `https://github.com/${org}/${item.repository}/${itemPath}/${item.number}`;
if (item.url !== canonicalItemUrl) throw new Error(`Detailed item ${item.repository}#${item.number} has a mismatched canonical URL`);
if (!item || typeof item !== 'object' || Array.isArray(item) || Object.keys(item).length !== DETAILED_ITEM_FIELDS.length || Object.keys(item).some((field) => !DETAILED_ITEM_FIELDS.includes(field))) throw new Error('Detailed item has unexpected fields');
if (typeof item.repository !== 'string' || !/^[A-Za-z0-9_.-]+$/.test(item.repository) || !Number.isSafeInteger(item.number) || item.number <= 0 || !['issue', 'pull_request'].includes(item.type)) throw new Error('Detailed item has an invalid identity');
const identity = `${item.repository}/${item.number}`;
if (identities.has(identity)) throw new Error(`Detailed item ${item.repository}#${item.number} is duplicated`);
identities.add(identity);
if (!finiteNonnegative(item.aic)) throw new Error(`Detailed item ${item.repository}#${item.number} has invalid AIC`);
if (!Array.isArray(item.aicRunIds) || !item.aicRunIds.length || item.aicRunIds.some((runId) => !Number.isSafeInteger(runId) || runId <= 0)) throw new Error(`Detailed item ${item.repository}#${item.number} has no valid AIC runs`);
const sortedUnique = [...new Set(item.aicRunIds)].sort((a, b) => a - b);
if (JSON.stringify(item.aicRunIds) !== JSON.stringify(sortedUnique)) throw new Error(`Detailed item ${item.repository}#${item.number} has unsorted or duplicate AIC runs`);
const itemPath = item.type === 'pull_request' ? 'pull' : 'issues';
const canonicalItemUrl = `https://github.com/${org}/${item.repository}/${itemPath}/${item.number}`;
let total = 0;
for (const runId of item.aicRunIds) {
const run = runById.get(runId);
const run = workflowRuns.find((candidate) => candidate && candidate.id === runId && candidate.repository === item.repository);
if (!run || run.repository !== item.repository || !finiteNonnegative(run.aic)) throw new Error(`Detailed item ${item.repository}#${item.number} references unknown-AIC run ${runId}`);
const commentEvidence = (commentAic || []).some((record) => record.repository === item.repository && record.runId === runId && record.itemNumber === item.number && record.itemUrl === item.url);
const pullRequestEvidence = item.type === 'pull_request' && (run.pullRequests || []).some((reference) => reference.number === item.number && reference.url === item.url);
total += run.aic;
const commentEvidence = (commentAic || []).some((record) => record.repository === item.repository && record.runId === runId && record.itemNumber === item.number && record.itemUrl === canonicalItemUrl);
const pullRequestEvidence = item.type === 'pull_request' && (run.pullRequests || []).some((reference) => reference.number === item.number && reference.url === canonicalItemUrl);
if (!commentEvidence && !pullRequestEvidence) throw new Error(`Detailed item ${item.repository}#${item.number} lacks exact evidence for run ${runId}`);
}
if (item.aic !== normalizeAic(total)) throw new Error(`Detailed item ${item.repository}#${item.number} has an incorrect AIC total`);
}
}

Expand Down
91 changes: 40 additions & 51 deletions tests/org-history.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,76 +53,65 @@ test('supports deterministic as-of week selection from the CLI', () => {
assert.throws(() => childProcess.execFileSync(process.execPath, [script, '--previous-iso-week', '--as-of', 'not-a-date'], { encoding: 'utf8' }));
});

test('links detailed items only through exact known-AIC comment or PR evidence', () => {
test('projects compact AIC-linked items from exact known-AIC evidence', () => {
const items = [
{ repository: 'repo', number: 1, type: 'issue', url: 'https://github.com/org/repo/issues/1' },
{ repository: 'repo', number: 2, type: 'pull_request', url: 'https://github.com/org/repo/pull/2' },
{ repository: 'repo', number: 1, type: 'issue', url: 'https://github.com/org/repo/issues/1', updatedAt: 'metadata' },
{ repository: 'repo', number: 2, type: 'pull_request', url: 'https://github.com/org/repo/pull/2', title: 'metadata' },
{ repository: 'repo', number: 3, type: 'issue', url: 'https://github.com/org/repo/issues/3' },
];
const run = (id, aic, pullRequests = []) => ({ id, repository: 'repo', kind: 'drafter', aic, pull_requests: pullRequests });
const pull = (number, repository = 'repo') => ({ url: `https://api.github.com/repos/org/${repository}/pulls/${number}` });
const comment = (runId, itemNumber) => ({ repository: 'repo', runId, itemNumber, itemUrl: items.find((item) => item.number === itemNumber).url });
for (const scenario of [
{ name: 'comment', runs: [run(10, 1)], comments: [{ repository: 'repo', runId: 10, itemNumber: 1 }], expected: [[1, [10]]] },
{ name: 'pull request', runs: [run(11, 2, [pull(2)])], comments: [], expected: [[2, [11]]] },
{ name: 'unrelated evidence', runs: [run(12, 3, [pull(3, 'other')]), run(13, null, [pull(2)])], comments: [{ repository: 'repo', runId: 12, itemNumber: 99 }], expected: [] },
{ name: 'deduplicates links', runs: [run(14, 4, [pull(2), pull(2)])], comments: [{ repository: 'repo', runId: 14, itemNumber: 2 }, { repository: 'repo', runId: 14, itemNumber: 2 }], expected: [[2, [14]]] },
{ name: 'keeps multiple comment targets', runs: [run(16, 5)], comments: [{ repository: 'repo', runId: 16, itemNumber: 1 }, { repository: 'repo', runId: 16, itemNumber: 3 }], expected: [[1, [16]], [3, [16]]] },
{ name: 'retains zero AIC', runs: [run(15, 0)], comments: [{ repository: 'repo', runId: 15, itemNumber: 1 }], expected: [[1, [15]]] },
]) {
const actual = aicLinkedItems(items, scenario.runs, scenario.comments, 'org').map((item) => [item.number, item.aicRunIds]);
assert.deepEqual(actual, scenario.expected, scenario.name);
}
{ name: 'comment', runs: [run(10, 1)], comments: [comment(10, 1)], expected: [{ repository: 'repo', number: 1, type: 'issue', aic: 1, aicRunIds: [10] }] },
{ name: 'pull request', runs: [run(11, 2, [pull(2)])], comments: [], expected: [{ repository: 'repo', number: 2, type: 'pull_request', aic: 2, aicRunIds: [11] }] },
{ name: 'unrelated or cross-repository evidence', runs: [run(12, 3, [pull(3, 'other')]), run(13, null, [pull(2)])], comments: [{ ...comment(12, 1), itemNumber: 99 }], expected: [] },
{ name: 'sums multiple runs', runs: [run(10, 1.25), run(14, 2.5)], comments: [comment(10, 1), comment(14, 1)], expected: [{ repository: 'repo', number: 1, type: 'issue', aic: 3.75, aicRunIds: [10, 14] }] },
{ name: 'deduplicates evidence', runs: [run(14, 4, [pull(2), pull(2)])], comments: [comment(14, 2), comment(14, 2)], expected: [{ repository: 'repo', number: 2, type: 'pull_request', aic: 4, aicRunIds: [14] }] },
{ name: 'attributes one run fully to multiple items', runs: [run(16, 5)], comments: [comment(16, 1), comment(16, 3)], expected: [{ repository: 'repo', number: 1, type: 'issue', aic: 5, aicRunIds: [16] }, { repository: 'repo', number: 3, type: 'issue', aic: 5, aicRunIds: [16] }] },
{ name: 'retains zero AIC', runs: [run(15, 0)], comments: [comment(15, 1)], expected: [{ repository: 'repo', number: 1, type: 'issue', aic: 0, aicRunIds: [15] }] },
]) assert.deepEqual(aicLinkedItems(items, scenario.runs, scenario.comments, 'org'), scenario.expected, scenario.name);
});

test('parses comment AIC with transparent same-repository item identity', () => {
const comment = parseCommentAic({
id: 1,
issue_url: 'https://api.github.com/repos/org/repo/issues/7', html_url: 'https://github.com/org/repo/pull/7#issuecomment-9',
body: '> Generated by [workflow](https://github.com/org/repo/actions/runs/42) · 0 AIC',
}, 'org', 'repo');
assert.deepEqual({ runId: comment.runId, itemNumber: comment.itemNumber, itemUrl: comment.itemUrl, aic: comment.aic }, {
runId: 42, itemNumber: 7, itemUrl: 'https://github.com/org/repo/pull/7', aic: 0,
});
const comment = parseCommentAic({ id: 1, issue_url: 'https://api.github.com/repos/org/repo/issues/7', html_url: 'https://github.com/org/repo/pull/7#issuecomment-9', body: '> Generated by [workflow](https://github.com/org/repo/actions/runs/42) · 0 AIC' }, 'org', 'repo');
assert.deepEqual({ runId: comment.runId, itemNumber: comment.itemNumber, itemUrl: comment.itemUrl, aic: comment.aic }, { runId: 42, itemNumber: 7, itemUrl: 'https://github.com/org/repo/pull/7', aic: 0 });
assert.equal(parseCommentAic({ id: 1, issue_url: 'https://api.github.com/repos/other/repo/issues/7', html_url: 'https://github.com/other/repo/issues/7#issuecomment-9', body: '> Generated by [workflow](https://github.com/org/repo/actions/runs/42) · 1 AIC' }, 'org', 'repo'), null);
assert.equal(parseCommentAic({ id: 1, issue_url: 'https://api.github.com/repos/org/repo/issues/7', html_url: 'https://github.com/org/repo/issues/7#issuecomment-9', body: '> Generated by [workflow](https://github.com/other/repo/actions/runs/42) · 1 AIC' }, 'org', 'repo'), null);
assert.equal(parseCommentAic({ id: 1, issue_url: 'https://api.github.com/repos/org/repo/issues/7', html_url: 'https://github.com/org/repo/issues/7#issuecomment-9', body: '> Generated by [workflow](https://github.com/org/repo/actions/runs/42) · 1 AIC\n<!-- gh-aw-agentic-workflow: id: 42, workflow_id: 1, run: https://github.com/other/repo/actions/runs/42 -->' }, 'org', 'repo'), null);
});

test('retains consistent comment evidence for every exact item target', () => {
const records = deduplicateCommentAic([
{ runId: 20, commentId: 1, commentUrl: 'one', agentAic: 1, detectionAic: null, aic: 1, repository: 'repo', itemNumber: 1 },
{ runId: 20, commentId: 2, commentUrl: 'two', agentAic: 1, detectionAic: null, aic: 1, repository: 'repo', itemNumber: 2 },
]).records;
const items = [{ repository: 'repo', number: 1, type: 'issue' }, { repository: 'repo', number: 2, type: 'issue' }];
assert.deepEqual(aicLinkedItems(items, [{ id: 20, repository: 'repo', kind: 'drafter', aic: 1 }], records, 'org').map((item) => [item.number, item.aicRunIds]), [[1, [20]], [2, [20]]]);
const records = deduplicateCommentAic([{ runId: 20, commentId: 1, commentUrl: 'one', agentAic: 1, detectionAic: null, aic: 1, repository: 'repo', itemNumber: 1 }, { runId: 20, commentId: 2, commentUrl: 'two', agentAic: 1, detectionAic: null, aic: 1, repository: 'repo', itemNumber: 2 }]).records;
const items = [{ repository: 'repo', number: 1, type: 'issue', url: 'https://github.com/org/repo/issues/1' }, { repository: 'repo', number: 2, type: 'issue', url: 'https://github.com/org/repo/issues/2' }];
for (const record of records) record.itemUrl = `https://github.com/org/repo/issues/${record.itemNumber}`;
assert.deepEqual(aicLinkedItems(items, [{ id: 20, repository: 'repo', kind: 'drafter', aic: 1 }], records, 'org'), [{ repository: 'repo', number: 1, type: 'issue', aic: 1, aicRunIds: [20] }, { repository: 'repo', number: 2, type: 'issue', aic: 1, aicRunIds: [20] }]);
});

test('normalizes only same-repository workflow PR references', () => {
assert.deepEqual(normalizedPullRequests({ pull_requests: [
{ url: 'https://api.github.com/repos/org/repo/pulls/2' },
{ url: 'https://api.github.com/repos/org/repo/pulls/1' },
{ url: 'https://api.github.com/repos/org/repo/pulls/2' },
{ url: 'https://api.github.com/repos/org/other/pulls/3' },
] }, 'org', 'repo'), [
{ number: 1, url: 'https://github.com/org/repo/pull/1' },
{ number: 2, url: 'https://github.com/org/repo/pull/2' },
]);
assert.deepEqual(normalizedPullRequests({ pull_requests: [{ url: 'https://api.github.com/repos/org/repo/pulls/2' }, { url: 'https://api.github.com/repos/org/repo/pulls/1' }, { url: 'https://api.github.com/repos/org/repo/pulls/2' }, { url: 'https://api.github.com/repos/org/other/pulls/3' }] }, 'org', 'repo'), [{ number: 1, url: 'https://github.com/org/repo/pull/1' }, { number: 2, url: 'https://github.com/org/repo/pull/2' }]);
});

test('validates detailed item links have exact serialized evidence', () => {
const issue = { repository: 'repo', number: 1, type: 'issue', url: 'https://github.com/org/repo/issues/1', aicRunIds: [1] };
const pullRequest = { repository: 'repo', number: 2, type: 'pull_request', url: 'https://github.com/org/repo/pull/2', aicRunIds: [2] };
const commentAic = [{ repository: 'repo', runId: 1, itemNumber: 1, itemUrl: issue.url }];
const workflowRuns = [
{ id: 1, repository: 'repo', aic: 0, pullRequests: [] },
{ id: 2, repository: 'repo', aic: 1, pullRequests: [{ number: 2, url: pullRequest.url }] },
];
test('validates compact detailed item records and exact serialized evidence', () => {
const issue = { repository: 'repo', number: 1, type: 'issue', aic: 0, aicRunIds: [1] };
const pullRequest = { repository: 'repo', number: 2, type: 'pull_request', aic: 1, aicRunIds: [2] };
const issueUrl = 'https://github.com/org/repo/issues/1';
const pullRequestUrl = 'https://github.com/org/repo/pull/2';
const commentAic = [{ repository: 'repo', runId: 1, itemNumber: 1, itemUrl: issueUrl }];
const workflowRuns = [{ id: 1, repository: 'repo', aic: 0, pullRequests: [] }, { id: 2, repository: 'repo', aic: 1, pullRequests: [{ number: 2, url: pullRequestUrl }] }];
assert.doesNotThrow(() => validateDetailedItems([issue, pullRequest], workflowRuns, commentAic, 'org'));
assert.throws(() => validateDetailedItems([issue], [{ id: 1, repository: 'repo', aic: 0, pullRequests: [] }], [], 'org'));
assert.throws(() => validateDetailedItems([pullRequest], [{ id: 2, repository: 'repo', aic: 1, pullRequests: [{ number: 2, url: issue.url }] }], [], 'org'));
assert.throws(() => validateDetailedItems([{ ...issue, type: 'pull_request' }], workflowRuns, commentAic, 'org'));
assert.throws(() => validateDetailedItems([{ ...issue, url: 'https://github.com/other/repo/issues/1' }], workflowRuns, [{ ...commentAic[0], itemUrl: 'https://github.com/other/repo/issues/1' }], 'org'));
assert.throws(() => validateDetailedItems([{ ...issue, aicRunIds: [1, 1] }], workflowRuns, commentAic, 'org'));
assert.throws(() => validateDetailedItems([{ ...issue, aicRunIds: [2] }], [{ id: 2, repository: 'repo', aic: null, pullRequests: [] }], commentAic, 'org'));
assert.throws(() => validateDetailedItems([{ ...issue, aicRunIds: [3] }], [{ id: 3, repository: 'other', aic: 1, pullRequests: [] }], commentAic, 'org'));
for (const [name, items, runs, comments] of [
['missing evidence', [issue], [{ id: 1, repository: 'repo', aic: 0, pullRequests: [] }], []],
['wrong PR evidence', [pullRequest], [{ id: 2, repository: 'repo', aic: 1, pullRequests: [{ number: 2, url: issueUrl }] }], []],
['wrong type', [{ ...issue, type: 'pull_request' }], workflowRuns, commentAic],
['cross-repository comment URL', [issue], workflowRuns, [{ ...commentAic[0], itemUrl: 'https://github.com/other/repo/issues/1' }]],
['duplicate run ID', [{ ...issue, aicRunIds: [1, 1] }], workflowRuns, commentAic],
['unknown AIC', [{ ...issue, aicRunIds: [2] }], [{ id: 2, repository: 'repo', aic: null, pullRequests: [] }], commentAic],
['cross-repository run', [{ ...issue, aicRunIds: [3] }], [{ id: 3, repository: 'other', aic: 1, pullRequests: [] }], commentAic],
['wrong total', [{ ...pullRequest, aic: 2 }], workflowRuns, commentAic],
['duplicate item identity', [issue, { ...issue, type: 'pull_request' }], workflowRuns, commentAic],
['unexpected metadata', [{ ...issue, updatedAt: '2026-09-01T00:00:00Z' }], workflowRuns, commentAic],
['non-finite AIC', [{ ...issue, aic: Number.NaN }], workflowRuns, commentAic],
['malformed identity', [{ ...issue, repository: 'bad/repo' }], workflowRuns, commentAic],
]) assert.throws(() => validateDetailedItems(items, runs, comments, 'org'), name);
});