Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion packages/warden/src/action/workflow/pr-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const RUNTIME_CLAUDE_FIXTURES_DIR = join(FIXTURES_DIR, 'runtime-claude');
const EMPTY_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'empty-auxiliary-model');
const LAYERED_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'layered-auxiliary-model');
const NO_MATCH_EMPTY_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'no-match-empty-auxiliary-model');
const SCHEDULE_ONLY_FIXTURES_DIR = join(FIXTURES_DIR, 'schedule');
const EVENT_PAYLOAD_PATH = join(FIXTURES_DIR, 'event-payloads/pull_request_opened.json');
const PR_HEAD_SHA = 'abc123def456';
const PREVIOUS_HEAD_SHA = 'previous123sha456';
Expand Down Expand Up @@ -392,6 +393,30 @@ describe('runPRWorkflow', () => {
}),
],
skillExecutions: [expect.objectContaining({ report, skillExecutionId: expect.any(String) })],
configuredSkills: [{ name: 'test-skill', triggered: true }],
})
);
});

it('analyze mode lists a schedule-only skill as configured but not triggered on a PR run', async () => {
await runPRWorkflow(
mockOctokit,
createDefaultInputs({ mode: 'analyze' }),
'pull_request',
EVENT_PAYLOAD_PATH,
SCHEDULE_ONLY_FIXTURES_DIR
);

expect(mockRunSkillTask).not.toHaveBeenCalled();
expect(mockWriteFindingsOutput).toHaveBeenCalledWith(
[],
expect.objectContaining({
repository: expect.objectContaining({ fullName: 'test-owner/test-repo' }),
}),
[],
expect.objectContaining({
triggerResults: [],
configuredSkills: [{ name: 'test-skill', triggered: false }],
})
);
});
Expand Down Expand Up @@ -422,6 +447,22 @@ describe('runPRWorkflow', () => {
);
});

it('analyze mode includes configuredSkills in the live findings snapshot', async () => {
mockRunSkillTask.mockResolvedValue({ name: 'test-trigger', report: createSkillReport({ skill: 'test-skill' }) });

await runPRWorkflow(
mockOctokit,
createDefaultInputs({ mode: 'analyze' }),
'pull_request',
EVENT_PAYLOAD_PATH,
FIXTURES_DIR
);

expect(mockWriteFindingsOutputLive).toHaveBeenCalledTimes(1);
const [, , , liveOptions] = mockWriteFindingsOutputLive.mock.calls[0]!;
expect(liveOptions?.configuredSkills).toEqual([{ name: 'test-skill', triggered: true }]);
});

it('report mode carries skillExecutionId and resolvedDefaults into the final findings output', async () => {
const finding = createFinding();
const report = createSkillReport({ findings: [finding] });
Expand Down Expand Up @@ -1643,6 +1684,7 @@ describe('runPRWorkflow', () => {
expect.objectContaining({ skillExecutionId: expect.any(String), triggerName: 'test-skill' }),
]);
expect(liveOptions?.skippedTriggers).toEqual([]);
expect(liveOptions?.configuredSkills).toEqual([{ name: 'test-skill', triggered: true }]);

// The final write happens after the live write and includes the same enrichment.
const [, , , finalOptions] = mockWriteFindingsOutput.mock.calls[0]!;
Expand Down Expand Up @@ -1857,6 +1899,16 @@ describe('runPRWorkflow', () => {
})
);
expect(mockRunSkillTask).not.toHaveBeenCalled();
expect(mockWriteFindingsOutput).toHaveBeenCalledWith(
[],
expect.objectContaining({
repository: expect.objectContaining({ fullName: 'test-owner/test-repo' }),
}),
[],
expect.objectContaining({
configuredSkills: [{ name: 'test-skill', triggered: true }],
})
);
});

it('fails when findings exceed fail-on threshold and failCheck is true', async () => {
Expand Down Expand Up @@ -2730,7 +2782,9 @@ describe('runPRWorkflow', () => {
resolvedReason: 'fix_evaluation',
}),
],
expect.any(Object)
expect.objectContaining({
configuredSkills: [{ name: 'test-skill', triggered: false }],
})
);
});

Expand Down
45 changes: 42 additions & 3 deletions packages/warden/src/action/workflow/pr-workflow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ import {
FindingsOutputSchema,
buildFindingsOutput,
buildBaseOutputOptions,
buildConfiguredSkillsList,
type SkippedTriggerReasonSchema,
type FindingsOutput,
type ReplayTriggerResult,
Expand All @@ -114,6 +115,7 @@ interface InitResult {
service?: ResolvedServiceOptions;
runnerConcurrency?: number;
auxiliaryOptions: AuxiliaryWorkflowOptions;
resolvedTriggers: ResolvedTrigger[];
matchedTriggers: ResolvedTrigger[];
skippedTriggers: ResolvedTrigger[];
memoryRecall?: ActionMemoryRecall;
Expand Down Expand Up @@ -469,6 +471,7 @@ async function initializeWorkflow(
service,
runnerConcurrency,
auxiliaryOptions,
resolvedTriggers,
matchedTriggers,
skippedTriggers,
memoryRecall,
Expand All @@ -486,6 +489,7 @@ async function initializeWorkflow(
service: resolveActionServiceOptions(inputs),
runnerConcurrency,
auxiliaryOptions,
resolvedTriggers: [],
matchedTriggers: [],
skippedTriggers: [],
skipCoreCheck: {
Expand Down Expand Up @@ -1165,6 +1169,8 @@ async function finalizeWorkflow(
inputs: ActionInputs,
service: ResolvedServiceOptions | undefined,
memoryRecall: ActionMemoryRecall | undefined,
matchedTriggers: ResolvedTrigger[],
resolvedTriggers: ResolvedTrigger[]
): Promise<void> {
await dismissPreviousReviewIfResolved(
octokit,
Expand All @@ -1189,6 +1195,7 @@ async function finalizeWorkflow(
skillExecutions: toSkillExecutions(results),
recalledMemories: memoryRecall?.memories.map(({ id, version }) => ({ id, version })),
memoryRecallId: memoryRecall?.clientRecallId,
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
};
try {
const findingsPath = writeFindingsOutput(reports, context, findingObservations, findingsOptions);
Expand Down Expand Up @@ -1800,6 +1807,8 @@ async function finalizeReportWorkflow(
service?: ResolvedServiceOptions;
recalledMemories?: readonly { id: string; version: number }[];
memoryRecallId?: string;
matchedTriggers: ResolvedTrigger[];
resolvedTriggers: ResolvedTrigger[];
}
): Promise<void> {
await dismissPreviousReviewIfResolved(
Expand All @@ -1824,6 +1833,10 @@ async function finalizeReportWorkflow(
skillExecutions: toSkillExecutions(results),
recalledMemories: options.recalledMemories,
memoryRecallId: options.memoryRecallId,
configuredSkills: buildConfiguredSkillsList({
allTriggers: options.resolvedTriggers,
matchedTriggers: options.matchedTriggers,
}),
};
try {
const findingsPath = writeFindingsOutput(reports, context, findingObservations, findingsOptions);
Expand Down Expand Up @@ -1953,6 +1966,7 @@ async function runAnalyzeMode(
const {
context,
runnerConcurrency,
resolvedTriggers,
matchedTriggers,
skippedTriggers,
skipCoreCheck,
Expand All @@ -1967,6 +1981,7 @@ async function runAnalyzeMode(
const findingsPath = writeFindingsOutput([], context, [], {
triggerResults: [],
...buildBaseOutputOptions(inputs, toSkippedTriggers(skippedTriggers, context)),
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
logAction(`Findings written to ${findingsPath}`);
} catch (error) {
Expand All @@ -1992,6 +2007,7 @@ async function runAnalyzeMode(
...toErroredSkippedTriggers(completedSoFar),
]),
skillExecutions: toSkillExecutions(completedSoFar),
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
},
}),
Expand All @@ -2012,6 +2028,7 @@ async function runAnalyzeMode(
skillExecutions: toSkillExecutions(results),
recalledMemories: memoryRecall?.memories.map(({ id, version }) => ({ id, version })),
memoryRecallId: memoryRecall?.clientRecallId,
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
logAction(`Findings written to ${findingsPath}`);
} catch (error) {
Expand All @@ -2037,6 +2054,7 @@ async function runReportMode(
context,
service,
auxiliaryOptions,
resolvedTriggers,
matchedTriggers,
skippedTriggers,
skipCoreCheck,
Expand Down Expand Up @@ -2065,6 +2083,7 @@ async function runReportMode(
triggerResults: [],
...buildBaseOutputOptions(inputs, toSkippedTriggers(skippedTriggers, context)),
...replayMemoryOptions,
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
} satisfies BuildFindingsOutputOptions;
try {
const findingsPath = writeFindingsOutput([], context, [], findingsOptions);
Expand Down Expand Up @@ -2104,6 +2123,7 @@ async function runReportMode(
triggerResults: [],
...buildBaseOutputOptions(inputs, toSkippedTriggers(skippedTriggers, context)),
...replayMemoryOptions,
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
} satisfies BuildFindingsOutputOptions;
try {
const findingsPath = writeFindingsOutput([], context, cleanupFindingObservations, findingsOptions);
Expand Down Expand Up @@ -2182,7 +2202,15 @@ async function runReportMode(
canResolveStale,
gate,
triggerErrors,
{ failOnWriteError: true, skippedTriggers, inputs, service, ...replayMemoryOptions },
{
failOnWriteError: true,
skippedTriggers,
inputs,
service,
...replayMemoryOptions,
matchedTriggers,
resolvedTriggers,
},
);
} catch (error) {
if (error instanceof ActionFailedError) {
Expand Down Expand Up @@ -2229,6 +2257,7 @@ export async function runPRWorkflow(
service,
runnerConcurrency,
auxiliaryOptions,
resolvedTriggers,
matchedTriggers,
skippedTriggers,
skipCoreCheck,
Expand Down Expand Up @@ -2279,7 +2308,10 @@ export async function runPRWorkflow(
setOutput('findings-count', 0);
setOutput('high-count', 0);
setOutput('summary', skipCoreCheck.title);
const findingsOptions = buildBaseOutputOptions(inputs, toSkippedTriggers(skippedTriggers, context));
const findingsOptions = {
...buildBaseOutputOptions(inputs, toSkippedTriggers(skippedTriggers, context)),
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
};
try {
writeFindingsOutput([], context, [], findingsOptions);
} catch (error) {
Expand All @@ -2301,7 +2333,10 @@ export async function runPRWorkflow(
setOutput('findings-count', 0);
setOutput('high-count', 0);
setOutput('summary', 'No triggers matched');
const findingsOptions = buildBaseOutputOptions(inputs, toSkippedTriggers(skippedTriggers, context));
const findingsOptions = {
...buildBaseOutputOptions(inputs, toSkippedTriggers(skippedTriggers, context)),
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
};
Comment thread
cursor[bot] marked this conversation as resolved.
try {
writeFindingsOutput([], context, cleanupFindingObservations, findingsOptions);
} catch (error) {
Expand Down Expand Up @@ -2338,6 +2373,7 @@ export async function runPRWorkflow(
...toErroredSkippedTriggers(completedSoFar),
]),
skillExecutions: toSkillExecutions(completedSoFar),
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
});
},
}),
Expand All @@ -2362,6 +2398,7 @@ export async function runPRWorkflow(
reason: 'error' as const,
})),
]),
configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }),
};
try {
writeFindingsOutput([], context, [], findingsOptions);
Expand Down Expand Up @@ -2429,6 +2466,8 @@ export async function runPRWorkflow(
inputs,
service,
memoryRecall,
matchedTriggers,
resolvedTriggers,
);

handleTriggerErrors(triggerErrors, matchedTriggers.length);
Expand Down
69 changes: 68 additions & 1 deletion packages/warden/src/reporting/output.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import { describe, expect, it } from 'vitest';
import type { EventContext, Finding, SkillReport } from '../types/index.js';
import { buildFindingsOutput, buildResolvedDefaults, FindingsOutputSchema } from './output.js';
import {
buildConfiguredSkillsList,
buildFindingsOutput,
buildResolvedDefaults,
FindingsOutputSchema,
} from './output.js';

describe('findings output schema', () => {
it('builds a schema-valid public findings payload', () => {
Expand Down Expand Up @@ -717,6 +722,68 @@ describe('findings output schema', () => {
{ skillName: 'style-skill', role: 'corroborating', matchType: 'semantic' },
]);
});

it('includes the configured skills roster when provided', () => {
const output = buildFindingsOutput([createReport()], createContext(), [], {
timestamp: '2026-01-01T00:00:00.000Z',
runId: '123',
configuredSkills: [
{ name: 'test-skill', triggered: true },
{ name: 'idle-skill', triggered: false },
],
});

expect(FindingsOutputSchema.parse(output)).toEqual(output);
expect(output.configuredSkills).toEqual([
{ name: 'test-skill', triggered: true },
{ name: 'idle-skill', triggered: false },
]);
});

it('omits the configured skills roster when not provided', () => {
const output = buildFindingsOutput([createReport()], createContext(), [], {
timestamp: '2026-01-01T00:00:00.000Z',
runId: '123',
});

expect(output.configuredSkills).toBeUndefined();
});
});

describe('buildConfiguredSkillsList', () => {
it('marks matched skills as triggered and unmatched skills as not', () => {
const result = buildConfiguredSkillsList({
allTriggers: [{ name: 'matched-skill' }, { name: 'skipped-skill' }],
matchedTriggers: [{ name: 'matched-skill' }],
});

expect(result).toEqual([
{ name: 'matched-skill', triggered: true },
{ name: 'skipped-skill', triggered: false },
]);
});

it('deduplicates multiple trigger blocks for the same skill', () => {
const result = buildConfiguredSkillsList({
allTriggers: [{ name: 'multi-trigger-skill' }, { name: 'multi-trigger-skill' }],
matchedTriggers: [{ name: 'multi-trigger-skill' }],
});

expect(result).toEqual([{ name: 'multi-trigger-skill', triggered: true }]);
});

it('returns an empty list when nothing is configured', () => {
expect(buildConfiguredSkillsList({ allTriggers: [], matchedTriggers: [] })).toEqual([]);
});

it('includes a skill whose only trigger is neither matched nor a PR-check skip', () => {
const result = buildConfiguredSkillsList({
allTriggers: [{ name: 'nightly-sweep' }],
matchedTriggers: [],
});

expect(result).toEqual([{ name: 'nightly-sweep', triggered: false }]);
});
});

describe('buildResolvedDefaults', () => {
Expand Down
Loading
Loading