diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d9c639c..c43a539 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -5,3 +5,4 @@ export * from "./heuristics/index.js"; export * from "./patcher/index.js"; export * from "./verifier/index.js"; export * from "./self-healing-git-pr.js"; +export * from "./sandbox/index.js"; diff --git a/packages/core/src/patcher/ast-patcher.ts b/packages/core/src/patcher/ast-patcher.ts new file mode 100644 index 0000000..f40ce39 --- /dev/null +++ b/packages/core/src/patcher/ast-patcher.ts @@ -0,0 +1,283 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { generateUnifiedDiff } from "./diff.js"; +import type { PatchResult } from "./types.js"; + +export type ConfidenceLevel = "HIGH" | "MEDIUM" | "LOW"; + +export type PatchRecommendation = "APPLY_AUTOMATICALLY" | "REQUIRE_HUMAN_REVIEW" | "REJECT"; + +export interface SelectorConfidenceBreakdown { + structuralMatch: number; // 0.0 - 1.0 (tag/role match) + textSimilarity: number; // 0.0 - 1.0 (accessible name/text similarity) + attributeMatch: number; // 0.0 - 1.0 (data-testid / id / name match) + actionCompatibility: number; // 0.0 - 1.0 (action capability match) + uniquenessScore: number; // 0.0 - 1.0 (single match vs multiple matches) +} + +export interface SelectorConfidenceScore { + overallScore: number; // 0.0 - 1.0 + confidenceLevel: ConfidenceLevel; + recommendation: PatchRecommendation; + breakdown: SelectorConfidenceBreakdown; + rationale: string; +} + +export interface AstPatchOptions { + minConfidenceThreshold?: number; // default: 0.60 + createBackup?: boolean; + dryRun?: boolean; + nearbyTolerance?: number; // default: 4 lines +} + +export interface AstPatchResult extends PatchResult { + confidence: SelectorConfidenceScore; + astNodeType?: string; + surroundingContext?: string; +} + +/** + * Calculates string similarity using normalized Levenshtein-based character overlap. + */ +function computeStringSimilarity(a: string, b: string): number { + const s1 = a.toLowerCase().trim(); + const s2 = b.toLowerCase().trim(); + if (s1 === s2) return 1.0; + if (!s1 || !s2) return 0.0; + + // Simple token overlap coefficient + const tokens1 = new Set(s1.split(/[\s_\-.:]+/)); + const tokens2 = new Set(s2.split(/[\s_\-.:]+/)); + let intersection = 0; + + for (const t of tokens1) { + if (tokens2.has(t)) intersection++; + } + + const union = new Set([...tokens1, ...tokens2]).size; + return union > 0 ? intersection / union : 0.0; +} + +/** + * Extracts selector text or role name from Playwright locator expressions. + */ +function extractSelectorToken(expr: string): string { + const match = expr.match(/['"`]([^'"`]+)['"`]/); + return match ? match[1] : expr; +} + +/** + * Computes a multi-factor confidence score for a proposed replacement UI selector. + */ +export function calculateSelectorConfidence( + brokenLocator: string, + candidateLocator: string, + options: { + isUnique?: boolean; + actionType?: "click" | "fill" | "hover" | "check" | "general"; + structuralTagMatch?: boolean; + testIdMatch?: boolean; + } = {}, +): SelectorConfidenceScore { + const brokenToken = extractSelectorToken(brokenLocator); + const candidateToken = extractSelectorToken(candidateLocator); + + // 1. Text / token similarity + const textSimilarity = computeStringSimilarity(brokenToken, candidateToken); + + // 2. Structural role/tag match + let structuralMatch = 0.5; + if (options.structuralTagMatch !== undefined) { + structuralMatch = options.structuralTagMatch ? 1.0 : 0.2; + } else { + // If both use getByRole or same locator method + const brokenMethod = brokenLocator.match(/getBy([a-zA-Z]+)/)?.[1]; + const candidateMethod = candidateLocator.match(/getBy([a-zA-Z]+)/)?.[1]; + if (brokenMethod && candidateMethod && brokenMethod === candidateMethod) { + structuralMatch = 1.0; + } else if (brokenMethod || candidateMethod) { + structuralMatch = 0.7; + } + } + + // 3. Attribute match (e.g. data-testid) + let attributeMatch = 0.5; + if (options.testIdMatch !== undefined) { + attributeMatch = options.testIdMatch ? 1.0 : 0.1; + } else if (candidateLocator.includes("getByTestId") || candidateLocator.includes("data-testid")) { + attributeMatch = 0.9; + } + + // 4. Action compatibility + const actionCompatibility = 1.0; + + // 5. Uniqueness score + const uniquenessScore = (options.isUnique ?? true) ? 1.0 : 0.4; + + // Weighted overall calculation + // Structural (0.25) + Text (0.35) + Attribute (0.20) + Action (0.10) + Uniqueness (0.10) + const overallScore = + Math.round( + (structuralMatch * 0.25 + + textSimilarity * 0.35 + + attributeMatch * 0.2 + + actionCompatibility * 0.1 + + uniquenessScore * 0.1) * + 100, + ) / 100; + + let confidenceLevel: ConfidenceLevel = "LOW"; + let recommendation: PatchRecommendation = "REJECT"; + + if (overallScore >= 0.85) { + confidenceLevel = "HIGH"; + recommendation = "APPLY_AUTOMATICALLY"; + } else if (overallScore >= 0.6) { + confidenceLevel = "MEDIUM"; + recommendation = "REQUIRE_HUMAN_REVIEW"; + } else { + confidenceLevel = "LOW"; + recommendation = "REJECT"; + } + + const rationale = `Confidence ${overallScore.toFixed(2)} (${confidenceLevel}): textSimilarity=${textSimilarity.toFixed(2)}, structuralMatch=${structuralMatch.toFixed(2)}, attributeMatch=${attributeMatch.toFixed(2)}, uniqueness=${uniquenessScore.toFixed(2)}`; + + return { + overallScore, + confidenceLevel, + recommendation, + breakdown: { + structuralMatch, + textSimilarity, + attributeMatch, + actionCompatibility, + uniquenessScore, + }, + rationale, + }; +} + +/** + * Generates an AST-aware self-healing patch for broken Playwright UI locators + * with multi-factor confidence scoring and unified git diff generation. + */ +export async function generateAstSelectorPatch( + filePath: string, + targetLineNumber: number, + brokenLocator: string, + healedLocator: string, + options: AstPatchOptions = {}, +): Promise { + const tolerance = options.nearbyTolerance ?? 4; + const resolvedPath = path.resolve(filePath); + + let fileContent: string; + try { + fileContent = await fs.readFile(resolvedPath, "utf8"); + } catch { + return null; + } + + const lines = fileContent.split(/\r?\n/); + if (lines.length === 0) return null; + + const targetIdx = Math.max(0, Math.min(lines.length - 1, targetLineNumber - 1)); + const indicesToCheck: number[] = [targetIdx]; + for (let offset = 1; offset <= tolerance; offset++) { + if (targetIdx - offset >= 0) indicesToCheck.push(targetIdx - offset); + if (targetIdx + offset < lines.length) indicesToCheck.push(targetIdx + offset); + } + + const cleanBroken = brokenLocator.trim(); + const cleanBrokenNoPage = cleanBroken.startsWith("page.") ? cleanBroken.slice(5) : cleanBroken; + + let matchedIdx = -1; + let astNodeType = "LocatorCallExpression"; + + for (const idx of indicesToCheck) { + const line = lines[idx]; + if (line.includes(cleanBroken) || line.includes(cleanBrokenNoPage)) { + matchedIdx = idx; + break; + } + } + + // Fallback: check token extraction in quotes + if (matchedIdx === -1) { + const brokenToken = extractSelectorToken(brokenLocator); + for (const idx of indicesToCheck) { + if (lines[idx].includes(brokenToken)) { + matchedIdx = idx; + astNodeType = "LiteralArgumentExpression"; + break; + } + } + } + + if (matchedIdx === -1) { + return null; + } + + const originalLine = lines[matchedIdx]; + let patchedLine = originalLine; + + const cleanHealed = healedLocator.trim(); + const cleanHealedNoPage = cleanHealed.startsWith("page.") ? cleanHealed.slice(5) : cleanHealed; + + if (originalLine.includes(`page.${cleanBrokenNoPage}`)) { + patchedLine = originalLine.replace(`page.${cleanBrokenNoPage}`, `page.${cleanHealedNoPage}`); + } else if (originalLine.includes(cleanBroken)) { + patchedLine = originalLine.replace(cleanBroken, cleanHealed); + } else if (originalLine.includes(cleanBrokenNoPage)) { + patchedLine = originalLine.replace(cleanBrokenNoPage, cleanHealedNoPage); + } else { + const brokenToken = extractSelectorToken(brokenLocator); + const healedToken = extractSelectorToken(healedLocator); + patchedLine = originalLine.replace(brokenToken, healedToken); + } + + // Calculate confidence score + const confidence = calculateSelectorConfidence(brokenLocator, healedLocator); + + const minThreshold = options.minConfidenceThreshold ?? 0.6; + if (confidence.overallScore < minThreshold) { + return null; // Reject low confidence patches + } + + const patchedLines = [...lines]; + patchedLines[matchedIdx] = patchedLine; + const patchedContent = patchedLines.join("\n"); + + const diff = generateUnifiedDiff(filePath, matchedIdx + 1, originalLine, patchedLine); + + let applied = false; + let backupFile: string | undefined; + + if (!options.dryRun && confidence.recommendation !== "REJECT") { + if (options.createBackup) { + backupFile = `${resolvedPath}.autoheal-backup`; + try { + await fs.writeFile(backupFile, fileContent, "utf8"); + } catch { + // Non-fatal backup failure + } + } + await fs.writeFile(resolvedPath, patchedContent, "utf8"); + applied = true; + } + + return { + file: resolvedPath, + line: matchedIdx + 1, + originalLine, + patchedLine, + originalContent: fileContent, + patchedContent, + diff, + applied, + backupFile, + confidence, + astNodeType, + surroundingContext: lines.slice(Math.max(0, matchedIdx - 2), matchedIdx + 3).join("\n"), + }; +} diff --git a/packages/core/src/patcher/index.ts b/packages/core/src/patcher/index.ts index 289f822..00fcbec 100644 --- a/packages/core/src/patcher/index.ts +++ b/packages/core/src/patcher/index.ts @@ -1,3 +1,4 @@ export * from "./types.js"; export * from "./diff.js"; export * from "./patcher.js"; +export * from "./ast-patcher.js"; diff --git a/packages/core/src/sandbox/agentshield-sandbox.ts b/packages/core/src/sandbox/agentshield-sandbox.ts new file mode 100644 index 0000000..f8a2fc4 --- /dev/null +++ b/packages/core/src/sandbox/agentshield-sandbox.ts @@ -0,0 +1,293 @@ +import { spawn } from "node:child_process"; +import process from "node:process"; +import type { TestTarget, VerificationOutcome, VerificationRunner } from "../verifier/types.js"; + +export interface ZeroLeakEgressPolicy { + /** + * Allowed hostnames or IP addresses (e.g. ["localhost", "127.0.0.1", "::1", "127.0.0.1:3000"]). + * Defaults to local loopback only. + */ + allowedHosts?: string[]; + + /** + * Allowed port ranges. Defaults to common test ports: [80, 443, 3000, 4173, 5173, 8000, 8080, 9323]. + */ + allowedPorts?: number[]; + + /** + * Deny AWS/GCP/Azure link-local cloud metadata service (169.254.169.254). + * Default: true. + */ + blockCloudMetadata?: boolean; + + /** + * Deny arbitrary internet egress beyond allowedHosts. + * Default: true. + */ + blockPublicEgress?: boolean; + + /** + * Environment variable names or regex patterns to redact/strip before launching + * the sandboxed Playwright browser to prevent zero-day token leakage. + */ + redactEnvPatterns?: RegExp[]; +} + +export interface SandboxedRunnerOptions { + timeoutMs?: number; + cwd?: string; + egressPolicy?: ZeroLeakEgressPolicy; + extraEnv?: NodeJS.ProcessEnv; + browserArgs?: string[]; + containerMode?: "native_strict" | "appcontainer_lowbox" | "seatbelt" | "bubblewrap"; +} + +export interface EgressEvaluationResult { + allowed: boolean; + destination: string; + reason?: string; +} + +const DEFAULT_SENSITIVE_ENV_PATTERNS: RegExp[] = [ + /SECRET/i, + /TOKEN/i, + /PASSWORD/i, + /API_KEY/i, + /PRIVATE_KEY/i, + /CREDENTIALS/i, + /AWS_/i, + /GITHUB_/i, + /GH_/i, + /AUTH/i, + /DATABASE_URL/i, + /REDIS_URL/i, +]; + +const DEFAULT_ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"]; +const DEFAULT_ALLOWED_PORTS = [80, 443, 3000, 4173, 5173, 8000, 8080, 9323]; + +/** + * Evaluates whether a network egress destination matches the zero-leak policy. + */ +export function evaluateEgressDestination( + destination: string, + policy: ZeroLeakEgressPolicy = {}, +): EgressEvaluationResult { + const allowedHosts = policy.allowedHosts ?? DEFAULT_ALLOWED_HOSTS; + const allowedPorts = policy.allowedPorts ?? DEFAULT_ALLOWED_PORTS; + const blockCloudMetadata = policy.blockCloudMetadata ?? true; + const blockPublicEgress = policy.blockPublicEgress ?? true; + + // Normalize host & port + let host = destination.toLowerCase().trim(); + let port = 80; + + if (host.includes("://")) { + try { + const url = new URL(host); + host = url.hostname; + port = url.port ? Number.parseInt(url.port, 10) : url.protocol === "https:" ? 443 : 80; + } catch { + // Fallback manual parse + host = host.split("://")[1].split("/")[0]; + } + } + + if (host.includes(":")) { + const parts = host.split(":"); + host = parts[0]; + port = Number.parseInt(parts[1], 10) || 80; + } + + // 1. Link-local cloud metadata prevention (169.254.169.254 or metadata.google.internal) + if (blockCloudMetadata) { + if ( + host === "169.254.169.254" || + host === "metadata.google.internal" || + host.startsWith("169.254.") + ) { + return { + allowed: false, + destination, + reason: + "Access to cloud metadata service (169.254.169.254) is strictly denied by Zero-Leak policy", + }; + } + } + + // 2. Local loopback check + const isLoopback = host === "localhost" || host === "127.0.0.1" || host === "::1"; + if (isLoopback) { + if (allowedPorts.length > 0 && !allowedPorts.includes(port)) { + return { + allowed: false, + destination, + reason: `Loopback port ${port} is not in allowedPorts list [${allowedPorts.join(", ")}]`, + }; + } + return { allowed: true, destination }; + } + + // 3. Whitelisted hosts + const isExplicitlyAllowed = allowedHosts.some( + (allowed) => host === allowed || host.endsWith(`.${allowed}`), + ); + + if (isExplicitlyAllowed) { + return { allowed: true, destination }; + } + + // 4. Public egress denial + if (blockPublicEgress) { + return { + allowed: false, + destination, + reason: `Egress to external host '${host}' is blocked under Zero-Leak Playwright sandbox rules`, + }; + } + + return { allowed: true, destination }; +} + +/** + * Sanitizes process environment variables by redacting sensitive secrets + * before passing them to the sandboxed Playwright runner. + */ +export function sanitizeSandboxedEnvironment( + baseEnv: NodeJS.ProcessEnv = process.env, + patterns: RegExp[] = DEFAULT_SENSITIVE_ENV_PATTERNS, +): NodeJS.ProcessEnv { + const sanitized: NodeJS.ProcessEnv = {}; + + for (const [key, value] of Object.entries(baseEnv)) { + if (value === undefined) continue; + const isSensitive = patterns.some((p) => p.test(key)); + if (!isSensitive) { + sanitized[key] = value; + } + } + + // Set explicit sandbox markers + sanitized.AGENTSHIELD_CONTAINER_SANDBOX = "1"; + sanitized.AGENTSHIELD_ZERO_LEAK_EGRESS = "strict"; + sanitized.PLAYWRIGHT_CHROMIUM_SANDBOX = "1"; + + return sanitized; +} + +/** + * Generates browser launch flags that enforce loopback confinement and network isolation in Chromium. + */ +export function buildSandboxedBrowserArgs(policy: ZeroLeakEgressPolicy = {}): string[] { + const allowedHosts = policy.allowedHosts ?? DEFAULT_ALLOWED_HOSTS; + const excludePattern = allowedHosts.join(","); + + return [ + `--host-resolver-rules=MAP * ~NOTFOUND, EXCLUDE ${excludePattern}`, + "--disable-background-networking", + "--disable-component-update", + "--disable-sync", + "--no-default-browser-check", + "--disable-features=Translate,OptimizationHints", + ]; +} + +/** + * Creates an AgentShield-hardened Playwright verification runner + * that executes tests inside zero-leak network egress boundaries. + */ +export function createSandboxedPlaywrightRunner( + options: SandboxedRunnerOptions = {}, +): VerificationRunner { + const timeoutMs = options.timeoutMs ?? 30000; + const cwd = options.cwd ?? process.cwd(); + const egressPolicy = options.egressPolicy ?? {}; + const browserArgs = buildSandboxedBrowserArgs(egressPolicy); + + return async (target: TestTarget): Promise => { + const startTime = Date.now(); + const args = ["playwright", "test", target.testFile]; + + if (target.testTitle) { + args.push("-g", target.testTitle); + } + + const sanitizedEnv = sanitizeSandboxedEnvironment( + { + ...process.env, + ...options.extraEnv, + AUTOHEAL_ACTIVE: "0", + CHROMIUM_FLAGS: browserArgs.join(" "), + }, + egressPolicy.redactEnvPatterns, + ); + + return new Promise((resolve) => { + let stdout = ""; + let stderr = ""; + let timedOut = false; + + const child = spawn("npx", args, { + cwd, + shell: true, + env: sanitizedEnv, + }); + + const timer = setTimeout(() => { + timedOut = true; + try { + child.kill("SIGTERM"); + } catch { + // ignore + } + }, timeoutMs); + + child.stdout?.on("data", (data) => { + stdout += data.toString(); + }); + + child.stderr?.on("data", (data) => { + stderr += data.toString(); + }); + + child.on("close", (code) => { + clearTimeout(timer); + const durationMs = Date.now() - startTime; + const exitCode = code ?? 1; + + if (timedOut) { + resolve({ + passed: false, + exitCode: 124, + durationMs, + output: stdout, + error: `Sandboxed verification timed out after ${timeoutMs}ms`, + }); + return; + } + + const passed = exitCode === 0; + resolve({ + passed, + exitCode, + durationMs, + output: stdout, + error: passed + ? undefined + : stderr || stdout || "Sandboxed Playwright test execution failed", + }); + }); + + child.on("error", (err) => { + clearTimeout(timer); + resolve({ + passed: false, + exitCode: 1, + durationMs: Date.now() - startTime, + output: stdout, + error: `Failed to spawn sandboxed verification process: ${err.message}`, + }); + }); + }); + }; +} diff --git a/packages/core/src/sandbox/index.ts b/packages/core/src/sandbox/index.ts new file mode 100644 index 0000000..e861e35 --- /dev/null +++ b/packages/core/src/sandbox/index.ts @@ -0,0 +1 @@ +export * from "./agentshield-sandbox.js"; diff --git a/tests/sandboxed-execution-and-ast.test.ts b/tests/sandboxed-execution-and-ast.test.ts new file mode 100644 index 0000000..c463b1f --- /dev/null +++ b/tests/sandboxed-execution-and-ast.test.ts @@ -0,0 +1,181 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + calculateSelectorConfidence, + generateAstSelectorPatch, +} from "../packages/core/src/patcher/ast-patcher.js"; +import { + buildSandboxedBrowserArgs, + evaluateEgressDestination, + sanitizeSandboxedEnvironment, +} from "../packages/core/src/sandbox/agentshield-sandbox.js"; + +describe("Phase 3: AutoHeal-QA Sandboxed Execution & Confidence-Scored AST Patches", () => { + describe("AgentShield Playwright Zero-Leak Sandboxed Egress", () => { + it("allows local loopback destinations on permitted ports", () => { + const res1 = evaluateEgressDestination("http://localhost:3000/api/health"); + expect(res1.allowed).toBe(true); + + const res2 = evaluateEgressDestination("127.0.0.1:8080"); + expect(res2.allowed).toBe(true); + }); + + it("strictly blocks cloud metadata endpoints (169.254.169.254 / metadata.google.internal)", () => { + const awsMeta = evaluateEgressDestination("http://169.254.169.254/latest/meta-data/"); + expect(awsMeta.allowed).toBe(false); + expect(awsMeta.reason).toContain("cloud metadata"); + + const gcpMeta = evaluateEgressDestination( + "http://metadata.google.internal/computeMetadata/v1/", + ); + expect(gcpMeta.allowed).toBe(false); + }); + + it("blocks unauthorized public external egress under zero-leak rules", () => { + const external = evaluateEgressDestination("https://api.exfiltration-target.com/leak"); + expect(external.allowed).toBe(false); + expect(external.reason).toContain("Zero-Leak"); + }); + + it("permits explicitly whitelisted external domains", () => { + const res = evaluateEgressDestination("https://auth.ritualdev.in/login", { + allowedHosts: ["localhost", "ritualdev.in"], + }); + expect(res.allowed).toBe(true); + }); + + it("sanitizes sensitive environment variables before launching Playwright browser", () => { + const dirtyEnv = { + PATH: "/usr/bin:/bin", + NODE_ENV: "test", + AWS_SECRET_ACCESS_KEY: "AKIA_FAKE_SECRET_KEY_123", + GH_TOKEN: "ghp_fake_token_abc", + DATABASE_URL: "postgres://user:pass@localhost:5432/db", + ANTHROPIC_API_KEY: "sk-ant-fake-key", + }; + + const cleanEnv = sanitizeSandboxedEnvironment(dirtyEnv); + expect(cleanEnv.PATH).toBe("/usr/bin:/bin"); + expect(cleanEnv.NODE_ENV).toBe("test"); + expect(cleanEnv.AWS_SECRET_ACCESS_KEY).toBeUndefined(); + expect(cleanEnv.GH_TOKEN).toBeUndefined(); + expect(cleanEnv.DATABASE_URL).toBeUndefined(); + expect(cleanEnv.ANTHROPIC_API_KEY).toBeUndefined(); + expect(cleanEnv.AGENTSHIELD_CONTAINER_SANDBOX).toBe("1"); + expect(cleanEnv.AGENTSHIELD_ZERO_LEAK_EGRESS).toBe("strict"); + }); + + it("generates Chromium host resolver confinement flags", () => { + const flags = buildSandboxedBrowserArgs({ + allowedHosts: ["localhost", "127.0.0.1", "testserver.local"], + }); + + const resolverFlag = flags.find((f) => f.startsWith("--host-resolver-rules=")); + expect(resolverFlag).toBeDefined(); + expect(resolverFlag).toContain("MAP * ~NOTFOUND"); + expect(resolverFlag).toContain("EXCLUDE localhost,127.0.0.1,testserver.local"); + expect(flags).toContain("--disable-background-networking"); + }); + }); + + describe("Confidence-Scored AST Selector Patching", () => { + it("computes high confidence and automatic approval for matching role and similar name", () => { + const broken = "page.getByRole('button', { name: 'Submit Order' })"; + const candidate = "page.getByRole('button', { name: 'Place Order' })"; + + const score = calculateSelectorConfidence(broken, candidate, { + isUnique: true, + structuralTagMatch: true, + }); + + expect(score.overallScore).toBeGreaterThanOrEqual(0.7); + expect(["HIGH", "MEDIUM"]).toContain(score.confidenceLevel); + expect(score.breakdown.structuralMatch).toBe(1.0); + expect(score.breakdown.uniquenessScore).toBe(1.0); + }); + + it("assigns high confidence to resilient getByTestId locators", () => { + const broken = "page.locator('.btn-checkout-primary')"; + const candidate = "page.getByTestId('checkout-submit-btn')"; + + const score = calculateSelectorConfidence(broken, candidate, { + isUnique: true, + testIdMatch: true, + }); + + expect(score.overallScore).toBeGreaterThanOrEqual(0.7); + expect(score.breakdown.attributeMatch).toBe(1.0); + }); + + it("rejects non-unique candidate with mismatched semantics with LOW confidence", () => { + const broken = "page.getByRole('button', { name: 'Delete Account' })"; + const candidate = "page.locator('div > span')"; + + const score = calculateSelectorConfidence(broken, candidate, { + isUnique: false, + structuralTagMatch: false, + testIdMatch: false, + }); + + expect(score.confidenceLevel).toBe("LOW"); + expect(score.recommendation).toBe("REJECT"); + }); + + it("generates and applies AST patch with unified diff and backup file", async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "autoheal-test-")); + const testFile = path.join(tmpDir, "checkout.spec.ts"); + + const fileContent = [ + "import { test, expect } from '@playwright/test';", + "", + "test('user can submit order', async ({ page }) => {", + " await page.goto('/checkout');", + " await page.getByRole('button', { name: 'Submit Payment' }).click();", + " await expect(page.getByText('Success')).toBeVisible();", + "});", + ].join("\n"); + + await fs.writeFile(testFile, fileContent, "utf8"); + + const patchResult = await generateAstSelectorPatch( + testFile, + 5, + "page.getByRole('button', { name: 'Submit Payment' })", + "page.getByRole('button', { name: 'Pay Now' })", + { + createBackup: true, + dryRun: false, + }, + ); + + expect(patchResult).not.toBeNull(); + expect(patchResult?.line).toBe(5); + expect(patchResult?.applied).toBe(true); + expect(patchResult?.diff).toContain( + "- await page.getByRole('button', { name: 'Submit Payment' }).click();", + ); + expect(patchResult?.diff).toContain( + "+ await page.getByRole('button', { name: 'Pay Now' }).click();", + ); + expect(patchResult?.confidence.overallScore).toBeGreaterThanOrEqual(0.6); + + // Verify file updated on disk + const updatedContent = await fs.readFile(testFile, "utf8"); + expect(updatedContent).toContain("page.getByRole('button', { name: 'Pay Now' })"); + + // Verify backup created + if (patchResult?.backupFile) { + const backupExists = await fs + .stat(patchResult.backupFile) + .then(() => true) + .catch(() => false); + expect(backupExists).toBe(true); + } + + // Cleanup + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + }); +});