Repository navigation
feat: central value-based secret redaction in the log publishers #1588
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,3 +2,4 @@ | |
| // Licensed under the MIT License. | ||
|
|
||
| export * from "./log-publisher.js"; | ||
| export * from "./redact-secrets.js"; | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| import { describe, it, expect } from "vitest"; | ||
| import { redactLogData, redactSecrets } from "./redact-secrets.js"; | ||
|
|
||
| // Fake secrets are assembled at runtime so the source file never contains a | ||
| // literal token shape (keeps secret scanners / push protection quiet). | ||
| const fake = { | ||
| aws: "AK" + "IA" + "ABCDEFGHIJKLMNOP", | ||
| github: "gh" + "p_" + "a1B2".repeat(9), | ||
| githubFineGrained: "github" + "_pat_" + "11ABCDEFG0".repeat(3), | ||
| slack: "xo" + "xb-" + "1234567890-abcdefghij", | ||
| openai: "sk" + "-proj-" + "Ab1".repeat(10), | ||
| anthropic: "sk" + "-ant-" + "api03-" + "Xy9".repeat(10), | ||
| google: "AI" + "za" + "S".repeat(35), | ||
| stripe: "sk" + "_live_" + "4eC39HqLyjWDarjtT1zdp7dc", | ||
| npm: "np" + "m_" + "z".repeat(36), | ||
| huggingface: "h" + "f_" + "Q".repeat(34), | ||
| jwt: "ey" + "JhbGciOiJIUzI1NiJ9" + ".ey" + "JzdWIiOiIxMjM0NTY3ODkwIn0" + ".dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U", | ||
| storageKey: "Q".repeat(86) + "==", | ||
| sasSig: "a%2Bb%2Fc" + "d".repeat(40) + "%3D", | ||
| bearer: "abc.DEF_ghi-" + "J".repeat(20), | ||
| pem: "-----BEGIN " + "RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA\nabc\n-----END RSA PRIVATE KEY-----", | ||
| }; | ||
|
|
||
| describe("redactSecrets", () => { | ||
| it.each([ | ||
| ["aws-access-key-id", `export AWS_ACCESS_KEY_ID=${fake.aws}`], | ||
| ["github-token", `git clone https://x:${fake.github}@github.com/o/r`], | ||
| ["github-fine-grained-pat", `token ${fake.githubFineGrained}`], | ||
| ["slack-token", `SLACK=${fake.slack}`], | ||
| ["openai-anthropic-key", `OPENAI_API_KEY=${fake.openai}`], | ||
| ["openai-anthropic-key", `ANTHROPIC_API_KEY=${fake.anthropic}`], | ||
| ["google-api-key", `key=${fake.google}`], | ||
| ["stripe-key", `stripe ${fake.stripe}`], | ||
| ["npm-token", `//registry.npmjs.org/:_authToken=${fake.npm}`], | ||
| ["huggingface-token", `HF_TOKEN=${fake.huggingface}`], | ||
| ["jwt", `cookie: session=${fake.jwt}`], | ||
| ["private-key", `wrote id_rsa:\n${fake.pem}\ndone`], | ||
| ])("redacts a %s found under an innocent context", (ruleId, text) => { | ||
| const result = redactSecrets(text); | ||
|
|
||
| expect(result).toContain(`[REDACTED:${ruleId}]`); | ||
| expect(result).not.toMatch(/AKIA[A-Z]{16}|ghp_|github_pat_|xoxb-|sk-proj-|sk-ant-|AIzaS|sk_live_|npm_z|hf_Q|eyJzdWIi|MIIEpA/); | ||
| }); | ||
|
|
||
| it("keeps the descriptive prefix for Bearer, AccountKey and SAS signatures", () => { | ||
| expect(redactSecrets(`command="curl -H 'Authorization: Bearer ${fake.bearer}'"`)) | ||
| .toBe(`command="curl -H 'Authorization: Bearer [REDACTED:authorization-bearer]'"`); | ||
| expect(redactSecrets(`DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=${fake.storageKey};EndpointSuffix=core.windows.net`)) | ||
| .toBe("DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=[REDACTED:azure-storage-key];EndpointSuffix=core.windows.net"); | ||
| expect(redactSecrets(`https://acct.blob.core.windows.net/c/b.tar.gz?sv=2024-01-01&sr=b&sig=${fake.sasSig}&se=2026`)) | ||
| .toBe("https://acct.blob.core.windows.net/c/b.tar.gz?sv=2024-01-01&sr=b&sig=[REDACTED:azure-sas-signature]&se=2026"); | ||
| }); | ||
|
|
||
| it("redacts every occurrence in a line", () => { | ||
| const result = redactSecrets(`${fake.github} and ${fake.github}`); | ||
| expect(result).toBe("[REDACTED:github-token] and [REDACTED:github-token]"); | ||
| }); | ||
|
|
||
| it.each([ | ||
| "Iteration 3 complete — 4/5 criteria passed", | ||
| "Uploaded snapshot https://acct.blob.core.windows.net/snapshots/req-1/iteration-1/workspace.tar.gz", | ||
| "commit 44d16d6a8f3e2b1c0d9e8f7a6b5c4d3e2f1a0b9c on branch high-priority-task-prompts", | ||
| "run 0199a3f2-7c4b-7d1e-9a2b-3c4d5e6f7a8b sk-learn style naming in task-skills", | ||
| "Authorization: Bearer <token>", | ||
| "GitHub ghp_short is not a token", | ||
| ])("leaves ordinary log text untouched: %s", (text) => { | ||
| expect(redactSecrets(text)).toBe(text); | ||
| }); | ||
| }); | ||
|
|
||
| describe("redactLogData", () => { | ||
| it("redacts strings nested in objects and arrays without mutating the input", () => { | ||
| const data = { | ||
| tool: "bash", | ||
| args: { command: `curl -H "Authorization: Bearer ${fake.bearer}"`, cwd: "/workspace" }, | ||
| outputs: ["ok", `AWS_ACCESS_KEY_ID=${fake.aws}`], | ||
| exitCode: 0, | ||
| }; | ||
| const snapshot = structuredClone(data); | ||
|
|
||
| const result = redactLogData(data); | ||
|
|
||
| expect(result).toEqual({ | ||
| tool: "bash", | ||
| args: { command: 'curl -H "Authorization: Bearer [REDACTED:authorization-bearer]"', cwd: "/workspace" }, | ||
| outputs: ["ok", "AWS_ACCESS_KEY_ID=[REDACTED:aws-access-key-id]"], | ||
| exitCode: 0, | ||
| }); | ||
| expect(data).toEqual(snapshot); | ||
| }); | ||
|
|
||
| it("leaves non-plain objects and primitives as they are", () => { | ||
| const when = new Date("2026-01-01T00:00:00Z"); | ||
| expect(redactLogData({ when, n: 1, ok: true, none: null }).when).toBe(when); | ||
| expect(redactLogData(42)).toBe(42); | ||
| }); | ||
|
|
||
| it("stops descending past the depth limit", () => { | ||
| let deep: Record<string, unknown> = { secret: fake.github }; | ||
| for (let i = 0; i < 10; i++) deep = { next: deep }; | ||
|
|
||
| // Deeper than the limit: returned by reference, not walked. | ||
| expect(JSON.stringify(redactLogData(deep))).toContain(fake.github); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| /** | ||
| * Value-based secret redaction for run logs (issue #1141). | ||
| * | ||
| * Name-based filters (e.g. the ACP workers' `formatToolArgs`) only catch a | ||
| * secret stored under a sensitive-looking key. This module inspects the | ||
| * *value*, so a token is caught wherever it appears — inside a `command` | ||
| * argument, a free-text message, or a nested `data` field. | ||
| * | ||
| * It is applied once, centrally, in `LogPublisher` / `RedisLogPublisher`, the | ||
| * boundary where log lines are persisted to Blob storage and fanned out to | ||
| * SSE subscribers via Redis. | ||
| * | ||
| * The rule set is a small, curated subset of high-confidence token shapes in | ||
| * the spirit of OWASP `secrets-patterns-db` (the corpus behind Gitleaks and | ||
| * TruffleHog): every rule matches a distinctive prefix or structure, so false | ||
| * positives on ordinary debug output stay rare. Generic high-entropy and PII | ||
| * detection are intentionally not included yet. | ||
| */ | ||
|
|
||
| /** One value-based detection rule. */ | ||
| interface SecretRule { | ||
| /** Stable rule id, shown in the replacement marker (`[REDACTED:<id>]`). */ | ||
| id: string; | ||
| /** Global regex. When it has a `prefix` named group, that part is kept. */ | ||
| pattern: RegExp; | ||
| } | ||
|
|
||
| /** | ||
| * Curated high-confidence secret patterns. Order matters: broader structures | ||
| * (PEM blocks, connection strings) run before the token shapes they contain. | ||
| */ | ||
| const SECRET_RULES: readonly SecretRule[] = [ | ||
| { | ||
| id: "private-key", | ||
| pattern: /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g, | ||
| }, | ||
| { id: "azure-storage-key", pattern: /(?<prefix>\bAccountKey=)[A-Za-z0-9+/]{40,}={0,2}/g }, | ||
| // SAS signature only — the rest of the blob URL stays readable for debugging. | ||
| { id: "azure-sas-signature", pattern: /(?<prefix>[?&]sig=)[A-Za-z0-9%+/=]{16,}/g }, | ||
| { id: "authorization-bearer", pattern: /(?<prefix>\bBearer\s+)[A-Za-z0-9._~+/-]{16,}=*/g }, | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2 / medium] Match HTTP bearer schemes case-insensitively in both the rule and prefilter. HTTP authentication schemes are case-insensitive, but this regex and the substring fast path only recognize For example, Please make both scheme detection and the fast-path condition case-insensitive while preserving the original prefix, and add lowercase/uppercase/mixed-case publisher regressions. Otherwise valid credentials with unrecognized value shapes still reach Redis/SSE and run.jsonl despite the new bearer protection. |
||
| { id: "jwt", pattern: /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g }, | ||
| { id: "aws-access-key-id", pattern: /\b(?:AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{16}\b/g }, | ||
| { id: "github-token", pattern: /\bgh[pousr]_[A-Za-z0-9]{36,255}\b/g }, | ||
| { id: "github-fine-grained-pat", pattern: /\bgithub_pat_[A-Za-z0-9_]{22,255}\b/g }, | ||
| { id: "slack-token", pattern: /\bxox[abposr]-[A-Za-z0-9-]{10,}\b/g }, | ||
| { id: "stripe-key", pattern: /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{20,}\b/g }, | ||
| // Covers OpenAI (`sk-`, `sk-proj-`) and Anthropic (`sk-ant-`) API keys. | ||
| { id: "openai-anthropic-key", pattern: /\bsk-(?:ant-|proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}/g }, | ||
| { id: "google-api-key", pattern: /\bAIza[0-9A-Za-z_-]{35}\b/g }, | ||
| { id: "npm-token", pattern: /\bnpm_[A-Za-z0-9]{36}\b/g }, | ||
| { id: "huggingface-token", pattern: /\bhf_[A-Za-z0-9]{34,}\b/g }, | ||
| ]; | ||
|
|
||
| /** | ||
| * Cheap substring pre-filter. A string containing none of these can't match | ||
| * any rule, so the common case (ordinary log text) skips every regex. | ||
| */ | ||
| const RULE_HINTS: readonly string[] = [ | ||
| "PRIVATE KEY", "AccountKey=", "sig=", "Bearer", "eyJ", "AKIA", "ASIA", "ABIA", "ACCA", | ||
| "ghp_", "gho_", "ghu_", "ghs_", "ghr_", "github_pat_", "xox", "_live_", "_test_", "sk-", "AIza", "npm_", "hf_", | ||
| ]; | ||
|
|
||
| /** Maximum nesting depth walked by {@link redactLogData}; deeper values are kept as-is. */ | ||
| const MAX_DATA_DEPTH = 8; | ||
|
|
||
| /** | ||
| * Replace every known secret shape in `text` with `[REDACTED:<rule-id>]`. | ||
| * A rule's `prefix` group (e.g. `Bearer `, `AccountKey=`) is preserved so the | ||
| * line still shows *what kind* of value was removed. | ||
| * | ||
| * @param text - Arbitrary log text. | ||
| * @returns The text with secrets replaced; the same string when nothing matched. | ||
| */ | ||
| export function redactSecrets(text: string): string { | ||
| if (!RULE_HINTS.some((hint) => text.includes(hint))) return text; | ||
|
|
||
| let result = text; | ||
| for (const rule of SECRET_RULES) { | ||
| result = result.replace(rule.pattern, (...args: unknown[]) => { | ||
| // The last replacer argument is the named-groups object when the regex has any. | ||
| const groups = args[args.length - 1] as { prefix?: string } | undefined; | ||
| const prefix = typeof groups === "object" && groups?.prefix ? groups.prefix : ""; | ||
| return `${prefix}[REDACTED:${rule.id}]`; | ||
| }); | ||
| } | ||
| return result; | ||
| } | ||
|
|
||
| /** | ||
| * Deep-copy `value`, applying {@link redactSecrets} to every string inside | ||
| * plain objects and arrays. Non-plain objects (Dates, class instances) and | ||
| * values nested deeper than {@link MAX_DATA_DEPTH} are returned unchanged. | ||
| * | ||
| * @param value - A log event's structured `data` payload (or any part of it). | ||
| * @param depth - Current nesting depth; callers omit it. | ||
| * @returns A redacted copy; never mutates the input. | ||
| */ | ||
| export function redactLogData<T>(value: T, depth = 0): T { | ||
| if (typeof value === "string") return redactSecrets(value) as T; | ||
| if (depth >= MAX_DATA_DEPTH || value === null || typeof value !== "object") return value; | ||
|
|
||
| if (Array.isArray(value)) { | ||
| return value.map((item) => redactLogData(item, depth + 1)) as T; | ||
| } | ||
|
|
||
| // Only walk plain objects; leave Dates, Buffers, etc. untouched. | ||
| const proto = Object.getPrototypeOf(value); | ||
| if (proto !== Object.prototype && proto !== null) return value; | ||
|
|
||
| const copy: Record<string, unknown> = {}; | ||
| for (const [key, item] of Object.entries(value as Record<string, unknown>)) { | ||
| copy[key] = redactLogData(item, depth + 1); | ||
| } | ||
| return copy as T; | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P2 / medium] Cover PEM blocks split across the real log stream. This rule requires BEGIN, key material, and END in one string, but the ACP subprocess stderr callback forwards each pipe chunk separately (
apps/workers/coder-acp-copilot/src/acp-client.ts:573-575). Independent writes of the three PEM lines therefore produce events with no redaction marker, and log readers can reconstruct the complete key.This was reproduced using a real subprocess stderr pipe and an ephemeral in-memory Ed25519 key: three chunks, zero redaction markers, and the reconstructed key still parsed. Passing the same key as one string was redacted. This is the forwarded stderr -> publisher path, not the explicitly excluded worker stdout or tool-call JSONL sink.
Please add stream-aware PEM suppression at an appropriate pre-publication boundary, including markers split between chunks and incomplete blocks, and test that path with separately emitted lines. If that requires a separately scoped change, the coverage/guarantee must at least be made explicit rather than claiming private-key protection for this delivery path.