Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/research/realtime-data-flow.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,16 @@ sequenceDiagram
- **SSE heartbeat** every 30s (`:\n\n` comment) to prevent proxy/LB disconnects
- **Inactivity timeout** of 5 minutes — closes stream if no log messages forwarded
- **Redis pub/sub** is the primary log transport (workers publish per-run)
- **Secret redaction at the boundary** — `LogPublisher` and `RedisLogPublisher` run every event's
`message` and (recursively) `data` strings through `redactSecrets` /
`redactLogData` (`packages/shared/src/logging/redact-secrets.ts`) before publishing to Redis
and appending to Blob, so SSE subscribers and `run.jsonl` never see the raw value (issue #1141).
Detection is **value-based** against a curated set of high-confidence token shapes (private-key
PEM blocks, Azure `AccountKey=` and SAS `sig=`, `Bearer` tokens, JWTs, AWS access key ids,
GitHub/Slack/OpenAI/Anthropic/Google/Stripe/npm/Hugging Face tokens); matches become
`[REDACTED:<rule-id>]`, keeping descriptive prefixes like `Bearer ` or `AccountKey=`. The
workers' name-based filter in `formatToolArgs` stays as defense-in-depth. Generic entropy and
PII detection are not included; tool-call JSONL (below) is not scrubbed by this layer.
- **MongoDB Change Stream** is a **completion detector only** — watches for status field updates to `completed`/`failed`/`exhausted`
- Cleanup on client disconnect (`req.on("close")`)

Expand Down
1 change: 1 addition & 0 deletions packages/shared/src/logging/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
// Licensed under the MIT License.

export * from "./log-publisher.js";
export * from "./redact-secrets.js";
31 changes: 31 additions & 0 deletions packages/shared/src/logging/log-publisher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,37 @@ describe("LogPublisher", () => {
});
});

describe("secret redaction (#1141)", () => {
// Assembled at runtime so the source never contains a literal token shape.
const githubToken = "gh" + "p_" + "a1B2".repeat(9);

it("redacts secrets in the message and data before both Redis and Blob", async () => {
const publisher = makePublisher();
await publisher.publish("run-abc", "attempt-1", "info", `cloning with ${githubToken}`, {
args: { command: `git push https://x:${githubToken}@github.com/o/r` },
});

const streamed = JSON.parse(mockRedis.publish.mock.calls[0][1]);
const persisted: LogEvent = mockAppendLogEvent.mock.calls[0][2];
for (const event of [streamed, persisted]) {
expect(event.message).toBe("cloning with [REDACTED:github-token]");
expect(event.data).toEqual({ args: { command: "git push https://x:[REDACTED:github-token]@github.com/o/r" } });
}
});

it("redacts secrets published through RedisLogPublisher", async () => {
const { RedisLogPublisher } = await import("./log-publisher.js");
const publisher = new RedisLogPublisher({ redisHost: "127.0.0.1", redisPort: 1, redisPassword: "" });
(publisher as any).redis?.disconnect?.();
(publisher as any).redis = mockRedis;
(publisher as any).redisBreaker = { execute: async (fn: () => Promise<unknown>) => fn() };

await publisher.publish("run-abc", "info", `token ${githubToken}`);

expect(JSON.parse(mockRedis.publish.mock.calls[0][1]).message).toBe("token [REDACTED:github-token]");
});
});

describe("close()", () => {
it("calls redis.quit()", async () => {
const publisher = makePublisher();
Expand Down
42 changes: 28 additions & 14 deletions packages/shared/src/logging/log-publisher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const Redis = require("ioredis");
import { circuitBreaker, handleAll, ConsecutiveBreaker, CircuitState } from "cockatiel";
import { LogEvent } from "../types/types.js";
import { BlobStorage } from "../storage/blob-storage.js";
import { redactLogData, redactSecrets } from "./redact-secrets.js";

export interface RedisConfig {
redisHost: string;
Expand All @@ -16,6 +17,31 @@ export interface RedisConfig {

export type LogPublisherConfig = RedisConfig;

/**
* Build a log event with value-based secret redaction applied to its message
* and data (issue #1141). Both publishers go through here, so every line that
* reaches Redis/SSE or Blob storage is scrubbed exactly once.
*
* @param level - Log level.
* @param source - Emitting service (e.g. `"coder"`, `"judge"`).
* @param message - Human-readable log line.
* @param data - Optional structured payload; string values are redacted recursively.
*/
function buildRedactedLogEvent(
level: LogEvent["level"],
source: string,
message: string,
data?: Record<string, unknown>,
): LogEvent {
return {
timestamp: new Date().toISOString(),
level,
source,
message: redactSecrets(message),
data: data === undefined ? undefined : redactLogData(data),
};
}

export class LogPublisher {
private redis: InstanceType<typeof Redis>;
private blobStorage: BlobStorage;
Expand Down Expand Up @@ -75,13 +101,7 @@ export class LogPublisher {
message: string,
data?: Record<string, unknown>
): Promise<void> {
const logEvent: LogEvent = {
timestamp: new Date().toISOString(),
level,
source: this.source,
message,
data,
};
const logEvent = buildRedactedLogEvent(level, this.source, message, data);

// Publish to Redis for real-time streaming (with circuit breaker).
// The Redis channel is keyed on requestId so SSE listeners receive logs
Expand Down Expand Up @@ -170,13 +190,7 @@ export class RedisLogPublisher {
message: string,
data?: Record<string, unknown>
): Promise<void> {
const logEvent: LogEvent = {
timestamp: new Date().toISOString(),
level,
source: this.source,
message,
data,
};
const logEvent = buildRedactedLogEvent(level, this.source, message, data);

const channel = `logs:${requestId}`;
try {
Expand Down
108 changes: 108 additions & 0 deletions packages/shared/src/logging/redact-secrets.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { describe, it, expect } from "vitest";
import { redactLogData, redactSecrets } from "./redact-secrets.js";

// Fake secrets are assembled at runtime so the source file never contains a
// literal token shape (keeps secret scanners / push protection quiet).
const fake = {
aws: "AK" + "IA" + "ABCDEFGHIJKLMNOP",
github: "gh" + "p_" + "a1B2".repeat(9),
githubFineGrained: "github" + "_pat_" + "11ABCDEFG0".repeat(3),
slack: "xo" + "xb-" + "1234567890-abcdefghij",
openai: "sk" + "-proj-" + "Ab1".repeat(10),
anthropic: "sk" + "-ant-" + "api03-" + "Xy9".repeat(10),
google: "AI" + "za" + "S".repeat(35),
stripe: "sk" + "_live_" + "4eC39HqLyjWDarjtT1zdp7dc",
npm: "np" + "m_" + "z".repeat(36),
huggingface: "h" + "f_" + "Q".repeat(34),
jwt: "ey" + "JhbGciOiJIUzI1NiJ9" + ".ey" + "JzdWIiOiIxMjM0NTY3ODkwIn0" + ".dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U",
storageKey: "Q".repeat(86) + "==",
sasSig: "a%2Bb%2Fc" + "d".repeat(40) + "%3D",
bearer: "abc.DEF_ghi-" + "J".repeat(20),
pem: "-----BEGIN " + "RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA\nabc\n-----END RSA PRIVATE KEY-----",
};

describe("redactSecrets", () => {
it.each([
["aws-access-key-id", `export AWS_ACCESS_KEY_ID=${fake.aws}`],
["github-token", `git clone https://x:${fake.github}@github.com/o/r`],
["github-fine-grained-pat", `token ${fake.githubFineGrained}`],
["slack-token", `SLACK=${fake.slack}`],
["openai-anthropic-key", `OPENAI_API_KEY=${fake.openai}`],
["openai-anthropic-key", `ANTHROPIC_API_KEY=${fake.anthropic}`],
["google-api-key", `key=${fake.google}`],
["stripe-key", `stripe ${fake.stripe}`],
["npm-token", `//registry.npmjs.org/:_authToken=${fake.npm}`],
["huggingface-token", `HF_TOKEN=${fake.huggingface}`],
["jwt", `cookie: session=${fake.jwt}`],
["private-key", `wrote id_rsa:\n${fake.pem}\ndone`],
])("redacts a %s found under an innocent context", (ruleId, text) => {
const result = redactSecrets(text);

expect(result).toContain(`[REDACTED:${ruleId}]`);
expect(result).not.toMatch(/AKIA[A-Z]{16}|ghp_|github_pat_|xoxb-|sk-proj-|sk-ant-|AIzaS|sk_live_|npm_z|hf_Q|eyJzdWIi|MIIEpA/);
});

it("keeps the descriptive prefix for Bearer, AccountKey and SAS signatures", () => {
expect(redactSecrets(`command="curl -H 'Authorization: Bearer ${fake.bearer}'"`))
.toBe(`command="curl -H 'Authorization: Bearer [REDACTED:authorization-bearer]'"`);
expect(redactSecrets(`DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=${fake.storageKey};EndpointSuffix=core.windows.net`))
.toBe("DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=[REDACTED:azure-storage-key];EndpointSuffix=core.windows.net");
expect(redactSecrets(`https://acct.blob.core.windows.net/c/b.tar.gz?sv=2024-01-01&sr=b&sig=${fake.sasSig}&se=2026`))
.toBe("https://acct.blob.core.windows.net/c/b.tar.gz?sv=2024-01-01&sr=b&sig=[REDACTED:azure-sas-signature]&se=2026");
});

it("redacts every occurrence in a line", () => {
const result = redactSecrets(`${fake.github} and ${fake.github}`);
expect(result).toBe("[REDACTED:github-token] and [REDACTED:github-token]");
});

it.each([
"Iteration 3 complete — 4/5 criteria passed",
"Uploaded snapshot https://acct.blob.core.windows.net/snapshots/req-1/iteration-1/workspace.tar.gz",
"commit 44d16d6a8f3e2b1c0d9e8f7a6b5c4d3e2f1a0b9c on branch high-priority-task-prompts",
"run 0199a3f2-7c4b-7d1e-9a2b-3c4d5e6f7a8b sk-learn style naming in task-skills",
"Authorization: Bearer <token>",
"GitHub ghp_short is not a token",
])("leaves ordinary log text untouched: %s", (text) => {
expect(redactSecrets(text)).toBe(text);
});
});

describe("redactLogData", () => {
it("redacts strings nested in objects and arrays without mutating the input", () => {
const data = {
tool: "bash",
args: { command: `curl -H "Authorization: Bearer ${fake.bearer}"`, cwd: "/workspace" },
outputs: ["ok", `AWS_ACCESS_KEY_ID=${fake.aws}`],
exitCode: 0,
};
const snapshot = structuredClone(data);

const result = redactLogData(data);

expect(result).toEqual({
tool: "bash",
args: { command: 'curl -H "Authorization: Bearer [REDACTED:authorization-bearer]"', cwd: "/workspace" },
outputs: ["ok", "AWS_ACCESS_KEY_ID=[REDACTED:aws-access-key-id]"],
exitCode: 0,
});
expect(data).toEqual(snapshot);
});

it("leaves non-plain objects and primitives as they are", () => {
const when = new Date("2026-01-01T00:00:00Z");
expect(redactLogData({ when, n: 1, ok: true, none: null }).when).toBe(when);
expect(redactLogData(42)).toBe(42);
});

it("stops descending past the depth limit", () => {
let deep: Record<string, unknown> = { secret: fake.github };
for (let i = 0; i < 10; i++) deep = { next: deep };

// Deeper than the limit: returned by reference, not walked.
expect(JSON.stringify(redactLogData(deep))).toContain(fake.github);
});
});
118 changes: 118 additions & 0 deletions packages/shared/src/logging/redact-secrets.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

/**
* Value-based secret redaction for run logs (issue #1141).
*
* Name-based filters (e.g. the ACP workers' `formatToolArgs`) only catch a
* secret stored under a sensitive-looking key. This module inspects the
* *value*, so a token is caught wherever it appears — inside a `command`
* argument, a free-text message, or a nested `data` field.
*
* It is applied once, centrally, in `LogPublisher` / `RedisLogPublisher`, the
* boundary where log lines are persisted to Blob storage and fanned out to
* SSE subscribers via Redis.
*
* The rule set is a small, curated subset of high-confidence token shapes in
* the spirit of OWASP `secrets-patterns-db` (the corpus behind Gitleaks and
* TruffleHog): every rule matches a distinctive prefix or structure, so false
* positives on ordinary debug output stay rare. Generic high-entropy and PII
* detection are intentionally not included yet.
*/

/** One value-based detection rule. */
interface SecretRule {
/** Stable rule id, shown in the replacement marker (`[REDACTED:<id>]`). */
id: string;
/** Global regex. When it has a `prefix` named group, that part is kept. */
pattern: RegExp;
}

/**
* Curated high-confidence secret patterns. Order matters: broader structures
* (PEM blocks, connection strings) run before the token shapes they contain.
*/
const SECRET_RULES: readonly SecretRule[] = [
{
id: "private-key",
pattern: /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2 / medium] Cover PEM blocks split across the real log stream. This rule requires BEGIN, key material, and END in one string, but the ACP subprocess stderr callback forwards each pipe chunk separately (apps/workers/coder-acp-copilot/src/acp-client.ts:573-575). Independent writes of the three PEM lines therefore produce events with no redaction marker, and log readers can reconstruct the complete key.

This was reproduced using a real subprocess stderr pipe and an ephemeral in-memory Ed25519 key: three chunks, zero redaction markers, and the reconstructed key still parsed. Passing the same key as one string was redacted. This is the forwarded stderr -> publisher path, not the explicitly excluded worker stdout or tool-call JSONL sink.

Please add stream-aware PEM suppression at an appropriate pre-publication boundary, including markers split between chunks and incomplete blocks, and test that path with separately emitted lines. If that requires a separately scoped change, the coverage/guarantee must at least be made explicit rather than claiming private-key protection for this delivery path.

},
{ id: "azure-storage-key", pattern: /(?<prefix>\bAccountKey=)[A-Za-z0-9+/]{40,}={0,2}/g },
// SAS signature only — the rest of the blob URL stays readable for debugging.
{ id: "azure-sas-signature", pattern: /(?<prefix>[?&]sig=)[A-Za-z0-9%+/=]{16,}/g },
{ id: "authorization-bearer", pattern: /(?<prefix>\bBearer\s+)[A-Za-z0-9._~+/-]{16,}=*/g },

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2 / medium] Match HTTP bearer schemes case-insensitively in both the rule and prefilter. HTTP authentication schemes are case-insensitive, but this regex and the substring fast path only recognize Bearer with that exact capitalization.

For example, redactSecrets("curl -H 'Authorization: bearer syntheticOpaqueCredential1234567890' https://example.invalid") leaves the entire synthetic credential visible; changing only bearer to Bearer redacts it. The production name-based argument filter also leaves this intact because the enclosing argument is command, not a sensitive-looking field name.

Please make both scheme detection and the fast-path condition case-insensitive while preserving the original prefix, and add lowercase/uppercase/mixed-case publisher regressions. Otherwise valid credentials with unrecognized value shapes still reach Redis/SSE and run.jsonl despite the new bearer protection.

{ id: "jwt", pattern: /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g },
{ id: "aws-access-key-id", pattern: /\b(?:AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{16}\b/g },
{ id: "github-token", pattern: /\bgh[pousr]_[A-Za-z0-9]{36,255}\b/g },
{ id: "github-fine-grained-pat", pattern: /\bgithub_pat_[A-Za-z0-9_]{22,255}\b/g },
{ id: "slack-token", pattern: /\bxox[abposr]-[A-Za-z0-9-]{10,}\b/g },
{ id: "stripe-key", pattern: /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{20,}\b/g },
// Covers OpenAI (`sk-`, `sk-proj-`) and Anthropic (`sk-ant-`) API keys.
{ id: "openai-anthropic-key", pattern: /\bsk-(?:ant-|proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}/g },
{ id: "google-api-key", pattern: /\bAIza[0-9A-Za-z_-]{35}\b/g },
{ id: "npm-token", pattern: /\bnpm_[A-Za-z0-9]{36}\b/g },
{ id: "huggingface-token", pattern: /\bhf_[A-Za-z0-9]{34,}\b/g },
];

/**
* Cheap substring pre-filter. A string containing none of these can't match
* any rule, so the common case (ordinary log text) skips every regex.
*/
const RULE_HINTS: readonly string[] = [
"PRIVATE KEY", "AccountKey=", "sig=", "Bearer", "eyJ", "AKIA", "ASIA", "ABIA", "ACCA",
"ghp_", "gho_", "ghu_", "ghs_", "ghr_", "github_pat_", "xox", "_live_", "_test_", "sk-", "AIza", "npm_", "hf_",
];

/** Maximum nesting depth walked by {@link redactLogData}; deeper values are kept as-is. */
const MAX_DATA_DEPTH = 8;

/**
* Replace every known secret shape in `text` with `[REDACTED:<rule-id>]`.
* A rule's `prefix` group (e.g. `Bearer `, `AccountKey=`) is preserved so the
* line still shows *what kind* of value was removed.
*
* @param text - Arbitrary log text.
* @returns The text with secrets replaced; the same string when nothing matched.
*/
export function redactSecrets(text: string): string {
if (!RULE_HINTS.some((hint) => text.includes(hint))) return text;

let result = text;
for (const rule of SECRET_RULES) {
result = result.replace(rule.pattern, (...args: unknown[]) => {
// The last replacer argument is the named-groups object when the regex has any.
const groups = args[args.length - 1] as { prefix?: string } | undefined;
const prefix = typeof groups === "object" && groups?.prefix ? groups.prefix : "";
return `${prefix}[REDACTED:${rule.id}]`;
});
}
return result;
}

/**
* Deep-copy `value`, applying {@link redactSecrets} to every string inside
* plain objects and arrays. Non-plain objects (Dates, class instances) and
* values nested deeper than {@link MAX_DATA_DEPTH} are returned unchanged.
*
* @param value - A log event's structured `data` payload (or any part of it).
* @param depth - Current nesting depth; callers omit it.
* @returns A redacted copy; never mutates the input.
*/
export function redactLogData<T>(value: T, depth = 0): T {
if (typeof value === "string") return redactSecrets(value) as T;
if (depth >= MAX_DATA_DEPTH || value === null || typeof value !== "object") return value;

if (Array.isArray(value)) {
return value.map((item) => redactLogData(item, depth + 1)) as T;
}

// Only walk plain objects; leave Dates, Buffers, etc. untouched.
const proto = Object.getPrototypeOf(value);
if (proto !== Object.prototype && proto !== null) return value;

const copy: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value as Record<string, unknown>)) {
copy[key] = redactLogData(item, depth + 1);
}
return copy as T;
}