diff --git a/NOTICE b/NOTICE index 9de9579..ed6629b 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,5 @@ Code Interpreter Copyright 2026 ClickHouse, Inc. + +This product includes RTK v0.45.0 (https://github.com/rtk-ai/rtk), +Copyright 2024 Patrick Szymkowiak, licensed under the Apache License 2.0. diff --git a/README.md b/README.md index 718db6a..d713d9b 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,28 @@ descriptors in the launcher. Setting `KVM_ENABLED=false` still selects the directory-root target and the host package mount automatically for direct NsJail development. +## Optional Bash output filtering + +Bash executions can opt into [RTK](https://github.com/rtk-ai/rtk) command +rewriting per request by setting `shell_output_filter` to `rtk`: + +```json +{ + "lang": "bash", + "code": "ls -la", + "shell_output_filter": "rtk" +} +``` + +Omitting the field (or setting it to `raw`) preserves the original execution +path, which makes side-by-side evaluation straightforward. RTK `v0.45.0` is +source-pinned to its immutable release commit and preinstalled in the sandbox +image; it can also be invoked directly from Bash. Rewriting happens inside +NsJail and fails open to the original script when RTK does not support a +command or cannot run. Prometheus +metrics expose Bash execution counts, outcomes, and stdout/stderr byte sizes by +the `raw` or `rtk` filter without adding request-specific labels. + Local Docker Compose files set `CODEAPI_INTERNAL_SERVICE_TOKEN` to a shared development value by default. Production deployments must override it with a strong secret; when it is unset, file object routes and Tool Call Server diff --git a/api/Dockerfile b/api/Dockerfile index f8d6713..6c486ee 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -60,9 +60,24 @@ RUN apt-install \ RUN mkdir -p /pkgs COPY docker/package-init.sh /package-init.sh +COPY docker/bash-run.sh /bash-run.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh +# ============================================================================ +# Stage 1c: Build the RTK binary against the sandbox's glibc baseline. The +# upstream arm64 release requires a newer glibc than Debian bookworm provides. +# ============================================================================ +FROM rust:1.91-bookworm AS rtk-builder + +ARG RTK_COMMIT=b34be37caf3796b69a50952a28e60e32b5daad43 +RUN git clone --depth 1 --branch v0.45.0 https://github.com/rtk-ai/rtk.git /rtk \ + && cd /rtk \ + && git checkout "$RTK_COMMIT" \ + && test "$(git rev-parse HEAD)" = "$RTK_COMMIT" \ + && cargo build --locked --release \ + && ./target/release/rtk --version + # ============================================================================ # Stage 2: Build sandbox rootfs # ============================================================================ @@ -108,6 +123,7 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=rtk-builder /rtk/target/release/rtk /usr/local/bin/rtk RUN chmod +x /usr/sbin/nsjail # The tool-call socket proxy must run under Node, not Bun. Bun's node:http diff --git a/api/README.md b/api/README.md index e0e8a6d..30f90db 100644 --- a/api/README.md +++ b/api/README.md @@ -109,3 +109,7 @@ curl -s http://localhost:2000/api/v2/execute \ -H 'Content-Type: application/json' \ -d '{"language":"python","version":"3.14.4","files":[{"content":"print(42)"}]}' | jq ``` + +For Bash, add `"shell_output_filter":"rtk"` to opt into compact RTK command +rewrites for that request. Omit the field or use `"raw"` to preserve the +unfiltered path. Other runtimes reject the field. diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index 895fec6..d5c2a21 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -13,7 +13,11 @@ import { import { EXECUTION_MANIFEST_HEADER, ExecutionManifestError, type ExecutionManifestClaims } from '../execution-manifest'; import { verifyExecuteRequestManifest } from '../execution-manifest-request'; import { EGRESS_GRANT_HEADER } from '../egress'; -import { activeSandboxExecutions, recordSandboxExecution } from '../metrics'; +import { + activeSandboxExecutions, + recordSandboxExecution, + recordSandboxOutputBytes, +} from '../metrics'; import { classifySandboxSafeError } from '../safe-error'; import { withSpan } from '../telemetry'; import { checkSandboxWorkspaceHealth } from '../workspace-isolation'; @@ -31,6 +35,10 @@ import { pruneInputCache, storeCachedInputs, } from '../session-inputs'; +import { + resolveShellOutputFilter, + type ShellOutputFilter, +} from '../../../shared/shell-output-filter'; const router = express.Router(); const SYNTHETIC_PRINCIPAL_SOURCE = 'synthetic_test'; @@ -132,6 +140,7 @@ export interface ExecuteRequestBody { output_session_id?: string; language: string; version: string; + shell_output_filter?: ShellOutputFilter; args?: string[]; stdin?: string; files: TFile[]; @@ -266,6 +275,7 @@ function getJob( if (!rt) { throw { message: `${language}-${version} runtime is unknown` }; } + const shellOutputFilter = resolveShellOutputFilter(body.shell_output_filter, rt.language); if ( rt.language !== 'file' && @@ -327,6 +337,7 @@ function getJob( compile: compile_memory_limit ?? rt.memory_limits.compile, }, extra_env_vars: sanitizeEnvVars(env_vars), + shell_output_filter: shellOutputFilter, output_session_id: body.output_session_id, egress_grant: egressGrantToken, tool_call_socket_enabled: toolCallSocketEnabled, @@ -398,6 +409,7 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn let cleanedUp = false; let activeExecution = false; let metricsLanguage = 'unknown'; + let metricsOutputFilter: ShellOutputFilter = 'raw'; let metricsOutcome: Parameters[0]['outcome'] = 'execution_error'; let primeCompleted = false; @@ -466,6 +478,7 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn req.headers[RUNTIME_SESSION_ID_HEADER], ); metricsLanguage = job.runtime.language; + metricsOutputFilter = job.shellOutputFilter; markActiveExecution(); } catch (error) { metricsOutcome = 'bad_request'; @@ -510,6 +523,13 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn result.run = result.compile; } + recordSandboxOutputBytes({ + language: job.runtime.language, + outputFilter: job.shellOutputFilter, + stdout: result.run?.stdout ?? '', + stderr: result.run?.stderr ?? '', + }); + if (result.files && result.files.length > 0) { /* Upload returns the set of file IDs that were actually transferred to * the file server. Files we minted IDs for but failed to ship (e.g. the @@ -584,6 +604,7 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn } recordSandboxExecution({ language: metricsLanguage, + outputFilter: metricsOutputFilter, outcome: metricsOutcome, durationSeconds: (performance.now() - started) / 1000, }); diff --git a/api/src/bash-run.test.ts b/api/src/bash-run.test.ts new file mode 100644 index 0000000..2638bf9 --- /dev/null +++ b/api/src/bash-run.test.ts @@ -0,0 +1,86 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { execFileSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +const wrapper = path.resolve(__dirname, '../../docker/bash-run.sh'); + +describe('Bash RTK run wrapper', () => { + let tempDir: string; + let sourcePath: string; + let mockBin: string; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codeapi-rtk-test-')); + sourcePath = path.join(tempDir, 'script.sh'); + mockBin = path.join(tempDir, 'bin'); + fs.mkdirSync(mockBin); + fs.writeFileSync(sourcePath, "printf 'raw:%s\\n' \"${1:-none}\"\n"); + fs.writeFileSync(path.join(mockBin, 'rtk'), `#!/bin/bash +case "\${MOCK_RTK_STATUS:-0}" in + 0|3) printf '%s\\n' 'printf '\"'\"'filtered:%s\\n'\"'\"' "$1"' ;; + *) printf '%s\\n' 'printf '\"'\"'must-not-run\\n'\"'\"'' ;; +esac +exit "\${MOCK_RTK_STATUS:-0}" +`); + fs.chmodSync(path.join(mockBin, 'rtk'), 0o755); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + function run(options: { + filter?: 'raw' | 'rtk'; + status?: number; + args?: string[]; + } = {}): string { + return execFileSync('bash', [wrapper, sourcePath, ...(options.args ?? [])], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${mockBin}:${process.env.PATH ?? '/usr/bin:/bin'}`, + CODEAPI_SHELL_OUTPUT_FILTER: options.filter ?? 'raw', + MOCK_RTK_STATUS: String(options.status ?? 0), + }, + }); + } + + it('preserves raw execution when the request does not opt in', () => { + expect(run()).toBe('raw:none\n'); + }); + + it('executes RTK rewrites and preserves user script arguments', () => { + expect(run({ filter: 'rtk', args: ['argument'] })).toBe('filtered:argument\n'); + }); + + it('preserves the submitted path as $0 and BASH_ARGV0 for rewrites', () => { + fs.writeFileSync(path.join(mockBin, 'rtk'), `#!/bin/bash +printf '%s\\n' 'printf "identity:%s:%s:%s\\n" "$0" "$BASH_ARGV0" "$1"' +`); + fs.chmodSync(path.join(mockBin, 'rtk'), 0o755); + + expect(run({ filter: 'rtk', args: ['argument'] })).toBe( + `identity:${sourcePath}:${sourcePath}:argument\n`, + ); + }); + + it('preserves file execution for scripts that inspect BASH_SOURCE', () => { + fs.writeFileSync( + sourcePath, + 'printf \'identity:%s:%s\\n\' "${BASH_SOURCE[0]}" "$0"\n', + ); + + expect(run({ filter: 'rtk' })).toBe(`identity:${sourcePath}:${sourcePath}\n`); + }); + + it('accepts RTK ask rewrites because the API request already authorizes execution', () => { + expect(run({ filter: 'rtk', status: 3 })).toBe('filtered:\n'); + }); + + it('fails open to the original script when RTK cannot rewrite the command', () => { + expect(run({ filter: 'rtk', status: 1, args: ['original'] })).toBe('raw:original\n'); + expect(run({ filter: 'rtk', status: 2, args: ['original'] })).toBe('raw:original\n'); + }); +}); diff --git a/api/src/job-helpers.test.ts b/api/src/job-helpers.test.ts index 3509f9d..a44bcc9 100644 --- a/api/src/job-helpers.test.ts +++ b/api/src/job-helpers.test.ts @@ -12,6 +12,7 @@ import { inputsLiveUnder, mapWithConcurrency, mimeTypeFor, + filterExtraEnvVars, } from './job'; import type { Runtime } from './runtime'; import type { TFile } from './job'; @@ -41,6 +42,17 @@ function makeRuntime(overrides: Partial & { language: string; pkgdir: s }; } +describe('filterExtraEnvVars', () => { + it('prevents callers from overriding internal RTK controls', () => { + expect(filterExtraEnvVars({ + CODEAPI_SHELL_OUTPUT_FILTER: 'rtk', + RTK_DB_PATH: '/mnt/data/history.db', + RTK_TEE: '1', + USER_VALUE: 'allowed', + })).toEqual({ USER_VALUE: 'allowed' }); + }); +}); + describe('resolveOriginalName', () => { function responseWithHeader(value?: string): Response { const headers = new Headers(); diff --git a/api/src/job.ts b/api/src/job.ts index 1193469..eb640e6 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -41,6 +41,7 @@ import { isValidFilePath, } from './validation'; import { cachedInputResponse, inputCacheKey, openCachedInput } from './session-inputs'; +import type { ShellOutputFilter } from '../../shared/shell-output-filter'; export { DIRKEEP, @@ -490,6 +491,9 @@ export const RESERVED_ENV_KEYS: ReadonlySet = new Set([ 'HOME', 'PATH', 'TOOL_CALL_SOCKET', + 'CODEAPI_SHELL_OUTPUT_FILTER', + 'RTK_DB_PATH', + 'RTK_TEE', 'PYTHONPATH', 'PYTHONSTARTUP', 'PYTHONHOME', @@ -689,6 +693,7 @@ export class Job { cpu_times: { run: number; compile: number }; memory_limits: { run: number; compile: number }; extra_env_vars?: Record; + shellOutputFilter: ShellOutputFilter; egressGrantToken?: string; toolCallSocketEnabled: boolean; isSynthetic: boolean; @@ -728,6 +733,7 @@ export class Job { cpu_times: { run: number; compile: number }; memory_limits: { run: number; compile: number }; extra_env_vars?: Record; + shell_output_filter?: ShellOutputFilter; output_session_id?: string; egress_grant?: string; tool_call_socket_enabled?: boolean; @@ -766,6 +772,7 @@ export class Job { this.cpu_times = opts.cpu_times; this.memory_limits = opts.memory_limits; this.extra_env_vars = opts.extra_env_vars; + this.shellOutputFilter = opts.shell_output_filter ?? 'raw'; this.egressGrantToken = opts.egress_grant; this.toolCallSocketEnabled = opts.tool_call_socket_enabled === true; this.isSynthetic = opts.is_synthetic === true; @@ -1486,6 +1493,18 @@ export class Job { HOME: '/mnt/data', }; + if ( + script === 'run' + && this.runtime.language === 'bash' + && this.shellOutputFilter === 'rtk' + ) { + envVars.CODEAPI_SHELL_OUTPUT_FILTER = 'rtk'; + /* Keep RTK bookkeeping ephemeral so a stateless run cannot create + * recoverable artifacts or pollute generated-file discovery. */ + envVars.RTK_DB_PATH = '/tmp/rtk-history.db'; + envVars.RTK_TEE = '0'; + } + let extraPkgdirs: string[] | undefined; if (this.runtime.language === 'bash') { const linkTarget: { nodeModulesPath?: string } = {}; diff --git a/api/src/metrics.ts b/api/src/metrics.ts index 2da33b2..1c126b4 100644 --- a/api/src/metrics.ts +++ b/api/src/metrics.ts @@ -1,5 +1,6 @@ import client, { Counter, Gauge, Histogram, register } from 'prom-client'; import type { NextFunction, Request, Response } from 'express'; +import type { ShellOutputFilter } from '../../shared/shell-output-filter'; client.collectDefaultMetrics({ register }); @@ -29,6 +30,19 @@ export const sandboxExecutionDuration = new Histogram({ buckets: [0.1, 0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 300], }); +export const sandboxShellOutputFilterExecutions = new Counter({ + name: 'codeapi_sandbox_shell_output_filter_executions_total', + help: 'Total Bash sandbox executions by request-scoped output filter and outcome', + labelNames: ['shell_output_filter', 'outcome'] as const, +}); + +export const sandboxShellOutputBytes = new Histogram({ + name: 'codeapi_sandbox_shell_output_bytes', + help: 'Bash sandbox output size by request-scoped output filter and stream', + labelNames: ['shell_output_filter', 'stream'] as const, + buckets: [0, 64, 256, 1024, 4096, 16_384, 65_536, 262_144, 1_048_576], +}); + export const activeSandboxExecutions = new Gauge({ name: 'codeapi_sandbox_active_executions', help: 'Number of sandbox executions currently past request validation', @@ -82,12 +96,36 @@ export function httpMetricsMiddleware(req: Request, res: Response, next: NextFun export function recordSandboxExecution(params: { language: string; + outputFilter?: ShellOutputFilter; outcome: 'success' | 'manifest_error' | 'bad_request' | 'validation_error' | 'execution_error'; durationSeconds: number; }): void { const labels = { language: params.language || 'unknown', outcome: params.outcome }; sandboxExecutions.inc(labels); sandboxExecutionDuration.observe(labels, params.durationSeconds); + if (params.language === 'bash') { + sandboxShellOutputFilterExecutions.inc({ + shell_output_filter: params.outputFilter ?? 'raw', + outcome: params.outcome, + }); + } +} + +export function recordSandboxOutputBytes(params: { + language: string; + outputFilter: ShellOutputFilter; + stdout: string; + stderr: string; +}): void { + if (params.language !== 'bash') return; + sandboxShellOutputBytes.observe( + { shell_output_filter: params.outputFilter, stream: 'stdout' }, + Buffer.byteLength(params.stdout), + ); + sandboxShellOutputBytes.observe( + { shell_output_filter: params.outputFilter, stream: 'stderr' }, + Buffer.byteLength(params.stderr), + ); } export async function metricsHandler(_req: Request, res: Response): Promise { diff --git a/docker/Dockerfile.package-init b/docker/Dockerfile.package-init index c94cb8a..c23eb27 100644 --- a/docker/Dockerfile.package-init +++ b/docker/Dockerfile.package-init @@ -24,6 +24,7 @@ RUN apt-install \ RUN mkdir -p /pkgs COPY docker/package-init.sh /package-init.sh +COPY docker/bash-run.sh /bash-run.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index cd3edd3..1f68aaf 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -72,6 +72,7 @@ RUN apt-install \ RUN mkdir -p /pkgs COPY docker/package-init.sh /package-init.sh +COPY docker/bash-run.sh /bash-run.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh @@ -99,6 +100,20 @@ COPY shared /shared COPY api/tsconfig.json ./ RUN bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' +# ============================================================================ +# Stage 3b: Build the RTK binary against the sandbox's glibc baseline. The +# upstream arm64 release requires a newer glibc than Debian bookworm provides. +# ============================================================================ +FROM rust:1.91-bookworm AS rtk-builder + +ARG RTK_COMMIT=b34be37caf3796b69a50952a28e60e32b5daad43 +RUN git clone --depth 1 --branch v0.45.0 https://github.com/rtk-ai/rtk.git /rtk \ + && cd /rtk \ + && git checkout "$RTK_COMMIT" \ + && test "$(git rev-parse HEAD)" = "$RTK_COMMIT" \ + && cargo build --locked --release \ + && ./target/release/rtk --version + # ============================================================================ # Stage 4: Build sandbox rootfs (full OS layer for the microVM guest) # ============================================================================ @@ -140,6 +155,7 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=rtk-builder /rtk/target/release/rtk /usr/local/bin/rtk RUN chmod +x /usr/sbin/nsjail WORKDIR /sandbox_api diff --git a/docker/bash-run.sh b/docker/bash-run.sh new file mode 100755 index 0000000..5604378 --- /dev/null +++ b/docker/bash-run.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# RTK is deliberately invoked here, inside NsJail, so user-controlled source +# never reaches a privileged service or worker process. Unsupported rewrites, +# denied commands, and tool failures preserve the existing raw execution path. +if [ "${CODEAPI_SHELL_OUTPUT_FILTER:-raw}" = "rtk" ] && [ "$#" -gt 0 ]; then + # `bash -c` preserves the submitted path as $0/BASH_ARGV0, but Bash does + # not populate BASH_SOURCE for command strings. Keep the original file + # execution path for scripts that inspect BASH_SOURCE so filtering cannot + # change source-relative imports or helper lookups. + if ! grep -q 'BASH_SOURCE' -- "$1" 2>/dev/null; then + rewritten="$(rtk rewrite "$(cat -- "$1")" 2>/dev/null)" + rewrite_status=$? + + if { [ "$rewrite_status" -eq 0 ] || [ "$rewrite_status" -eq 3 ]; } && [ -n "$rewritten" ]; then + exec bash -c "$rewritten" "$1" "${@:2}" + fi + fi +fi + +exec bash "$@" diff --git a/docker/package-init.sh b/docker/package-init.sh index 60855b9..daa57fc 100644 --- a/docker/package-init.sh +++ b/docker/package-init.sh @@ -23,8 +23,10 @@ UV_VERSION="${UV_VERSION:-0.11.26}" NODE_VERSION="${NODE_VERSION:-24.15.0}" BUN_VERSION="${BUN_VERSION:-1.3.14}" BASH_PACKAGE_VERSION="${BASH_PACKAGE_VERSION:-5.2.0}" +BASH_DEST="/pkgs/bash/${BASH_PACKAGE_VERSION}" INSTALL_FAILED=false JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" +BASH_RUN_SCRIPT="${BASH_RUN_SCRIPT:-${SCRIPT_DIR}/bash-run.sh}" load_js_packages() { if [ ! -f "$JS_PACKAGE_MANIFEST" ]; then @@ -100,8 +102,23 @@ packages_ready() { [ -f "/pkgs/bash/${BASH_PACKAGE_VERSION}/.package-installed" ] } +sync_bash_run_wrapper() { + if [ ! -f "$BASH_RUN_SCRIPT" ]; then + echo "ERROR: Missing Bash run wrapper: $BASH_RUN_SCRIPT" >&2 + return 1 + fi + if ! cmp -s "$BASH_RUN_SCRIPT" "$BASH_DEST/run"; then + echo "Updating Bash run wrapper" + if ! install -m 0755 "$BASH_RUN_SCRIPT" "$BASH_DEST/run"; then + return 1 + fi + fi + return 0 +} + if [ -f "$MARKER_FILE" ] && [ "$FORCE_REBUILD" != "true" ]; then if packages_ready; then + sync_bash_run_wrapper echo "Packages already initialized (marker file exists)" echo "Set FORCE_REBUILD=true to force reinstall" echo "" @@ -498,7 +515,6 @@ echo "==============================================" echo "" SYSTEM_BASH_VERSION=$(bash --version | sed -nE '1s/.* ([0-9]+[.][0-9]+[.][0-9]+).*/\1/p') -BASH_DEST="/pkgs/bash/${BASH_PACKAGE_VERSION}" mkdir -p "$BASH_DEST" cat > "$BASH_DEST/pkg-info.json" << EOF @@ -511,11 +527,9 @@ cat > "$BASH_DEST/pkg-info.json" << EOF } EOF -cat > "$BASH_DEST/run" << 'EOF' -#!/bin/bash -bash "$@" -EOF -chmod +x "$BASH_DEST/run" +if ! sync_bash_run_wrapper; then + INSTALL_FAILED=true +fi echo "PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:." > "$BASH_DEST/.env" echo "$(date +%s)000" > "$BASH_DEST/.package-installed" diff --git a/launcher/Dockerfile b/launcher/Dockerfile index 0d252c2..e0f6393 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -36,6 +36,17 @@ COPY api/src/spec-guard.c /tmp/spec-guard.c RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && chmod 0111 /usr/local/bin/spec-guard +# Build RTK against the same Debian bookworm glibc baseline as the sandbox. +FROM rust:1.91-bookworm AS rtk-builder + +ARG RTK_COMMIT=b34be37caf3796b69a50952a28e60e32b5daad43 +RUN git clone --depth 1 --branch v0.45.0 https://github.com/rtk-ai/rtk.git /rtk \ + && cd /rtk \ + && git checkout "$RTK_COMMIT" \ + && test "$(git rev-parse HEAD)" = "$RTK_COMMIT" \ + && cargo build --locked --release \ + && ./target/release/rtk --version + FROM oven/bun:1.3.14-debian AS sandbox-build ENV DEBIAN_FRONTEND=noninteractive @@ -62,6 +73,7 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=rtk-builder /rtk/target/release/rtk /usr/local/bin/rtk RUN chmod +x /usr/sbin/nsjail WORKDIR /sandbox_api diff --git a/service/openapi.yml b/service/openapi.yml index c1f8f6e..7f5fc29 100644 --- a/service/openapi.yml +++ b/service/openapi.yml @@ -100,6 +100,12 @@ components: type: array items: type: string + shell_output_filter: + type: string + enum: [raw, rtk] + description: >- + Optional Bash-only output filter. Omit or use raw for unchanged + execution; use rtk to rewrite supported commands for compact output. user_id: type: string entity_id: diff --git a/service/rollup.config.js b/service/rollup.config.js index 2400f72..3bc9160 100644 --- a/service/rollup.config.js +++ b/service/rollup.config.js @@ -38,7 +38,7 @@ export default { commonjs(), typescript({ tsconfig: './tsconfig.esm.json', - include: ['src/**/*.ts', '../shared/telemetry-core.ts'], + include: ['src/**/*.ts', '../shared/telemetry-core.ts', '../shared/shell-output-filter.ts'], sourceMap: true, declaration: false, declarationMap: false, diff --git a/service/src/config.spec.ts b/service/src/config.spec.ts index 44b64b8..27e6b5a 100644 --- a/service/src/config.spec.ts +++ b/service/src/config.spec.ts @@ -11,6 +11,10 @@ import { import { Languages } from './enum'; import { createPayload } from './payload'; import type { AuthenticatedRequest } from './types'; +import { + resolveShellOutputFilter, + ShellOutputFilterError, +} from '../../shared/shell-output-filter'; describe('node language configuration', () => { it('resolves Node.js aliases', () => { @@ -58,6 +62,35 @@ describe('node language configuration', () => { }); }); +describe('request-scoped shell output filtering', () => { + it('propagates an opted-in RTK filter to the Bash sandbox payload', () => { + const req = { + body: { + lang: 'bash', + code: 'git status', + shell_output_filter: 'rtk', + }, + } as unknown as AuthenticatedRequest; + + expect(createPayload({ req, session_id: 'session-bash' })).toMatchObject({ + language: 'bash', + version: '5.2.0', + shell_output_filter: 'rtk', + }); + }); + + it('keeps omitted filters raw by leaving the wire field absent', () => { + expect(resolveShellOutputFilter(undefined, 'bash')).toBeUndefined(); + }); + + it('rejects unknown filters and non-Bash runtimes', () => { + expect(() => resolveShellOutputFilter('compact', 'bash')).toThrow(ShellOutputFilterError); + expect(() => resolveShellOutputFilter('rtk', 'python')).toThrow( + 'shell_output_filter is only supported for Bash executions', + ); + }); +}); + describe('runtime version configuration', () => { it('maps Python requests to Python 3.14.4', () => { expect(languageConfig[Languages.py]).toMatchObject({ diff --git a/service/src/payload.ts b/service/src/payload.ts index 8a849e4..8c3c6da 100644 --- a/service/src/payload.ts +++ b/service/src/payload.ts @@ -10,7 +10,13 @@ export function createPayload({ isPyPlot, session_id, }: t.CreatePayload): t.PayloadBody { - const { lang: rawLang, code: userCode, args, files } = req.body as t.RequestBody; + const { + lang: rawLang, + code: userCode, + args, + files, + shell_output_filter, + } = req.body as t.RequestBody; const language = resolveLanguage(rawLang); if (language === undefined) { throw new Error(`Unsupported language: ${rawLang}`); @@ -52,6 +58,10 @@ export function createPayload({ payload.args = args; } + if (shell_output_filter) { + payload.shell_output_filter = shell_output_filter; + } + if (files && files.length > 0) { files.forEach(obj => { /* The sandbox downloads files by `(storage_session_id, id)`; @@ -66,4 +76,4 @@ export function createPayload({ } return payload; -} \ No newline at end of file +} diff --git a/service/src/service/router.ts b/service/src/service/router.ts index e0f8f44..6f2ad81 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -25,6 +25,10 @@ import { Jobs, Languages } from '../enum'; import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; import { prepareSandboxJobSecurity } from '../sandbox-egress'; import logger from '../logger'; +import { + resolveShellOutputFilter, + ShellOutputFilterError, +} from '../../../shared/shell-output-filter'; const { INSTANCE_ID } = env; const JOB_COMPLETION_WAIT_TIMEOUT_MS = jobCompletionWaitTimeoutMs( @@ -138,6 +142,14 @@ router.post('/exec', executionLimiter, async (req: t.AuthenticatedRequest, res) if (language == null) { return res.status(400).json({ error: `Unsupported language: ${rawLang}` }); } + try { + body.shell_output_filter = resolveShellOutputFilter(body.shell_output_filter, language); + } catch (error) { + if (error instanceof ShellOutputFilterError) { + return res.status(400).json({ error: error.message }); + } + throw error; + } let runtimeSessionId: string | undefined; try { diff --git a/service/src/types/service.ts b/service/src/types/service.ts index d298298..ca03a93 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -4,6 +4,7 @@ import type { ExecutionManifestClaims } from '../execution-manifest'; import type { ExecutionIdentity } from '../execution-identity'; import type { CodeApiPrincipal } from '../auth/principal'; import { Jobs } from '@/enum/service'; +import type { ShellOutputFilter } from '../../../shared/shell-output-filter'; /** * Per-file vs. top-level session distinction @@ -126,6 +127,8 @@ export interface RequestBody { code: string; lang: string; args?: string[]; + /** Optional per-request Bash output filtering. Omitted requests stay raw. */ + shell_output_filter?: ShellOutputFilter; user_id?: string; files?: RequestFile[]; /** @@ -172,6 +175,8 @@ export type PayloadFileRef = { export interface PayloadBody { language: string; version: string; + /** Intra-monorepo request-scoped Bash output filter. */ + shell_output_filter?: ShellOutputFilter; run_memory_limit?: number; run_timeout?: number; run_cpu_time?: number; diff --git a/service/tsconfig.esm.json b/service/tsconfig.esm.json index f251a02..4965fb0 100644 --- a/service/tsconfig.esm.json +++ b/service/tsconfig.esm.json @@ -18,7 +18,7 @@ "@/*": ["src/*"] } }, - "include": ["src/**/*.ts", "../shared/telemetry-core.ts"], + "include": ["src/**/*.ts", "../shared/telemetry-core.ts", "../shared/shell-output-filter.ts"], "exclude": [ "node_modules", "**/*.spec.ts", diff --git a/service/tsconfig.json b/service/tsconfig.json index c3e8863..ea54e12 100644 --- a/service/tsconfig.json +++ b/service/tsconfig.json @@ -13,7 +13,7 @@ "@/*": ["src/*"] } }, - "include": ["src/**/*.ts", "../shared/telemetry-core.ts"], + "include": ["src/**/*.ts", "../shared/telemetry-core.ts", "../shared/shell-output-filter.ts"], "exclude": [ "node_modules", "**/*.spec.ts", diff --git a/shared/shell-output-filter.ts b/shared/shell-output-filter.ts new file mode 100644 index 0000000..eb5c796 --- /dev/null +++ b/shared/shell-output-filter.ts @@ -0,0 +1,32 @@ +export const SHELL_OUTPUT_FILTERS = ['raw', 'rtk'] as const; + +export type ShellOutputFilter = typeof SHELL_OUTPUT_FILTERS[number]; + +export class ShellOutputFilterError extends Error { + constructor(message: string) { + super(message); + this.name = 'ShellOutputFilterError'; + } +} + +/** + * Validates the request-scoped shell output filter at each trust boundary. + * RTK rewrites shell commands, so exposing it for non-Bash runtimes would be + * misleading and would make future runtime behavior ambiguous. + */ +export function resolveShellOutputFilter( + value: unknown, + language: string, +): ShellOutputFilter | undefined { + if (value === undefined) return undefined; + if ( + typeof value !== 'string' + || !SHELL_OUTPUT_FILTERS.includes(value as ShellOutputFilter) + ) { + throw new ShellOutputFilterError('shell_output_filter must be one of: raw, rtk'); + } + if (language !== 'bash') { + throw new ShellOutputFilterError('shell_output_filter is only supported for Bash executions'); + } + return value as ShellOutputFilter; +}