diff --git a/cli/package.json b/cli/package.json index ce10e13..51bc8e0 100644 --- a/cli/package.json +++ b/cli/package.json @@ -13,7 +13,8 @@ "dev": "bun run src/index.ts", "build:bin": "bun build src/index.ts --compile --outfile dist/hoist --target=bun-darwin-arm64", "build:all": "bun run scripts/build-all.ts", - "typecheck": "tsc -p tsconfig.json --noEmit" + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "bun test" }, "dependencies": { "@clack/prompts": "^0.10.0", diff --git a/cli/src/index.ts b/cli/src/index.ts index 5cb343b..2bd59ee 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -12,6 +12,7 @@ import { listSecrets, maskSecret, secretIdFor, + secretIdForProviderConfig, setSecret, } from './lib/vault' import { @@ -114,13 +115,26 @@ program process.exit(1) } const value = await getSecret(secretIdFor(provider.id)) - if (!value) { + const needsKey = provider.probeKind !== 'aws' + if (!value && needsKey) { console.log(`No key stored for ${provider.id}. Run \`hoist keys set ${provider.id}\` first.`) process.exit(1) } + const storedProject = + provider.probeKind === 'gcp' ? await getSecret(secretIdForProviderConfig(provider.id, 'project')).catch(() => null) : null + const storedRegion = + provider.probeKind === 'gcp' || provider.probeKind === 'aws' + ? await getSecret(secretIdForProviderConfig(provider.id, 'region')).catch(() => null) + : null + const envEndpoint = + provider.probeKind === 'azure' && provider.baseUrlEnv ? process.env[provider.baseUrlEnv] : undefined + const baseUrl = provider.defaultBaseUrl ?? envEndpoint const s = p.spinner() s.start(`Probing ${provider.label}…`) - const result = await probeProvider(provider, value, provider.defaultBaseUrl) + const result = await probeProvider(provider, value ?? '', baseUrl, { + project: storedProject ?? undefined, + region: storedRegion ?? undefined, + }) s.stop(`${result.valid ? '✓' : '✗'} ${result.status}${result.detail ? ` · ${result.detail}` : ''}`) process.exit(result.valid ? 0 : 1) } diff --git a/cli/src/lib/probes.ts b/cli/src/lib/probes.ts index 12531b7..f9810e7 100644 --- a/cli/src/lib/probes.ts +++ b/cli/src/lib/probes.ts @@ -1,3 +1,4 @@ +import { createHash, createHmac } from 'node:crypto' import type { ProviderEntry } from './providers' const ANTHROPIC_VERSION = '2023-06-01' @@ -10,6 +11,13 @@ export interface ProbeResult { checkedAt: string } +export interface ProbeOptions { + project?: string + region?: string + /** Endpoint override (tests). */ + endpoint?: string +} + /** Providers that share the OpenAI-compatible /models probe (mirrors app). */ const OPENAI_COMPAT = new Set([ 'openai', @@ -24,17 +32,33 @@ const OPENAI_COMPAT = new Set([ 'custom-openai', ]) -export async function probeProvider(provider: ProviderEntry, apiKey: string, baseUrl?: string): Promise { +export async function probeProvider( + provider: ProviderEntry, + apiKey: string, + baseUrl?: string, + opts: ProbeOptions = {}, +): Promise { const checkedAt = new Date().toISOString() - if (!apiKey) { - return { valid: false, status: 'invalid', detail: 'No API key supplied.', checkedAt } - } if (provider.id === 'anthropic' || provider.probeKind === 'anthropicModels') { + if (!apiKey) return { valid: false, status: 'invalid', detail: 'No API key supplied.', checkedAt } return probeAnthropic({ apiKey, baseUrl: baseUrl ?? provider.defaultBaseUrl ?? 'https://api.anthropic.com' }) } if (provider.probeKind === 'openaiModels' || OPENAI_COMPAT.has(provider.id)) { + if (!apiKey) return { valid: false, status: 'invalid', detail: 'No API key supplied.', checkedAt } return probeOpenAI({ apiKey, baseUrl: baseUrl ?? provider.defaultBaseUrl }) } + if (provider.probeKind === 'aws') { + return probeBedrock({ region: opts.region, endpoint: opts.endpoint }) + } + if (provider.probeKind === 'gcp') { + return probeVertex({ apiKey, project: opts.project, region: opts.region, endpoint: opts.endpoint }) + } + if (provider.probeKind === 'azure') { + return probeAzure({ apiKey, baseUrl: baseUrl ?? provider.defaultBaseUrl }) + } + if (!apiKey) { + return { valid: false, status: 'invalid', detail: 'No API key supplied.', checkedAt } + } return { valid: false, status: 'error', detail: `No probe implemented for "${provider.id}".`, checkedAt } } @@ -94,3 +118,256 @@ export async function probeOpenAI({ apiKey, baseUrl }: { apiKey: string; baseUrl clearTimeout(timer) } } + +const AWS_SERVICE = 'bedrock' +const AWS_DEFAULT_REGION = 'us-east-1' + +export interface AwsCredentials { + accessKeyId: string + secretAccessKey: string + sessionToken?: string +} + +export interface BedrockProbeOptions { + credentials?: AwsCredentials + region?: string + endpoint?: string + now?: Date +} + +function credentialsFromEnv(): AwsCredentials | null { + const accessKeyId = process.env.AWS_ACCESS_KEY_ID + const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY + if (!accessKeyId || !secretAccessKey) return null + return { accessKeyId, secretAccessKey, sessionToken: process.env.AWS_SESSION_TOKEN } +} + +function regionFromEnv(): string { + return process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? AWS_DEFAULT_REGION +} + +function sha256Hex(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex') +} + +function hmac(key: Buffer | string, value: string): Buffer { + return createHmac('sha256', key).update(value, 'utf8').digest() +} + +export function signBedrockRequest(opts: { + endpoint: string + region: string + credentials: AwsCredentials + now: Date +}): { url: string; headers: Record } { + const url = new URL('/foundation-models', opts.endpoint) + const host = url.host + const amzDate = opts.now.toISOString().replace(/[:-]|\.\d{3}/g, '') + const dateStamp = amzDate.slice(0, 8) + const headers: Record = { host, 'x-amz-date': amzDate } + if (opts.credentials.sessionToken) { + headers['x-amz-security-token'] = opts.credentials.sessionToken + } + const signedHeaderNames = Object.keys(headers).sort() + const canonicalHeaders = signedHeaderNames.map((name) => `${name}:${headers[name]}\n`).join('') + const signedHeaders = signedHeaderNames.join(';') + const payloadHash = sha256Hex('') + const canonicalRequest = ['GET', url.pathname, '', canonicalHeaders, signedHeaders, payloadHash].join('\n') + const scope = `${dateStamp}/${opts.region}/${AWS_SERVICE}/aws4_request` + const stringToSign = ['AWS4-HMAC-SHA256', amzDate, scope, sha256Hex(canonicalRequest)].join('\n') + const kDate = hmac(`AWS4${opts.credentials.secretAccessKey}`, dateStamp) + const kRegion = hmac(kDate, opts.region) + const kService = hmac(kRegion, AWS_SERVICE) + const kSigning = hmac(kService, 'aws4_request') + const signature = createHmac('sha256', kSigning).update(stringToSign, 'utf8').digest('hex') + headers.Authorization = + `AWS4-HMAC-SHA256 Credential=${opts.credentials.accessKeyId}/${scope}, ` + + `SignedHeaders=${signedHeaders}, Signature=${signature}` + return { url: url.toString(), headers } +} + +export function parseAwsError(body: string): { code?: string; message?: string } { + const code = body.match(/([^<]+)<\/Code>/)?.[1] ?? body.match(/"__type"\s*:\s*"[^"#]*#([^"]+)"/)?.[1] + const message = + body.match(/([^<]+)<\/Message>/)?.[1] ?? body.match(/"message"\s*:\s*"([^"]+)"/i)?.[1] + return { code, message } +} + +export async function probeBedrock(opts: BedrockProbeOptions = {}): Promise { + const checkedAt = new Date().toISOString() + const credentials = opts.credentials ?? credentialsFromEnv() + if (!credentials) { + return { + valid: false, + status: 'invalid', + detail: + 'AWS error: NoCredentialProviders: no AWS credentials found. Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or configure the standard AWS credential chain.', + checkedAt, + } + } + const region = opts.region ?? regionFromEnv() + const endpoint = opts.endpoint ?? `https://bedrock.${region}.amazonaws.com` + const { url, headers } = signBedrockRequest({ endpoint, region, credentials, now: opts.now ?? new Date() }) + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), TIMEOUT_MS) + try { + const res = await fetch(url, { method: 'GET', headers, signal: controller.signal }) + if (res.ok) { + return { valid: true, status: 'ok', detail: 'Credentials validated against Bedrock ListFoundationModels.', checkedAt } + } + const body = await res.text().catch(() => '') + const { code, message } = parseAwsError(body) + const awsDetail = code ? `AWS error: ${code}${message ? `: ${message}` : ''}` : `AWS error: Unexpected HTTP ${res.status}.` + if (res.status === 429 || code === 'ThrottlingException') { + return { valid: true, status: 'quota_exceeded', detail: awsDetail, checkedAt } + } + if (res.status === 401 || res.status === 403) { + return { valid: false, status: 'invalid', detail: awsDetail, checkedAt } + } + return { valid: false, status: 'error', detail: awsDetail, checkedAt } + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + if (message.includes('abort')) { + return { valid: false, status: 'error', detail: `Timed out after ${TIMEOUT_MS}ms.`, checkedAt } + } + return { valid: false, status: 'error', detail: `AWS error: ${message}`, checkedAt } + } finally { + clearTimeout(timer) + } +} + +export interface VertexProbeOptions { + apiKey: string + project?: string + region?: string + endpoint?: string +} + +export function vertexModelsUrl(project: string, region: string, endpoint?: string): string { + const base = + endpoint ?? + (region === 'global' ? 'https://aiplatform.googleapis.com' : `https://${region}-aiplatform.googleapis.com`) + return `${base.replace(/\/$/, '')}/v1/projects/${project}/locations/${region}/publishers/google/models` +} + +function parseGoogleError(body: string): string | undefined { + try { + const parsed = JSON.parse(body) as { error?: { status?: string; message?: string } } + if (parsed.error?.message) { + return parsed.error.status ? `${parsed.error.status}: ${parsed.error.message}` : parsed.error.message + } + } catch { + // fall through to the raw body snippet + } + return body.trim() ? body.trim().slice(0, 200) : undefined +} + +export async function probeVertex(opts: VertexProbeOptions): Promise { + const checkedAt = new Date().toISOString() + if (!opts.project || !opts.region) { + return { + valid: false, + status: 'error', + detail: 'Vertex probe requires a GCP project and region. Save a project id and region for this key first.', + checkedAt, + } + } + if (!opts.apiKey) { + return { + valid: false, + status: 'invalid', + detail: 'No access token supplied. Run `gcloud auth application-default login` and try again.', + checkedAt, + } + } + const url = vertexModelsUrl(opts.project, opts.region, opts.endpoint) + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), TIMEOUT_MS) + try { + const res = await fetch(url, { + method: 'GET', + headers: { Authorization: `Bearer ${opts.apiKey}` }, + signal: controller.signal, + }) + if (res.ok) { + return { valid: true, status: 'ok', detail: 'Credentials validated against Vertex AI publishers/google/models.', checkedAt } + } + const body = await res.text().catch(() => '') + const googleDetail = parseGoogleError(body) + if (res.status === 401 || res.status === 403) { + return { + valid: false, + status: 'invalid', + detail: googleDetail ? `Vertex error: ${googleDetail}` : `Authentication failed (${res.status}).`, + checkedAt, + } + } + if (res.status === 429) { + return { valid: true, status: 'quota_exceeded', detail: 'Vertex rate limited; credential accepted.', checkedAt } + } + return { + valid: false, + status: 'error', + detail: googleDetail ? `Vertex error: ${googleDetail}` : `Unexpected HTTP ${res.status}.`, + checkedAt, + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + if (message.includes('abort')) { + return { valid: false, status: 'error', detail: `Timed out after ${TIMEOUT_MS}ms.`, checkedAt } + } + return { valid: false, status: 'error', detail: message, checkedAt } + } finally { + clearTimeout(timer) + } +} + +export interface AzureProbeOptions { + apiKey: string + baseUrl?: string + apiVersion?: string +} + +export async function probeAzure(opts: AzureProbeOptions): Promise { + const checkedAt = new Date().toISOString() + if (!opts.baseUrl) { + return { + valid: false, + status: 'error', + detail: 'Azure probe requires an endpoint. Set the Azure OpenAI endpoint (AZURE_OPENAI_ENDPOINT) for this key first.', + checkedAt, + } + } + if (!opts.apiKey) { + return { valid: false, status: 'invalid', detail: 'No API key supplied.', checkedAt } + } + const apiVersion = opts.apiVersion ?? '2024-02-01' + const url = `${opts.baseUrl.replace(/\/$/, '')}/openai/deployments?api-version=${apiVersion}` + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), TIMEOUT_MS) + try { + const res = await fetch(url, { + method: 'GET', + headers: { 'api-key': opts.apiKey }, + signal: controller.signal, + }) + if (res.ok) { + return { valid: true, status: 'ok', detail: 'Key validated against Azure OpenAI deployments.', checkedAt } + } + if (res.status === 401 || res.status === 403) { + return { valid: false, status: 'invalid', detail: `Authentication failed (${res.status}).`, checkedAt } + } + if (res.status === 429) { + return { valid: true, status: 'quota_exceeded', detail: 'Rate limited; credential accepted.', checkedAt } + } + return { valid: false, status: 'error', detail: `Unexpected HTTP ${res.status}.`, checkedAt } + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + if (message.includes('abort')) { + return { valid: false, status: 'error', detail: `Timed out after ${TIMEOUT_MS}ms.`, checkedAt } + } + return { valid: false, status: 'error', detail: message, checkedAt } + } finally { + clearTimeout(timer) + } +} diff --git a/cli/src/lib/vault.ts b/cli/src/lib/vault.ts index d5c34d9..5504016 100644 --- a/cli/src/lib/vault.ts +++ b/cli/src/lib/vault.ts @@ -170,6 +170,13 @@ export function secretIdFor(providerId: string): string { return `provider:${providerId}:api_key` } +export type ProviderConfigField = 'project' | 'region' + +/** Canonical vault secret id for a provider config field, matching src/shared/secrets.ts. */ +export function secretIdForProviderConfig(providerId: string, field: ProviderConfigField): string { + return `provider:${providerId}:${field}` +} + export function maskSecret(value: string): string { if (!value) return '' if (value.length <= 8) return '…' diff --git a/cli/test/probes-cloud.test.ts b/cli/test/probes-cloud.test.ts new file mode 100644 index 0000000..87e83e2 --- /dev/null +++ b/cli/test/probes-cloud.test.ts @@ -0,0 +1,140 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http' +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' +import { probeAzure, probeBedrock, probeProvider, probeVertex, signBedrockRequest, vertexModelsUrl } from '../src/lib/probes' +import { findProvider } from '../src/lib/providers' +import { secretIdForProviderConfig } from '../src/lib/vault' + +let handler: (req: IncomingMessage, res: ServerResponse) => void = (_req, res) => { + res.writeHead(200) + res.end('{}') +} +const requests: { method?: string; url?: string; headers: IncomingMessage['headers'] }[] = [] +const server = createServer((req, res) => { + requests.push({ method: req.method, url: req.url, headers: req.headers }) + handler(req, res) +}) +let endpoint = '' +const lastRequest = () => requests[requests.length - 1] +const fakeCreds = { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY' } + +beforeAll(async () => { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const addr = server.address() + endpoint = `http://127.0.0.1:${typeof addr === 'object' && addr ? addr.port : 0}` + for (const key of ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN', 'AWS_REGION', 'AWS_DEFAULT_REGION']) { + delete process.env[key] + } +}) + +afterAll(() => { + server.close() +}) + +describe('cli cloud probes (mocked HTTP, no live keys)', () => { + test('vault sibling config secret ids match the app convention', () => { + expect(secretIdForProviderConfig('vertex', 'project')).toBe('provider:vertex:project') + expect(secretIdForProviderConfig('vertex', 'region')).toBe('provider:vertex:region') + }) + + test('aws no credentials returns invalid with AWS error', async () => { + const r = await probeBedrock({ endpoint }) + expect(r.valid).toBe(false) + expect(r.status).toBe('invalid') + expect(r.detail).toContain('AWS error:') + }) + + test('aws mocked 200 returns ok and sends a SigV4 signed request', async () => { + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end('{"modelSummaries":[]}') + } + const before = requests.length + const r = await probeBedrock({ credentials: fakeCreds, region: 'us-east-1', endpoint }) + expect(r.valid).toBe(true) + expect(r.status).toBe('ok') + expect(requests.length).toBe(before + 1) + const req = lastRequest() + expect(req.url).toBe('/foundation-models') + expect(req.headers['x-amz-date']).toBeTruthy() + const auth = String(req.headers.authorization ?? '') + expect(auth.startsWith('AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/')).toBe(true) + expect(auth).toContain('/us-east-1/bedrock/aws4_request') + expect(/Signature=[0-9a-f]{64}/.test(auth)).toBe(true) + }) + + test('aws mocked 403 surfaces the AWS error code', async () => { + handler = (_req, res) => { + res.writeHead(403, { 'content-type': 'application/xml' }) + res.end('AccessDeniedExceptionnot entitled') + } + const r = await probeBedrock({ credentials: fakeCreds, region: 'us-east-1', endpoint }) + expect(r.valid).toBe(false) + expect(r.status).toBe('invalid') + expect(r.detail).toContain('AWS error: AccessDeniedException') + }) + + test('aws SigV4 signature is deterministic for a fixed clock', () => { + const now = new Date('2026-01-02T03:04:05Z') + const a = signBedrockRequest({ endpoint: 'https://bedrock.us-east-1.amazonaws.com', region: 'us-east-1', credentials: fakeCreds, now }) + const b = signBedrockRequest({ endpoint: 'https://bedrock.us-east-1.amazonaws.com', region: 'us-east-1', credentials: fakeCreds, now }) + expect(a.headers.Authorization).toBe(b.headers.Authorization) + }) + + test('gcp missing project and region returns error without HTTP', async () => { + const before = requests.length + const r = await probeVertex({ apiKey: 'ya29.fake-token' }) + expect(r.valid).toBe(false) + expect(r.status).toBe('error') + expect(r.detail).toContain('project and region') + expect(requests.length).toBe(before) + }) + + test('gcp mocked 200 returns ok with project and region in the path', async () => { + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end('{"models":[]}') + } + const r = await probeVertex({ apiKey: 'ya29.fake-token', project: 'my-project', region: 'us-central1', endpoint }) + expect(r.valid).toBe(true) + expect(lastRequest().url).toBe('/v1/projects/my-project/locations/us-central1/publishers/google/models') + expect(lastRequest().headers.authorization).toBe('Bearer ya29.fake-token') + expect(vertexModelsUrl('p', 'global')).toBe( + 'https://aiplatform.googleapis.com/v1/projects/p/locations/global/publishers/google/models', + ) + }) + + test('azure missing endpoint returns error, mocked 200 validates', async () => { + const noEndpoint = await probeAzure({ apiKey: 'fake-key' }) + expect(noEndpoint.valid).toBe(false) + expect(noEndpoint.detail).toContain('endpoint') + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end('{"data":[]}') + } + const r = await probeAzure({ apiKey: 'fake-key', baseUrl: endpoint }) + expect(r.valid).toBe(true) + expect(lastRequest().url).toBe('/openai/deployments?api-version=2024-02-01') + expect(lastRequest().headers['api-key']).toBe('fake-key') + }) + + test('probeProvider dispatches cloud kinds instead of No probe implemented', async () => { + const bedrock = findProvider('bedrock') + const vertex = findProvider('vertex') + const azure = findProvider('azure-openai') + expect(bedrock).toBeTruthy() + expect(vertex).toBeTruthy() + expect(azure).toBeTruthy() + const b = await probeProvider(bedrock!, '', undefined, { endpoint }) + expect(b.detail).toContain('AWS error:') + const v = await probeProvider(vertex!, 'ya29.fake-token') + expect(v.detail).toContain('project and region') + const a = await probeProvider(azure!, 'fake-key') + expect(a.detail).toContain('endpoint') + const withConfig = await probeProvider(vertex!, 'ya29.fake-token', undefined, { + project: 'my-project', + region: 'us-central1', + endpoint, + }) + expect(withConfig.valid).toBe(true) + }) +}) diff --git a/scripts/ci/cli.sh b/scripts/ci/cli.sh index e1169b9..025a2e7 100755 --- a/scripts/ci/cli.sh +++ b/scripts/ci/cli.sh @@ -8,4 +8,5 @@ cd "$MISE_PROJECT_ROOT/cli" bun install --frozen-lockfile bun run typecheck +bun test bun build src/index.ts --compile --outfile="$HOIST_CLI_OUTFILE" --target="$HOIST_CLI_TARGET"