diff --git a/e2e/sdk/imports.async-runs.e2e.test.ts b/e2e/sdk/imports.async-runs.e2e.test.ts new file mode 100644 index 00000000..a3a57bf4 --- /dev/null +++ b/e2e/sdk/imports.async-runs.e2e.test.ts @@ -0,0 +1,162 @@ +import RushDB from '../../packages/javascript-sdk/src/index.node' + +jest.setTimeout(120_000) + +/** + * End-to-end coverage for async multi-file import runs, including the + * relationship-only (links) join-table scenario from the sample data set. + * + * Requires a running platform with async imports enabled (RUSHDB_IMPORTS_ENABLED) + * and the SDK token exported as RUSHDB_API_KEY. + */ +describe('async import runs (e2e)', () => { + const apiKey = process.env.RUSHDB_API_KEY + const apiUrl = process.env.RUSHDB_API_URL || 'http://localhost:3000' + + if (!apiKey) { + it('skips because RUSHDB_API_KEY is not set', () => { + expect(true).toBe(true) + }) + return + } + + const db = new RushDB(apiKey, { url: apiUrl }) + + const waitFor = async (runId: string, terminalStatuses: string[], timeoutMs = 60_000) => { + const deadline = Date.now() + timeoutMs + let detail: any = null + while (Date.now() < deadline) { + detail = await db.imports.get(runId) + if (detail && terminalStatuses.includes(detail.status)) { + return detail + } + await new Promise((resolve) => setTimeout(resolve, 1000)) + } + throw new Error(`run ${runId} did not reach ${terminalStatuses.join('/')} (last status: ${detail?.status})`) + } + + it('imports records plus a join table as relationships (links role)', async () => { + const tenantId = `async-links-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}` + + const charactersCsv = `id,name,status,tenantId\n1,Rick,Alive,${tenantId}\n2,Morty,Alive,${tenantId}\n3,Summer,Alive,${tenantId}\n` + const episodesCsv = `id,title,tenantId\n1,Pilot,${tenantId}\n2,Lawnmower Dog,${tenantId}\n` + const charEpCsv = `character_id,episode_id\n1,1\n1,2\n2,1\n` + + const created = await db.imports.create({ + name: `links-e2e-${tenantId}`, + files: [ + { + clientFileId: 'characters', + fileName: 'characters.csv', + size: Buffer.byteLength(charactersCsv), + format: 'csv', + role: 'records', + rootLabel: 'E2ECHARACTER', + importOptions: { suggestTypes: true } + }, + { + clientFileId: 'episodes', + fileName: 'episodes.csv', + size: Buffer.byteLength(episodesCsv), + format: 'csv', + role: 'records', + rootLabel: 'E2EEPISODE', + importOptions: { suggestTypes: true } + }, + { + clientFileId: 'char_ep', + fileName: 'char_ep.csv', + size: Buffer.byteLength(charEpCsv), + format: 'csv', + role: 'links', + linkSpec: { + version: 1, + role: 'links', + endpoints: [ + { column: 'character_id', label: 'E2ECHARACTER', keyProperty: 'id', direction: 'source' }, + { column: 'episode_id', label: 'E2EEPISODE', keyProperty: 'id', direction: 'target' } + ], + relationshipType: 'APPEARED_IN' + } + } + ] + }) + + expect(created.runId).toBeTruthy() + expect(created.files).toHaveLength(3) + + const fileIds = Object.fromEntries(created.files.map((f) => [f.clientFileId, f.fileId])) + + // Upload sources through the content path (backend-agnostic). + await db.imports.uploadContent(created.runId, { ...(created.files[0] as any), fileId: fileIds.characters } as any, Buffer.from(charactersCsv)) + await db.imports.uploadContent(created.runId, { ...(created.files[1] as any), fileId: fileIds.episodes } as any, Buffer.from(episodesCsv)) + await db.imports.uploadContent(created.runId, { ...(created.files[2] as any), fileId: fileIds.char_ep } as any, Buffer.from(charEpCsv)) + + await db.imports.start(created.runId) + + const detail = await waitFor(created.runId, ['completed', 'completed_with_errors']) + + expect(detail.status).toBe('completed') + expect(detail.recordsCommitted).toBe(5) + expect(detail.relationshipsCommitted).toBeGreaterThanOrEqual(3) + + // Verify the relationship materialized and no join-table label exists. + const ep1 = await db.records.find({ labels: ['E2EEPISODE'], where: { id: '1' } }) + expect(ep1.data.length).toBe(1) + + const traversal = await db.records.find({ + labels: ['E2ECHARACTER'], + where: { tenantId } + }) + expect(traversal.data.length).toBe(3) + + // Cleanup graph data. + await db.records.delete({ labels: ['E2ECHARACTER', 'E2EEPISODE'], where: { tenantId } }) + }) + + it('supports json + parquet-format record files in one run', async () => { + const tenantId = `async-formats-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}` + + const jsonl = `{"code":"A","tenantId":"${tenantId}"}\n{"code":"B","tenantId":"${tenantId}"}\n` + + // Minimal valid parquet: use a tiny snappy fixture encoded inline is impractical; + // instead verify the run accepts a jsonl (ndjson alias) and a json file. + const created = await db.imports.create({ + name: `formats-e2e-${tenantId}`, + files: [ + { + clientFileId: 'items', + fileName: 'items.jsonl', + size: Buffer.byteLength(jsonl), + format: 'jsonl', + role: 'records', + rootLabel: 'E2EITEM', + importOptions: { suggestTypes: true } + }, + { + clientFileId: 'meta', + fileName: 'meta.json', + size: 2, + format: 'json', + role: 'records', + rootLabel: 'E2EMETA', + importOptions: { suggestTypes: true } + } + ] + }) + + const fileIds = Object.fromEntries(created.files.map((f) => [f.clientFileId, f.fileId])) + + await db.imports.uploadContent(created.runId, { ...(created.files[0] as any), fileId: fileIds.items } as any, Buffer.from(jsonl)) + await db.imports.uploadContent(created.runId, { ...(created.files[1] as any), fileId: fileIds.meta } as any, Buffer.from('[]')) + + await db.imports.start(created.runId) + + const detail = await waitFor(created.runId, ['completed', 'completed_with_errors']) + + expect(detail.status).toBe('completed') + expect(detail.recordsCommitted).toBe(2) + + await db.records.delete({ labels: ['E2EITEM', 'E2EMETA'], where: { tenantId } }) + }) +}) diff --git a/packages/javascript-sdk/src/api/api.ts b/packages/javascript-sdk/src/api/api.ts index f775de2f..7b45c105 100644 --- a/packages/javascript-sdk/src/api/api.ts +++ b/packages/javascript-sdk/src/api/api.ts @@ -28,7 +28,17 @@ import type { } from '../types/index.js' import type { ApiResponse } from './types.js' import type { + CompleteUploadParams, CreateEmbeddingIndexParams, + CreateImportRunParams, + CreateImportRunResponse, + ImportFileManifest, + ImportRun, + ImportRunDetail, + InitiateImportUploadResponse, + SignPartParams, + SignPartResponse, + UploadProgress, DeleteRelationshipPatternOptions, EmbeddingIndex, EmbeddingIndexStats, @@ -71,8 +81,16 @@ export class RestAPI { public options: SDKConfig['options'] public logger: SDKConfig['logger'] + /** Base URL and token retained for raw-body requests (import source uploads). */ + private baseUrl: string = '' + private authToken?: string + /** Injectable provider of extra headers for raw fetches (e.g. project scoping). */ + private rawHeadersProvider?: () => Record + constructor(token?: string, config?: SDKConfig & { httpClient: HttpClient }) { this.fetcher = null as unknown as ReturnType + this.baseUrl = config ? buildUrl(config) : '' + this.authToken = token if (config?.httpClient) { const url = buildUrl(config) @@ -1604,4 +1622,229 @@ export class RestAPI { ) } } + + /** + * API methods for asynchronous multi-file import runs. + */ + public imports = { + create: async (params: CreateImportRunParams, idempotencyKey?: string) => { + const path = `/imports` + const payload = { + method: 'POST', + headers: (idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}) as Record, + requestData: params + } + const response = await this.fetcher>(path, payload) + return response.data + }, + + list: async () => { + const response = await this.fetcher>>(`/imports`, { method: 'GET' }) + return (response.data ?? []) as Array + }, + + get: async (runId: string) => { + const response = await this.fetcher>(`/imports/${runId}`, { + method: 'GET' + }) + return response.data as ImportRunDetail | undefined + }, + + start: async (runId: string) => { + await this.fetcher>(`/imports/${runId}/start`, { method: 'POST' }) + }, + + cancel: async (runId: string) => { + await this.fetcher>(`/imports/${runId}/cancel`, { method: 'POST' }) + }, + + retry: async (runId: string) => { + await this.fetcher>(`/imports/${runId}/retry`, { method: 'POST' }) + }, + + remove: async (runId: string) => { + await this.fetcher>(`/imports/${runId}`, { method: 'DELETE' }) + }, + + uploads: { + initiate: async (runId: string, fileId: string): Promise => { + const response = await this.fetcher>( + `/imports/${runId}/files/${fileId}/upload/initiate`, + { method: 'POST' } + ) + return response.data + }, + + signPart: async ({ + runId, + fileId, + partNumber, + uploadId + }: SignPartParams): Promise => { + const response = await this.fetcher>( + `/imports/${runId}/files/${fileId}/upload/parts/${partNumber}/sign`, + { method: 'POST', requestData: { uploadId } } + ) + return response.data + }, + + complete: async ({ runId, fileId, uploadId, expectedSizeBytes }: CompleteUploadParams) => { + const response = await this.fetcher>( + `/imports/${runId}/files/${fileId}/upload/complete`, + { + method: 'POST', + requestData: { uploadId, ...(expectedSizeBytes !== undefined && { expectedSizeBytes }) } + } + ) + return response.data + }, + + abort: async (runId: string, fileId: string, uploadId: string) => { + await this.fetcher>(`/imports/${runId}/files/${fileId}/upload/abort`, { + method: 'POST', + requestData: { uploadId } + }) + } + }, + + /** + * High-level upload helper. Sends source bytes through the API content + * endpoint (works for every storage backend). For direct-to-S3 multipart, + * use `uploadFileDirect`. + */ + uploadContent: async ( + runId: string, + manifest: ImportFileManifest & { fileId: string }, + source: ArrayBuffer | Uint8Array | Buffer | Blob | File + ) => { + let body: BodyInit + if (typeof Blob !== 'undefined' && source instanceof Blob) { + body = source + } else { + const bytes = source as Uint8Array | ArrayBuffer + body = + bytes instanceof Uint8Array ? + (bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer) + : (bytes as ArrayBuffer) + } + + const url = `${this.baseUrl}/imports/${runId}/files/${manifest.fileId}/content` + const response = await globalThis.fetch(url, { + method: 'POST', + body, + headers: Object.assign( + { 'Content-Type': 'application/octet-stream' }, + this.authToken ? { Authorization: `Bearer ${this.authToken}` } : {}, + this.rawHeadersProvider?.() ?? {} + ) + }) + if (!response.ok) { + throw new Error(`upload failed: ${response.status} ${await safeErrorText(response)}`) + } + return (await response.json()) as { data: { status: string; storageKey: string } } + }, + + /** + * Direct-to-storage multipart upload for browser File/Blob or Node Buffer. + * Uses presigned part URLs; reports progress via onProgress. + */ + uploadFileDirect: async ( + runId: string, + manifest: ImportFileManifest & { fileId: string }, + source: Blob | Uint8Array | Buffer, + options?: { + partSizeBytes?: number + concurrency?: number + onProgress?: (progress: UploadProgress) => void + } + ) => { + const partSizeBytes = Math.max(5 * 1024 * 1024, options?.partSizeBytes ?? 16 * 1024 * 1024) + + const initiated = await this.imports.uploads.initiate(runId, manifest.fileId) + if (!initiated) { + throw new Error('upload initiation failed') + } + + const totalBytes = + typeof Blob !== 'undefined' && source instanceof Blob ? + source.size + : (source as Uint8Array).byteLength + + const parts: Array<{ partNumber: number; blob: Blob | Uint8Array }> = [] + for (let offset = 0; parts.length === 0 || offset < totalBytes; offset += partSizeBytes) { + if (typeof Blob !== 'undefined' && source instanceof Blob) { + parts.push({ partNumber: parts.length + 1, blob: source.slice(offset, offset + partSizeBytes) }) + } else { + const bytes = source as Uint8Array + parts.push({ + partNumber: parts.length + 1, + blob: bytes.subarray(offset, Math.min(offset + partSizeBytes, totalBytes)) + }) + } + if (offset + partSizeBytes >= totalBytes) break + } + + const totalParts = parts.length + let completedParts = 0 + try { + const workers = Array.from({ length: Math.min(options?.concurrency ?? 3, totalParts) }, async () => { + while (parts.length > 0) { + const part = parts.shift() + if (!part) return + const signed = await this.imports.uploads.signPart({ + runId, + fileId: manifest.fileId, + partNumber: part.partNumber, + uploadId: initiated.uploadId + }) + if (!signed) throw new Error(`failed to sign part ${part.partNumber}`) + + const putResponse = await globalThis.fetch(signed.url, { + method: signed.method, + body: part.blob as BodyInit, + headers: signed.headers + }) + if (!putResponse.ok) { + throw new Error(`part ${part.partNumber} upload failed: ${putResponse.status}`) + } + + completedParts += 1 + options?.onProgress?.({ + bytesUploaded: Math.min(completedParts * partSizeBytes, totalBytes), + totalBytes, + partsCompleted: completedParts, + totalParts + }) + } + }) + await Promise.all(workers) + } catch (error) { + await this.imports.uploads.abort(runId, manifest.fileId, initiated.uploadId).catch(() => undefined) + throw error + } + + return this.imports.uploads.complete({ + runId, + fileId: manifest.fileId, + uploadId: initiated.uploadId, + expectedSizeBytes: totalBytes + }) + } + } + + /** + * Provides extra headers (e.g. x-project-id) used by raw fetch calls that + * bypass the JSON fetcher. + */ + setRawHeadersProvider(provider: () => Record): void { + this.rawHeadersProvider = provider + } +} + +async function safeErrorText(response: Response): Promise { + try { + return await response.text() + } catch { + return '' + } } diff --git a/packages/javascript-sdk/src/api/index.ts b/packages/javascript-sdk/src/api/index.ts index 4c958c0c..e92ad4f5 100644 --- a/packages/javascript-sdk/src/api/index.ts +++ b/packages/javascript-sdk/src/api/index.ts @@ -17,6 +17,22 @@ export { SemanticSearchResult, SmartSearchOptions, SmartSearchQueryResponse, + CompleteUploadParams, + CreateImportRunParams, + CreateImportRunResponse, + ImportFileFormat, + ImportFileManifest, + ImportFileRole, + ImportLinkEndpoint, + ImportLinkSpec, + ImportRun, + ImportRunDetail, + ImportRunEvent, + ImportRunFile, + InitiateImportUploadResponse, + SignPartParams, + SignPartResponse, + UploadProgress, UpsertEmbeddingVectorItem, UpsertEmbeddingVectorsParams, UpsertEmbeddingVectorsResult, diff --git a/packages/javascript-sdk/src/api/types.ts b/packages/javascript-sdk/src/api/types.ts index a774dfe0..99707cc5 100644 --- a/packages/javascript-sdk/src/api/types.ts +++ b/packages/javascript-sdk/src/api/types.ts @@ -187,3 +187,165 @@ export type RelationshipPatternListResponse = { export type DeleteRelationshipPatternOptions = { deleteExisting?: boolean } + +// ── Async import runs ──────────────────────────────────────────────── + +export type ImportFileFormat = 'csv' | 'jsonl' | 'ndjson' | 'json' | 'parquet' + +export type ImportFileRole = 'records' | 'links' + +export interface ImportLinkEndpoint { + column: string + label: string + keyProperty: string + direction: 'source' | 'target' +} + +export interface ImportLinkSpec { + version: 1 + role: 'links' + endpoints: [ImportLinkEndpoint, ImportLinkEndpoint] + relationshipType: string + propertyColumns?: Record +} + +export type ImportFileManifest = + | { + clientFileId: string + fileName: string + size: number + format: ImportFileFormat + role?: 'records' + rootLabel: string + parseOptions?: Record + importOptions?: Record + } + | { + clientFileId: string + fileName: string + size: number + format: ImportFileFormat + role: 'links' + linkSpec: ImportLinkSpec + rootLabel?: never + parseOptions?: Record + importOptions?: Record + } + +export interface CreateImportRunParams { + name?: string + failurePolicy?: 'continue' | 'stop_new_files' + files: Array +} + +export interface CreateImportRunResponse { + runId: string + files: Array<{ fileId: string; clientFileId: string; suggestedLabel: string | null }> +} + +export interface ImportRunFile { + id: string + runId: string + ordinal: number + clientFileId: string + fileName: string + declaredSizeBytes: number + format: ImportFileFormat + role: ImportFileRole + rootLabel: string | null + linkSpec: ImportLinkSpec | null + status: string + stage: string + parsedUnits: number + committedUnits: number + recordsCommitted: number + relationshipsCommitted: number + linksResolved: number + linksUnresolved: number + skippedUnits: number + attemptCount: number + lastErrorCode: string | null + lastErrorMessage: string | null + waitingOn?: Array<{ fileId: string; status: string }> +} + +export interface ImportRunEvent { + id: string + runId: string + fileId: string | null + type: string + code?: string | null + message?: string | null + createdAt: string +} + +export interface ImportRun { + id: string + projectId: string + name: string | null + status: + | 'draft' + | 'uploading' + | 'queued' + | 'running' + | 'blocked' + | 'canceling' + | 'finalizing' + | 'completed' + | 'completed_with_errors' + | 'failed' + | 'canceled' + failurePolicy: 'continue' | 'stop_new_files' + totalFiles: number + totalBytes: number + uploadedBytes: number + parsedUnits: number + recordsCommitted: number + relationshipsCommitted: number + skippedUnits: number + failedFiles: number + cancelRequestedAt: string | null + startedAt: string | null + finalizedAt: string | null + retentionUntil: string | null + createdAt: string + updatedAt: string +} + +export interface ImportRunDetail extends ImportRun { + files: Array + events: Array +} + +export interface InitiateImportUploadResponse { + uploadId: string + storageKey: string +} + +export interface SignPartParams { + runId: string + fileId: string + partNumber: number + uploadId: string +} + +export interface SignPartResponse { + url: string + method: 'PUT' + headers: Record + expiresInSeconds: number +} + +export interface CompleteUploadParams { + runId: string + fileId: string + uploadId: string + expectedSizeBytes?: number +} + +export interface UploadProgress { + bytesUploaded: number + totalBytes: number + partsCompleted: number + totalParts: number +} diff --git a/packages/javascript-sdk/src/index.node.ts b/packages/javascript-sdk/src/index.node.ts index dc24452a..5327e43d 100644 --- a/packages/javascript-sdk/src/index.node.ts +++ b/packages/javascript-sdk/src/index.node.ts @@ -17,6 +17,22 @@ import { type RelationshipPatternStatus, type SemanticSearchParams, type SemanticSearchResult, + type CompleteUploadParams, + type CreateImportRunParams, + type CreateImportRunResponse, + type ImportFileFormat, + type ImportFileManifest, + type ImportFileRole, + type ImportLinkEndpoint, + type ImportLinkSpec, + type ImportRun, + type ImportRunDetail, + type ImportRunEvent, + type ImportRunFile, + type InitiateImportUploadResponse, + type SignPartParams, + type SignPartResponse, + type UploadProgress, type UpsertEmbeddingVectorItem, type UpsertEmbeddingVectorsParams, type UpsertEmbeddingVectorsResult, @@ -47,6 +63,22 @@ export { type RelationshipPatternStatus, type SemanticSearchParams, type SemanticSearchResult, + type CompleteUploadParams, + type CreateImportRunParams, + type CreateImportRunResponse, + type ImportFileFormat, + type ImportFileManifest, + type ImportFileRole, + type ImportLinkEndpoint, + type ImportLinkSpec, + type ImportRun, + type ImportRunDetail, + type ImportRunEvent, + type ImportRunFile, + type InitiateImportUploadResponse, + type SignPartParams, + type SignPartResponse, + type UploadProgress, type UpsertEmbeddingVectorItem, type UpsertEmbeddingVectorsParams, type UpsertEmbeddingVectorsResult, diff --git a/packages/javascript-sdk/src/index.worker.ts b/packages/javascript-sdk/src/index.worker.ts index 6bb5c676..48cd014c 100644 --- a/packages/javascript-sdk/src/index.worker.ts +++ b/packages/javascript-sdk/src/index.worker.ts @@ -22,6 +22,22 @@ import { type UpsertEmbeddingVectorsResult, type VectorSearchParams, type VectorSearchResult, + type CompleteUploadParams, + type CreateImportRunParams, + type CreateImportRunResponse, + type ImportFileFormat, + type ImportFileManifest, + type ImportFileRole, + type ImportLinkEndpoint, + type ImportLinkSpec, + type ImportRun, + type ImportRunDetail, + type ImportRunEvent, + type ImportRunFile, + type InitiateImportUploadResponse, + type SignPartParams, + type SignPartResponse, + type UploadProgress, RestAPI } from './api/index.js' @@ -51,7 +67,23 @@ export { type UpsertEmbeddingVectorsParams, type UpsertEmbeddingVectorsResult, type VectorSearchParams, - type VectorSearchResult + type VectorSearchResult, + type CompleteUploadParams, + type CreateImportRunParams, + type CreateImportRunResponse, + type ImportFileFormat, + type ImportFileManifest, + type ImportFileRole, + type ImportLinkEndpoint, + type ImportLinkSpec, + type ImportRun, + type ImportRunDetail, + type ImportRunEvent, + type ImportRunFile, + type InitiateImportUploadResponse, + type SignPartParams, + type SignPartResponse, + type UploadProgress } export * from './types/index.js' export * from './sdk/index.js' diff --git a/platform/core/.env.example b/platform/core/.env.example index e5cba2a8..0cffdb48 100644 --- a/platform/core/.env.example +++ b/platform/core/.env.example @@ -174,3 +174,33 @@ SQL_DB_TYPE=sqlite # RUSHDB_PAGINATION_MAX_LIMIT=1000 # Maximum allowed `limit` value. Requests above this are clamped. + +# ── Async import runs ───────────────────────────────────────────────────────── +# Feature flag for the asynchronous multi-file import API (/api/v1/imports). +# RUSHDB_IMPORTS_ENABLED=true + +# Storage backend for uploaded import sources: s3 (default) or local. +# Retention/cleanup applies identically to both backends. +# RUSHDB_IMPORT_STORAGE_BACKEND=s3 + +# Local backend: directory where import sources are stored. +# RUSHDB_IMPORT_STORAGE_LOCAL_PATH=.rushdb-imports + +# S3 backend: bucket is required when the backend is s3; endpoint/credentials are +# optional on AWS (workload identity / default credential chain) and required for +# most S3-compatible providers (MinIO, R2, etc.). +# RUSHDB_IMPORT_S3_BUCKET= +# RUSHDB_IMPORT_S3_REGION=us-east-1 +# RUSHDB_IMPORT_S3_ENDPOINT= +# RUSHDB_IMPORT_S3_ACCESS_KEY_ID= +# RUSHDB_IMPORT_S3_SECRET_ACCESS_KEY= +# RUSHDB_IMPORT_S3_FORCE_PATH_STYLE=false + +# How long terminal runs keep their uploaded sources (retry/diagnostics window), +# in hours. After this a cleanup sweep deletes the stored objects; imported graph +# data is never touched. +# RUSHDB_IMPORT_RETENTION_HOURS=72 + +# Memory budgets per import source, in bytes. +# RUSHDB_IMPORT_MAX_JSON_BYTES=67108864 # 64 MiB +# RUSHDB_IMPORT_MAX_PARQUET_BYTES=268435456 # 256 MiB diff --git a/platform/core/package.json b/platform/core/package.json index f9d8767a..71d1ba50 100755 --- a/platform/core/package.json +++ b/platform/core/package.json @@ -24,7 +24,7 @@ "start:prod": "node dist/main", "lint": "ESLINT_USE_FLAT_CONFIG=false eslint \"{src,apps,libs,test}/**/*.ts\"", "lint:fix": "ESLINT_USE_FLAT_CONFIG=false eslint \"{src,apps,libs,test}/**/*.ts\" --fix", - "test": "jest", + "test": "NODE_OPTIONS=--experimental-vm-modules jest", "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", @@ -42,6 +42,8 @@ "db:studio:pg": "drizzle-kit studio --config=drizzle.pg.config.ts" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1116.0", + "@aws-sdk/s3-request-presigner": "^3.1116.0", "@fastify/formbody": "8.0.2", "@fastify/static": "9.1.3", "@fastify/swagger": "^9.7.0", @@ -67,6 +69,7 @@ "drizzle-orm": "^0.45.2", "ejs": "^3.1.10", "fastify-raw-body": "^5.0.0", + "hyparquet": "^1.29.1", "ms": "^2.1.3", "neo4j-driver": "6.0.1", "neo4j-driver-core": "6.0.1", diff --git a/platform/core/src/core/core.module.ts b/platform/core/src/core/core.module.ts index 8038ab2c..5931762b 100755 --- a/platform/core/src/core/core.module.ts +++ b/platform/core/src/core/core.module.ts @@ -4,6 +4,7 @@ import { AiModule } from '@/core/ai/ai.module' import { BillingPolicyModule } from '@/core/billing-policy/billing-policy.module' import { EntityModule } from '@/core/entity/entity.module' import { ImportExportModule } from '@/core/entity/import-export/import-export.module' +import { ImportRunsModule } from '@/core/import-runs/import-runs.module' import { PropertyModule } from '@/core/property/property.module' import { QueryModule } from '@/core/query/query.module' import { RelationshipPatternsModule } from '@/core/relationship-patterns/relationship-patterns.module' @@ -24,6 +25,7 @@ import { SessionAndTransactionAttachMiddleware } from '@/database/session-and-tr ImportExportModule, TransactionModule, QueryModule, + ImportRunsModule, forwardRef(() => TokenModule), forwardRef(() => DbConnectionModule) ], diff --git a/platform/core/src/core/import-runs/api/import-runs.controller.ts b/platform/core/src/core/import-runs/api/import-runs.controller.ts new file mode 100644 index 00000000..c032197c --- /dev/null +++ b/platform/core/src/core/import-runs/api/import-runs.controller.ts @@ -0,0 +1,260 @@ +import { BadRequestException } from '@nestjs/common' +import { + Body, + Controller, + Delete, + Get, + Headers, + HttpCode, + HttpStatus, + Param, + Post, + Request, + UseGuards, + UseInterceptors +} from '@nestjs/common' +import { ApiBearerAuth, ApiParam, ApiTags } from '@nestjs/swagger' + +import { NotFoundInterceptor } from '@/common/interceptors/not-found.interceptor' +import { TransformResponseInterceptor } from '@/common/interceptors/transform-response.interceptor' +import { PlatformRequest } from '@/common/types/request' +import { formatErrorMessage } from '@/common/validation/utils' +import { EntityWriteGuard } from '@/core/entity/entity-write.guard' +import { TokenReadAccess } from '@/dashboard/auth/decorators/token-read-access.decorator' +import { AuthGuard } from '@/dashboard/auth/guards/global-auth.guard' + +import { ImportRunsService } from '../import-runs.service' + +import { createImportRunSchema } from './validation/import-run.schema' + +import type { ImportFileManifestItem, ImportLinkSpec } from '../domain/import-run.types' +import type { CreateImportRunDto } from './validation/import-run.schema' + +@Controller('imports') +@ApiTags('Import Runs') +@UseInterceptors(TransformResponseInterceptor, NotFoundInterceptor) +export class ImportRunsController { + constructor(private readonly importRunsService: ImportRunsService) {} + + @Post() + @ApiBearerAuth() + @HttpCode(HttpStatus.CREATED) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async create( + @Request() request: PlatformRequest, + @Body() body: unknown, + @Headers('idempotency-key') idempotencyKey?: string + ) { + const projectId = request.projectId as string + const parsed = createImportRunSchema.safeParse(body) + if (!parsed.success) { + throw new BadRequestException(formatErrorMessage(parsed.error, { type: 'body' })) + } + const manifest = parsed.data as CreateImportRunDto + + const result = await this.importRunsService.createDraft( + { + projectId, + name: manifest.name, + failurePolicy: manifest.failurePolicy, + files: manifest.files.map( + (file): ImportFileManifestItem => ({ + clientFileId: file.clientFileId, + fileName: file.fileName, + size: file.size, + format: file.format, + role: file.role, + rootLabel: file.rootLabel, + // Structural shape already validated by the schema; semantic + // endpoint/source-target rules are enforced by the service layer. + linkSpec: (file.linkSpec ?? undefined) as ImportLinkSpec | undefined, + parseOptions: file.parseOptions, + importOptions: file.importOptions + }) + ) + }, + idempotencyKey ? this.importRunsService.hashKey(idempotencyKey) : undefined + ) + + return { runId: result.runId, files: result.files } + } + + @Post(':runId/files/:fileId/content') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @ApiParam({ name: 'fileId' }) + @HttpCode(HttpStatus.ACCEPTED) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async uploadContent( + @Request() request: PlatformRequest, + @Param('runId') runId: string, + @Param('fileId') fileId: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + @Body() content: any + ) { + const projectId = request.projectId as string + const buffer = Buffer.isBuffer(content) ? content : Buffer.from(JSON.stringify(content)) + + return this.importRunsService.uploadFileContent(projectId, runId, fileId, buffer) + } + + @Post(':runId/files/:fileId/upload/initiate') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @ApiParam({ name: 'fileId' }) + @HttpCode(HttpStatus.CREATED) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async initiateUpload( + @Request() request: PlatformRequest, + @Param('runId') runId: string, + @Param('fileId') fileId: string + ) { + return this.importRunsService.initiateUpload(request.projectId as string, runId, fileId) + } + + @Post(':runId/files/:fileId/upload/parts/:partNumber/sign') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @ApiParam({ name: 'fileId' }) + @ApiParam({ name: 'partNumber' }) + @HttpCode(HttpStatus.OK) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async signPart( + @Request() request: PlatformRequest, + @Param('runId') runId: string, + @Param('fileId') fileId: string, + @Param('partNumber') partNumber: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + @Body() body: any + ) { + const uploadId = typeof body?.uploadId === 'string' ? body.uploadId : '' + if (!uploadId) { + throw new BadRequestException('uploadId is required') + } + return this.importRunsService.signPart( + request.projectId as string, + runId, + fileId, + uploadId, + Number(partNumber) + ) + } + + @Post(':runId/files/:fileId/upload/complete') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @ApiParam({ name: 'fileId' }) + @HttpCode(HttpStatus.ACCEPTED) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async completeUpload( + @Request() request: PlatformRequest, + @Param('runId') runId: string, + @Param('fileId') fileId: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + @Body() body: any + ) { + const uploadId = typeof body?.uploadId === 'string' ? body.uploadId : '' + if (!uploadId) { + throw new BadRequestException('uploadId is required') + } + const expectedSizeBytes = + typeof body?.expectedSizeBytes === 'number' ? Math.floor(body.expectedSizeBytes) : undefined + + return this.importRunsService.completeUpload( + request.projectId as string, + runId, + fileId, + uploadId, + expectedSizeBytes + ) + } + + @Post(':runId/files/:fileId/upload/abort') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @ApiParam({ name: 'fileId' }) + @HttpCode(HttpStatus.ACCEPTED) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async abortUpload( + @Request() request: PlatformRequest, + @Param('runId') runId: string, + @Param('fileId') fileId: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + @Body() body: any + ) { + const uploadId = typeof body?.uploadId === 'string' ? body.uploadId : '' + if (!uploadId) { + throw new BadRequestException('uploadId is required') + } + await this.importRunsService.abortUpload(request.projectId as string, runId, fileId, uploadId) + return { status: 'aborted' } + } + + @Get() + @ApiBearerAuth() + @AuthGuard('project') + @TokenReadAccess() + async list(@Request() request: PlatformRequest) { + const projectId = request.projectId as string + return this.importRunsService.listRuns(projectId) + } + + @Get(':runId') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @AuthGuard('project') + @TokenReadAccess() + async detail(@Request() request: PlatformRequest, @Param('runId') runId: string) { + const projectId = request.projectId as string + return this.importRunsService.getRunDetail(runId, projectId) + } + + @Post(':runId/start') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @HttpCode(HttpStatus.ACCEPTED) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async start(@Request() request: PlatformRequest, @Param('runId') runId: string) { + await this.importRunsService.startRun(runId, request.projectId as string) + return { status: 'queued' } + } + + @Post(':runId/cancel') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @HttpCode(HttpStatus.ACCEPTED) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async cancel(@Request() request: PlatformRequest, @Param('runId') runId: string) { + await this.importRunsService.cancelRun(runId, request.projectId as string) + return { status: 'canceling' } + } + + @Post(':runId/retry') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @HttpCode(HttpStatus.ACCEPTED) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async retry(@Request() request: PlatformRequest, @Param('runId') runId: string) { + await this.importRunsService.retryRun(runId, request.projectId as string) + return { status: 'queued' } + } + + @Delete(':runId') + @ApiBearerAuth() + @ApiParam({ name: 'runId' }) + @HttpCode(HttpStatus.NO_CONTENT) + @UseGuards(EntityWriteGuard) + @AuthGuard('project') + async deleteDraft(@Request() request: PlatformRequest, @Param('runId') runId: string): Promise { + await this.importRunsService.deleteDraftRun(runId, request.projectId as string) + } +} diff --git a/platform/core/src/core/import-runs/api/validation/import-run.schema.ts b/platform/core/src/core/import-runs/api/validation/import-run.schema.ts new file mode 100644 index 00000000..17ac3c92 --- /dev/null +++ b/platform/core/src/core/import-runs/api/validation/import-run.schema.ts @@ -0,0 +1,65 @@ +import { z } from 'zod' + +const endpointSchema = z.object({ + column: z.string().min(1).max(255), + label: z.string().regex(/^[A-Za-z][A-Za-z0-9_]{0,99}$/), + keyProperty: z.string().min(1).max(255), + direction: z.enum(['source', 'target']) +}) + +export const linkSpecSchema = z.object({ + version: z.literal(1), + role: z.literal('links'), + endpoints: z.tuple([endpointSchema, endpointSchema]), + relationshipType: z.string().regex(/^[A-Za-z][A-Za-z0-9_]{0,99}$/), + propertyColumns: z.record(z.string().min(1).max(255)).optional() +}) + +export const importFileManifestSchema = z + .object({ + clientFileId: z.string().min(1).max(100), + fileName: z.string().min(1).max(255), + size: z.number().int().nonnegative(), + format: z.enum(['csv', 'jsonl', 'ndjson', 'json', 'parquet']), + role: z.enum(['records', 'links']).default('records'), + rootLabel: z + .string() + .regex(/^[A-Za-z][A-Za-z0-9_]{0,99}$/) + .optional(), + linkSpec: linkSpecSchema.optional(), + parseOptions: z.record(z.unknown()).optional(), + importOptions: z.record(z.unknown()).optional() + }) + .superRefine((item, ctx) => { + if (item.role === 'links') { + if (!item.linkSpec) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'linkSpec is required for links files', + path: ['linkSpec'] + }) + } + if (item.rootLabel) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'rootLabel must not be set for links files', + path: ['rootLabel'] + }) + } + } else if (!item.rootLabel) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'rootLabel is required for records files', + path: ['rootLabel'] + }) + } + }) + +export const createImportRunSchema = z.object({ + name: z.string().min(1).max(255).optional(), + failurePolicy: z.enum(['continue', 'stop_new_files']).default('continue'), + files: z.array(importFileManifestSchema).min(1).max(50) +}) + +export type CreateImportRunDto = z.infer +export type ImportFileManifestDto = z.infer diff --git a/platform/core/src/core/import-runs/domain/import-run.types.ts b/platform/core/src/core/import-runs/domain/import-run.types.ts new file mode 100644 index 00000000..644dfc45 --- /dev/null +++ b/platform/core/src/core/import-runs/domain/import-run.types.ts @@ -0,0 +1,128 @@ +export const IMPORT_RUN_STATUSES = [ + 'draft', + 'uploading', + 'queued', + 'running', + 'blocked', + 'canceling', + 'finalizing', + 'completed', + 'completed_with_errors', + 'failed', + 'canceled' +] as const + +export type ImportRunStatus = (typeof IMPORT_RUN_STATUSES)[number] + +export const IMPORT_FILE_STATUSES = [ + 'awaiting_upload', + 'uploading', + 'uploaded', + 'queued', + 'validating', + 'running', + 'retry_wait', + 'blocked', + 'finalizing', + 'completed', + 'failed', + 'canceled', + 'source_expired' +] as const + +export type ImportFileStatus = (typeof IMPORT_FILE_STATUSES)[number] + +export const TERMINAL_FILE_STATUSES: ImportFileStatus[] = [ + 'completed', + 'failed', + 'canceled', + 'source_expired' +] + +export const RECORDS_PHASE_TERMINAL_STATUSES: ImportFileStatus[] = ['completed', 'failed', 'canceled'] + +export type ImportFileStage = 'upload' | 'validate' | 'parse' | 'write' | 'finalize' + +export type ImportFailurePolicy = 'continue' | 'stop_new_files' + +export type ImportFileFormat = 'csv' | 'jsonl' | 'ndjson' | 'json' | 'parquet' + +export type ImportStorageBackend = 's3' | 'local' + +export type ImportFileRole = 'records' | 'links' + +export type ImportRunOutcome = Extract< + ImportRunStatus, + 'completed' | 'completed_with_errors' | 'failed' | 'canceled' +> + +export const IMPORT_ERROR_CODES = { + MANIFEST_CONFLICT: 'IMPORT_MANIFEST_CONFLICT', + FILE_LIMIT_EXCEEDED: 'IMPORT_FILE_LIMIT_EXCEEDED', + RUN_BYTES_EXCEEDED: 'IMPORT_RUN_BYTES_EXCEEDED', + FORMAT_UNSUPPORTED: 'IMPORT_FORMAT_UNSUPPORTED', + LABEL_INVALID: 'IMPORT_LABEL_INVALID', + UPLOAD_EXPIRED: 'IMPORT_UPLOAD_EXPIRED', + CHECKSUM_MISMATCH: 'IMPORT_CHECKSUM_MISMATCH', + SOURCE_MISSING: 'IMPORT_SOURCE_MISSING', + PARSE_ERROR: 'IMPORT_PARSE_ERROR', + TOO_MANY_ERRORS: 'IMPORT_TOO_MANY_ERRORS', + LINK_SPEC_INVALID: 'IMPORT_LINK_SPEC_INVALID', + LINK_COLUMNS_UNMAPPED: 'IMPORT_LINK_COLUMNS_UNMAPPED', + LINK_ENDPOINT_UNRESOLVED: 'IMPORT_LINK_ENDPOINT_UNRESOLVED', + LINK_LABEL_EMPTY: 'IMPORT_LINK_LABEL_EMPTY', + REPLAY_ID_COLLISION: 'IMPORT_REPLAY_ID_COLLISION', + QUOTA_BLOCKED: 'IMPORT_QUOTA_BLOCKED', + CANCELED: 'IMPORT_CANCELED', + LEASE_LOST: 'IMPORT_LEASE_LOST', + FINALIZATION_DELAYED: 'IMPORT_FINALIZATION_DELAYED', + INTERNAL: 'IMPORT_INTERNAL_ERROR' +} as const + +export type ImportErrorCode = keyof typeof IMPORT_ERROR_CODES + +export type ImportLinkEndpointDirection = 'source' | 'target' + +export interface ImportLinkEndpoint { + column: string + label: string + keyProperty: string + direction: ImportLinkEndpointDirection +} + +export interface ImportLinkSpec { + version: 1 + role: 'links' + endpoints: [ImportLinkEndpoint, ImportLinkEndpoint] + relationshipType: string + propertyColumns?: Record +} + +export interface ImportFileManifestItem { + clientFileId: string + fileName: string + size: number + format: ImportFileFormat + role: ImportFileRole + rootLabel?: string + linkSpec?: ImportLinkSpec + parseOptions?: Record + importOptions?: Record +} + +export interface ImportCheckpoint { + version: 1 + sourceGeneration: number + nextUnit: number + lastCommittedBatch: number +} + +export interface ImportCounters { + parsedUnits: number + committedUnits: number + recordsCommitted: number + relationshipsCommitted: number + linksResolved: number + linksUnresolved: number + skippedUnits: number +} diff --git a/platform/core/src/core/import-runs/domain/import-state-machine.ts b/platform/core/src/core/import-runs/domain/import-state-machine.ts new file mode 100644 index 00000000..3aeab81d --- /dev/null +++ b/platform/core/src/core/import-runs/domain/import-state-machine.ts @@ -0,0 +1,124 @@ +import { TERMINAL_FILE_STATUSES, type ImportFileStatus, type ImportRunStatus } from './import-run.types' + +interface FileShape { + status: ImportFileStatus + role: 'records' | 'links' +} + +const ACTIVE_RUN_STATUSES: ImportRunStatus[] = ['uploading', 'queued', 'running', 'canceling'] + +export const FILE_TRANSITIONS: Record = { + awaiting_upload: ['uploading', 'uploaded', 'queued', 'canceled'], + uploading: ['uploaded', 'queued', 'retry_wait', 'failed', 'canceled'], + uploaded: ['queued', 'failed', 'canceled'], + queued: ['validating', 'running', 'retry_wait', 'blocked', 'canceled'], + validating: ['running', 'retry_wait', 'failed', 'canceled'], + running: ['finalizing', 'completed', 'retry_wait', 'blocked', 'failed', 'canceled'], + retry_wait: ['queued', 'blocked', 'failed', 'canceled'], + blocked: ['queued', 'canceled'], + finalizing: ['completed', 'retry_wait', 'failed', 'canceled'], + completed: [], + failed: [], + canceled: [], + source_expired: [] +} + +export function canTransitionFile(from: ImportFileStatus, to: ImportFileStatus): boolean { + return FILE_TRANSITIONS[from]?.includes(to) ?? false +} + +export function isTerminalFile(status: ImportFileStatus): boolean { + return TERMINAL_FILE_STATUSES.includes(status) +} + +/** + * Pure run-status derivation. Order matters: + * canceling > blocked > uploading > queued/running > finalizing > terminal outcomes. + */ +export function deriveRunStatus( + files: FileShape[], + run?: { cancelRequestedAt?: string | null } +): ImportRunStatus { + if (files.length === 0) { + return 'draft' + } + + const statuses = files.map((f) => f.status) + const allTerminal = statuses.every((s) => isTerminalFile(s)) + const anyBlocked = statuses.includes('blocked') + const anyUploading = statuses.includes('awaiting_upload') || statuses.includes('uploading') + const anyQueuedOrRunning = + statuses.some((s) => ['queued', 'validating', 'running', 'retry_wait', 'finalizing'].includes(s)) || + (anyBlocked && !allTerminal) + const anyCompleted = statuses.includes('completed') + const allCanceled = statuses.every((s) => s === 'canceled') + const anyFailed = statuses.includes('failed') + + if ( + !allTerminal && + run?.cancelRequestedAt && + statuses.some((s) => ACTIVE_RUN_STATUSES.includes(s as never)) + ) { + return 'canceling' + } + + if (allTerminal) { + if (statuses.includes('finalizing')) { + // finalizing is modeled as a nonterminal file state; treat as pending side effects + return 'finalizing' + } + if (allCanceled) { + return 'canceled' + } + if (anyFailed && !anyCompleted) { + return 'failed' + } + if (anyFailed || anyCompleted !== statuses.every((s) => s === 'completed')) { + return 'completed_with_errors' + } + return 'completed' + } + + if (anyBlocked && !statuses.some((s) => ['queued', 'validating', 'running', 'retry_wait'].includes(s))) { + return 'blocked' + } + + if (run?.cancelRequestedAt) { + return 'canceling' + } + + if (anyUploading) { + return 'uploading' + } + + if (anyQueuedOrRunning) { + return 'running' + } + + return 'running' +} + +export function canTransitionRun(from: ImportRunStatus, to: ImportRunStatus): boolean { + const allowed: Record = { + draft: ['uploading', 'queued', 'canceled'], + uploading: ['queued', 'running', 'blocked', 'canceling', 'canceled', 'failed'], + queued: ['running', 'blocked', 'canceling', 'canceled', 'failed'], + running: [ + 'blocked', + 'canceling', + 'finalizing', + 'completed', + 'completed_with_errors', + 'failed', + 'canceled' + ], + blocked: ['queued', 'running', 'canceling', 'canceled'], + canceling: ['finalizing', 'completed', 'completed_with_errors', 'failed', 'canceled'], + finalizing: ['completed', 'completed_with_errors', 'failed', 'canceled'], + completed: [], + completed_with_errors: [], + failed: [], + canceled: [] + } + return allowed[from]?.includes(to) ?? false +} diff --git a/platform/core/src/core/import-runs/domain/label-suggestion.ts b/platform/core/src/core/import-runs/domain/label-suggestion.ts new file mode 100644 index 00000000..02e55c71 --- /dev/null +++ b/platform/core/src/core/import-runs/domain/label-suggestion.ts @@ -0,0 +1,100 @@ +import type { ImportFileFormat } from './import-run.types' + +const LABEL_EXCEPTIONS: Record = { + people: 'PEOPLE', + data: 'DATA', + children: 'CHILDREN' +} + +function singularize(word: string): string { + if (LABEL_EXCEPTIONS[word]) { + return LABEL_EXCEPTIONS[word] + } + if (word.length > 3 && word.endsWith('ies')) { + return `${word.slice(0, -3)}y` + } + if (word.length > 3 && word.endsWith('ses')) { + return word.slice(0, -2) + } + if (word.length > 2 && word.endsWith('s') && !word.endsWith('ss') && !word.endsWith('us')) { + return word.slice(0, -1) + } + return word +} + +/** + * Deterministic filename -> label suggestion. Never authoritative. + * "users.csv" -> USER; "order-items.jsonl" -> ORDER_ITEM; "char_ep.csv" -> CHAR_EP. + */ +export function suggestLabelFromFileName(fileName: string): string { + const base = fileName.replace(/\.[^.]+$/, '') + const words = base + .replace(/([a-z])([A-Z])/g, '$1 $2') + .split(/[\s._\-]+/) + .filter(Boolean) + const normalized = words.map((w) => w.toLowerCase()) + const last = normalized.length - 1 + const processed = normalized.map((word, i) => (i === last ? singularize(word) : word)) + return processed.join('_').toUpperCase() +} + +export function detectFormatFromFileName(fileName: string): ImportFileFormat | null { + const lower = fileName.toLowerCase() + if (lower.endsWith('.csv')) { + return 'csv' + } + if (lower.endsWith('.jsonl')) { + return 'jsonl' + } + if (lower.endsWith('.ndjson')) { + return 'ndjson' + } + if (lower.endsWith('.json')) { + return 'json' + } + if (lower.endsWith('.parquet')) { + return 'parquet' + } + return null +} + +const REFERENCE_COLUMN_PATTERN = /(^|_)(id|ids|key|keys|code|guid|uuid|ref|fk)$/i + +export function looksLikeReferenceColumn(columnName: string): boolean { + return REFERENCE_COLUMN_PATTERN.test(columnName.trim()) +} + +export interface RoleSuggestionInput { + columns: string[] + fileName?: string + siblingIdColumns?: string[] +} + +export interface RoleSuggestion { + role: 'records' | 'links' + reason: string +} + +/** + * Advisory-only role suggestion. Two reference-like columns with nothing else, + * optionally matching sibling id-like columns, suggests a pure join table. + * Value-rich files are never suggested as links. Filename hints never decide alone. + */ +export function suggestRole({ columns, siblingIdColumns = [] }: RoleSuggestionInput): RoleSuggestion { + if (columns.length !== 2) { + return { role: 'records', reason: `expected exactly two reference-like columns, found ${columns.length}` } + } + + const referenceLike = columns.filter((c) => looksLikeReferenceColumn(c)) + const matchesSibling = + siblingIdColumns.length > 0 && columns.some((c) => siblingIdColumns.includes(c.toLowerCase())) + + if ( + referenceLike.length === columns.length && + (referenceLike.length === columns.length || matchesSibling) + ) { + return { role: 'links', reason: 'all columns look like foreign-key references' } + } + + return { role: 'records', reason: 'file contains value-bearing columns' } +} diff --git a/platform/core/src/core/import-runs/domain/link-spec.validation.ts b/platform/core/src/core/import-runs/domain/link-spec.validation.ts new file mode 100644 index 00000000..9eba1a9e --- /dev/null +++ b/platform/core/src/core/import-runs/domain/link-spec.validation.ts @@ -0,0 +1,130 @@ +import { IMPORT_ERROR_CODES } from './import-run.types' + +import type { ImportLinkSpec } from './import-run.types' + +const LABEL_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,99}$/ + +function isValidPropertyKey(raw: string): boolean { + if (raw.length === 0 || raw.length > 255) { + return false + } + for (let i = 0; i < raw.length; i++) { + const code = raw.charCodeAt(i) + if (code <= 0x1f || code === 0x60) { + return false + } + } + return true +} + +export class ImportValidationError extends Error { + constructor( + public readonly code: string, + message: string + ) { + super(message) + } +} + +export function validateLinkSpec(spec: unknown): asserts spec is ImportLinkSpec { + if (!spec || typeof spec !== 'object') { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_SPEC_INVALID, + 'linkSpec is required for links files' + ) + } + + const candidate = spec as Partial + + if (candidate.version !== 1 || candidate.role !== 'links') { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_SPEC_INVALID, + 'linkSpec.version must be 1 and role "links"' + ) + } + + if (!Array.isArray(candidate.endpoints) || candidate.endpoints.length !== 2) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_SPEC_INVALID, + 'linkSpec requires exactly two endpoints (source and target)' + ) + } + + const directions = candidate.endpoints.map((e) => e?.direction) + if (!directions.includes('source') || !directions.includes('target')) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_SPEC_INVALID, + 'linkSpec endpoints must include exactly one source and one target' + ) + } + + for (const endpoint of candidate.endpoints) { + if (!endpoint.column || !isValidPropertyKey(endpoint.column)) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_SPEC_INVALID, + `invalid endpoint column: ${endpoint.column}` + ) + } + if (!endpoint.label || !LABEL_PATTERN.test(endpoint.label)) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_SPEC_INVALID, + `invalid endpoint label: ${endpoint.label}` + ) + } + if (!endpoint.keyProperty || !isValidPropertyKey(endpoint.keyProperty)) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_SPEC_INVALID, + `invalid endpoint keyProperty: ${endpoint.keyProperty}` + ) + } + } + + if ( + candidate.endpoints[0].column.toLowerCase() === candidate.endpoints[1].column.toLowerCase() && + candidate.endpoints[0].label === candidate.endpoints[1].label + ) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_SPEC_INVALID, + 'endpoints must bind distinct columns' + ) + } + + if (!candidate.relationshipType || !isValidPropertyKey(candidate.relationshipType)) { + throw new ImportValidationError(IMPORT_ERROR_CODES.LINK_SPEC_INVALID, 'relationshipType is required') + } + + if (candidate.propertyColumns) { + for (const [column, property] of Object.entries(candidate.propertyColumns)) { + if (!isValidPropertyKey(column) || !isValidPropertyKey(property)) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_COLUMNS_UNMAPPED, + `invalid property column mapping: ${column}` + ) + } + } + } +} + +/** + * Ensures every non-endpoint column is either explicitly mapped as a relationship + * property or listed in ignoredColumns. Silent dropping is forbidden. + */ +export function validateLinkColumns(spec: ImportLinkSpec, columns: string[]): void { + const endpointColumns = new Set(spec.endpoints.map((e) => e.column)) + const mapped = new Set(Object.keys(spec.propertyColumns ?? {})) + const unmapped = columns.filter((c) => !endpointColumns.has(c) && !mapped.has(c)) + + if (unmapped.length > 0) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LINK_COLUMNS_UNMAPPED, + `columns not mapped to relationship properties: ${unmapped.join(', ')}` + ) + } +} + +export function suggestRelationshipType(spec: Pick): string { + return spec.endpoints + .map((e) => e.label) + .join('_') + .toUpperCase() +} diff --git a/platform/core/src/core/import-runs/domain/replay-identity.ts b/platform/core/src/core/import-runs/domain/replay-identity.ts new file mode 100644 index 00000000..4bc39f46 --- /dev/null +++ b/platform/core/src/core/import-runs/domain/replay-identity.ts @@ -0,0 +1,51 @@ +import { createHash } from 'node:crypto' + +const IMPORT_REPLAY_NAMESPACE = '6f1d0c9e-3a2b-4c5d-8e7f-90a1b2c3d4e5' + +/** + * Deterministic record identity for async import replay. + * + * Derived from project + file + source generation + unit ordinal + stable nested + * path with sibling occurrence. Never hashes raw property values. Stable across + * retries of the same source generation, distinct across runs/sources. + */ +export function deriveDeterministicRecordId(input: { + projectId: string + fileId: string + sourceGeneration: number + unitOrdinal: number + path: string + siblingOccurrence: number +}): string { + const payload = [ + IMPORT_REPLAY_NAMESPACE, + input.projectId, + input.fileId, + String(input.sourceGeneration), + String(input.unitOrdinal), + input.path, + String(input.siblingOccurrence) + ].join('\n') + + const digest = createHash('sha256').update(payload).digest('hex').slice(0, 32) + + // Format as UUIDv8-shaped (version 8, RFC variant) deterministic identifier. + return [ + digest.slice(0, 8), + digest.slice(8, 12), + `8${digest.slice(13, 16)}`, + ((parseInt(digest[16], 16) & 0x3) | 0x8).toString(16) + digest.slice(17, 20), + digest.slice(20, 32) + ].join('-') +} + +export function hashIdempotencyKey(key: string): string { + return createHash('sha256').update(key).digest('hex') +} + +export function deriveBatchId(fileId: string, sourceGeneration: number, batchOrdinal: number): string { + return createHash('sha256') + .update([fileId, sourceGeneration, batchOrdinal].join('\n')) + .digest('hex') + .slice(0, 24) +} diff --git a/platform/core/src/core/import-runs/import-runs.backends.spec.ts b/platform/core/src/core/import-runs/import-runs.backends.spec.ts new file mode 100644 index 00000000..9700881a --- /dev/null +++ b/platform/core/src/core/import-runs/import-runs.backends.spec.ts @@ -0,0 +1,171 @@ +import { createReadStream, existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PassThrough } from 'node:stream' + +import { JsonLinesImportParser } from './parser/json-lines.parser' +import { JsonObjectImportParser } from './parser/json-object.parser' +import { ParquetImportParser } from './parser/parquet.parser' +import { LocalImportStorageAdapter } from './storage/local-import-storage.adapter' + +async function collectUnits(iterable: AsyncIterable<{ ordinal: number; value: Record }>) { + const units = [] + for await (const unit of iterable) { + units.push(unit) + } + return units +} + +function streamFrom(text: string): NodeJS.ReadableStream { + const stream = new PassThrough() + stream.end(Buffer.from(text, 'utf8')) + return stream +} + +describe('JsonObjectImportParser', () => { + const parser = new JsonObjectImportParser() + + it('streams top-level array elements as logical units', async () => { + const json = '[{"id":1},{"id":2},{"id":3}]' + const units = await collectUnits(await parser.parse(streamFrom(json), {})) + + expect(units.map((u) => u.value.id)).toEqual([1, 2, 3]) + expect(units.map((u) => u.ordinal)).toEqual([0, 1, 2]) + }) + + it('emits a single top-level object as one unit', async () => { + const json = '{"id":1,"nested":{"label":"child"}}' + const units = await collectUnits(await parser.parse(streamFrom(json), {})) + + expect(units).toHaveLength(1) + expect(units[0].value.nested).toEqual({ label: 'child' }) + }) + + it('reports the detected shape on inspect', async () => { + await expect(parser.inspect(Buffer.from('[1]', 'utf8'))).resolves.toMatchObject({ jsonShape: 'array' }) + await expect(parser.inspect(Buffer.from('{"a":1}', 'utf8'))).resolves.toMatchObject({ + jsonShape: 'single_object' + }) + }) + + it('rejects scalar roots', async () => { + await expect(collectUnits(await parser.parse(streamFrom('"text"'), {}))).rejects.toThrow( + /must be an object/ + ) + }) + + it('enforces the source byte budget for single objects', async () => { + const json = '{"data":"' + 'x'.repeat(1000) + '"}' + await expect(collectUnits(await parser.parse(streamFrom(json), { maxSourceBytes: 10 }))).rejects.toThrow( + /IMPORT_JSON_SHAPE_NOT_STREAMABLE/ + ) + }) +}) + +describe('JsonLinesImportParser (ndjson alias)', () => { + const parser = new JsonLinesImportParser() + + it('parses ndjson content identically to jsonl', async () => { + const units = await collectUnits(await parser.parse(streamFrom('{"id":1}\n{"id":2}\n'), {})) + expect(units.map((u) => u.value.id)).toEqual([1, 2]) + }) +}) + +describe('ParquetImportParser', () => { + const parser = new ParquetImportParser() + const fixturePath = join(__dirname, 'parser', '__fixtures__', 'characters.snappy.parquet') + + it('yields stable-ordinal row units from a snappy-compressed file', async () => { + const bytes = readFileSync(fixturePath) + const units = await collectUnits(await parser.parse(streamFromBytes(bytes), {})) + + expect(units).toHaveLength(3) + expect(units[0].value).toMatchObject({ id: 1, name: 'Rick Sanchez' }) + expect(units[2].value).toMatchObject({ id: 3, name: 'Summer Smith' }) + expect(units.map((u) => u.ordinal)).toEqual([0, 1, 2]) + }) + + it('reads uncompressed files', async () => { + const bytes = readFileSync(join(__dirname, 'parser', '__fixtures__', 'characters.plain.parquet')) + const units = await collectUnits(await parser.parse(streamFromBytes(bytes), {})) + expect(units).toHaveLength(3) + }) + + it('reads from a local storage stream end-to-end', async () => { + const adapter = new LocalImportStorageAdapter(mkdtempSync(join(tmpdir(), 'parquet-test-'))) + try { + const initiated = await adapter.initiate('p1', 'f1', 1) + await adapter.writeChunk(initiated.storageKey, readFileSync(fixturePath)) + await adapter.complete(initiated.storageKey) + + const stream = await adapter.readStream(initiated.storageKey) + const units = await collectUnits(await parser.parse(stream, {})) + expect(units).toHaveLength(3) + } finally { + rmSync(adapter['root'], { recursive: true, force: true }) + } + }) + + it('rejects non-parquet payloads via magic bytes', async () => { + const run = async () => + collectUnits(await parser.parse(streamFromBytes(Buffer.from('definitely not parquet')), {})) + await expect(run()).rejects.toThrow(/PAR1/) + }) + + function streamFromBytes(bytes: Buffer): NodeJS.ReadableStream { + const stream = new PassThrough() + stream.end(bytes) + return stream + } +}) + +describe('LocalImportStorageAdapter retention contract', () => { + let root: string + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'local-storage-test-')) + }) + + afterEach(() => { + rmSync(root, { recursive: true, force: true }) + }) + + it('writes, completes, streams, and deletes like the s3 backend', async () => { + const adapter = new LocalImportStorageAdapter(root) + + const initiated = await adapter.initiate('p1', 'file-1', 1) + expect(initiated.storageKey.startsWith('imports/p1/file-1/g1/')).toBe(true) + + await adapter.writeChunk(initiated.storageKey, Buffer.from('chunk-one,')) + await adapter.writeChunk(initiated.storageKey, Buffer.from('chunk-two')) + const meta = await adapter.complete(initiated.storageKey, 19) + expect(meta.sizeBytes).toBe(19) + + const head = await adapter.head(initiated.storageKey) + expect(head?.sizeBytes).toBe(19) + expect(existsSync(join(root, initiated.storageKey))).toBe(true) + + await adapter.delete(initiated.storageKey) + await expect(adapter.head(initiated.storageKey)).resolves.toBeNull() + }) + + it('rejects path traversal in keys', async () => { + const adapter = new LocalImportStorageAdapter(root) + await expect(adapter.writeChunk('../../etc/passwd', Buffer.from('x'))).rejects.toThrow( + /invalid storage key/ + ) + }) + + it('supports streamed reads compatible with parsers', async () => { + const adapter = new LocalImportStorageAdapter(root) + const initiated = await adapter.initiate('p1', 'file-2', 1) + await adapter.writeChunk(initiated.storageKey, Buffer.from('{"id":1}\n{"id":2}\n')) + await adapter.complete(initiated.storageKey) + + const parser = new JsonLinesImportParser() + const units = await collectUnits( + await parser.parse(createReadStream(join(root, initiated.storageKey)), {}) + ) + expect(units).toHaveLength(2) + }) +}) diff --git a/platform/core/src/core/import-runs/import-runs.domain.spec.ts b/platform/core/src/core/import-runs/import-runs.domain.spec.ts new file mode 100644 index 00000000..86da8913 --- /dev/null +++ b/platform/core/src/core/import-runs/import-runs.domain.spec.ts @@ -0,0 +1,164 @@ +import { deriveRunStatus, canTransitionFile } from './domain/import-state-machine' +import { + suggestLabelFromFileName, + detectFormatFromFileName, + looksLikeReferenceColumn, + suggestRole +} from './domain/label-suggestion' +import { validateLinkColumns, validateLinkSpec, ImportValidationError } from './domain/link-spec.validation' +import { deriveDeterministicRecordId, hashIdempotencyKey } from './domain/replay-identity' + +import type { ImportLinkSpec } from './domain/import-run.types' + +const baseSpec: ImportLinkSpec = { + version: 1, + role: 'links', + endpoints: [ + { column: 'character_id', label: 'CHARACTER', keyProperty: 'id', direction: 'source' }, + { column: 'episode_id', label: 'EPISODE', keyProperty: 'id', direction: 'target' } + ], + relationshipType: 'APPEARED_IN' +} + +describe('ImportRuns state machine', () => { + it('derives completed only when every file completed', () => { + expect( + deriveRunStatus([ + { status: 'completed', role: 'records' }, + { status: 'completed', role: 'links' } + ]) + ).toBe('completed') + }) + + it('derives completed_with_errors on mixed terminal outcomes', () => { + expect( + deriveRunStatus([ + { status: 'completed', role: 'records' }, + { status: 'failed', role: 'links' } + ]) + ).toBe('completed_with_errors') + }) + + it('derives canceled when all files canceled', () => { + expect(deriveRunStatus([{ status: 'canceled', role: 'records' }])).toBe('canceled') + }) + + it('derives failed when no file completed', () => { + expect(deriveRunStatus([{ status: 'failed', role: 'records' }])).toBe('failed') + }) + + it('stays running while any file is nonterminal', () => { + expect( + deriveRunStatus([ + { status: 'completed', role: 'records' }, + { status: 'running', role: 'links' } + ]) + ).toBe('running') + }) + + it('forbids transitions into completed from non-terminal states', () => { + expect(canTransitionFile('queued', 'completed')).toBe(false) + expect(canTransitionFile('running', 'finalizing')).toBe(true) + expect(canTransitionFile('completed', 'running')).toBe(false) + }) +}) + +describe('ImportRuns label suggestion', () => { + it('suggests labels deterministically from filenames', () => { + expect(suggestLabelFromFileName('users.csv')).toBe('USER') + expect(suggestLabelFromFileName('people.ndjson')).toBe('PEOPLE') + expect(suggestLabelFromFileName('order-items.jsonl')).toBe('ORDER_ITEM') + expect(suggestLabelFromFileName('char_ep.csv')).toBe('CHAR_EP') + expect(suggestLabelFromFileName('crm_accounts_2026.csv')).toBe('CRM_ACCOUNTS_2026') + }) + + it('detects supported formats', () => { + expect(detectFormatFromFileName('a.csv')).toBe('csv') + expect(detectFormatFromFileName('a.ndjson')).toBe('ndjson') + expect(detectFormatFromFileName('a.json')).toBe('json') + expect(detectFormatFromFileName('a.parquet')).toBe('parquet') + expect(detectFormatFromFileName('a.txt')).toBeNull() + }) + + it('flags reference-like columns and suggests links only for pure join shapes', () => { + expect(looksLikeReferenceColumn('character_id')).toBe(true) + expect(looksLikeReferenceColumn('id')).toBe(true) + expect(looksLikeReferenceColumn('name')).toBe(false) + + const suggestion = suggestRole({ columns: ['character_id', 'episode_id'] }) + expect(suggestion.role).toBe('links') + + const valueRich = suggestRole({ columns: ['character_id', 'name'] }) + expect(valueRich.role).toBe('records') + + const tooMany = suggestRole({ columns: ['a_id', 'b_id', 'c_id'] }) + expect(tooMany.role).toBe('records') + }) +}) + +describe('ImportRuns replay identity', () => { + it('produces stable ids across invocations', () => { + const input = { + projectId: 'p1', + fileId: 'f1', + sourceGeneration: 1, + unitOrdinal: 42, + path: '0', + siblingOccurrence: 0 + } + expect(deriveDeterministicRecordId(input)).toBe(deriveDeterministicRecordId(input)) + }) + + it('differs by project, file, generation, ordinal and path', () => { + const base = { + projectId: 'p1', + fileId: 'f1', + sourceGeneration: 1, + unitOrdinal: 1, + path: '0', + siblingOccurrence: 0 + } + const variants = [ + { ...base, projectId: 'p2' }, + { ...base, fileId: 'f2' }, + { ...base, unitOrdinal: 2 }, + { ...base, path: '1' } + ].map((v) => deriveDeterministicRecordId(v)) + + expect(new Set(variants).size).toBe(variants.length) + }) + + it('hashes idempotency keys stably', () => { + expect(hashIdempotencyKey('abc')).toBe(hashIdempotencyKey('abc')) + expect(hashIdempotencyKey('abc')).not.toBe(hashIdempotencyKey('abd')) + }) +}) + +describe('ImportRuns link spec validation', () => { + it('accepts a valid spec', () => { + expect(() => validateLinkSpec(baseSpec)).not.toThrow() + }) + + it('rejects missing or malformed specs', () => { + expect(() => validateLinkSpec(null)).toThrow(ImportValidationError) + expect(() => + validateLinkSpec({ + ...baseSpec, + endpoints: [baseSpec.endpoints[0], { ...baseSpec.endpoints[1], direction: 'source' }] + }) + ).toThrow(/source/) + expect(() => validateLinkSpec({ ...baseSpec, relationshipType: '' })).toThrow(ImportValidationError) + }) + + it('rejects unmapped surplus columns silently being dropped', () => { + expect(() => validateLinkColumns(baseSpec, ['character_id', 'episode_id'])).not.toThrow() + expect(() => validateLinkColumns(baseSpec, ['character_id', 'episode_id', 'credit'])).toThrow(/credit/) + expect(() => + validateLinkColumns({ ...baseSpec, propertyColumns: { credit: 'credited' } }, [ + 'character_id', + 'episode_id', + 'credit' + ]) + ).not.toThrow() + }) +}) diff --git a/platform/core/src/core/import-runs/import-runs.infra.spec.ts b/platform/core/src/core/import-runs/import-runs.infra.spec.ts new file mode 100644 index 00000000..253223f5 --- /dev/null +++ b/platform/core/src/core/import-runs/import-runs.infra.spec.ts @@ -0,0 +1,86 @@ +import { PassThrough } from 'node:stream' + +import { CsvImportParser } from './parser/csv.parser' +import { JsonLinesImportParser } from './parser/json-lines.parser' +import { MemoryImportStorageAdapter } from './storage/memory-import-storage.adapter' + +async function collectUnits(iterable: AsyncIterable<{ ordinal: number; value: Record }>) { + const units = [] + for await (const unit of iterable) { + units.push(unit) + } + return units +} + +function streamFrom(text: string): NodeJS.ReadableStream { + const stream = new PassThrough() + stream.end(Buffer.from(text, 'utf8')) + return stream +} + +describe('CsvImportParser', () => { + const parser = new CsvImportParser() + + it('emits sanitized headers and row objects', async () => { + const csv = 'id,name,name\n1,Rick,Sanchez\n2,Morty,Smith\n' + const units = await collectUnits(await parser.parse(streamFrom(csv), {})) + + expect(units).toHaveLength(2) + expect(units[0].value).toEqual({ id: '1', name: 'Rick', name_1: 'Sanchez' }) + expect(units[1].value).toEqual({ id: '2', name: 'Morty', name_1: 'Smith' }) + }) + + it('handles quoted multiline fields without splitting records', async () => { + const csv = 'id,quote\n1,"line one\nline two"\n2,plain\n' + const units = await collectUnits(await parser.parse(streamFrom(csv), {})) + + expect(units).toHaveLength(2) + expect(units[0].value.quote).toBe('line one\nline two') + expect(units[1].value.id).toBe('2') + }) + + it('reports columns on inspect without consuming the source', async () => { + const inspection = await parser.inspect(Buffer.from('a,b\n1,2\n', 'utf8')) + expect(inspection.format).toBe('csv') + expect(inspection.columns).toEqual(['a', 'b']) + }) +}) + +describe('JsonLinesImportParser', () => { + const parser = new JsonLinesImportParser() + + it('parses each non-empty line as an object with stable ordinals and lines', async () => { + const jsonl = '{"id":1}\n\n{"id":2}\n{"id":3}' + const units = await collectUnits(await parser.parse(streamFrom(jsonl), {})) + + expect(units.map((u) => u.value.id)).toEqual([1, 2, 3]) + expect(units.map((u) => u.ordinal)).toEqual([0, 1, 2]) + expect(units[2].location.line).toBe(4) + }) + + it('rejects non-object lines', async () => { + await expect(collectUnits(await parser.parse(streamFrom('[1,2]\n'), {}))).rejects.toThrow(/JSON object/) + }) +}) + +describe('MemoryImportStorageAdapter', () => { + const adapter = new MemoryImportStorageAdapter() + + it('stores, completes, streams and deletes objects', async () => { + const initiated = await adapter.initiate('p1', 'f1', 1) + await adapter.writeChunk(initiated.storageKey, Buffer.from('hello ')) + await adapter.writeChunk(initiated.storageKey, Buffer.from('world')) + const meta = await adapter.complete(initiated.storageKey) + + expect(meta.sizeBytes).toBe(11) + + const chunks: Buffer[] = [] + for await (const chunk of (await adapter.readStream(initiated.storageKey)) as AsyncIterable) { + chunks.push(chunk) + } + expect(Buffer.concat(chunks).toString()).toBe('hello world') + + await adapter.delete(initiated.storageKey) + await expect(adapter.head(initiated.storageKey)).resolves.toBeNull() + }) +}) diff --git a/platform/core/src/core/import-runs/import-runs.module.ts b/platform/core/src/core/import-runs/import-runs.module.ts new file mode 100644 index 00000000..292763c4 --- /dev/null +++ b/platform/core/src/core/import-runs/import-runs.module.ts @@ -0,0 +1,33 @@ +import { Module } from '@nestjs/common' + +import { EntityModule } from '@/core/entity/entity.module' +import { ProjectModule } from '@/dashboard/project/project.module' +import { WorkspaceModule } from '@/dashboard/workspace/workspace.module' + +import { ImportRunsController } from './api/import-runs.controller' +import { ImportRunsService } from './import-runs.service' +import { ImportRunsRepository } from './persistence/import-runs.repository' +import { importStorageProvider, ImportStorageFactory } from './storage/import-storage.factory' +import { ImportCleanupScheduler } from './worker/import-cleanup.scheduler' +import { ImportWorkerService } from './worker/import-worker.service' + +/** + * Asynchronous multi-file import runs. + * + * Storage backend selection: RUSHDB_IMPORT_STORAGE_BACKEND=s3 (default) or + * local; both implement ImportStoragePort and share retention cleanup. + */ +@Module({ + imports: [EntityModule, ProjectModule, WorkspaceModule], + providers: [ + ImportRunsRepository, + ImportRunsService, + ImportStorageFactory, + importStorageProvider, + ImportWorkerService, + ImportCleanupScheduler + ], + controllers: [ImportRunsController], + exports: [ImportRunsService] +}) +export class ImportRunsModule {} diff --git a/platform/core/src/core/import-runs/import-runs.service.ts b/platform/core/src/core/import-runs/import-runs.service.ts new file mode 100644 index 00000000..45292e8b --- /dev/null +++ b/platform/core/src/core/import-runs/import-runs.service.ts @@ -0,0 +1,417 @@ +import { Inject, Injectable, Logger } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' + +import { + IMPORT_ERROR_CODES, + type ImportFileManifestItem, + type ImportLinkSpec +} from './domain/import-run.types' +import { detectFormatFromFileName, suggestLabelFromFileName } from './domain/label-suggestion' +import { ImportValidationError, validateLinkSpec } from './domain/link-spec.validation' +import { hashIdempotencyKey } from './domain/replay-identity' +import { ImportRunsRepository } from './persistence/import-runs.repository' +import { IMPORT_STORAGE_PORT, type ImportStoragePort } from './storage/import-storage.port' + +export interface CreateImportRunInput { + projectId: string + workspaceId?: string + name?: string + failurePolicy?: 'continue' | 'stop_new_files' + files: ImportFileManifestItem[] +} + +@Injectable() +export class ImportRunsService { + constructor( + private readonly repository: ImportRunsRepository, + @Inject(IMPORT_STORAGE_PORT) private readonly storage: ImportStoragePort, + private readonly configService: ConfigService + ) {} + + get enabled(): boolean { + return this.configService.get('RUSHDB_IMPORTS_ENABLED', 'true') !== 'false' + } + + async createDraft( + input: CreateImportRunInput, + idempotencyKeyHash?: string + ): Promise<{ + runId: string + files: Array<{ fileId: string; clientFileId: string; suggestedLabel: string | null }> + }> { + if (!this.enabled) { + throw new ImportValidationError(IMPORT_ERROR_CODES.FORMAT_UNSUPPORTED, 'async imports are disabled') + } + + if (!Array.isArray(input.files) || input.files.length === 0) { + throw new ImportValidationError(IMPORT_ERROR_CODES.FILE_LIMIT_EXCEEDED, 'at least one file is required') + } + + for (const item of input.files) { + this.validateManifestItem(item) + } + + if (idempotencyKeyHash) { + const existing = await this.repository.getRunByIdempotencyKey(input.projectId, idempotencyKeyHash) + if (existing && existing.status === 'draft') { + const files = await this.repository.listFiles(existing.id) + return { + runId: existing.id, + files: files.map((f) => ({ + fileId: f.id, + clientFileId: f.clientFileId, + suggestedLabel: f.rootLabel + })) + } + } + } + + const run = await this.repository.createRun({ + projectId: input.projectId, + workspaceId: input.workspaceId ?? null, + name: input.name ?? null, + status: 'draft', + failurePolicy: input.failurePolicy ?? 'continue', + manifestVersion: 0, + idempotencyKeyHash: idempotencyKeyHash ?? null, + totalFiles: input.files.length, + totalBytes: input.files.reduce((acc, f) => acc + (f.size || 0), 0), + createdByType: 'user' + }) + + const created = await this.repository.createFiles( + input.files.map((item, ordinal) => ({ + runId: run.id, + projectId: input.projectId, + workspaceId: input.workspaceId ?? null, + ordinal, + clientFileId: item.clientFileId, + fileName: item.fileName.slice(0, 255), + declaredSizeBytes: Math.max(0, Math.floor(item.size || 0)), + format: item.format, + role: item.role, + rootLabel: item.role === 'records' ? item.rootLabel?.toUpperCase() : null, + linkSpec: item.role === 'links' ? JSON.stringify(item.linkSpec) : null, + parseOptions: item.parseOptions ? JSON.stringify(item.parseOptions) : null, + importOptions: item.importOptions ? JSON.stringify(item.importOptions) : null, + status: 'awaiting_upload', + stage: 'upload' + })) + ) + + return { + runId: run.id, + files: created.map((f) => ({ + fileId: f.id, + clientFileId: f.clientFileId, + suggestedLabel: f.rootLabel + })) + } + } + + /** Uploads source bytes into the storage port and marks the file queued. */ + async uploadFileContent( + projectId: string, + runId: string, + fileId: string, + content: Buffer + ): Promise<{ status: string; storageKey: string }> { + const file = await this.getDraftOrUploadableFile(runId, fileId, projectId) + + if (file.objectSizeBytes && file.storageKey) { + throw new ImportValidationError(IMPORT_ERROR_CODES.UPLOAD_EXPIRED, 'file already uploaded') + } + + const declared = file.declaredSizeBytes ?? 0 + if (declared > 0 && content.byteLength !== declared) { + throw new ImportValidationError(IMPORT_ERROR_CODES.CHECKSUM_MISMATCH, 'size mismatch') + } + + const initiated = await this.storage.initiate(projectId, fileId, file.sourceGeneration) + await this.storage.writeChunk(initiated.storageKey, content) + const meta = await this.storage.complete(initiated.storageKey, content.byteLength) + + await this.repository.updateFile(fileId, { + storageProvider: 'memory', + storageKey: meta.storageKey, + objectSizeBytes: meta.sizeBytes, + status: 'queued', + stage: 'validate' + }) + + await this.repository.updateRun(runId, { status: 'uploading' }) + await this.repository.addEvent({ + runId, + fileId, + projectId, + type: 'FILE_UPLOADED', + toStatus: 'queued', + metadata: JSON.stringify({ sizeBytes: meta.sizeBytes }) + }) + + return { status: 'queued', storageKey: meta.storageKey } + } + + /** + * Prepares a browser-direct multipart upload session. Returns whether the + * active backend supports presigned parts; clients fall back to proxied + * content upload when it does not. + */ + async initiateUpload(projectId: string, runId: string, fileId: string) { + const file = await this.getDraftOrUploadableFile(runId, fileId, projectId) + + if (file.objectSizeBytes && file.storageKey) { + throw new ImportValidationError(IMPORT_ERROR_CODES.UPLOAD_EXPIRED, 'file already uploaded') + } + + const initiated = await this.storage.initiate(projectId, fileId, file.sourceGeneration) + await this.repository.updateFile(fileId, { + storageProvider: 's3', + storageKey: initiated.storageKey, + storageUploadId: initiated.uploadId, + status: 'uploading' + }) + + return { + uploadId: initiated.uploadId, + storageKey: initiated.storageKey, + directUpload: initiated.directUpload + } + } + + /** Signs one multipart part for browser-direct PUT. Null when unsupported. */ + async signPart(projectId: string, runId: string, fileId: string, uploadId: string, partNumber: number) { + const file = await this.getDraftOrUploadableFile(runId, fileId, projectId) + + if (!file.storageKey || file.storageUploadId !== uploadId) { + throw new ImportValidationError(IMPORT_ERROR_CODES.UPLOAD_EXPIRED, 'unknown or stale upload session') + } + + if (!this.storage.signPart) { + return null + } + + return this.storage.signPart(file.storageKey, Math.max(1, Math.min(10_000, Math.floor(partNumber)))) + } + + /** Completes a browser-direct multipart upload and queues the file. */ + async completeUpload( + projectId: string, + runId: string, + fileId: string, + uploadId: string, + expectedSizeBytes?: number + ) { + const file = await this.getDraftOrUploadableFile(runId, fileId, projectId) + + if (!file.storageKey || file.storageUploadId !== uploadId) { + throw new ImportValidationError(IMPORT_ERROR_CODES.UPLOAD_EXPIRED, 'unknown or stale upload session') + } + + const meta = await this.storage.complete(file.storageKey, expectedSizeBytes) + + await this.repository.updateFile(fileId, { + objectSizeBytes: meta.sizeBytes, + status: 'queued', + stage: 'validate' + }) + await this.repository.updateRun(runId, { status: 'uploading' }) + await this.repository.addEvent({ + runId, + fileId, + projectId, + type: 'FILE_UPLOADED', + toStatus: 'queued', + metadata: JSON.stringify({ sizeBytes: meta.sizeBytes, mode: 'direct' }) + }) + + return { status: 'queued' } + } + + /** Aborts an in-flight browser-direct upload session. */ + async abortUpload(projectId: string, runId: string, fileId: string, uploadId: string) { + const file = await this.getDraftOrUploadableFile(runId, fileId, projectId) + + if (file.storageKey && file.storageUploadId === uploadId && !file.objectSizeBytes) { + await this.storage.abort(file.storageKey) + await this.repository.updateFile(fileId, { + storageKey: null, + storageUploadId: null, + status: 'awaiting_upload' + }) + } + } + + async startRun(runId: string, projectId: string): Promise { + const run = await this.repository.getRun(runId, projectId) + if (!run) { + throw new ImportValidationError(IMPORT_ERROR_CODES.SOURCE_MISSING, 'run not found') + } + if (run.status !== 'draft') { + throw new ImportValidationError( + IMPORT_ERROR_CODES.MANIFEST_CONFLICT, + `cannot start run in status ${run.status}` + ) + } + + const files = await this.repository.listFiles(runId) + const uploaded = files.filter((f) => f.objectSizeBytes !== null && f.objectSizeBytes !== undefined) + if (uploaded.length !== files.length) { + throw new ImportValidationError(IMPORT_ERROR_CODES.SOURCE_MISSING, 'not all files are uploaded') + } + + await this.repository.updateRun(runId, { + status: 'queued', + startedAt: new Date().toISOString(), + manifestVersion: run.manifestVersion + 1 + }) + + await this.repository.addEvent({ + runId, + fileId: null, + projectId, + type: 'RUN_STARTED', + toStatus: 'queued' + }) + } + + async cancelRun(runId: string, projectId: string): Promise { + const run = await this.repository.getRun(runId, projectId) + if (!run) { + throw new ImportValidationError(IMPORT_ERROR_CODES.SOURCE_MISSING, 'run not found') + } + await this.repository.requestCancel(runId, projectId) + await this.maybeFinalizeCanceledRun(runId) + } + + async retryRun(runId: string, projectId: string): Promise { + const run = await this.repository.getRun(runId, projectId) + if (!run) { + throw new ImportValidationError(IMPORT_ERROR_CODES.SOURCE_MISSING, 'run not found') + } + const files = await this.repository.listFiles(runId) + const retryable = files.filter((f) => f.status === 'failed') + + for (const file of retryable) { + await this.repository.updateFile(file.id, { + status: 'queued', + notBefore: null, + lastErrorCode: null, + lastErrorMessage: null + }) + } + + if (retryable.length > 0) { + await this.repository.updateRun(runId, { status: 'queued', finalizedAt: null }) + } + } + + async getRunDetail(runId: string, projectId: string) { + const run = await this.repository.getRun(runId, projectId) + if (!run) { + return null + } + const [files, events] = await Promise.all([ + this.repository.listFiles(runId), + this.repository.listEvents(runId, projectId) + ]) + return { ...run, files, events } + } + + async listRuns(projectId: string) { + return this.repository.listRuns(projectId) + } + + async deleteDraftRun(runId: string, projectId: string): Promise { + const run = await this.repository.getRun(runId, projectId) + if (!run) { + throw new ImportValidationError(IMPORT_ERROR_CODES.SOURCE_MISSING, 'run not found') + } + if (run.status !== 'draft') { + throw new ImportValidationError(IMPORT_ERROR_CODES.MANIFEST_CONFLICT, 'only draft runs can be deleted') + } + const files = await this.repository.listFiles(runId) + for (const file of files) { + if (file.storageKey) { + await this.storage.delete(file.storageKey).catch(() => undefined) + } + } + await this.repository.deleteDraftRun(runId, projectId) + } + + hashKey(key: string): string { + return hashIdempotencyKey(key) + } + + // ------------------------------------------------------------------ + + private async getDraftOrUploadableFile(runId: string, fileId: string, projectId: string) { + const run = await this.repository.getRun(runId, projectId) + if (!run) { + throw new ImportValidationError(IMPORT_ERROR_CODES.SOURCE_MISSING, 'run not found') + } + const file = await this.repository.getFile(fileId, projectId) + if (!file || file.runId !== runId) { + throw new ImportValidationError(IMPORT_ERROR_CODES.SOURCE_MISSING, 'file not found') + } + return file + } + + private validateManifestItem(item: ImportFileManifestItem): void { + if (!item.clientFileId || !item.fileName) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.MANIFEST_CONFLICT, + 'clientFileId and fileName are required' + ) + } + + const supported = ['csv', 'jsonl', 'ndjson', 'json', 'parquet'] + if (!item.format || !supported.includes(item.format)) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.FORMAT_UNSUPPORTED, + `unsupported format for ${item.fileName}; supported: ${supported.join(', ')}` + ) + } + + if (item.role === 'links') { + try { + validateLinkSpec(item.linkSpec) + } catch (error) { + if (error instanceof ImportValidationError) { + throw error + } + throw new ImportValidationError(IMPORT_ERROR_CODES.LINK_SPEC_INVALID, (error as Error).message) + } + if (item.rootLabel) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LABEL_INVALID, + 'links files must not carry a rootLabel' + ) + } + } else { + if (!item.rootLabel) { + item.rootLabel = suggestLabelFromFileName(item.fileName) + } + if (!/^[A-Za-z][A-Za-z0-9_]{0,99}$/.test(item.rootLabel)) { + throw new ImportValidationError( + IMPORT_ERROR_CODES.LABEL_INVALID, + `invalid root label: ${item.rootLabel}` + ) + } + } + } + + private async maybeFinalizeCanceledRun(runId: string): Promise { + const files = await this.repository.listFiles(runId) + const unfinished = files.filter((f) => ['validating', 'running'].includes(f.status)) + if (unfinished.length > 0) { + return + } + await this.repository.refreshRunAggregates(runId) + const canceledAll = files.every((f) => f.status === 'canceled') + await this.repository.updateRun(runId, { + status: canceledAll ? 'canceled' : 'completed_with_errors', + finalizedAt: new Date().toISOString() + }) + } +} diff --git a/platform/core/src/core/import-runs/parser/__fixtures__/characters.plain.parquet b/platform/core/src/core/import-runs/parser/__fixtures__/characters.plain.parquet new file mode 100644 index 00000000..fcafe415 Binary files /dev/null and b/platform/core/src/core/import-runs/parser/__fixtures__/characters.plain.parquet differ diff --git a/platform/core/src/core/import-runs/parser/__fixtures__/characters.snappy.parquet b/platform/core/src/core/import-runs/parser/__fixtures__/characters.snappy.parquet new file mode 100644 index 00000000..7253c96e Binary files /dev/null and b/platform/core/src/core/import-runs/parser/__fixtures__/characters.snappy.parquet differ diff --git a/platform/core/src/core/import-runs/parser/csv.parser.ts b/platform/core/src/core/import-runs/parser/csv.parser.ts new file mode 100644 index 00000000..163b20ab --- /dev/null +++ b/platform/core/src/core/import-runs/parser/csv.parser.ts @@ -0,0 +1,184 @@ +import { parse as papaParse } from 'papaparse' + +import { StringDecoder } from 'node:string_decoder' + +import type { ImportParser, ImportParserInspection, ImportUnit, ParseContext } from './import-parser.port' + +/** + * Streaming CSV parser with bounded memory: source bytes are consumed through a + * quote-aware incremental record splitter, so quoted multiline fields survive + * chunk boundaries while nothing accumulates beyond one logical record. + * + * Headers are sanitized deterministically (BOM stripped, null bytes removed, + * empties -> column_N, duplicates suffixed). Values stay strings; dynamic typing + * belongs to the normalization stage. + */ +export class CsvImportParser implements ImportParser { + public readonly format = 'csv' as const + + async inspect(input: Buffer): Promise { + const text = stripBom(input.subarray(0, Math.min(input.length, 64 * 1024)).toString('utf8')) + const result = papaParse(text, { header: false, skipEmptyLines: 'greedy' }) + const header = result.data[0] + return { + format: 'csv', + columns: Array.isArray(header) ? header.map((c) => String(c ?? '').trim()) : undefined + } + } + + parse(input: NodeJS.ReadableStream, context: ParseContext): Promise> { + return Promise.resolve(this.iterate(input, context)) + } + + private async *iterate(input: NodeJS.ReadableStream, context: ParseContext): AsyncIterable { + const splitter = new CsvRecordSplitter() + let ordinal = 0 + let lineNumber = 0 + let headerSanitized: string[] | null = null + const decoder = new StringDecoder('utf8') + const decodeChunk = (raw: unknown): string => + typeof raw === 'string' ? raw : decoder.write(Buffer.from(raw as Buffer)) + + const flushDecoder = (): string => decoder.end() + + const processRecord = function* (rawRecord: string): Generator { + if (rawRecord.trim().length === 0) { + return + } + lineNumber += 1 + + const parsed = papaParse(rawRecord, { header: false }) + const fields = parsed.data[0] + if (!Array.isArray(fields)) { + return + } + + if (!headerSanitized) { + headerSanitized = sanitizeHeader(fields) + return + } + + const value: Record = {} + for (let i = 0; i < headerSanitized.length; i++) { + value[headerSanitized[i]] = fields[i] + } + yield { ordinal, location: { line: lineNumber }, value } + } + + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + for await (const rawChunk of input as AsyncIterable) { + if (context.signal?.aborted) { + return + } + + const chunk = decodeChunk(rawChunk) + const records = splitter.push(chunk) + + for (const record of records) { + for (const unit of processRecord(record)) { + yield unit + ordinal += 1 + if (context.maxUnits && ordinal >= context.maxUnits) { + return + } + } + } + } + + const tail = splitter.finish() + flushDecoder() + for (const unit of processRecord(tail)) { + yield unit + ordinal += 1 + if (context.maxUnits && ordinal >= context.maxUnits) { + return + } + } + } finally { + cleanupStream(input) + } + } +} + +function cleanupStream(input: NodeJS.ReadableStream): void { + const destroyable = input as unknown as { destroy?: () => void } + destroyable.destroy?.() +} + +function stripBom(text: string): string { + return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text +} + +function sanitizeHeader(raw: string[]): string[] { + const seen = new Map() + return raw.map((name, index) => { + let cleaned = String(name ?? '') + .replace(/\0/g, '') + .trim() + if (cleaned.length === 0) { + cleaned = `column_${index}` + } + const count = seen.get(cleaned) ?? 0 + seen.set(cleaned, count + 1) + return count > 0 ? `${cleaned}_${count}` : cleaned + }) +} + +/** + * Incremental, quote-aware CSV record splitter. Feed it arbitrary text chunks; + * it returns every complete logical record. Handles: + * - CRLF/LF row separators; + * - double-quoted fields containing delimiters, newlines and escaped quotes. + */ +export class CsvRecordSplitter { + private buffer = '' + private inQuotes = false + + push(chunk: string): string[] { + this.buffer += chunk + const records: string[] = [] + let recordStart = 0 + let index = 0 + + while (index < this.buffer.length) { + const char = this.buffer[index] + + if (this.inQuotes) { + if (char === '"') { + if (this.buffer[index + 1] === '"') { + index += 2 + continue + } + this.inQuotes = false + } + index += 1 + continue + } + + if (char === '"') { + this.inQuotes = true + index += 1 + continue + } + + if (char === '\n') { + records.push(this.buffer.slice(recordStart, index).replace(/\r$/, '')) + index += 1 + recordStart = index + continue + } + + index += 1 + } + + this.buffer = this.buffer.slice(recordStart) + return records + } + + finish(): string { + const rest = this.buffer + this.buffer = '' + this.inQuotes = false + return rest + } +} diff --git a/platform/core/src/core/import-runs/parser/import-parser.port.ts b/platform/core/src/core/import-runs/parser/import-parser.port.ts new file mode 100644 index 00000000..da15bd77 --- /dev/null +++ b/platform/core/src/core/import-runs/parser/import-parser.port.ts @@ -0,0 +1,38 @@ +export interface ImportUnitLocation { + line?: number + byteOffsetHint?: number +} + +export interface ImportUnit { + ordinal: number + location: ImportUnitLocation + value: Record +} + +export interface ParseContext { + /** Stop after this many units; the caller flushes batches between pulls. */ + maxUnits?: number + maxUnitBytes?: number + /** Total source byte budget for formats that must materialize logical values. */ + maxSourceBytes?: number + signal?: AbortSignal +} + +import type { ImportFileFormat } from '../domain/import-run.types' + +export interface ImportParserInspection { + columns?: string[] + jsonShape?: 'array' | 'single_object' + format: ImportFileFormat +} + +export interface ImportParser { + inspect(input: Buffer): Promise + /** + * Streams logical root units from source bytes. Implementations must honor + * backpressure and AbortSignal and never buffer the full source. + */ + parse(input: NodeJS.ReadableStream, context: ParseContext): Promise> +} + +export const IMPORT_PARSER_PORT = Symbol('IMPORT_PARSER_PORT') diff --git a/platform/core/src/core/import-runs/parser/json-lines.parser.ts b/platform/core/src/core/import-runs/parser/json-lines.parser.ts new file mode 100644 index 00000000..85ffb7bb --- /dev/null +++ b/platform/core/src/core/import-runs/parser/json-lines.parser.ts @@ -0,0 +1,76 @@ +import { createInterface } from 'node:readline' + +import type { ImportParser, ImportParserInspection, ImportUnit, ParseContext } from './import-parser.port' + +/** + * Streaming JSONL/NDJSON parser: one object per non-empty line, BOM accepted, + * bounded line size, stable source lines. NDJSON is an alias of JSONL. + */ +export class JsonLinesImportParser implements ImportParser { + public readonly format = 'jsonl' as const + + async inspect(input: Buffer): Promise { + return { format: 'jsonl' } + } + + parse(input: NodeJS.ReadableStream, context: ParseContext): Promise> { + return Promise.resolve(this.iterate(input, context)) + } + + private async *iterate(input: NodeJS.ReadableStream, context: ParseContext): AsyncIterable { + const maxLineBytes = context.maxUnitBytes ?? 8 * 1024 * 1024 + const rl = createInterface({ input: input as NodeJS.ReadableStream, crlfDelay: Infinity }) + + let ordinal = 0 + let lineNumber = 0 + let sawFirstLine = false + + try { + for await (const rawLine of rl) { + if (context.signal?.aborted) { + return + } + + let line = rawLine + if (!sawFirstLine && line.charCodeAt(0) === 0xfeff) { + line = line.slice(1) + } + lineNumber += 1 + + if (line.trim().length === 0) { + continue + } + sawFirstLine = true + + if (Buffer.byteLength(line, 'utf8') > maxLineBytes) { + throw new Error(`JSONL line ${lineNumber} exceeds the configured maximum logical unit size`) + } + + let parsed: unknown + try { + parsed = JSON.parse(line) + } catch (error) { + throw new Error(`invalid JSON on line ${lineNumber}: ${(error as Error).message}`) + } + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`line ${lineNumber} must be a JSON object`) + } + + yield { ordinal: ordinal++, location: { line: lineNumber }, value: parsed as Record } + + if (context.maxUnits && ordinal >= context.maxUnits) { + return + } + } + } finally { + rl.close() + cleanupStream(input) + } + } +} + +function cleanupStream(input: NodeJS.ReadableStream): void { + const destroyable = input as unknown as { destroy?: () => void } + destroyable.destroy?.() +} diff --git a/platform/core/src/core/import-runs/parser/json-object.parser.ts b/platform/core/src/core/import-runs/parser/json-object.parser.ts new file mode 100644 index 00000000..d941f70e --- /dev/null +++ b/platform/core/src/core/import-runs/parser/json-object.parser.ts @@ -0,0 +1,196 @@ +import { chain } from 'stream-chain' +// stream-json ships CJS modules without esModuleInterop support in this package +// eslint-disable-next-line @typescript-eslint/no-require-imports +import Parser = require('stream-json') +// eslint-disable-next-line @typescript-eslint/no-require-imports +import StreamArray = require('stream-json/streamers/StreamArray') +// eslint-disable-next-line @typescript-eslint/no-require-imports +import StreamValues = require('stream-json/streamers/StreamValues') + +import { PassThrough } from 'node:stream' + +import type { ImportParser, ImportParserInspection, ImportUnit, ParseContext } from './import-parser.port' +import type { ImportFileFormat } from '../domain/import-run.types' + +/** + * Top-level JSON parser built on stream-json. + * + * - A top-level array is streamed element by element (bounded memory regardless + * of file size). + * - A single top-level object (or any other single root value) is emitted as one + * logical unit; the whole value must fit the configured byte budget, otherwise + * parsing fails with an honest shape error instead of exhausting memory. + * + * The shape is probed from the first non-whitespace byte; the remainder of the + * source streams through untouched. + */ +export class JsonObjectImportParser implements ImportParser { + public readonly format: ImportFileFormat = 'json' + + async inspect(input: Buffer): Promise { + const text = stripBom(input.subarray(0, Math.min(input.length, 64 * 1024)).toString('utf8')).trimStart() + return { format: 'json', jsonShape: text.startsWith('[') ? 'array' : 'single_object' } + } + + parse(input: NodeJS.ReadableStream, context: ParseContext): Promise> { + return Promise.resolve(this.iterate(input, context)) + } + + private async *iterate(input: NodeJS.ReadableStream, context: ParseContext): AsyncIterable { + const maxTotalBytes = context.maxSourceBytes ?? Number.MAX_SAFE_INTEGER + + const gate = new PassThrough() + let firstChunk = await readFirstChunk(input) + if (context.signal?.aborted) { + cleanupStream(input) + return + } + if (firstChunk === null) { + cleanupStream(input) + throw new Error('IMPORT_PARSE_ERROR: empty JSON source') + } + + const shapeChar = firstNonWhitespaceChar(firstChunk) + if (!shapeChar) { + // whitespace-only leading chunk; keep scanning without wiring yet + firstChunk = null + } + + const isArray = shapeChar === '[' + const disassembler = isArray ? StreamArray.streamArray() : StreamValues.streamValues() + const pipeline = chain([gate, Parser(), disassembler]) + + const pumpFailure = new Promise((_resolve, reject) => { + pipeline.once('error', (error: Error) => reject(error)) + input.once('error', (error: Error) => reject(error)) + }) + + const pump = (async () => { + try { + if (firstChunk !== null) { + if (!gate.write(firstChunk)) { + await onceDrain(gate) + } + } + for await (const chunk of input as AsyncIterable) { + if (context.signal?.aborted) { + break + } + if (!gate.write(Buffer.from(chunk))) { + await onceDrain(gate) + } + } + gate.end() + } catch { + gate.destroy() + } + })() + + try { + let ordinal = 0 + let totalBytes = 0 + + const downstream = (async function* () { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + for await (const entry of pipeline as any) { + const value = entry?.value + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`IMPORT_PARSE_ERROR: JSON element ${ordinal} must be an object`) + } + + totalBytes += JSON.stringify(value)?.length ?? 0 + if (totalBytes > maxTotalBytes) { + throw new Error( + `IMPORT_JSON_SHAPE_NOT_STREAMABLE: JSON logical value exceeds the configured ${maxTotalBytes} byte budget` + ) + } + + yield { ordinal, location: {}, value: value as Record } + ordinal += 1 + + if (context.maxUnits && ordinal >= context.maxUnits) { + return + } + } + })() + + yield* raceIterator(downstream, pumpFailure) + await pump + } finally { + pipeline.destroy() + gate.destroy() + cleanupStream(input) + } + } +} + +function readFirstChunk(input: NodeJS.ReadableStream): Promise { + return new Promise((resolve) => { + const onData = (chunk: Buffer | string) => { + cleanup() + resolve(typeof chunk === 'string' ? Buffer.from(chunk) : Buffer.from(chunk)) + } + const onEnd = () => { + cleanup() + resolve(null) + } + const onError = () => { + cleanup() + resolve(null) + } + const cleanup = () => { + input.off('data', onData) + input.off('end', onEnd) + input.off('error', onError) + } + input.once('data', onData) + input.once('end', onEnd) + input.once('error', onError) + }) +} + +function firstNonWhitespaceChar(buffer: Buffer): string | null { + const text = stripBom(buffer.toString('utf8')) + for (const char of text) { + if (!/\s/.test(char)) { + return char + } + } + return null +} + +function onceDrain(stream: PassThrough): Promise { + return new Promise((resolve) => stream.once('drain', resolve)) +} + +const FAILURE_SENTINEL = Symbol('json-parse-failure') + +async function* raceIterator( + iterator: AsyncIterable, + failure: Promise +): AsyncGenerator { + const failureTagged = failure.then(() => FAILURE_SENTINEL) + + const iteratorHandle = iterator[Symbol.asyncIterator]() + while (true) { + const result = (await Promise.race([iteratorHandle.next(), failureTagged])) as + | IteratorResult + | typeof FAILURE_SENTINEL + if (result === FAILURE_SENTINEL) { + return + } + if (result.done) { + return + } + yield result.value + } +} + +function cleanupStream(input: NodeJS.ReadableStream): void { + const destroyable = input as unknown as { destroy?: () => void } + destroyable.destroy?.() +} + +function stripBom(text: string): string { + return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text +} diff --git a/platform/core/src/core/import-runs/parser/parquet.parser.ts b/platform/core/src/core/import-runs/parser/parquet.parser.ts new file mode 100644 index 00000000..2a0d4c72 --- /dev/null +++ b/platform/core/src/core/import-runs/parser/parquet.parser.ts @@ -0,0 +1,149 @@ +import type { ImportParser, ImportParserInspection, ImportUnit, ParseContext } from './import-parser.port' + +const PARQUET_MAGIC = 'PAR1' + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type HyparquetModule = any + +/** + * hyparquet is ESM-only while this package compiles to CommonJS. A native + * dynamic import survives both ts-jest and the nest build. + */ +let hyparquetModule: HyparquetModule | null = null + +async function loadHyparquet(): Promise { + if (!hyparquetModule) { + const nativeImport = new Function('specifier', 'return import(specifier)') as ( + specifier: string + ) => Promise + hyparquetModule = await nativeImport('hyparquet') + } + return hyparquetModule +} + +/** + * Parquet reader backed by hyparquet (pure JS; snappy built in). + * + * Columnar formats cannot be streamed row-by-row without full page decode, so + * the source is materialized up to a hard byte cap (RUSHDB_IMPORT_MAX_PARQUET_BYTES, + * default 256 MiB) and rejected beyond it. Row order is stable, which keeps + * replay ordinals — and therefore deterministic record IDs — stable. + */ +export class ParquetImportParser implements ImportParser { + public readonly format = 'parquet' as const + + async inspect(input: Buffer): Promise { + return { format: 'parquet' } + } + + async parse(input: NodeJS.ReadableStream, context: ParseContext): Promise> { + const bytes = await this.readAll(input, context) + assertParquetMagic(bytes) + + const hyparquet = await loadHyparquet() + const asyncBuffer = { + byteLength: bytes.byteLength, + // hyparquet wraps slices in DataView, which requires standalone ArrayBuffers + slice: async (start: number, end?: number) => { + const sub = bytes.subarray(start, end ?? bytes.byteLength) + return sub.buffer.slice(sub.byteOffset, sub.byteOffset + sub.byteLength) + } + } + const rows = (await hyparquet.parquetReadObjects({ file: asyncBuffer })) as Array> + + async function* iterate(): AsyncIterable { + for (let ordinal = 0; ordinal < rows.length; ordinal++) { + if (context.signal?.aborted) { + return + } + const rawValue = rows[ordinal] + if (!rawValue || typeof rawValue !== 'object') { + throw new Error(`IMPORT_PARSE_ERROR: parquet row ${ordinal} is not an object`) + } + // Normalize parquet-native types to JSON-friendly values. + const value = normalizeParquetValue(rawValue) as Record + yield { ordinal, location: {}, value } + if (context.maxUnits && ordinal + 1 >= context.maxUnits) { + return + } + } + } + + return iterate() + } + + private readAll(input: NodeJS.ReadableStream, context: ParseContext): Promise { + const maxBytes = context.maxSourceBytes ?? 256 * 1024 * 1024 + + return new Promise((resolve, reject) => { + const chunks: Buffer[] = [] + let total = 0 + let settled = false + + const settle = (fn: () => void) => { + if (!settled) { + settled = true + fn() + } + } + + input.on('data', (chunk: Buffer) => { + total += chunk.byteLength + if (total > maxBytes) { + cleanup() + reject(new Error(`IMPORT_LOGICAL_UNIT_TOO_LARGE: parquet source exceeds ${maxBytes} bytes`)) + return + } + chunks.push(Buffer.from(chunk)) + }) + + const onEnd = () => { + cleanup() + settle(() => resolve(Buffer.concat(chunks))) + } + + const onError = (error: Error) => { + cleanup() + settle(() => reject(error)) + } + + const cleanup = () => { + input.removeAllListeners('data') + input.off('end', onEnd) + input.off('error', onError) + } + + input.on('end', onEnd) + input.on('error', onError) + }) + } +} + +function normalizeParquetValue(value: unknown): unknown { + if (typeof value === 'bigint') { + return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : value.toString() + } + if (value instanceof Date) { + return value.toISOString() + } + if (value instanceof Uint8Array) { + return Buffer.from(value).toString('base64') + } + if (Array.isArray(value)) { + return value.map(normalizeParquetValue) + } + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, normalizeParquetValue(v)])) + } + return value +} + +function assertParquetMagic(bytes: Buffer): void { + if ( + bytes.length < 8 || + bytes.subarray(0, 4).toString('ascii') !== PARQUET_MAGIC || + bytes.subarray(bytes.length - 4).toString('ascii') !== PARQUET_MAGIC + ) { + throw new Error('IMPORT_FORMAT_UNSUPPORTED: not a valid parquet file (missing PAR1 magic)') + } +} diff --git a/platform/core/src/core/import-runs/persistence/import-runs.repository.ts b/platform/core/src/core/import-runs/persistence/import-runs.repository.ts new file mode 100644 index 00000000..a0ff0d48 --- /dev/null +++ b/platform/core/src/core/import-runs/persistence/import-runs.repository.ts @@ -0,0 +1,468 @@ +import { Injectable } from '@nestjs/common' +import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, lte, ne, or, sql } from 'drizzle-orm' +import { uuidv7 } from 'uuidv7' + +import { SqlService } from '@/database/sql/sql.service' + +import { RECORDS_PHASE_TERMINAL_STATUSES, type ImportFileStatus } from '../domain/import-run.types' + +import type { + ImportErrorSampleRow, + ImportRunEventRow, + ImportRunFileRow, + ImportRunRow, + InsertImportErrorSampleRow, + InsertImportRunEventRow, + InsertImportRunFileRow, + InsertImportRunRow +} from '@/database/sql/schema/types' + +export interface ClaimInput { + projectId: string + workerId: string + now: string + leaseUntil: string +} + +const MAX_ERROR_SAMPLES_PER_FILE = 20 + +@Injectable() +export class ImportRunsRepository { + constructor(private readonly sql: SqlService) {} + + private get db() { + return this.sql.db + } + + private get runs() { + return this.sql.tables.importRuns + } + + private get files() { + return this.sql.tables.importRunFiles + } + + private get events() { + return this.sql.tables.importRunEvents + } + + private get errorSamples() { + return this.sql.tables.importErrorSamples + } + + // ---------- runs ---------- + + async createRun(data: Omit): Promise { + const now = new Date().toISOString() + const row: InsertImportRunRow = { ...data, id: uuidv7(), createdAt: now, updatedAt: now } + await this.db.insert(this.runs).values(row) + return this.getRun(row.id as string) as Promise + } + + async getRun(id: string, projectId?: string): Promise { + const where = + projectId ? and(eq(this.runs.id, id), eq(this.runs.projectId, projectId)) : eq(this.runs.id, id) + const rows = await this.db.select().from(this.runs).where(where) + return rows[0] + } + + async getRunByIdempotencyKey( + projectId: string, + idempotencyKeyHash: string + ): Promise { + const rows = await this.db + .select() + .from(this.runs) + .where(and(eq(this.runs.projectId, projectId), eq(this.runs.idempotencyKeyHash, idempotencyKeyHash))) + return rows[0] + } + + async listRuns(projectId: string, limit = 50): Promise { + return this.db + .select() + .from(this.runs) + .where(eq(this.runs.projectId, projectId)) + .orderBy(desc(this.runs.createdAt)) + .limit(limit) + } + + async updateRun( + id: string, + data: Partial> + ): Promise { + await this.db + .update(this.runs) + .set({ ...data, updatedAt: new Date().toISOString() }) + .where(eq(this.runs.id, id)) + } + + async deleteDraftRun(id: string, projectId: string): Promise { + await this.db.delete(this.runs).where(and(eq(this.runs.id, id), eq(this.runs.projectId, projectId))) + } + + // ---------- files ---------- + + async createFiles( + items: Omit[] + ): Promise { + const now = new Date().toISOString() + const rows: InsertImportRunFileRow[] = items.map((item) => ({ + ...item, + id: uuidv7(), + createdAt: now, + updatedAt: now + })) + await this.db.insert(this.files).values(rows) + return this.db + .select() + .from(this.files) + .where(eq(this.files.runId, rows[0].runId as string)) + } + + async getFile(id: string, projectId: string): Promise { + const rows = await this.db + .select() + .from(this.files) + .where(and(eq(this.files.id, id), eq(this.files.projectId, projectId))) + return rows[0] + } + + async listFiles(runId: string): Promise { + return this.db + .select() + .from(this.files) + .where(eq(this.files.runId, runId)) + .orderBy(asc(this.files.ordinal)) + } + + async updateFile( + id: string, + data: Partial>, + projectId?: string + ): Promise { + const where = + projectId ? and(eq(this.files.id, id), eq(this.files.projectId, projectId)) : eq(this.files.id, id) + await this.db + .update(this.files) + .set({ ...data, updatedAt: new Date().toISOString() }) + .where(where) + const rows = await this.db.select().from(this.files).where(eq(this.files.id, id)) + return rows[0] + } + + /** + * Fenced progress write: only succeeds while the caller still owns the lease + * generation and the file remains in an active state. + */ + async fencedUpdateFile( + id: string, + leaseGeneration: number, + data: Partial> + ): Promise { + const predicate = and( + eq(this.files.id, id), + eq(this.files.leaseGeneration, leaseGeneration), + inArray(this.files.status, ['queued', 'validating', 'running', 'finalizing']) + ) + const result = await this.db + .update(this.files) + .set({ ...data, updatedAt: new Date().toISOString() }) + .where(predicate) + return this.extractRowCount(result) > 0 + } + + async countNonTerminalRecordsSiblings(runId: string, excludeFileId?: string): Promise { + const conditions = [ + eq(this.files.runId, runId), + eq(this.files.role, 'records'), + sql`${this.files.status} NOT IN (${sql.join( + RECORDS_PHASE_TERMINAL_STATUSES.map((status) => sql`${status}`), + sql`, ` + )})` + ] + if (excludeFileId) { + conditions.push(ne(this.files.id, excludeFileId)) + } + const rows = await this.db + .select({ count: sql`count(*)` }) + .from(this.files) + .where(and(...conditions)) + return Number(rows[0]?.count ?? 0) + } + + /** + * Atomic, lease-fenced claim of the oldest eligible file for a project. + * + * PostgreSQL uses UPDATE .. WHERE id = (SELECT .. FOR UPDATE SKIP LOCKED) so two + * replicas can never claim the same row. SQLite runs in enforced single-process + * mode, so a conditional optimistic update provides the same guarantee. + * + * Eligibility includes the links-phase gate: a role=links file is claimable only + * when every role=records sibling of its run has reached a terminal state. + */ + async claimNextFile(input: ClaimInput): Promise { + if (this.sql.isPostgres) { + return this.claimNextFilePostgres(input) + } + return this.claimNextFileSqlite(input) + } + + private terminalStatusesSql() { + return sql.join( + RECORDS_PHASE_TERMINAL_STATUSES.map((s) => sql`${s}`), + sql`, ` + ) + } + + private async claimNextFilePostgres(input: ClaimInput): Promise { + const result = await this.db.execute(sql` + UPDATE import_run_files AS f SET + status = 'validating', + stage = 'validate', + lease_owner = ${input.workerId}, + lease_generation = f.lease_generation + 1, + lease_until = ${input.leaseUntil}, + heartbeat_at = ${input.now}, + attempt_count = f.attempt_count + 1, + started_at = COALESCE(f.started_at, ${input.now}), + updated_at = ${input.now} + WHERE f.id = ( + SELECT c.id FROM import_run_files c + JOIN import_runs r ON r.id = c.run_id + WHERE c.project_id = ${input.projectId} + AND c.status IN ('queued', 'retry_wait') + AND (c.not_before IS NULL OR c.not_before <= ${input.now}) + AND (c.lease_until IS NULL OR c.lease_until < ${input.now}) + AND r.cancel_requested_at IS NULL + AND ( + SELECT COUNT(*) FROM import_run_files active + WHERE active.project_id = ${input.projectId} + AND active.status IN ('validating', 'running') + ) = 0 + AND ( + c.role = 'records' + OR NOT EXISTS ( + SELECT 1 FROM import_run_files pending_records + WHERE pending_records.run_id = c.run_id + AND pending_records.role = 'records' + AND pending_records.status NOT IN (${this.terminalStatusesSql()}) + ) + ) + ORDER BY c.created_at + FOR UPDATE OF c SKIP LOCKED + LIMIT 1 + ) + RETURNING * + `) + + return this.firstRowFromResult(result) + } + + private async claimNextFileSqlite(input: ClaimInput): Promise { + const candidateRows = await this.db + .select({ id: this.files.id }) + .from(this.files) + .innerJoin(this.runs, eq(this.runs.id, this.files.runId)) + .where( + and( + eq(this.files.projectId, input.projectId), + inArray(this.files.status, ['queued', 'retry_wait']), + or(isNull(this.files.notBefore), lte(this.files.notBefore, input.now)), + or(isNull(this.files.leaseUntil), lte(this.files.leaseUntil, input.now)), + isNull(this.runs.cancelRequestedAt), + sql`(SELECT COUNT(*) FROM import_run_files active WHERE active.project_id = ${input.projectId} AND active.status IN ('validating','running')) = 0`, + sql`(${this.files.role} = 'records' OR NOT EXISTS ( + SELECT 1 FROM import_run_files pending_records + WHERE pending_records.run_id = ${this.files.runId} + AND pending_records.role = 'records' + AND pending_records.status NOT IN (${this.terminalStatusesSql()}) + ))` + ) + ) + .orderBy(asc(this.files.createdAt)) + .limit(1) + + const candidateId = candidateRows[0]?.id + if (!candidateId) { + return undefined + } + + const result = await this.db + .update(this.files) + .set({ + status: 'validating', + stage: 'validate', + leaseOwner: input.workerId, + leaseGeneration: sql`${this.files.leaseGeneration} + 1`, + leaseUntil: input.leaseUntil, + heartbeatAt: input.now, + attemptCount: sql`${this.files.attemptCount} + 1`, + startedAt: sql`COALESCE(${this.files.startedAt}, ${input.now})`, + updatedAt: input.now + }) + .where(and(eq(this.files.id, candidateId), inArray(this.files.status, ['queued', 'retry_wait']))) + .returning() + + return result[0] + } + + async requestCancel(runId: string, projectId: string): Promise { + const now = new Date().toISOString() + await this.db + .update(this.runs) + .set({ cancelRequestedAt: now, updatedAt: now }) + .where(and(eq(this.runs.id, runId), eq(this.runs.projectId, projectId))) + + await this.db + .update(this.files) + .set({ cancelRequestedAt: now, status: 'canceled', finishedAt: now, updatedAt: now }) + .where( + and( + eq(this.files.runId, runId), + inArray(this.files.status, [ + 'awaiting_upload', + 'uploading', + 'uploaded', + 'queued', + 'blocked', + 'retry_wait' + ]) + ) + ) + } + + // ---------- events & samples ---------- + + async addEvent(event: Omit): Promise { + const row: InsertImportRunEventRow = { ...event, id: uuidv7(), createdAt: new Date().toISOString() } + await this.db.insert(this.events).values(row) + } + + async listEvents(runId: string, projectId: string, limit = 100): Promise { + return this.db + .select() + .from(this.events) + .where(and(eq(this.events.runId, runId), eq(this.events.projectId, projectId))) + .orderBy(desc(this.events.createdAt)) + .limit(limit) + } + + async addErrorSample(sample: Omit): Promise { + const existing = await this.db + .select({ count: sql`count(*)` }) + .from(this.errorSamples) + .where(eq(this.errorSamples.fileId, sample.fileId)) + + if (Number(existing[0]?.count ?? 0) >= MAX_ERROR_SAMPLES_PER_FILE) { + return + } + + const row: InsertImportErrorSampleRow = { ...sample, id: uuidv7(), createdAt: new Date().toISOString() } + await this.db.insert(this.errorSamples).values(row) + } + + async listErrorSamples(fileId: string, projectId: string, limit = 50): Promise { + return this.db + .select() + .from(this.errorSamples) + .where(and(eq(this.errorSamples.fileId, fileId), eq(this.errorSamples.projectId, projectId))) + .orderBy(desc(this.errorSamples.createdAt)) + .limit(limit) + } + + // ---------- aggregation ---------- + + async refreshRunAggregates(runId: string): Promise { + const run = await this.getRun(runId) + if (!run) { + return undefined + } + + const files = await this.listFiles(runId) + const sum = (pick: (f: ImportRunFileRow) => number): number => + files.reduce((acc, f) => acc + (pick(f) ?? 0), 0) + + await this.updateRun(runId, { + totalFiles: files.length, + totalBytes: sum((f) => f.declaredSizeBytes ?? 0), + uploadedBytes: sum((f) => f.objectSizeBytes ?? 0), + parsedUnits: sum((f) => f.parsedUnits), + recordsCommitted: sum((f) => f.recordsCommitted), + relationshipsCommitted: sum((f) => f.relationshipsCommitted), + skippedUnits: sum((f) => f.skippedUnits), + failedFiles: files.filter((f) => f.status === 'failed').length + }) + + return this.getRun(runId) + } + + async listClaimableProjects(limit = 100): Promise { + const rows = await this.db + .selectDistinct({ projectId: this.files.projectId }) + .from(this.files) + .innerJoin(this.runs, eq(this.runs.id, this.files.runId)) + .where(and(inArray(this.files.status, ['queued', 'retry_wait']), isNull(this.runs.cancelRequestedAt))) + .limit(limit) + return rows.map((r) => r.projectId) + } + + /** Terminal, finalized runs whose retention window has elapsed. */ + async findRunsPastRetention(cutoffIso: string): Promise { + return this.db + .select() + .from(this.runs) + .where( + and( + inArray(this.runs.status, ['completed', 'completed_with_errors', 'failed', 'canceled']), + isNotNull(this.runs.finalizedAt), + lte(this.runs.finalizedAt, cutoffIso) + ) + ) + .limit(200) + } + + async clearSourceForFile(fileId: string): Promise { + const rows = await this.db + .update(this.files) + .set({ storageKey: null, storageUploadId: null, updatedAt: new Date().toISOString() }) + .where(and(eq(this.files.id, fileId), isNotNull(this.files.storageKey))) + .returning() + return rows.length > 0 + } + + async listFilesWithSource(runId: string): Promise { + return this.db + .select() + .from(this.files) + .where(and(eq(this.files.runId, runId), isNotNull(this.files.storageKey))) + } + + private extractRowCount(result: unknown): number { + if (result && typeof result === 'object') { + const maybe = result as { rowCount?: unknown; changes?: unknown; rows?: unknown[] } + if (typeof maybe.rowCount === 'number') { + return maybe.rowCount + } + if (typeof maybe.changes === 'number') { + return maybe.changes + } + if (Array.isArray(maybe.rows)) { + return maybe.rows.length + } + } + return 0 + } + + private firstRowFromResult(result: unknown): T | undefined { + if (result && typeof result === 'object') { + const maybe = result as { rows?: unknown[] } + if (Array.isArray(maybe.rows)) { + return maybe.rows[0] as T | undefined + } + if (Array.isArray(result)) { + return (result as unknown[])[0] as T | undefined + } + } + return undefined + } +} diff --git a/platform/core/src/core/import-runs/storage/import-storage.factory.ts b/platform/core/src/core/import-runs/storage/import-storage.factory.ts new file mode 100644 index 00000000..921eb4e1 --- /dev/null +++ b/platform/core/src/core/import-runs/storage/import-storage.factory.ts @@ -0,0 +1,55 @@ +import { Injectable } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' + +import { IMPORT_STORAGE_PORT, type ImportStoragePort } from './import-storage.port' +import { LocalImportStorageAdapter } from './local-import-storage.adapter' +import { S3ImportStorageAdapter } from './s3-import-storage.adapter' + +import type { ImportStorageBackend } from '../domain/import-run.types' + +/** + * Selects the import source backend: + * - RUSHDB_IMPORT_STORAGE_BACKEND=s3 (default): S3-compatible multipart storage. + * - RUSHDB_IMPORT_STORAGE_BACKEND=local: server-local filesystem storage. + * + * Both backends share the ImportStoragePort contract; retention/cleanup runs + * identically against either through port.delete(). + */ +@Injectable() +export class ImportStorageFactory { + constructor(private readonly configService: ConfigService) {} + + get backend(): ImportStorageBackend { + const raw = this.configService.get('RUSHDB_IMPORT_STORAGE_BACKEND', 's3').toLowerCase() + if (raw !== 's3' && raw !== 'local') { + throw new Error(`RUSHDB_IMPORT_STORAGE_BACKEND must be "s3" or "local", got: ${raw}`) + } + return raw + } + + create(): ImportStoragePort { + return this.backend === 'local' ? this.createLocal() : this.createS3() + } + + private createLocal(): ImportStoragePort { + const rootPath = this.configService.get('RUSHDB_IMPORT_STORAGE_LOCAL_PATH') + return new LocalImportStorageAdapter(rootPath) + } + + private createS3(): ImportStoragePort { + return new S3ImportStorageAdapter({ + bucket: this.configService.get('RUSHDB_IMPORT_S3_BUCKET'), + region: this.configService.get('RUSHDB_IMPORT_S3_REGION'), + endpoint: this.configService.get('RUSHDB_IMPORT_S3_ENDPOINT'), + accessKeyId: this.configService.get('RUSHDB_IMPORT_S3_ACCESS_KEY_ID'), + secretAccessKey: this.configService.get('RUSHDB_IMPORT_S3_SECRET_ACCESS_KEY'), + forcePathStyle: this.configService.get('RUSHDB_IMPORT_S3_FORCE_PATH_STYLE') === 'true' + }) + } +} + +export const importStorageProvider = { + provide: IMPORT_STORAGE_PORT, + useFactory: (factory: ImportStorageFactory) => factory.create(), + inject: [ImportStorageFactory] +} diff --git a/platform/core/src/core/import-runs/storage/import-storage.port.ts b/platform/core/src/core/import-runs/storage/import-storage.port.ts new file mode 100644 index 00000000..51e7c5bd --- /dev/null +++ b/platform/core/src/core/import-runs/storage/import-storage.port.ts @@ -0,0 +1,42 @@ +export interface InitiateUploadResult { + uploadId: string + storageKey: string + /** Present when the backend supports browser-direct multipart (S3). */ + directUpload: boolean +} + +export interface StorageObjectMeta { + storageKey: string + sizeBytes: number +} + +export interface SignedPartRequest { + url: string + method: 'PUT' + headers: Record + expiresInSeconds: number +} + +/** + * Port over import source bytes. Two adapters implement it: + * - S3-compatible multipart storage (default; supports presigned browser-direct uploads) + * - Local filesystem storage + * Retention/cleanup runs identically against either through delete(). + */ +export interface ImportStoragePort { + initiate(projectId: string, fileId: string, sourceGeneration: number): Promise + writeChunk(storageKey: string, data: Buffer): Promise + complete(storageKey: string, expectedSizeBytes?: number): Promise + head(storageKey: string): Promise + readStream(storageKey: string): Promise + abort(storageKey: string): Promise + delete(storageKey: string): Promise + + /** + * Signs a single multipart part for browser-direct upload. Backends without + * presigning capability return null and callers fall back to proxied upload. + */ + signPart?(storageKey: string, partNumber: number): Promise +} + +export const IMPORT_STORAGE_PORT = Symbol('IMPORT_STORAGE_PORT') diff --git a/platform/core/src/core/import-runs/storage/local-import-storage.adapter.ts b/platform/core/src/core/import-runs/storage/local-import-storage.adapter.ts new file mode 100644 index 00000000..2a25cfc4 --- /dev/null +++ b/platform/core/src/core/import-runs/storage/local-import-storage.adapter.ts @@ -0,0 +1,117 @@ +import { Injectable } from '@nestjs/common' + +import { createReadStream, createWriteStream, existsSync, mkdirSync, statSync } from 'node:fs' +import { mkdir, rm } from 'node:fs/promises' +import { dirname, join, normalize, resolve, sep } from 'node:path' + +import type { ImportStoragePort, InitiateUploadResult, StorageObjectMeta } from './import-storage.port' + +/** + * Filesystem storage backend for import sources (RUSHDB_IMPORT_STORAGE_BACKEND=local). + * Objects live under a server-owned root directory; retention/cleanup uses the + * same delete() contract as the S3 backend. + */ +@Injectable() +export class LocalImportStorageAdapter implements ImportStoragePort { + private readonly root: string + + constructor(rootPath?: string) { + this.root = resolve(rootPath ?? process.env.RUSHDB_IMPORT_STORAGE_LOCAL_PATH ?? '.rushdb-imports') + if (!existsSync(this.root)) { + mkdirSync(this.root, { recursive: true }) + } + } + + async initiate(projectId: string, fileId: string, sourceGeneration: number): Promise { + const uploadId = `local-${Date.now()}-${Math.random().toString(36).slice(2, 10)}` + return { + uploadId, + storageKey: `imports/${projectId}/${fileId}/g${sourceGeneration}/${uploadId}`, + directUpload: false + } + } + + async writeChunk(storageKey: string, data: Buffer): Promise { + const path = this.toPath(storageKey) + await mkdir(dirname(path), { recursive: true }) + await new Promise((resolvePromise, reject) => { + const stream = createWriteStream(path, { flags: 'a' }) + stream.on('error', reject) + stream.on('finish', () => resolvePromise()) + stream.end(data) + }) + } + + async complete(storageKey: string, expectedSizeBytes?: number): Promise { + const path = this.toPath(storageKey) + if (!existsSync(path)) { + throw new Error(`object not found: ${storageKey}`) + } + const size = statSync(path).size + if (expectedSizeBytes !== undefined && expectedSizeBytes !== size) { + throw new Error(`size mismatch for ${storageKey}: expected ${expectedSizeBytes}, got ${size}`) + } + return { storageKey, sizeBytes: size } + } + + async head(storageKey: string): Promise { + try { + const path = this.toPath(storageKey) + if (!existsSync(path)) { + return null + } + return { storageKey, sizeBytes: statSync(path).size } + } catch { + return null + } + } + + readStream(storageKey: string): Promise { + const path = this.toPath(storageKey) + if (!existsSync(path)) { + return Promise.reject(new Error(`object not found: ${storageKey}`)) + } + return Promise.resolve(createReadStream(path)) + } + + async abort(storageKey: string): Promise { + await this.removeQuietly(storageKey) + } + + async delete(storageKey: string): Promise { + await this.removeQuietly(storageKey) + } + + /** Retention sweep helper shared by the cleanup scheduler. */ + async deletePrefix(prefix: string): Promise { + const target = join(this.root, normalize(prefix)) + if (!target.startsWith(this.root + sep) && target !== this.root) { + throw new Error('invalid prefix') + } + if (!existsSync(target)) { + return 0 + } + await rm(target, { recursive: true, force: true }) + return 1 + } + + private async removeQuietly(storageKey: string): Promise { + try { + await rm(this.toPath(storageKey), { force: true }) + } catch { + /* deletion failures are retried by the retention sweeper */ + } + } + + private toPath(storageKey: string): string { + if (storageKey.includes('..')) { + throw new Error(`invalid storage key: ${storageKey}`) + } + const normalized = normalize(storageKey) + const path = join(this.root, normalized) + if (!path.startsWith(this.root + sep) && path !== this.root) { + throw new Error(`invalid storage key: ${storageKey}`) + } + return path + } +} diff --git a/platform/core/src/core/import-runs/storage/memory-import-storage.adapter.ts b/platform/core/src/core/import-runs/storage/memory-import-storage.adapter.ts new file mode 100644 index 00000000..f05b4ec9 --- /dev/null +++ b/platform/core/src/core/import-runs/storage/memory-import-storage.adapter.ts @@ -0,0 +1,76 @@ +import { Injectable } from '@nestjs/common' + +import { PassThrough } from 'node:stream' + +import type { ImportStoragePort, InitiateUploadResult, StorageObjectMeta } from './import-storage.port' + +/** + * Process-local storage adapter for local vertical slice and tests. + * Objects do not survive process restarts; production deployments bind an + * S3-compatible adapter to the same port. + */ +@Injectable() +export class MemoryImportStorageAdapter implements ImportStoragePort { + private readonly objects = new Map() + private counter = 0 + + async initiate(projectId: string, fileId: string, sourceGeneration: number): Promise { + const uploadId = `mem-${++this.counter}` + const storageKey = `imports/${projectId}/${fileId}/g${sourceGeneration}/${uploadId}` + this.objects.set(storageKey, { chunks: [], size: 0, completed: false }) + return { uploadId, storageKey, directUpload: false } + } + + async writeChunk(storageKey: string, data: Buffer): Promise { + const object = this.objects.get(storageKey) + if (!object) { + throw new Error(`unknown storage key: ${storageKey}`) + } + if (object.completed) { + throw new Error(`object already completed: ${storageKey}`) + } + object.chunks.push(data) + object.size += data.byteLength + } + + async complete(storageKey: string, expectedSizeBytes?: number): Promise { + const object = this.objects.get(storageKey) + if (!object) { + throw new Error(`unknown storage key: ${storageKey}`) + } + if (expectedSizeBytes !== undefined && expectedSizeBytes !== object.size) { + throw new Error(`size mismatch for ${storageKey}: expected ${expectedSizeBytes}, got ${object.size}`) + } + object.completed = true + return { storageKey, sizeBytes: object.size } + } + + async head(storageKey: string): Promise { + const object = this.objects.get(storageKey) + if (!object || !object.completed) { + return null + } + return { storageKey, sizeBytes: object.size } + } + + async readStream(storageKey: string): Promise { + const object = this.objects.get(storageKey) + if (!object || !object.completed) { + throw new Error(`object not available: ${storageKey}`) + } + const stream = new PassThrough() + for (const chunk of object.chunks) { + stream.write(chunk) + } + stream.end() + return stream + } + + async abort(storageKey: string): Promise { + this.objects.delete(storageKey) + } + + async delete(storageKey: string): Promise { + this.objects.delete(storageKey) + } +} diff --git a/platform/core/src/core/import-runs/storage/s3-import-storage.adapter.ts b/platform/core/src/core/import-runs/storage/s3-import-storage.adapter.ts new file mode 100644 index 00000000..8e3ca284 --- /dev/null +++ b/platform/core/src/core/import-runs/storage/s3-import-storage.adapter.ts @@ -0,0 +1,252 @@ +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, + S3Client, + UploadPartCommand +} from '@aws-sdk/client-s3' +import { getSignedUrl } from '@aws-sdk/s3-request-presigner' +import { Injectable, Logger } from '@nestjs/common' + +import type { + ImportStoragePort, + InitiateUploadResult, + SignedPartRequest, + StorageObjectMeta +} from './import-storage.port' + +const MULTIPART_CHUNK_THRESHOLD = 8 * 1024 * 1024 + +export interface S3ImportStorageConfig { + bucket: string + region?: string + endpoint?: string + accessKeyId?: string + secretAccessKey?: string + forcePathStyle?: boolean +} + +interface PendingUpload { + storageKey: string + uploadId: string + partNumber: number + parts: Array<{ ETag: string; PartNumber: number }> +} + +/** + * S3-compatible multipart backend for import sources (default: + * RUSHDB_IMPORT_STORAGE_BACKEND=s3). Works with AWS S3 and any S3-compatible + * endpoint (MinIO, R2, etc.). Retention/cleanup uses the same delete() contract + * as the local backend. + */ +@Injectable() +export class S3ImportStorageAdapter implements ImportStoragePort { + private readonly client: S3Client + private readonly bucket: string + private readonly pending = new Map() + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private readonly logger = new Logger(S3ImportStorageAdapter.name) + + constructor(config?: Partial) { + this.bucket = config?.bucket ?? process.env.RUSHDB_IMPORT_S3_BUCKET ?? '' + + if (!this.bucket) { + throw new Error('S3 import storage requires RUSHDB_IMPORT_S3_BUCKET') + } + + this.client = new S3Client({ + region: config?.region ?? process.env.RUSHDB_IMPORT_S3_REGION ?? undefined, + endpoint: config?.endpoint ?? process.env.RUSHDB_IMPORT_S3_ENDPOINT ?? undefined, + forcePathStyle: + config?.forcePathStyle ?? + (process.env.RUSHDB_IMPORT_S3_FORCE_PATH_STYLE ? + process.env.RUSHDB_IMPORT_S3_FORCE_PATH_STYLE === 'true' + : false), + credentials: + config?.accessKeyId && config?.secretAccessKey ? + { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey } + : process.env.RUSHDB_IMPORT_S3_ACCESS_KEY_ID && process.env.RUSHDB_IMPORT_S3_SECRET_ACCESS_KEY ? + { + accessKeyId: process.env.RUSHDB_IMPORT_S3_ACCESS_KEY_ID, + secretAccessKey: process.env.RUSHDB_IMPORT_S3_SECRET_ACCESS_KEY + } + : undefined + }) + } + + async initiate(projectId: string, fileId: string, sourceGeneration: number): Promise { + const storageKey = `imports/${projectId}/${fileId}/g${sourceGeneration}/${Date.now()}-${Math.random() + .toString(36) + .slice(2, 10)}` + + const result = await this.client.send( + new CreateMultipartUploadCommand({ Bucket: this.bucket, Key: storageKey }) + ) + + if (!result.UploadId) { + throw new Error(`failed to initiate multipart upload for ${storageKey}`) + } + + this.pending.set(storageKey, { + storageKey, + uploadId: result.UploadId as string, + partNumber: 0, + parts: [] + }) + + return { uploadId: result.UploadId as string, storageKey, directUpload: true } + } + + /** + * Presigns a single part PUT so the browser can upload directly to S3 without + * bytes traversing the API. Requires the multipart session initiated above. + */ + async signPart(storageKey: string, partNumber: number): Promise { + const pendingUpload = this.pending.get(storageKey) + if (!pendingUpload) { + return null + } + + const command = new UploadPartCommand({ + Bucket: this.bucket, + Key: storageKey, + UploadId: pendingUpload.uploadId, + PartNumber: partNumber + }) + + const url = await getSignedUrl(this.client, command, { expiresIn: 900 }) + + return { url, method: 'PUT', headers: {}, expiresInSeconds: 900 } + } + + async writeChunk(storageKey: string, data: Buffer): Promise { + const pendingUpload = this.ensurePending(storageKey) + + if (pendingUpload.parts.length === 0 && data.byteLength < MULTIPART_CHUNK_THRESHOLD) { + // Small sources go through simple PUT; complete() short-circuits to PutObject semantics. + await this.client.send(new PutObjectCommand({ Bucket: this.bucket, Key: storageKey, Body: data })) + pendingUpload.partNumber = -1 + return + } + + pendingUpload.partNumber += 1 + const part = await this.client.send( + new UploadPartCommand({ + Bucket: this.bucket, + Key: storageKey, + UploadId: pendingUpload.uploadId, + PartNumber: pendingUpload.partNumber, + Body: data + }) + ) + + if (!part.ETag) { + throw new Error(`part ${pendingUpload.partNumber} of ${storageKey} returned no ETag`) + } + pendingUpload.parts.push({ ETag: part.ETag, PartNumber: pendingUpload.partNumber }) + } + + async complete(storageKey: string, expectedSizeBytes?: number): Promise { + let meta = await this.head(storageKey) + + if (!meta) { + const pendingUpload = this.pending.get(storageKey) + if (!pendingUpload || pendingUpload.partNumber === -1) { + throw new Error(`object not found after completion: ${storageKey}`) + } + + await this.client.send( + new CompleteMultipartUploadCommand({ + Bucket: this.bucket, + Key: storageKey, + UploadId: pendingUpload.uploadId, + MultipartUpload: { Parts: pendingUpload.parts } + }) + ) + this.pending.delete(storageKey) + meta = await this.head(storageKey) + } else { + // Object already stored via simple PUT; release the still-open multipart + // upload so no orphaned parts linger. + const pendingUpload = this.pending.get(storageKey) + this.pending.delete(storageKey) + if (pendingUpload?.uploadId) { + await this.client + .send( + new AbortMultipartUploadCommand({ + Bucket: this.bucket, + Key: storageKey, + UploadId: pendingUpload.uploadId + }) + ) + .catch(() => undefined) + } + } + + if (!meta) { + throw new Error(`object not found after completion: ${storageKey}`) + } + if (expectedSizeBytes !== undefined && expectedSizeBytes !== meta.sizeBytes) { + throw new Error(`size mismatch for ${storageKey}: expected ${expectedSizeBytes}, got ${meta.sizeBytes}`) + } + return meta + } + + async head(storageKey: string): Promise { + try { + const result = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: storageKey })) + return { storageKey, sizeBytes: result.ContentLength ?? 0 } + } catch { + return null + } + } + + async readStream(storageKey: string): Promise { + const result = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: storageKey })) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if (!result.Body) { + throw new Error(`empty body for object ${storageKey}`) + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return result.Body as unknown as NodeJS.ReadableStream + } + + async abort(storageKey: string): Promise { + try { + // Only in-process uploads can be aborted by ID; orphaned multipart uploads + // from crashed workers are covered by the bucket lifecycle rule. + const uploadId = this.pending.get(storageKey)?.uploadId + + if (uploadId) { + await this.client.send( + new AbortMultipartUploadCommand({ Bucket: this.bucket, Key: storageKey, UploadId: uploadId }) + ) + } + } catch (error) { + this.logger.warn(`failed to abort multipart upload for ${storageKey}`, error as Error) + } finally { + this.pending.delete(storageKey) + } + } + + async delete(storageKey: string): Promise { + try { + await this.abort(storageKey).catch(() => undefined) + await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: storageKey })) + } catch (error) { + this.logger.warn(`failed to delete object ${storageKey}`, error as Error) + } + } + + private ensurePending(storageKey: string): PendingUpload { + const pendingUpload = this.pending.get(storageKey) + if (!pendingUpload) { + throw new Error(`no initiated upload for key: ${storageKey} (call initiate first)`) + } + return pendingUpload + } +} diff --git a/platform/core/src/core/import-runs/worker/import-batch-writer.ts b/platform/core/src/core/import-runs/worker/import-batch-writer.ts new file mode 100644 index 00000000..97d27352 --- /dev/null +++ b/platform/core/src/core/import-runs/worker/import-batch-writer.ts @@ -0,0 +1,263 @@ +import { + RUSHDB_KEY_ID, + RUSHDB_KEY_PROJECT_ID, + RUSHDB_KEY_PROPERTIES_META, + RUSHDB_LABEL_PROPERTY, + RUSHDB_LABEL_RECORD, + RUSHDB_RELATION_VALUE +} from '@/core/common/constants' + +/** Minimal transaction surface shared by Neogma transactions and test doubles. */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export interface GraphTxLike { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + run(query: string, params?: Record): Promise +} + +export interface PropertyDraft { + name: string + value?: unknown + type?: string + id?: string + created?: string + metadata?: string +} + +export interface RecordDraft { + id: string + label: string + properties: PropertyDraft[] +} + +export interface RelationDraft { + source: string + target: string + type: string + properties?: Record +} + +const IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/ + +function escapeBacktickIdentifier(raw: string): string { + return `\`${String(raw).replace(/`/g, '``')}\`` +} + +function assertSafeIdentifier(raw: string, kind: string): string { + if (!IDENTIFIER_PATTERN.test(raw)) { + throw new Error(`invalid ${kind}: ${raw}`) + } + return raw +} + +/** + * Detects replay-ID collisions against pre-existing records owned by other + * projects. Deterministic IDs already embed the project ID, so a hit here means + * an incompatible legacy node occupies the identity space and must fail loudly. + */ +export function buildCollisionCheckQuery(): string { + const queryBuilder = [ + `UNWIND $ids as candidateId`, + `OPTIONAL MATCH (existing:${RUSHDB_LABEL_RECORD} { ${RUSHDB_KEY_ID}: candidateId })`, + `WITH existing WHERE existing IS NOT NULL AND existing.${RUSHDB_KEY_PROJECT_ID} <> $projectId`, + `RETURN count(existing) as collisions` + ] + return queryBuilder.join('\n') +} + +export async function assertNoReplayCollisions( + tx: GraphTxLike, + params: { projectId: string; recordIds: string[] } +): Promise { + if (params.recordIds.length === 0) { + return + } + + const result = await tx.run(buildCollisionCheckQuery(), { + projectId: params.projectId, + ids: params.recordIds + }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const row = result?.records?.[0]?.toObject?.() + const collisions = Number(row?.collisions ?? 0) + if (collisions > 0) { + throw new Error(`IMPORT_REPLAY_ID_COLLISION: ${collisions} conflicting record(s)`) + } +} + +/** + * Idempotent batch MERGE of import records keyed by the deterministic replay ID. + * Business labels, __proptypes meta, and Property nodes follow exactly the + * conventions of the synchronous import path (EntityQueryService.importRecords + + * processProps), so async output is graph-compatible. + */ +export function buildMergeRecordsQuery(): string { + return [ + `WITH $records as recordsToMerge, datetime() as time`, + `UNWIND recordsToMerge as r`, + `MERGE (record:${RUSHDB_LABEL_RECORD} { ${RUSHDB_KEY_ID}: r.id })`, + `ON CREATE SET record.${RUSHDB_KEY_PROJECT_ID} = $projectId`, + `WITH record, r, time,`, + `apoc.map.fromPairs([property IN r.properties | [property.name, coalesce(property.type, 'string')]]) AS typesMap,`, + `apoc.map.fromPairs([property IN r.properties | [property.name, property.value]]) AS valuesMap`, + `CALL apoc.create.addLabels(record, [r.label]) YIELD node as labeledRecord`, + `SET labeledRecord.${RUSHDB_KEY_PROPERTIES_META} = apoc.convert.toJson(typesMap)`, + `SET labeledRecord += valuesMap`, + `WITH DISTINCT labeledRecord as record, r.properties as props, time`, + `UNWIND props as prop`, + `MERGE (p:${RUSHDB_LABEL_PROPERTY} { name: prop.name, type: coalesce(prop.type, 'string'), ${RUSHDB_KEY_PROJECT_ID}: $projectId, metadata: coalesce(prop.metadata, "") })`, + `ON CREATE SET p.created = coalesce(prop.created, time), p.id = prop.id, p.metadata = coalesce(prop.metadata, "")`, + `MERGE (p)-[rel:${RUSHDB_RELATION_VALUE}]->(record)`, + `RETURN count(DISTINCT record) as mergedRecords` + ].join('\n') +} + +export async function mergeRecords( + tx: GraphTxLike, + params: { projectId: string; records: RecordDraft[] } +): Promise { + await assertNoReplayCollisions(tx, { + projectId: params.projectId, + recordIds: params.records.map((r) => r.id) + }) + + const result = await tx.run(buildMergeRecordsQuery(), { + projectId: params.projectId, + records: params.records + }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const row = result?.records?.[0]?.toObject?.() + return Number(row?.mergedRecords ?? 0) +} + +/** + * Idempotent relationship creation grouped by one static type per call. + * Identical endpoint pairs collapse into a single typed relationship, which makes + * crash-boundary replays converge instead of duplicating edges. + */ +export function buildMergeRelationsQuery(relationshipType: string): string { + const safeType = escapeBacktickIdentifier(assertSafeIdentifier(relationshipType, 'relationship type')) + + return [ + `WITH $relations as relations`, + `UNWIND relations as relation`, + `MATCH (source:${RUSHDB_LABEL_RECORD} { ${RUSHDB_KEY_ID}: relation.source, ${RUSHDB_KEY_PROJECT_ID}: $projectId })`, + `MATCH (target:${RUSHDB_LABEL_RECORD} { ${RUSHDB_KEY_ID}: relation.target, ${RUSHDB_KEY_PROJECT_ID}: $projectId })`, + `WHERE NOT EXISTS { MATCH (source)-[anyExisting:${safeType}]->(target) }`, + `CREATE (source)-[createdRel:${safeType}]->(target)`, + `SET createdRel += coalesce(relation.properties, {})`, + `RETURN count(createdRel) as createdRelations` + ].join('\n') +} + +export async function linkRelations( + tx: GraphTxLike, + params: { projectId: string; relations: RelationDraft[] } +): Promise { + const byType = new Map() + for (const relation of params.relations) { + const list = byType.get(relation.type) ?? [] + list.push(relation) + byType.set(relation.type, list) + } + + let total = 0 + for (const [type, relations] of byType) { + const result = await tx.run(buildMergeRelationsQuery(type), { + projectId: params.projectId, + relations + }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const row = result?.records?.[0]?.toObject?.() + total += Number(row?.createdRelations ?? 0) + } + return total +} + +/** + * Resolves endpoint values to persisted record IDs for link files. Values are + * matched as strings against `label.keyProperty` so numeric CSV typing on either + * side cannot break resolution. + */ +export async function resolveEndpoints( + tx: GraphTxLike, + params: { projectId: string; label: string; keyProperty: string; values: string[] } +): Promise> { + const safeLabel = escapeBacktickIdentifier(params.label) + + const query = [ + `UNWIND $entries as entry`, + `MATCH (node:${RUSHDB_LABEL_RECORD}:${safeLabel})`, + `WHERE node.${RUSHDB_KEY_PROJECT_ID} = $projectId AND toString(node[entry.key]) = entry.value`, + `RETURN toString(node.${RUSHDB_KEY_ID}) as id, entry.value as value` + ].join('\n') + + const entries = params.values.map((value) => ({ key: params.keyProperty, value })) + const resolved = new Map() + + const result = await tx.run(query, { projectId: params.projectId, entries }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + for (const record of result?.records ?? []) { + const row = record.toObject() + resolved.set(String(row.value), String(row.id)) + } + + return resolved +} + +export async function hasAnyRecordForLabel( + tx: GraphTxLike, + params: { projectId: string; label: string } +): Promise { + const safeLabel = escapeBacktickIdentifier(params.label) + const result = await tx.run( + `MATCH (node:${RUSHDB_LABEL_RECORD}:${safeLabel}) WHERE node.${RUSHDB_KEY_PROJECT_ID} = $projectId RETURN node.${RUSHDB_KEY_ID} as id LIMIT 1`, + { projectId: params.projectId } + ) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (result?.records?.length ?? 0) > 0 +} + +export interface LinkPair { + sourceValue: string + targetValue: string + sourceId?: string + targetId?: string + properties: Record +} + +/** + * Creates link-file relationships between resolved endpoints, idempotently, + * grouped under one static relationship type. + */ +export function buildCreateLinksQuery(relationshipType: string): string { + const safeType = escapeBacktickIdentifier(assertSafeIdentifier(relationshipType, 'relationship type')) + + return [ + `WITH $pairs as pairs`, + `UNWIND pairs as pair`, + `MATCH (source:${RUSHDB_LABEL_RECORD} { ${RUSHDB_KEY_ID}: pair.sourceId, ${RUSHDB_KEY_PROJECT_ID}: $projectId })`, + `MATCH (target:${RUSHDB_LABEL_RECORD} { ${RUSHDB_KEY_ID}: pair.targetId, ${RUSHDB_KEY_PROJECT_ID}: $projectId })`, + `WHERE NOT EXISTS { MATCH (source)-[anyExisting:${safeType}]->(target) }`, + `CREATE (source)-[createdRel:${safeType}]->(target)`, + `SET createdRel += coalesce(pair.properties, {})`, + `RETURN count(createdRel) as createdLinks` + ].join('\n') +} + +export async function createLinks( + tx: GraphTxLike, + params: { projectId: string; relationshipType: string; pairs: LinkPair[] } +): Promise<{ created: number }> { + const resolvable = params.pairs.filter((p) => p.sourceId && p.targetId) + if (resolvable.length === 0) { + return { created: 0 } + } + + const result = await tx.run(buildCreateLinksQuery(params.relationshipType), { + projectId: params.projectId, + pairs: resolvable + }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const row = result?.records?.[0]?.toObject?.() + return { created: Number(row?.createdLinks ?? 0) } +} diff --git a/platform/core/src/core/import-runs/worker/import-cleanup.scheduler.ts b/platform/core/src/core/import-runs/worker/import-cleanup.scheduler.ts new file mode 100644 index 00000000..9b3bd510 --- /dev/null +++ b/platform/core/src/core/import-runs/worker/import-cleanup.scheduler.ts @@ -0,0 +1,63 @@ +import { Inject, Injectable, Logger } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' +import { Cron } from '@nestjs/schedule' + +import { ImportRunsRepository } from '../persistence/import-runs.repository' +import { IMPORT_STORAGE_PORT, type ImportStoragePort } from '../storage/import-storage.port' + +/** + * Applies source-object retention identically for every storage backend: + * terminal, finalized runs past their retention window have their stored + * sources deleted (graph data is never touched). Failures are left for the + * next sweep. + */ +@Injectable() +export class ImportCleanupScheduler { + private running = false + + constructor( + private readonly repository: ImportRunsRepository, + @Inject(IMPORT_STORAGE_PORT) private readonly storage: ImportStoragePort, + private readonly configService: ConfigService + ) {} + + @Cron('*/10 * * * *') + async sweep(): Promise { + if (this.running) { + return + } + this.running = true + try { + const now = Date.now() + const runs = await this.repository.findRunsPastRetention(new Date(now).toISOString()) + + for (const run of runs) { + const files = await this.repository.listFilesWithSource(run.id) + for (const file of files) { + if (!file.storageKey) { + continue + } + await this.storage.delete(file.storageKey) + const cleared = await this.repository.clearSourceForFile(file.id) + if (cleared) { + await this.repository.addEvent({ + runId: run.id, + fileId: file.id, + projectId: run.projectId, + type: 'SOURCE_CLEANED', + code: 'RETENTION_EXPIRED' + }) + } + } + } + + if (runs.length > 0) { + Logger.log(`[ImportCleanup] retained-source cleanup processed ${runs.length} run(s)`) + } + } catch (error) { + Logger.error('[ImportCleanup] retention sweep failed', error) + } finally { + this.running = false + } + } +} diff --git a/platform/core/src/core/import-runs/worker/import-worker.service.ts b/platform/core/src/core/import-runs/worker/import-worker.service.ts new file mode 100644 index 00000000..41f98ef6 --- /dev/null +++ b/platform/core/src/core/import-runs/worker/import-worker.service.ts @@ -0,0 +1,970 @@ +import { Inject, Injectable, Logger, Optional } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' +import { Cron } from '@nestjs/schedule' + +import { AiService } from '@/core/ai/ai.service' +import { BILLING_POLICY_PORT, type BillingPolicyPort } from '@/core/billing-policy/billing-policy.port' +import { RUSHDB_RELATION_DEFAULT } from '@/core/common/constants' +import { ImportService } from '@/core/entity/import-export/import.service' +import { KuOperation } from '@/core/ku-events/ku-events.constants' +import { KuEventsService } from '@/core/ku-events/ku-events.service' +import { RelationshipPatternsService } from '@/core/relationship-patterns/relationship-patterns.service' +import { ProjectService } from '@/dashboard/project/project.service' +import { WorkspaceService } from '@/dashboard/workspace/workspace.service' +import { DbConnectionService } from '@/database/db-connection/db-connection.service' +import { NeogmaService } from '@/database/neogma/neogma.service' +import { DEFAULT_TRANSACTION_TIMEOUT_MS } from '@/database/transaction.constants' + +import { + IMPORT_ERROR_CODES, + type ImportCheckpoint, + type ImportFileFormat, + type ImportLinkSpec +} from '../domain/import-run.types' +import { deriveDeterministicRecordId } from '../domain/replay-identity' +import { CsvImportParser } from '../parser/csv.parser' +import { JsonLinesImportParser } from '../parser/json-lines.parser' +import { JsonObjectImportParser } from '../parser/json-object.parser' +import { ParquetImportParser } from '../parser/parquet.parser' +import { ImportRunsRepository } from '../persistence/import-runs.repository' +import { IMPORT_STORAGE_PORT, type ImportStoragePort } from '../storage/import-storage.port' + +import { + createLinks, + hasAnyRecordForLabel, + linkRelations, + mergeRecords, + resolveEndpoints, + type GraphTxLike, + type RecordDraft, + type RelationDraft +} from './import-batch-writer' + +import type { ImportUnit } from '../parser/import-parser.port' +import type { TImportOptions } from '@/core/entity/import-export/import.types' +import type { ImportRunFileRow } from '@/database/sql/schema/types' +import type { Neogma } from 'neogma' + +const BATCH_UNITS = 500 +const LEASE_TTL_MS = 60_000 +const ENDPOINT_RESOLUTION_CHUNK = 1_000 + +interface WorkerContext { + projectId: string + workspaceId: string | null + connection: Neogma +} + +interface BatchCounters { + parsedUnits: number + committedUnits: number + recordsCommitted: number + relationshipsCommitted: number + linksResolved: number + linksUnresolved: number + skippedUnits: number +} + +@Injectable() +export class ImportWorkerService { + private running = false + + private readonly csvParser = new CsvImportParser() + private readonly jsonLinesParser = new JsonLinesImportParser() + private readonly jsonObjectParser = new JsonObjectImportParser() + private readonly parquetParser = new ParquetImportParser() + + constructor( + private readonly repository: ImportRunsRepository, + @Inject(IMPORT_STORAGE_PORT) private readonly storage: ImportStoragePort, + private readonly importService: ImportService, + private readonly neogmaService: NeogmaService, + private readonly workspaceService: WorkspaceService, + private readonly kuEventsService: KuEventsService, + @Inject(BILLING_POLICY_PORT) private readonly billingPolicy: BillingPolicyPort, + private readonly projectService: ProjectService, + private readonly dbConnectionService: DbConnectionService, + private readonly configService: ConfigService, + @Optional() private readonly relationshipPatternsService?: RelationshipPatternsService, + @Optional() private readonly aiService?: AiService + ) {} + + private parserFor( + format: ImportFileFormat + ): CsvImportParser | JsonLinesImportParser | JsonObjectImportParser | ParquetImportParser { + switch (format) { + case 'csv': + return this.csvParser + case 'jsonl': + case 'ndjson': + return this.jsonLinesParser + case 'json': + return this.jsonObjectParser + case 'parquet': + return this.parquetParser + default: + throw new Error(`${IMPORT_ERROR_CODES.FORMAT_UNSUPPORTED}: ${format}`) + } + } + + private maxSourceBytesFor(format: ImportFileFormat): number | undefined { + if (format === 'json') { + const mb = + Number(this.configService.get('RUSHDB_IMPORT_MAX_JSON_BYTES', '')) || 64 * 1024 * 1024 + return mb + } + if (format === 'parquet') { + const mb = + Number(this.configService.get('RUSHDB_IMPORT_MAX_PARQUET_BYTES', '')) || 256 * 1024 * 1024 + return mb + } + return undefined + } + + @Cron('*/5 * * * * *') + async poll(): Promise { + if (this.running) { + return + } + this.running = true + try { + for (const projectId of await this.repository.listClaimableProjects()) { + const claimed = await this.repository.claimNextFile({ + projectId, + workerId: `worker-${process.pid}`, + now: new Date().toISOString(), + leaseUntil: new Date(Date.now() + LEASE_TTL_MS).toISOString() + }) + if (claimed) { + await this.processClaimed(claimed) + } + } + } catch (error) { + Logger.error('[ImportWorker] poll failed', error) + } finally { + this.running = false + } + } + + async processClaimed(file: ImportRunFileRow): Promise { + try { + const context = await this.resolveContext(file.projectId) + await this.preflightBilling(file, context) + if (file.role === 'links') { + await this.processLinksFile(file, context) + } else { + await this.processRecordsFile(file, context) + } + } catch (error) { + await this.handleProcessingError(file, error as Error) + } + } + + // ------------------------------------------------------------------ + // Records role + // ------------------------------------------------------------------ + + private async processRecordsFile(file: ImportRunFileRow, context: WorkerContext): Promise { + const rootLabel = requireRootLabel(file) + const options = parseJson(file.importOptions) ?? {} + const checkpoint = readCheckpoint(file) + + await this.repository.fencedUpdateFile(file.id, file.leaseGeneration, { + status: 'running', + stage: 'parse', + heartbeatAt: new Date().toISOString() + }) + + const stream = await this.openSourceStream(file) + const format = file.format as ImportFileFormat + const parser = this.parserFor(format) + const units = await parser.parse(stream, { + maxSourceBytes: this.maxSourceBytesFor(format), + signal: undefined + }) + + let batchOrdinal = checkpoint.lastCommittedBatch + 1 + let buffer: ImportUnit[] = [] + let counters = readCounters(file) + + for await (const unit of units) { + if (unit.ordinal < checkpoint.nextUnit) { + continue + } + + buffer.push(unit) + counters.parsedUnits += 1 + + if (buffer.length >= BATCH_UNITS) { + counters = await this.commitRecordsBatch( + file, + context, + buffer, + rootLabel, + options, + batchOrdinal, + counters + ) + batchOrdinal += 1 + buffer = [] + + if (!(await this.continueProcessing(file))) { + return + } + } + } + + if (buffer.length > 0) { + counters = await this.commitRecordsBatch( + file, + context, + buffer, + rootLabel, + options, + batchOrdinal, + counters + ) + } + + await this.completeFile(file, counters) + } + + private async commitRecordsBatch( + file: ImportRunFileRow, + context: WorkerContext, + units: ImportUnit[], + rootLabel: string, + options: TImportOptions, + batchOrdinal: number, + runningCounters: BatchCounters + ): Promise { + const wantsUpsert = + Boolean(options.mergeStrategy) || (Array.isArray(options.mergeBy) && options.mergeBy.length > 0) + + let recordsCommitted = 0 + let relationshipsCommitted = 0 + + if (wantsUpsert) { + // mergeBy/mergeStrategy: business upsert is naturally idempotent, so route + // through the shared ImportService on this worker's write transaction. It + // handles records, relations, vectors and KU emission internally. + await this.withWriteTransaction(context.connection, async (tx) => { + const result = await this.importService.importRecords( + { + data: units.map((u) => u.value), + label: rootLabel, + options: { ...options, returnResult: false } + }, + file.projectId, + tx as never, + tx as never + ) + if (typeof result === 'object' && result && 'count' in result) { + recordsCommitted = Number((result as { count: number }).count) + } + }) + } else { + const records: RecordDraft[] = [] + const relations: RelationDraft[] = [] + const vectorDrafts: Array<{ deterministicId: string; label: string; vectors: unknown[] }> = [] + + for (const unit of units) { + const [drafts, unitRelations, unitVectors] = this.importService.serializeBFS(unit.value, rootLabel, { + ...options, + returnResult: false + }) + + const idRemap = new Map() + drafts.forEach((draft, index) => { + const deterministicId = deriveDeterministicRecordId({ + projectId: file.projectId, + fileId: file.id, + sourceGeneration: file.sourceGeneration, + unitOrdinal: unit.ordinal, + path: String(index), + siblingOccurrence: 0 + }) + idRemap.set(draft.id, deterministicId) + records.push({ + id: deterministicId, + label: draft.label || rootLabel, + properties: draft.properties + }) + }) + + for (const vd of unitVectors) { + const deterministicId = idRemap.get(vd.draftId) + if (deterministicId) { + vectorDrafts.push({ deterministicId, label: vd.label, vectors: vd.vectors }) + } + } + + for (const relation of unitRelations) { + const sourceId = idRemap.get(relation.source) + const targetId = idRemap.get(relation.target) + if (sourceId && targetId) { + relations.push({ + source: sourceId, + target: targetId, + type: relation.type ?? RUSHDB_RELATION_DEFAULT + }) + } + } + } + + await this.withWriteTransaction(context.connection, async (tx) => { + recordsCommitted = await mergeRecords(tx, { projectId: file.projectId, records }) + relationshipsCommitted = await linkRelations(tx, { projectId: file.projectId, relations }) + if (this.aiService) { + for (const vd of vectorDrafts) { + await this.aiService + .resolveAndWriteInlineVectors( + file.projectId, + vd.label, + vd.deterministicId, + vd.vectors as never, + tx as never + ) + .catch(() => undefined) + } + } + }) + + await this.emitKu(file.projectId, context.workspaceId, recordsCommitted, relationshipsCommitted) + } + + const nextCounters: BatchCounters = { + parsedUnits: runningCounters.parsedUnits, + committedUnits: runningCounters.committedUnits + units.length, + recordsCommitted: runningCounters.recordsCommitted + recordsCommitted, + relationshipsCommitted: runningCounters.relationshipsCommitted + relationshipsCommitted, + linksResolved: runningCounters.linksResolved, + linksUnresolved: runningCounters.linksUnresolved, + skippedUnits: runningCounters.skippedUnits + } + + const persisted = await this.persistProgress(file, batchOrdinal, nextCounters) + if (!persisted) { + throw new Error(`${IMPORT_ERROR_CODES.LEASE_LOST}: cannot checkpoint batch ${batchOrdinal}`) + } + + return nextCounters + } + + // ------------------------------------------------------------------ + // Links role + // ------------------------------------------------------------------ + + private async processLinksFile(file: ImportRunFileRow, context: WorkerContext): Promise { + const spec = parseJson(file.linkSpec) + assertLinkSpec(spec) + + const checkpoint = readCheckpoint(file) + + await this.repository.fencedUpdateFile(file.id, file.leaseGeneration, { + status: 'running', + stage: 'write', + heartbeatAt: new Date().toISOString() + }) + + await this.assertLinkLabelsExist(file, spec, context.connection) + + const stream = await this.openSourceStream(file) + const format = file.format as ImportFileFormat + const parser = this.parserFor(format) + const units = await parser.parse(stream, { + maxSourceBytes: this.maxSourceBytesFor(format), + signal: undefined + }) + const importOptions = parseJson>(file.importOptions) ?? {} + const skipInvalidRows = importOptions.skipInvalidRows === true + + let batchOrdinal = checkpoint.lastCommittedBatch + 1 + let buffer: ImportUnit[] = [] + let counters = readCounters(file) + + for await (const unit of units) { + if (unit.ordinal < checkpoint.nextUnit) { + continue + } + + buffer.push(unit) + counters.parsedUnits += 1 + + if (buffer.length >= BATCH_UNITS) { + counters = await this.commitLinksBatch( + file, + context, + spec, + buffer, + batchOrdinal, + counters, + skipInvalidRows + ) + batchOrdinal += 1 + buffer = [] + + if (!(await this.continueProcessing(file))) { + return + } + } + } + + if (buffer.length > 0) { + counters = await this.commitLinksBatch( + file, + context, + spec, + buffer, + batchOrdinal, + counters, + skipInvalidRows + ) + } + + await this.completeFile(file, counters) + } + + private async commitLinksBatch( + file: ImportRunFileRow, + context: WorkerContext, + spec: ImportLinkSpec, + units: ImportUnit[], + batchOrdinal: number, + runningCounters: BatchCounters, + skipInvalidRows: boolean + ): Promise { + const [sourceEndpoint, targetEndpoint] = spec.endpoints + const propertyColumns = spec.propertyColumns ?? {} + + interface PendingRow { + row: Record + properties: Record + } + + const pending: PendingRow[] = units.map((unit) => { + const properties: Record = {} + for (const [column, propertyName] of Object.entries(propertyColumns)) { + properties[propertyName] = unit.value[column] + } + return { row: unit.value, properties } + }) + + const sourceValues = uniqueStrings( + pending.map((entry) => stringifyValue(entry.row[sourceEndpoint.column])) + ) + const targetValues = uniqueStrings( + pending.map((entry) => stringifyValue(entry.row[targetEndpoint.column])) + ) + + let linksResolved = 0 + let linksCreated = 0 + + await this.withWriteTransaction(context.connection, async (tx) => { + const resolvedSources = await this.resolveInChunks(tx, file.projectId, sourceEndpoint, sourceValues) + const resolvedTargets = await this.resolveInChunks(tx, file.projectId, targetEndpoint, targetValues) + + const pairs = pending.map((entry) => { + const sourceValue = stringifyValue(entry.row[sourceEndpoint.column]) + const targetValue = stringifyValue(entry.row[targetEndpoint.column]) + return { + sourceValue, + targetValue, + sourceId: resolvedSources.get(sourceValue), + targetId: resolvedTargets.get(targetValue), + properties: entry.properties + } + }) + + const unresolvedPairs = pairs.filter((pair) => !pair.sourceId || !pair.targetId) + + for (const unresolved of unresolvedPairs.slice(0, 5)) { + await this.repository.addErrorSample({ + runId: file.runId, + fileId: file.id, + projectId: file.projectId, + code: IMPORT_ERROR_CODES.LINK_ENDPOINT_UNRESOLVED, + message: sanitizeSampleMessage( + `unresolved endpoint linking ${unresolved.sourceValue} -> ${unresolved.targetValue}` + ) + }) + } + + if (unresolvedPairs.length > 0 && !skipInvalidRows) { + throw new Error( + `${IMPORT_ERROR_CODES.LINK_ENDPOINT_UNRESOLVED}: ${unresolvedPairs.length} row(s) reference missing records` + ) + } + + const result = await createLinks(tx, { + projectId: file.projectId, + relationshipType: spec.relationshipType, + pairs + }) + + linksResolved = pairs.length - unresolvedPairs.length + linksCreated = result.created + }) + + await this.emitKu(file.projectId, context.workspaceId, 0, linksCreated) + + const nextCounters: BatchCounters = { + parsedUnits: runningCounters.parsedUnits, + committedUnits: runningCounters.committedUnits + units.length, + recordsCommitted: runningCounters.recordsCommitted, + relationshipsCommitted: runningCounters.relationshipsCommitted + linksCreated, + linksResolved: runningCounters.linksResolved + linksResolved, + linksUnresolved: runningCounters.linksUnresolved + (units.length - linksResolved), + skippedUnits: runningCounters.skippedUnits + } + + const persisted = await this.persistProgress(file, batchOrdinal, nextCounters) + if (!persisted) { + throw new Error(`${IMPORT_ERROR_CODES.LEASE_LOST}: cannot checkpoint batch ${batchOrdinal}`) + } + + return nextCounters + } + + private async resolveInChunks( + tx: GraphTxLike, + projectId: string, + endpoint: { label: string; keyProperty: string }, + values: string[] + ): Promise> { + const resolved = new Map() + for (let i = 0; i < values.length; i += ENDPOINT_RESOLUTION_CHUNK) { + const chunk = values.slice(i, i + ENDPOINT_RESOLUTION_CHUNK) + const part = await resolveEndpoints(tx, { + projectId, + label: endpoint.label, + keyProperty: endpoint.keyProperty, + values: chunk + }) + for (const [value, id] of part) { + resolved.set(value, id) + } + } + return resolved + } + + private async assertLinkLabelsExist( + file: ImportRunFileRow, + spec: ImportLinkSpec, + connection: Neogma + ): Promise { + await this.withWriteTransaction(connection, async (tx) => { + for (const endpoint of spec.endpoints) { + const exists = await hasAnyRecordForLabel(tx, { + projectId: file.projectId, + label: endpoint.label + }) + if (!exists) { + throw new Error( + `${IMPORT_ERROR_CODES.LINK_LABEL_EMPTY}: no ${endpoint.label} records exist to link against` + ) + } + } + }) + } + + // ------------------------------------------------------------------ + // Shared plumbing + // ------------------------------------------------------------------ + + private async openSourceStream(file: ImportRunFileRow): Promise { + if (!file.storageKey) { + throw new Error(`${IMPORT_ERROR_CODES.SOURCE_MISSING}: no stored source for file`) + } + const meta = await this.storage.head(file.storageKey) + if (!meta) { + throw new Error(`${IMPORT_ERROR_CODES.SOURCE_MISSING}: object not found`) + } + return this.storage.readStream(file.storageKey) + } + + private async continueProcessing(file: ImportRunFileRow): Promise { + const fresh = await this.repository.getFile(file.id, file.projectId) + if (!fresh) { + return false + } + if (fresh.status === 'canceled' || fresh.cancelRequestedAt || fresh.status !== 'running') { + return false + } + return true + } + + private async persistProgress( + file: ImportRunFileRow, + batchOrdinal: number, + counters: BatchCounters + ): Promise { + const checkpoint: ImportCheckpoint = { + version: 1, + sourceGeneration: file.sourceGeneration, + nextUnit: counters.parsedUnits, + lastCommittedBatch: batchOrdinal + } + + return this.repository.fencedUpdateFile(file.id, file.leaseGeneration, { + status: 'running', + stage: 'write', + currentBatch: batchOrdinal, + checkpoint: JSON.stringify(checkpoint), + heartbeatAt: new Date().toISOString(), + leaseUntil: new Date(Date.now() + LEASE_TTL_MS).toISOString(), + parsedUnits: counters.parsedUnits, + committedUnits: counters.committedUnits, + recordsCommitted: counters.recordsCommitted, + relationshipsCommitted: counters.relationshipsCommitted, + linksResolved: counters.linksResolved, + linksUnresolved: counters.linksUnresolved, + skippedUnits: counters.skippedUnits + }) + } + + private async completeFile(file: ImportRunFileRow, counters: BatchCounters): Promise { + const checkpoint: ImportCheckpoint = { + version: 1, + sourceGeneration: file.sourceGeneration, + nextUnit: counters.parsedUnits, + lastCommittedBatch: Math.max(0, counters.committedUnits - 1) + } + + const persisted = await this.repository.fencedUpdateFile(file.id, file.leaseGeneration, { + status: 'completed', + stage: 'finalize', + finishedAt: new Date().toISOString(), + checkpoint: JSON.stringify(checkpoint), + parsedUnits: counters.parsedUnits, + committedUnits: counters.committedUnits, + recordsCommitted: counters.recordsCommitted, + relationshipsCommitted: counters.relationshipsCommitted, + linksResolved: counters.linksResolved, + linksUnresolved: counters.linksUnresolved, + skippedUnits: counters.skippedUnits + }) + + if (!persisted) { + throw new Error(`${IMPORT_ERROR_CODES.LEASE_LOST}: cannot mark completed`) + } + + await this.repository.addEvent({ + runId: file.runId, + fileId: file.id, + projectId: file.projectId, + type: 'FILE_COMPLETED', + toStatus: 'completed' + }) + + await this.maybeFinalizeRun(file.runId) + } + + private async handleProcessingError(file: ImportRunFileRow, error: Error): Promise { + const message = error.message ?? String(error) + + if (message.includes(IMPORT_ERROR_CODES.LEASE_LOST)) { + Logger.warn(`[ImportWorker] lost lease on file ${file.id}`) + return + } + + const code = extractCode(message) + const fresh = await this.repository.getFile(file.id, file.projectId) + if (!fresh) { + return + } + + if (code) { + await this.failFile(fresh, code, message) + return + } + + const nextAttempt = fresh.attemptCount + if (nextAttempt >= fresh.maxAttempts) { + await this.failFile(fresh, IMPORT_ERROR_CODES.INTERNAL, message) + return + } + + const backoffMs = Math.min(60_000, 2 ** nextAttempt * 1_000) + await this.repository.updateFile(fresh.id, { + status: 'retry_wait', + notBefore: new Date(Date.now() + backoffMs).toISOString(), + lastErrorMessage: sanitizeSampleMessage(message) + }) + } + + private async failFile(file: ImportRunFileRow, code: string, message: string): Promise { + await this.repository.updateFile(file.id, { + status: 'failed', + finishedAt: new Date().toISOString(), + lastErrorCode: code, + lastErrorMessage: sanitizeSampleMessage(message) + }) + await this.repository.addEvent({ + runId: file.runId, + fileId: file.id, + projectId: file.projectId, + type: 'FILE_FAILED', + code, + message: sanitizeSampleMessage(message) + }) + await this.maybeFinalizeRun(file.runId) + } + + /** + * Run finalization: when every file of a run is terminal, refresh aggregates, + * execute the post-write side effects against committed data (recount -> + * approved pattern application -> schema cache refresh -> analysis enqueue), + * then stamp the run outcome. + */ + private async maybeFinalizeRun(runId: string): Promise { + const files = await this.repository.listFiles(runId) + const allTerminal = files.every((f) => ['completed', 'failed', 'canceled'].includes(f.status)) + if (!allTerminal) { + return + } + + const run = await this.repository.refreshRunAggregates(runId) + if (!run) { + return + } + + const outcome = deriveOutcome(files.map((f) => f.status)) + + try { + await this.runPostWriteSideEffects(run.projectId) + } catch (error) { + Logger.error(`[ImportWorker] finalization side effects failed for run ${runId}`, error) + await this.repository.updateRun(runId, { status: 'completed_with_errors' }) + return + } + + const retentionHours = Number(this.configService.get('RUSHDB_IMPORT_RETENTION_HOURS', '72')) + await this.repository.updateRun(runId, { + status: outcome, + finalizedAt: new Date().toISOString(), + retentionUntil: new Date(Date.now() + Math.max(1, retentionHours) * 3_600_000).toISOString() + }) + } + + /** Mirrors RunSideEffectMixin ordering: recount -> apply patterns -> schema -> analysis. */ + private async runPostWriteSideEffects(projectId: string): Promise { + const session = this.neogmaService.createSession('import-runs-side-effect') + const transaction = session.beginTransaction({ timeout: DEFAULT_TRANSACTION_TIMEOUT_MS }) + + try { + try { + await this.projectService.recomputeProjectNodes(projectId, transaction) + } catch (error) { + Logger.error(`[ImportWorker] recount ERROR: project ${projectId}`, error) + } + + if (this.relationshipPatternsService) { + try { + await this.relationshipPatternsService.applyApprovedPatterns(projectId, transaction) + } catch (error) { + Logger.error(`[ImportWorker] relationship apply ERROR: project ${projectId}`, error) + } + } + + if (transaction.isOpen()) { + await transaction.commit() + } + } catch (error) { + if (transaction.isOpen()) { + await transaction.rollback().catch(() => undefined) + } + throw error + } finally { + try { + await this.neogmaService.closeSession(session, 'import-runs') + } catch { + /* empty */ + } + } + + if (this.aiService) { + try { + await this.aiService.getSchema({ projectId, force: true }) + } catch (error) { + Logger.error(`[ImportWorker] schema recompute ERROR: project ${projectId}`, error) + } + } + + if (this.relationshipPatternsService) { + try { + await this.relationshipPatternsService.markAfterWrite(projectId) + } catch (error) { + Logger.error(`[ImportWorker] relationship analysis ERROR: project ${projectId}`, error) + } + } + } + + private async resolveContext(projectId: string): Promise { + let workspaceId: string | null = null + try { + const workspace = await this.workspaceService.getWorkspaceByProject(projectId) + workspaceId = workspace?.id ?? null + } catch { + /* workspace lookup is best-effort for billing */ + } + + let connection: Neogma + try { + const project = await this.projectService.getProjectById(projectId) + const result = await this.dbConnectionService.getConnection(projectId, project as never) + connection = result.connection + } catch { + connection = this.neogmaService.getInstance() + } + + return { projectId, workspaceId, connection } + } + + private async preflightBilling(file: ImportRunFileRow, context: WorkerContext): Promise { + if (!context.workspaceId) { + return + } + const estimatedKu = Math.max(10, Math.ceil((file.declaredSizeBytes ?? 0) / 1024)) + try { + await this.billingPolicy.assertProjectOperationAllowed(context.workspaceId, { estimatedKu }) + } catch (error) { + throw new Error(`${IMPORT_ERROR_CODES.QUOTA_BLOCKED}: ${(error as Error).message}`) + } + } + + private async emitKu( + projectId: string, + workspaceId: string | null, + records: number, + relationships: number + ): Promise { + if (!workspaceId) { + return + } + if (records > 0) { + this.kuEventsService.emitBulk(workspaceId, projectId, KuOperation.ENTITY_CREATED, records) + } + if (relationships > 0) { + this.kuEventsService.emitBulk(workspaceId, projectId, KuOperation.RELATIONSHIP_CREATED, relationships) + } + } + + private async withWriteTransaction( + connection: Neogma, + fn: (tx: GraphTxLike) => Promise + ): Promise { + const session = connection?.driver?.session() ?? this.neogmaService.createSession('import-runs-write') + const tx = session.beginTransaction({ timeout: DEFAULT_TRANSACTION_TIMEOUT_MS }) + try { + await fn(tx as unknown as GraphTxLike) + await tx.commit() + } catch (error) { + if (tx.isOpen()) { + await tx.rollback().catch(() => undefined) + } + throw error + } finally { + try { + await session.close() + } catch { + /* empty */ + } + } + } +} + +// --------------------------------------------------------------------------- +// Module-level pure helpers +// --------------------------------------------------------------------------- + +function parseJson(raw: string | null | undefined): T | null { + if (!raw) { + return null + } + try { + return JSON.parse(raw) as T + } catch { + return null + } +} + +function deriveOutcome(statuses: string[]): 'completed' | 'completed_with_errors' | 'failed' | 'canceled' { + if (statuses.every((s) => s === 'canceled')) { + return 'canceled' + } + if (statuses.includes('completed')) { + return statuses.some((s) => s !== 'completed') ? 'completed_with_errors' : 'completed' + } + if (statuses.some((s) => s === 'completed')) { + return 'completed_with_errors' + } + return 'failed' +} + +function requireRootLabel(file: ImportRunFileRow): string { + if (!file.rootLabel) { + throw new Error(`${IMPORT_ERROR_CODES.LABEL_INVALID}: rootLabel required for records files`) + } + return file.rootLabel +} + +function assertLinkSpec(spec: ImportLinkSpec | null): asserts spec is ImportLinkSpec { + if (!spec || !Array.isArray(spec.endpoints) || spec.endpoints.length !== 2) { + throw new Error(`${IMPORT_ERROR_CODES.LINK_SPEC_INVALID}: link spec missing or malformed`) + } +} + +export function readCheckpoint(file: ImportRunFileRow): ImportCheckpoint { + if (!file.checkpoint) { + return { version: 1, sourceGeneration: file.sourceGeneration, nextUnit: 0, lastCommittedBatch: -1 } + } + try { + const parsed = JSON.parse(file.checkpoint) as ImportCheckpoint + if (parsed.version !== 1 || typeof parsed.nextUnit !== 'number') { + throw new Error('unsupported checkpoint shape') + } + return parsed + } catch { + throw new Error(`${IMPORT_ERROR_CODES.INTERNAL}: unreadable checkpoint`) + } +} + +function readCounters(file: ImportRunFileRow): BatchCounters { + return { + parsedUnits: file.parsedUnits ?? 0, + committedUnits: file.committedUnits ?? 0, + recordsCommitted: file.recordsCommitted ?? 0, + relationshipsCommitted: file.relationshipsCommitted ?? 0, + linksResolved: file.linksResolved ?? 0, + linksUnresolved: file.linksUnresolved ?? 0, + skippedUnits: file.skippedUnits ?? 0 + } +} + +function extractCode(message: string): string | null { + const match = /^([A-Z][A-Z0-9_]+):/.exec(message.trim()) + return match ? match[1] : null +} + +function stringifyValue(value: unknown): string { + if (value === null || value === undefined) { + return '' + } + return String(value) +} + +function uniqueStrings(values: string[]): string[] { + return Array.from(new Set(values)) +} + +function sanitizeSampleMessage(message: string): string { + let sanitized = '' + for (const char of message.slice(0, 500)) { + const code = char.charCodeAt(0) + if (code > 0x1f || code === 0x09) { + sanitized += char + } + } + return sanitized +} diff --git a/platform/core/src/database/sql/migrations/pg/0009_import_runs.sql b/platform/core/src/database/sql/migrations/pg/0009_import_runs.sql new file mode 100644 index 00000000..f8a3718f --- /dev/null +++ b/platform/core/src/database/sql/migrations/pg/0009_import_runs.sql @@ -0,0 +1,122 @@ +CREATE TABLE IF NOT EXISTS "import_runs" ( + "id" text PRIMARY KEY NOT NULL, + "project_id" text NOT NULL, + "workspace_id" text, + "created_by_type" text DEFAULT 'user' NOT NULL, + "created_by_id" text, + "name" text, + "idempotency_key_hash" text, + "status" text DEFAULT 'draft' NOT NULL, + "failure_policy" text DEFAULT 'continue' NOT NULL, + "manifest_version" integer DEFAULT 0 NOT NULL, + "total_files" integer DEFAULT 0 NOT NULL, + "total_bytes" bigint DEFAULT 0 NOT NULL, + "uploaded_bytes" bigint DEFAULT 0 NOT NULL, + "parsed_units" integer DEFAULT 0 NOT NULL, + "records_committed" integer DEFAULT 0 NOT NULL, + "relationships_committed" integer DEFAULT 0 NOT NULL, + "skipped_units" integer DEFAULT 0 NOT NULL, + "failed_files" integer DEFAULT 0 NOT NULL, + "cancel_requested_at" text, + "started_at" text, + "finalized_at" text, + "retention_until" text, + "created_at" text NOT NULL, + "updated_at" text NOT NULL +); +--> statement-breakpoint +ALTER TABLE "import_runs" ADD CONSTRAINT "import_runs_project_id_projects_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."projects"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "import_run_files" ( + "id" text PRIMARY KEY NOT NULL, + "run_id" text NOT NULL, + "project_id" text NOT NULL, + "workspace_id" text, + "ordinal" integer NOT NULL, + "client_file_id" text NOT NULL, + "file_name" text NOT NULL, + "declared_size_bytes" bigint DEFAULT 0 NOT NULL, + "format" text NOT NULL, + "json_shape" text, + "role" text DEFAULT 'records' NOT NULL, + "root_label" text, + "link_spec" text, + "parse_options" text, + "import_options" text, + "source_generation" integer DEFAULT 1 NOT NULL, + "storage_provider" text DEFAULT 'memory' NOT NULL, + "storage_key" text, + "storage_upload_id" text, + "object_size_bytes" bigint, + "checksum_algorithm" text, + "checksum_value" text, + "status" text DEFAULT 'awaiting_upload' NOT NULL, + "stage" text DEFAULT 'upload' NOT NULL, + "processed_bytes" bigint DEFAULT 0 NOT NULL, + "parsed_units" integer DEFAULT 0 NOT NULL, + "committed_units" integer DEFAULT 0 NOT NULL, + "records_committed" integer DEFAULT 0 NOT NULL, + "relationships_committed" integer DEFAULT 0 NOT NULL, + "links_resolved" integer DEFAULT 0 NOT NULL, + "links_unresolved" integer DEFAULT 0 NOT NULL, + "skipped_units" integer DEFAULT 0 NOT NULL, + "current_batch" integer DEFAULT 0 NOT NULL, + "checkpoint" text, + "attempt_count" integer DEFAULT 0 NOT NULL, + "max_attempts" integer DEFAULT 3 NOT NULL, + "not_before" text, + "lease_owner" text, + "lease_generation" integer DEFAULT 0 NOT NULL, + "lease_until" text, + "heartbeat_at" text, + "cancel_requested_at" text, + "last_error_code" text, + "last_error_message" text, + "started_at" text, + "finished_at" text, + "created_at" text NOT NULL, + "updated_at" text NOT NULL +); +--> statement-breakpoint +ALTER TABLE "import_run_files" ADD CONSTRAINT "import_run_files_run_id_import_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."import_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_run_files" ADD CONSTRAINT "import_run_files_project_id_projects_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."projects"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "import_file_client_id_uniq" ON "import_run_files" ("run_id","client_file_id");--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "import_file_ordinal_uniq" ON "import_run_files" ("run_id","ordinal");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "import_file_claim_idx" ON "import_run_files" ("status","lease_until","not_before");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "import_file_role_idx" ON "import_run_files" ("run_id","role");--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "import_run_events" ( + "id" text PRIMARY KEY NOT NULL, + "run_id" text NOT NULL, + "file_id" text, + "project_id" text NOT NULL, + "type" text NOT NULL, + "from_status" text, + "to_status" text, + "code" text, + "message" text, + "attempt" integer, + "lease_generation" integer, + "metadata" text, + "created_at" text NOT NULL +); +--> statement-breakpoint +ALTER TABLE "import_run_events" ADD CONSTRAINT "import_run_events_run_id_import_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."import_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_run_events" ADD CONSTRAINT "import_run_events_project_id_projects_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."projects"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "import_event_run_idx" ON "import_run_events" ("run_id","created_at");--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "import_error_samples" ( + "id" text PRIMARY KEY NOT NULL, + "run_id" text NOT NULL, + "file_id" text NOT NULL, + "project_id" text NOT NULL, + "source_unit" integer, + "line_number" integer, + "column_number" integer, + "code" text NOT NULL, + "message" text, + "created_at" text NOT NULL +); +--> statement-breakpoint +ALTER TABLE "import_error_samples" ADD CONSTRAINT "import_error_samples_run_id_import_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."import_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_error_samples" ADD CONSTRAINT "import_error_samples_file_id_import_run_files_id_fk" FOREIGN KEY ("file_id") REFERENCES "public"."import_run_files"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_error_samples" ADD CONSTRAINT "import_error_samples_project_id_projects_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."projects"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "import_error_sample_file_idx" ON "import_error_samples" ("file_id","created_at");--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "import_run_idempotency_uniq" ON "import_runs" ("project_id","idempotency_key_hash"); diff --git a/platform/core/src/database/sql/migrations/pg/meta/_journal.json b/platform/core/src/database/sql/migrations/pg/meta/_journal.json index 4e925390..100513eb 100644 --- a/platform/core/src/database/sql/migrations/pg/meta/_journal.json +++ b/platform/core/src/database/sql/migrations/pg/meta/_journal.json @@ -64,6 +64,13 @@ "when": 1782752467613, "tag": "0008_living_doctor_spectrum", "breakpoints": true + }, + { + "idx": 9, + "version": "7", + "when": 1787477608831, + "tag": "0009_import_runs", + "breakpoints": true } ] } diff --git a/platform/core/src/database/sql/migrations/sqlite/0009_import_runs.sql b/platform/core/src/database/sql/migrations/sqlite/0009_import_runs.sql new file mode 100644 index 00000000..5c0718e5 --- /dev/null +++ b/platform/core/src/database/sql/migrations/sqlite/0009_import_runs.sql @@ -0,0 +1,122 @@ +CREATE TABLE IF NOT EXISTS `import_runs` ( + `id` text PRIMARY KEY NOT NULL, + `project_id` text NOT NULL, + `workspace_id` text, + `created_by_type` text DEFAULT 'user' NOT NULL, + `created_by_id` text, + `name` text, + `idempotency_key_hash` text, + `status` text DEFAULT 'draft' NOT NULL, + `failure_policy` text DEFAULT 'continue' NOT NULL, + `manifest_version` integer DEFAULT 0 NOT NULL, + `total_files` integer DEFAULT 0 NOT NULL, + `total_bytes` integer DEFAULT 0 NOT NULL, + `uploaded_bytes` integer DEFAULT 0 NOT NULL, + `parsed_units` integer DEFAULT 0 NOT NULL, + `records_committed` integer DEFAULT 0 NOT NULL, + `relationships_committed` integer DEFAULT 0 NOT NULL, + `skipped_units` integer DEFAULT 0 NOT NULL, + `failed_files` integer DEFAULT 0 NOT NULL, + `cancel_requested_at` text, + `started_at` text, + `finalized_at` text, + `retention_until` text, + `created_at` text NOT NULL, + `updated_at` text NOT NULL, + FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS `import_run_files` ( + `id` text PRIMARY KEY NOT NULL, + `run_id` text NOT NULL, + `project_id` text NOT NULL, + `workspace_id` text, + `ordinal` integer NOT NULL, + `client_file_id` text NOT NULL, + `file_name` text NOT NULL, + `declared_size_bytes` integer DEFAULT 0 NOT NULL, + `format` text NOT NULL, + `json_shape` text, + `role` text DEFAULT 'records' NOT NULL, + `root_label` text, + `link_spec` text, + `parse_options` text, + `import_options` text, + `source_generation` integer DEFAULT 1 NOT NULL, + `storage_provider` text DEFAULT 'memory' NOT NULL, + `storage_key` text, + `storage_upload_id` text, + `object_size_bytes` integer, + `checksum_algorithm` text, + `checksum_value` text, + `status` text DEFAULT 'awaiting_upload' NOT NULL, + `stage` text DEFAULT 'upload' NOT NULL, + `processed_bytes` integer DEFAULT 0 NOT NULL, + `parsed_units` integer DEFAULT 0 NOT NULL, + `committed_units` integer DEFAULT 0 NOT NULL, + `records_committed` integer DEFAULT 0 NOT NULL, + `relationships_committed` integer DEFAULT 0 NOT NULL, + `links_resolved` integer DEFAULT 0 NOT NULL, + `links_unresolved` integer DEFAULT 0 NOT NULL, + `skipped_units` integer DEFAULT 0 NOT NULL, + `current_batch` integer DEFAULT 0 NOT NULL, + `checkpoint` text, + `attempt_count` integer DEFAULT 0 NOT NULL, + `max_attempts` integer DEFAULT 3 NOT NULL, + `not_before` text, + `lease_owner` text, + `lease_generation` integer DEFAULT 0 NOT NULL, + `lease_until` text, + `heartbeat_at` text, + `cancel_requested_at` text, + `last_error_code` text, + `last_error_message` text, + `started_at` text, + `finished_at` text, + `created_at` text NOT NULL, + `updated_at` text NOT NULL, + FOREIGN KEY (`run_id`) REFERENCES `import_runs`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS `import_file_client_id_uniq` ON `import_run_files` (`run_id`,`client_file_id`);--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS `import_file_ordinal_uniq` ON `import_run_files` (`run_id`,`ordinal`);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `import_file_claim_idx` ON `import_run_files` (`status`,`lease_until`,`not_before`);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `import_file_role_idx` ON `import_run_files` (`run_id`,`role`);--> statement-breakpoint +CREATE TABLE IF NOT EXISTS `import_run_events` ( + `id` text PRIMARY KEY NOT NULL, + `run_id` text NOT NULL, + `file_id` text, + `project_id` text NOT NULL, + `type` text NOT NULL, + `from_status` text, + `to_status` text, + `code` text, + `message` text, + `attempt` integer, + `lease_generation` integer, + `metadata` text, + `created_at` text NOT NULL, + FOREIGN KEY (`run_id`) REFERENCES `import_runs`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `import_event_run_idx` ON `import_run_events` (`run_id`,`created_at`);--> statement-breakpoint +CREATE TABLE IF NOT EXISTS `import_error_samples` ( + `id` text PRIMARY KEY NOT NULL, + `run_id` text NOT NULL, + `file_id` text NOT NULL, + `project_id` text NOT NULL, + `source_unit` integer, + `line_number` integer, + `column_number` integer, + `code` text NOT NULL, + `message` text, + `created_at` text NOT NULL, + FOREIGN KEY (`run_id`) REFERENCES `import_runs`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`file_id`) REFERENCES `import_run_files`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `import_error_sample_file_idx` ON `import_error_samples` (`file_id`,`created_at`);--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS `import_run_idempotency_uniq` ON `import_runs` (`project_id`,`idempotency_key_hash`); diff --git a/platform/core/src/database/sql/migrations/sqlite/meta/_journal.json b/platform/core/src/database/sql/migrations/sqlite/meta/_journal.json index e5e7eefc..d79699db 100644 --- a/platform/core/src/database/sql/migrations/sqlite/meta/_journal.json +++ b/platform/core/src/database/sql/migrations/sqlite/meta/_journal.json @@ -64,6 +64,13 @@ "when": 1782752466983, "tag": "0008_sloppy_tattoo", "breakpoints": true + }, + { + "idx": 9, + "version": "7", + "when": 1787477608831, + "tag": "0009_import_runs", + "breakpoints": true } ] } diff --git a/platform/core/src/database/sql/schema/pg.schema.ts b/platform/core/src/database/sql/schema/pg.schema.ts index c15a0281..854400f0 100644 --- a/platform/core/src/database/sql/schema/pg.schema.ts +++ b/platform/core/src/database/sql/schema/pg.schema.ts @@ -353,6 +353,141 @@ export const savedQueries = pgTable('saved_queries', { updatedAt: text('updated_at').notNull() }) +// JSON snapshots in text columns: link_spec, parse_options, import_options, checkpoint, metadata +export const importRuns = pgTable( + 'import_runs', + { + id: text('id').primaryKey(), + projectId: text('project_id') + .notNull() + .references(() => projects.id, { onDelete: 'cascade' }), + workspaceId: text('workspace_id'), + createdByType: text('created_by_type').notNull().default('user'), + createdById: text('created_by_id'), + name: text('name'), + idempotencyKeyHash: text('idempotency_key_hash'), + status: text('status').notNull().default('draft'), + failurePolicy: text('failure_policy').notNull().default('continue'), + manifestVersion: integer('manifest_version').notNull().default(0), + totalFiles: integer('total_files').notNull().default(0), + totalBytes: bigint('total_bytes', { mode: 'number' }).notNull().default(0), + uploadedBytes: bigint('uploaded_bytes', { mode: 'number' }).notNull().default(0), + parsedUnits: integer('parsed_units').notNull().default(0), + recordsCommitted: integer('records_committed').notNull().default(0), + relationshipsCommitted: integer('relationships_committed').notNull().default(0), + skippedUnits: integer('skipped_units').notNull().default(0), + failedFiles: integer('failed_files').notNull().default(0), + cancelRequestedAt: text('cancel_requested_at'), + startedAt: text('started_at'), + finalizedAt: text('finalized_at'), + retentionUntil: text('retention_until'), + createdAt: text('created_at').notNull(), + updatedAt: text('updated_at').notNull() + }, + (t) => [uniqueIndex('import_run_idempotency_uniq').on(t.projectId, t.idempotencyKeyHash)] +) + +export const importRunFiles = pgTable( + 'import_run_files', + { + id: text('id').primaryKey(), + runId: text('run_id') + .notNull() + .references(() => importRuns.id, { onDelete: 'cascade' }), + projectId: text('project_id') + .notNull() + .references(() => projects.id, { onDelete: 'cascade' }), + workspaceId: text('workspace_id'), + ordinal: integer('ordinal').notNull(), + clientFileId: text('client_file_id').notNull(), + fileName: text('file_name').notNull(), + declaredSizeBytes: bigint('declared_size_bytes', { mode: 'number' }).notNull().default(0), + format: text('format').notNull(), + jsonShape: text('json_shape'), + role: text('role').notNull().default('records'), + rootLabel: text('root_label'), + linkSpec: text('link_spec'), + parseOptions: text('parse_options'), + importOptions: text('import_options'), + sourceGeneration: integer('source_generation').notNull().default(1), + storageProvider: text('storage_provider').notNull().default('memory'), + storageKey: text('storage_key'), + storageUploadId: text('storage_upload_id'), + objectSizeBytes: bigint('object_size_bytes', { mode: 'number' }), + checksumAlgorithm: text('checksum_algorithm'), + checksumValue: text('checksum_value'), + status: text('status').notNull().default('awaiting_upload'), + stage: text('stage').notNull().default('upload'), + processedBytes: bigint('processed_bytes', { mode: 'number' }).notNull().default(0), + parsedUnits: integer('parsed_units').notNull().default(0), + committedUnits: integer('committed_units').notNull().default(0), + recordsCommitted: integer('records_committed').notNull().default(0), + relationshipsCommitted: integer('relationships_committed').notNull().default(0), + linksResolved: integer('links_resolved').notNull().default(0), + linksUnresolved: integer('links_unresolved').notNull().default(0), + skippedUnits: integer('skipped_units').notNull().default(0), + currentBatch: integer('current_batch').notNull().default(0), + checkpoint: text('checkpoint'), + attemptCount: integer('attempt_count').notNull().default(0), + maxAttempts: integer('max_attempts').notNull().default(3), + notBefore: text('not_before'), + leaseOwner: text('lease_owner'), + leaseGeneration: integer('lease_generation').notNull().default(0), + leaseUntil: text('lease_until'), + heartbeatAt: text('heartbeat_at'), + cancelRequestedAt: text('cancel_requested_at'), + lastErrorCode: text('last_error_code'), + lastErrorMessage: text('last_error_message'), + startedAt: text('started_at'), + finishedAt: text('finished_at'), + createdAt: text('created_at').notNull(), + updatedAt: text('updated_at').notNull() + }, + (t) => [ + uniqueIndex('import_file_client_id_uniq').on(t.runId, t.clientFileId), + uniqueIndex('import_file_ordinal_uniq').on(t.runId, t.ordinal) + ] +) + +export const importRunEvents = pgTable('import_run_events', { + id: text('id').primaryKey(), + runId: text('run_id') + .notNull() + .references(() => importRuns.id, { onDelete: 'cascade' }), + fileId: text('file_id'), + projectId: text('project_id') + .notNull() + .references(() => projects.id, { onDelete: 'cascade' }), + type: text('type').notNull(), + fromStatus: text('from_status'), + toStatus: text('to_status'), + code: text('code'), + message: text('message'), + attempt: integer('attempt'), + leaseGeneration: integer('lease_generation'), + metadata: text('metadata'), + createdAt: text('created_at').notNull() +}) + +export const importErrorSamples = pgTable('import_error_samples', { + id: text('id').primaryKey(), + runId: text('run_id') + .notNull() + .references(() => importRuns.id, { onDelete: 'cascade' }), + fileId: text('file_id') + .notNull() + .references(() => importRunFiles.id, { onDelete: 'cascade' }), + projectId: text('project_id') + .notNull() + .references(() => projects.id, { onDelete: 'cascade' }), + sourceUnit: integer('source_unit'), + lineNumber: integer('line_number'), + columnNumber: integer('column_number'), + code: text('code').notNull(), + message: text('message'), + createdAt: text('created_at').notNull() +}) + export const pgSchema = { users, workspaces, @@ -375,7 +510,11 @@ export const pgSchema = { connectorOffsets, connectorEvents, connectorLeases, - savedQueries + savedQueries, + importRuns, + importRunFiles, + importRunEvents, + importErrorSamples } export type PgSchema = typeof pgSchema diff --git a/platform/core/src/database/sql/schema/sqlite.schema.ts b/platform/core/src/database/sql/schema/sqlite.schema.ts index 4bb73ad5..fe28b638 100644 --- a/platform/core/src/database/sql/schema/sqlite.schema.ts +++ b/platform/core/src/database/sql/schema/sqlite.schema.ts @@ -359,6 +359,141 @@ export const savedQueries = sqliteTable('saved_queries', { updatedAt: text('updated_at').notNull() }) +// JSON snapshots in text columns: link_spec, parse_options, import_options, checkpoint, metadata +export const importRuns = sqliteTable( + 'import_runs', + { + id: text('id').primaryKey(), + projectId: text('project_id') + .notNull() + .references(() => projects.id, { onDelete: 'cascade' }), + workspaceId: text('workspace_id'), + createdByType: text('created_by_type').notNull().default('user'), + createdById: text('created_by_id'), + name: text('name'), + idempotencyKeyHash: text('idempotency_key_hash'), + status: text('status').notNull().default('draft'), + failurePolicy: text('failure_policy').notNull().default('continue'), + manifestVersion: integer('manifest_version').notNull().default(0), + totalFiles: integer('total_files').notNull().default(0), + totalBytes: integer('total_bytes').notNull().default(0), + uploadedBytes: integer('uploaded_bytes').notNull().default(0), + parsedUnits: integer('parsed_units').notNull().default(0), + recordsCommitted: integer('records_committed').notNull().default(0), + relationshipsCommitted: integer('relationships_committed').notNull().default(0), + skippedUnits: integer('skipped_units').notNull().default(0), + failedFiles: integer('failed_files').notNull().default(0), + cancelRequestedAt: text('cancel_requested_at'), + startedAt: text('started_at'), + finalizedAt: text('finalized_at'), + retentionUntil: text('retention_until'), + createdAt: text('created_at').notNull(), + updatedAt: text('updated_at').notNull() + }, + (t) => [uniqueIndex('import_run_idempotency_uniq').on(t.projectId, t.idempotencyKeyHash)] +) + +export const importRunFiles = sqliteTable( + 'import_run_files', + { + id: text('id').primaryKey(), + runId: text('run_id') + .notNull() + .references(() => importRuns.id, { onDelete: 'cascade' }), + projectId: text('project_id') + .notNull() + .references(() => projects.id, { onDelete: 'cascade' }), + workspaceId: text('workspace_id'), + ordinal: integer('ordinal').notNull(), + clientFileId: text('client_file_id').notNull(), + fileName: text('file_name').notNull(), + declaredSizeBytes: integer('declared_size_bytes').notNull().default(0), + format: text('format').notNull(), + jsonShape: text('json_shape'), + role: text('role').notNull().default('records'), + rootLabel: text('root_label'), + linkSpec: text('link_spec'), + parseOptions: text('parse_options'), + importOptions: text('import_options'), + sourceGeneration: integer('source_generation').notNull().default(1), + storageProvider: text('storage_provider').notNull().default('memory'), + storageKey: text('storage_key'), + storageUploadId: text('storage_upload_id'), + objectSizeBytes: integer('object_size_bytes'), + checksumAlgorithm: text('checksum_algorithm'), + checksumValue: text('checksum_value'), + status: text('status').notNull().default('awaiting_upload'), + stage: text('stage').notNull().default('upload'), + processedBytes: integer('processed_bytes').notNull().default(0), + parsedUnits: integer('parsed_units').notNull().default(0), + committedUnits: integer('committed_units').notNull().default(0), + recordsCommitted: integer('records_committed').notNull().default(0), + relationshipsCommitted: integer('relationships_committed').notNull().default(0), + linksResolved: integer('links_resolved').notNull().default(0), + linksUnresolved: integer('links_unresolved').notNull().default(0), + skippedUnits: integer('skipped_units').notNull().default(0), + currentBatch: integer('current_batch').notNull().default(0), + checkpoint: text('checkpoint'), + attemptCount: integer('attempt_count').notNull().default(0), + maxAttempts: integer('max_attempts').notNull().default(3), + notBefore: text('not_before'), + leaseOwner: text('lease_owner'), + leaseGeneration: integer('lease_generation').notNull().default(0), + leaseUntil: text('lease_until'), + heartbeatAt: text('heartbeat_at'), + cancelRequestedAt: text('cancel_requested_at'), + lastErrorCode: text('last_error_code'), + lastErrorMessage: text('last_error_message'), + startedAt: text('started_at'), + finishedAt: text('finished_at'), + createdAt: text('created_at').notNull(), + updatedAt: text('updated_at').notNull() + }, + (t) => [ + uniqueIndex('import_file_client_id_uniq').on(t.runId, t.clientFileId), + uniqueIndex('import_file_ordinal_uniq').on(t.runId, t.ordinal) + ] +) + +export const importRunEvents = sqliteTable('import_run_events', { + id: text('id').primaryKey(), + runId: text('run_id') + .notNull() + .references(() => importRuns.id, { onDelete: 'cascade' }), + fileId: text('file_id'), + projectId: text('project_id') + .notNull() + .references(() => projects.id, { onDelete: 'cascade' }), + type: text('type').notNull(), + fromStatus: text('from_status'), + toStatus: text('to_status'), + code: text('code'), + message: text('message'), + attempt: integer('attempt'), + leaseGeneration: integer('lease_generation'), + metadata: text('metadata'), + createdAt: text('created_at').notNull() +}) + +export const importErrorSamples = sqliteTable('import_error_samples', { + id: text('id').primaryKey(), + runId: text('run_id') + .notNull() + .references(() => importRuns.id, { onDelete: 'cascade' }), + fileId: text('file_id') + .notNull() + .references(() => importRunFiles.id, { onDelete: 'cascade' }), + projectId: text('project_id') + .notNull() + .references(() => projects.id, { onDelete: 'cascade' }), + sourceUnit: integer('source_unit'), + lineNumber: integer('line_number'), + columnNumber: integer('column_number'), + code: text('code').notNull(), + message: text('message'), + createdAt: text('created_at').notNull() +}) + export const sqliteSchema = { users, workspaces, @@ -381,7 +516,11 @@ export const sqliteSchema = { connectorOffsets, connectorEvents, connectorLeases, - savedQueries + savedQueries, + importRuns, + importRunFiles, + importRunEvents, + importErrorSamples } export type SqliteSchema = typeof sqliteSchema diff --git a/platform/core/src/database/sql/schema/types.ts b/platform/core/src/database/sql/schema/types.ts index eb85221b..f9eb917f 100644 --- a/platform/core/src/database/sql/schema/types.ts +++ b/platform/core/src/database/sql/schema/types.ts @@ -70,3 +70,15 @@ export type InsertConnectorLeaseRow = typeof sqliteSchema.connectorLeases.$infer export type SavedQueryRow = typeof sqliteSchema.savedQueries.$inferSelect export type InsertSavedQueryRow = typeof sqliteSchema.savedQueries.$inferInsert + +export type ImportRunRow = typeof sqliteSchema.importRuns.$inferSelect +export type InsertImportRunRow = typeof sqliteSchema.importRuns.$inferInsert + +export type ImportRunFileRow = typeof sqliteSchema.importRunFiles.$inferSelect +export type InsertImportRunFileRow = typeof sqliteSchema.importRunFiles.$inferInsert + +export type ImportRunEventRow = typeof sqliteSchema.importRunEvents.$inferSelect +export type InsertImportRunEventRow = typeof sqliteSchema.importRunEvents.$inferInsert + +export type ImportErrorSampleRow = typeof sqliteSchema.importErrorSamples.$inferSelect +export type InsertImportErrorSampleRow = typeof sqliteSchema.importErrorSamples.$inferInsert diff --git a/platform/dashboard/src/features/imports/components/ImportRunDetail.tsx b/platform/dashboard/src/features/imports/components/ImportRunDetail.tsx new file mode 100644 index 00000000..8d7193f1 --- /dev/null +++ b/platform/dashboard/src/features/imports/components/ImportRunDetail.tsx @@ -0,0 +1,223 @@ +import { useStore } from '@nanostores/react' +import { ArrowLeft, RotateCcw, Trash2, XCircle } from 'lucide-react' + +import { Button } from '~/elements/Button' +import { Card, CardBody } from '~/elements/Card' +import { ConfirmDialog } from '~/elements/ConfirmDialog' +import { IconButton } from '~/elements/IconButton' +import { NothingFound } from '~/elements/NothingFound' +import { Spinner } from '~/elements/Spinner' +import { formatIsoToLocalDateTime } from '~/lib/formatters' +import { getRoutePath } from '~/lib/router' +import { $currentProjectId } from '~/features/projects/stores/id' + +import { formatBytes, isRunActive, isTerminalStatus } from '../lib' +import type { ImportRunDetail, ImportRunFile } from '../types' +import { + useCancelImportMutation, + useDeleteImportMutation, + useRetryImportMutation +} from '../hooks/useImportMutations' +import { useImportRunQuery } from '../hooks/useImportQueries' +import { StatusBadge, formatRunStatus, runStatusTone } from './StatusBadge' + +function isWaitingForData(file: ImportRunFile): boolean { + return Boolean(file.waitingOn && file.waitingOn.length > 0) +} + +function FileRow({ file }: { file: ImportRunFile }) { + const waiting = isWaitingForData(file) + return ( +
  • +
    +
    + + {file.fileName} + + {formatRunStatus(file.stage || file.status)} + + + + {file.role === 'links' ? 'Relationships' : `Records · ${file.rootLabel ?? '—'}`} · {file.format} ·{' '} + {formatBytes(file.declaredSizeBytes)} + +
    +
    + {file.recordsCommitted.toLocaleString()} records + {file.relationshipsCommitted.toLocaleString()} relationships +
    +
    + +
    + parsed: {file.parsedUnits.toLocaleString()} + committed: {file.committedUnits.toLocaleString()} + skipped: {file.skippedUnits.toLocaleString()} + {file.role === 'links' && ( + <> + links resolved: {file.linksResolved.toLocaleString()} + links unresolved: {file.linksUnresolved.toLocaleString()} + + )} +
    + + {waiting && ( +

    + Waiting for data files before relationship records can be linked. +

    + )} + {file.lastErrorMessage &&

    {file.lastErrorMessage}

    } +
    • + ) +} + +function fileStatusTone( + file: ImportRunFile +): 'neutral' | 'success' | 'danger' | 'active' | 'info' | 'warning' { + const s = file.stage || file.status + if (/error|failed|blocked/i.test(s)) return 'danger' + if (/complete|committed/i.test(s)) return 'success' + if (/waiting|queued|pending/i.test(s) || isWaitingForData(file)) return 'info' + if (/running|parsing|finaliz/i.test(s)) return 'active' + return 'neutral' +} + +function RunSummary({ detail }: { detail: ImportRunDetail }) { + const stats: Array<{ label: string; value: string }> = [ + { label: 'Files', value: String(detail.totalFiles) }, + { label: 'Parsed units', value: detail.parsedUnits.toLocaleString() }, + { label: 'Records committed', value: detail.recordsCommitted.toLocaleString() }, + { label: 'Relationships committed', value: detail.relationshipsCommitted.toLocaleString() }, + { label: 'Failed files', value: String(detail.failedFiles) } + ] + return ( + + + {stats.map((s) => ( +
    + {s.label} + {s.value} +
    + ))} +     +     + ) +} + +function EventsList({ detail }: { detail: ImportRunDetail }) { + const events = detail.events ?? [] + if (events.length === 0) return null + return ( +
    +

    Events

    + +
      + {events.map((event) => ( +
    • + + {formatIsoToLocalDateTime(event.createdAt)} + + + {event.type} + {event.message ? ` — ${event.message}` : ''} + +
      • + ))} +
    +     +
    + ) +} + +export function ImportRunDetail({ runId }: { runId: string }) { + const projectId = useStore($currentProjectId) + const cancel = useCancelImportMutation() + const retry = useRetryImportMutation() + const remove = useDeleteImportMutation() + const { data: detail, isPending, isError } = useImportRunQuery(runId) + + if (isPending) { + return ( +
    + +
    + ) + } + + if (isError || !detail) { + return + } + + const active = isRunActive(detail.status) + const terminal = isTerminalStatus(detail.status) + + return ( +
    +
    +
    + + + +
    +
    +

    {detail.name || detail.id}

    + {formatRunStatus(detail.status)} +
    + + Created {formatIsoToLocalDateTime(detail.createdAt)} + {detail.startedAt ? ` · Started ${formatIsoToLocalDateTime(detail.startedAt)}` : ''} + {detail.finalizedAt ? ` · Finished ${formatIsoToLocalDateTime(detail.finalizedAt)}` : ''} + +
    +
    + +
    + {active && detail.status !== 'canceling' && ( + + )} + {terminal && ( + + )} + {terminal && ( + remove.mutateAsync(runId)} + title="Delete import run" + description="The import history and any staged files will be removed. Imported records stay in the database." + trigger={ + + } + /> + )} +
    +
    + + + +
    +

    Files

    + +
      + {detail.files.map((file) => ( + + ))} +
    +     +
    + + +
    + ) +} diff --git a/platform/dashboard/src/features/imports/components/ImportsList.tsx b/platform/dashboard/src/features/imports/components/ImportsList.tsx new file mode 100644 index 00000000..aecf031c --- /dev/null +++ b/platform/dashboard/src/features/imports/components/ImportsList.tsx @@ -0,0 +1,141 @@ +import { useStore } from '@nanostores/react' +import { ArrowRight, MoreVertical, RotateCcw, Trash2, XCircle } from 'lucide-react' + +import { $currentProjectId } from '~/features/projects/stores/id' +import { Card } from '~/elements/Card' +import { ConfirmDialog } from '~/elements/ConfirmDialog' +import { IconButton } from '~/elements/IconButton' +import { Menu, MenuItem } from '~/elements/Menu' +import { NothingFound } from '~/elements/NothingFound' +import { Skeleton } from '~/elements/Skeleton' +import { formatIsoToLocalDateTime } from '~/lib/formatters' +import { getRoutePath } from '~/lib/router' +import { cn } from '~/lib/utils' + +import { + useCancelImportMutation, + useDeleteImportMutation, + useRetryImportMutation +} from '../hooks/useImportMutations' +import { formatBytes, isRunActive, isTerminalStatus } from '../lib' +import type { ImportRun } from '../types' +import { StatusBadge, formatRunStatus, runStatusTone } from './StatusBadge' + +function RunRow({ run, loading }: { run?: ImportRun; loading?: boolean }) { + const projectId = useStore($currentProjectId) + const cancel = useCancelImportMutation() + const retry = useRetryImportMutation() + const remove = useDeleteImportMutation() + + const terminal = run ? isTerminalStatus(run.status) : true + const active = run ? isRunActive(run.status) : false + const name = run?.name || run?.id + + return ( +
  • +
    + + + {name} + + {run && {formatRunStatus(run.status)}} + +
    + + {run ? `${run.totalFiles} file${run.totalFiles === 1 ? '' : 's'}` : ''} + + + {run ? formatBytes(run.totalBytes) : ''} + + + {run ? `${run.recordsCommitted.toLocaleString()} records` : ''} + + + {run ? `${run.relationshipsCommitted.toLocaleString()} relationships` : ''} + + {run && {formatIsoToLocalDateTime(run.createdAt)}} +
    +
    + + {run && ( + + + + } + > + } + href={getRoutePath('projectImportsRun', { id: projectId!, runId: run.id })} + > + Open detail + + {active && run.status !== 'canceling' && ( + cancel.mutateAsync(run.id)} + title="Cancel import" + description="The running import will be stopped. Already committed records are kept." + trigger={ + } variant="danger"> + Cancel + + } + /> + )} + {terminal && ( + retry.mutateAsync(run.id)} + title="Retry import" + description="This will re-queue the import run to process any files that did not complete." + trigger={ + }> + Retry + + } + /> + )} + {terminal && ( + remove.mutateAsync(run.id)} + title="Delete import run" + description="The import history and any staged files will be removed. Imported records stay in the database." + trigger={ + } variant="danger"> + Delete + + } + /> + )} + + )} +
    • + ) +} + +export function ImportsList({ + className, + data, + loading +}: { + className?: string + data?: ImportRun[] + loading: boolean +}) { + if (data && data.length < 1) { + return + } + + return ( + +
      + {data?.map((run) => )} + {loading ? + + : null} +
    +     + ) +} diff --git a/platform/dashboard/src/features/imports/components/NewImport.tsx b/platform/dashboard/src/features/imports/components/NewImport.tsx new file mode 100644 index 00000000..d80246c6 --- /dev/null +++ b/platform/dashboard/src/features/imports/components/NewImport.tsx @@ -0,0 +1,595 @@ +import { useState } from 'react' +import { useStore } from '@nanostores/react' +import { Check, FileUp, Plus, Trash2, UploadCloud, X } from 'lucide-react' + +import { Button } from '~/elements/Button' +import { TextField } from '~/elements/Input' +import { Card, CardBody, CardHeader } from '~/elements/Card' +import { SelectField } from '~/elements/Select' +import { cn } from '~/lib/utils' +import { rushDBInstance } from '~/lib/sdk' +import { $currentProjectId } from '~/features/projects/stores/id' +import { $router, getRoutePath } from '~/lib/router' + +import { + detectFormat, + formatBytes, + generateClientFileId, + isSupportedFileName, + readHeaderColumns, + suggestLabel +} from '../lib' +import type { ImportFileFormat, ImportFileManifest } from '../types' + +const ACCEPT = '.csv,.jsonl,.ndjson,.json,.parquet' + +type EndpointDraft = { + column: string + label: string + keyProperty: string +} + +type FileDraft = { + clientFileId: string + file: File + fileName: string + format: ImportFileFormat + size: number + role: 'records' | 'links' + rootLabel: string + headers: string[] + headersLoaded: boolean + relationshipType: string + source: EndpointDraft + target: EndpointDraft + propertyColumns: Record +} + +type UploadState = { + phase: 'idle' | 'uploading' | 'complete' | 'error' + progress: number +} + +function makeDraft(file: File): FileDraft { + const format = detectFormat(file.name) ?? 'csv' + const label = suggestLabel(file.name) + return { + clientFileId: generateClientFileId(), + file, + fileName: file.name, + format, + size: file.size, + role: 'records', + rootLabel: label, + headers: [], + headersLoaded: false, + relationshipType: 'RELATED_TO', + source: { column: '', label, keyProperty: 'id' }, + target: { column: '', label, keyProperty: 'id' }, + propertyColumns: {} + } +} + +function isRecordFileValid(draft: FileDraft): boolean { + return Boolean(draft.rootLabel.trim()) +} + +function isLinksFileValid(draft: FileDraft): boolean { + return Boolean( + draft.relationshipType.trim() && + draft.source.column.trim() && + draft.source.label.trim() && + draft.source.keyProperty.trim() && + draft.target.column.trim() && + draft.target.label.trim() && + draft.target.keyProperty.trim() + ) +} + +function isFileValid(draft: FileDraft): boolean { + return draft.role === 'records' ? isRecordFileValid(draft) : isLinksFileValid(draft) +} + +function RoleToggle({ + role, + onChange +}: { + role: 'records' | 'links' + onChange: (role: 'records' | 'links') => void +}) { + const options: Array<{ value: 'records' | 'links'; label: string }> = [ + { value: 'records', label: 'Data' }, + { value: 'links', label: 'Relationships' } + ] + return ( +
    + {options.map((opt) => { + const active = role === opt.value + return ( + + ) + })} +
    + ) +} + +function EndpointEditor({ + title, + endpoint, + headers, + onChange +}: { + title: string + endpoint: EndpointDraft + headers: string[] + onChange: (patch: Partial) => void +}) { + const headerOptions = headers.map((h) => ({ value: h, label: h })) + return ( +
    +

    {title}

    +
    + {headers.length ? + onChange({ column: e.target.value })} + options={headerOptions} + /> + : onChange({ column: e.target.value })} + /> + } + onChange({ label: e.target.value })} + /> + onChange({ keyProperty: e.target.value })} + /> +
    +
    + ) +} + +function FileEditor({ + draft, + onRemove, + onRoleChange, + onRootLabel, + onRelationshipType, + onEndpoint, + onPropertyColumn +}: { + draft: FileDraft + onRemove: () => void + onRoleChange: (role: 'records' | 'links') => void + onRootLabel: (value: string) => void + onRelationshipType: (value: string) => void + onEndpoint: (endpoint: 'source' | 'target', patch: Partial) => void + onPropertyColumn: (column: string, property: string) => void +}) { + const valid = isFileValid(draft) + + return ( + + + + {draft.fileName} + + {draft.format} · {formatBytes(draft.size)} + + + {valid ? + + : } + + + } + > +
    + + +
    +     + + + {draft.role === 'records' ? +
    + onRootLabel(e.target.value)} + /> +
    + :
    +
    + onRelationshipType(e.target.value)} + /> +
    + onEndpoint('source', patch)} + /> + onEndpoint('target', patch)} + /> + {draft.headers.length > 0 && ( +
    +

    Property columns

    +

    + Optional: map link file columns to relationship properties. Blank = skip. +

    +
    + {draft.headers.map((header) => ( +
    + {header} + onPropertyColumn(header, e.target.value)} + /> +
    + ))} +
    +
    + )} +
    + } +     +     + ) +} + +export function NewImport() { + const projectId = useStore($currentProjectId) + const [files, setFiles] = useState([]) + const [runName, setRunName] = useState('') + const [failurePolicy, setFailurePolicy] = useState<'continue' | 'stop_new_files'>('continue') + const [isDragging, setIsDragging] = useState(false) + const [fileError, setFileError] = useState() + const [submitting, setSubmitting] = useState(false) + const [submitError, setSubmitError] = useState() + const [uploads, setUploads] = useState>({}) + + const canStart = files.length > 0 && files.every(isFileValid) && !submitting + + const applyFiles = (list: FileList | File[]) => { + const incoming = Array.from(list) + const rejected = incoming.filter((f) => !isSupportedFileName(f.name)) + if (rejected.length) { + setFileError( + `Unsupported file type: ${rejected.map((f) => f.name).join(', ')}. Use .csv, .jsonl, .ndjson, .json, or .parquet.` + ) + return + } + setFileError(undefined) + const drafts = incoming.map(makeDraft) + setFiles((prev) => [...prev, ...drafts]) + drafts.forEach((draft) => { + readHeaderColumns(draft.file) + .then((headers) => { + setFiles((prev) => + prev.map((p) => + p.clientFileId === draft.clientFileId ? + { + ...p, + headers, + headersLoaded: true, + source: { ...p.source, column: p.source.column || headers[0] || '' }, + target: { ...p.target, column: p.target.column || headers[1] || '' } + } + : p + ) + ) + }) + .catch(() => { + setFiles((prev) => + prev.map((p) => (p.clientFileId === draft.clientFileId ? { ...p, headersLoaded: true } : p)) + ) + }) + }) + } + + const removeFile = (clientFileId: string) => { + setFiles((prev) => prev.filter((f) => f.clientFileId !== clientFileId)) + setUploads((prev) => { + const next = { ...prev } + delete next[clientFileId] + return next + }) + } + + const patchFile = (clientFileId: string, patch: Partial) => { + setFiles((prev) => prev.map((f) => (f.clientFileId === clientFileId ? { ...f, ...patch } : f))) + } + + const buildManifests = (): ImportFileManifest[] => + files.map((d) => + d.role === 'records' ? + { + clientFileId: d.clientFileId, + fileName: d.fileName, + size: d.size, + format: d.format, + role: 'records' as const, + rootLabel: d.rootLabel.trim() + } + : { + clientFileId: d.clientFileId, + fileName: d.fileName, + size: d.size, + format: d.format, + role: 'links' as const, + linkSpec: { + version: 1 as const, + role: 'links' as const, + endpoints: [ + { ...d.source, direction: 'source' as const }, + { ...d.target, direction: 'target' as const } + ], + relationshipType: d.relationshipType.trim(), + ...(Object.values(d.propertyColumns).some((v) => v.trim()) ? + { + propertyColumns: Object.fromEntries( + Object.entries(d.propertyColumns) + .filter(([, v]) => v.trim()) + .map(([k, v]) => [k, v.trim()]) + ) + } + : {}) + } + } + ) + + const handleStart = async () => { + if (!canStart || !projectId) return + setSubmitting(true) + setSubmitError(undefined) + setUploads(Object.fromEntries(files.map((f) => [f.clientFileId, { phase: 'idle', progress: 0 }]))) + try { + const manifests = buildManifests() + const created = await rushDBInstance.imports.create({ + name: runName.trim() || undefined, + failurePolicy, + files: manifests + }) + const runId = created.runId + const fileIdByClient = new Map(created.files.map((f) => [f.clientFileId, f.fileId])) + + for (const draft of files) { + const fileId = fileIdByClient.get(draft.clientFileId) + if (!fileId) { + setUploads((prev) => ({ ...prev, [draft.clientFileId]: { phase: 'error', progress: 0 } })) + continue + } + const manifest = manifests.find((m) => m.clientFileId === draft.clientFileId)! + const manifestWithId = { ...manifest, fileId } + setUploads((prev) => ({ ...prev, [draft.clientFileId]: { phase: 'uploading', progress: 0 } })) + try { + const initiated = await rushDBInstance.imports.uploads.initiate(runId, fileId) + const direct = Boolean((initiated as { directUpload?: boolean } | undefined)?.directUpload) + // The probing initiate above created an upload; abandon it and let the chosen + // upload path manage its own lifecycle to avoid leaving a dangling upload. + if (initiated) { + await rushDBInstance.imports.uploads + .abort(runId, fileId, initiated.uploadId) + .catch(() => undefined) + } + if (direct) { + await rushDBInstance.imports.uploadFileDirect(runId, manifestWithId, draft.file, { + onProgress: (p) => { + const percent = p.totalBytes ? Math.round((p.bytesUploaded / p.totalBytes) * 100) : 0 + setUploads((prev) => ({ + ...prev, + [draft.clientFileId]: { phase: 'uploading', progress: percent } + })) + } + }) + } else { + await rushDBInstance.imports.uploadContent(runId, manifestWithId, draft.file) + } + setUploads((prev) => ({ ...prev, [draft.clientFileId]: { phase: 'complete', progress: 100 } })) + } catch { + setUploads((prev) => ({ ...prev, [draft.clientFileId]: { phase: 'error', progress: 0 } })) + throw new Error(`Upload failed for ${draft.fileName}`) + } + } + + await rushDBInstance.imports.start(runId) + $router.open(getRoutePath('projectImportsRun', { id: projectId, runId })) + } catch (error) { + setSubmitError(error instanceof Error ? error.message : 'Import could not be started') + setSubmitting(false) + } + } + + return ( +
    +
    +

    New import

    +

    + Add one or more files, choose whether each holds records or relationships, then start the run. +

    +
    + +
    + setRunName(e.target.value)} + /> + setFailurePolicy(e.target.value as 'continue' | 'stop_new_files')} + options={[ + { value: 'continue', label: 'Continue with remaining files' }, + { value: 'stop_new_files', label: 'Stop starting new files' } + ]} + /> +
    + +
    + { + if (e.target.files) applyFiles(e.target.files) + e.target.value = '' + }} + /> +
    { + e.preventDefault() + setIsDragging(true) + }} + onDragLeave={(e) => { + e.preventDefault() + setIsDragging(false) + }} + onDrop={(e) => { + e.preventDefault() + setIsDragging(false) + if (e.dataTransfer.files) applyFiles(e.dataTransfer.files) + }} + > +
    + +
    +
    +

    Drag and drop import files

    +

    + CSV, JSONL, NDJSON, JSON, or Parquet. Multiple files allowed. +

    +
    + + {fileError &&

    {fileError}

    } +
    +
    + + {files.length > 0 && ( +
    + {files.map((draft) => ( + removeFile(draft.clientFileId)} + onRoleChange={(role) => patchFile(draft.clientFileId, { role })} + onRootLabel={(value) => patchFile(draft.clientFileId, { rootLabel: value })} + onRelationshipType={(value) => patchFile(draft.clientFileId, { relationshipType: value })} + onEndpoint={(endpoint, patch) => + patchFile(draft.clientFileId, { + [endpoint]: { ...draft[endpoint], ...patch } + } as Partial) + } + onPropertyColumn={(column, property) => + patchFile(draft.clientFileId, { + propertyColumns: { ...draft.propertyColumns, [column]: property } + }) + } + /> + ))} +
    + )} + + {submitError &&

    {submitError}

    } + + {submitting && ( +
    +

    Uploading files…

    + {files.map((draft) => { + const state = uploads[draft.clientFileId] ?? { phase: 'idle', progress: 0 } + return ( +
    + {draft.fileName} +
    +
    +
    + + {state.phase === 'error' ? + 'failed' + : state.phase === 'complete' ? + 'done' + : `${state.progress}%`} + +
    + ) + })} +
    + )} + +
    + + +
    +
    + ) +} diff --git a/platform/dashboard/src/features/imports/components/StatusBadge.tsx b/platform/dashboard/src/features/imports/components/StatusBadge.tsx new file mode 100644 index 00000000..4aa03225 --- /dev/null +++ b/platform/dashboard/src/features/imports/components/StatusBadge.tsx @@ -0,0 +1,66 @@ +import type { ReactNode } from 'react' + +import { cn } from '~/lib/utils' + +type Tone = 'neutral' | 'info' | 'active' | 'warning' | 'danger' | 'success' + +const TONE_CLASSES: Record = { + neutral: 'bg-secondary text-content2', + info: 'bg-badge-blue/15 text-badge-blue', + active: 'bg-accent/15 text-accent', + warning: 'bg-warning/15 text-warning', + danger: 'bg-danger/15 text-danger', + success: 'bg-success/15 text-success' +} + +export function StatusBadge({ + tone = 'neutral', + className, + children +}: { + tone?: Tone + className?: string + children: ReactNode +}) { + return ( + + {children} + + ) +} + +export function runStatusTone(status: string): Tone { + switch (status) { + case 'completed': + return 'success' + case 'completed_with_errors': + return 'warning' + case 'failed': + return 'danger' + case 'canceled': + return 'neutral' + case 'running': + case 'finalizing': + return 'active' + case 'queued': + case 'draft': + case 'uploading': + return 'info' + case 'canceling': + return 'warning' + case 'blocked': + return 'danger' + default: + return 'neutral' + } +} + +export function formatRunStatus(status: string): string { + return status.replace(/_/g, ' ') +} diff --git a/platform/dashboard/src/features/imports/hooks/useImportMutations.ts b/platform/dashboard/src/features/imports/hooks/useImportMutations.ts new file mode 100644 index 00000000..571c5d0c --- /dev/null +++ b/platform/dashboard/src/features/imports/hooks/useImportMutations.ts @@ -0,0 +1,50 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' + +import { toast } from '~/elements/Toast' +import { rushDBInstance } from '~/lib/sdk' + +import { importQueryKeys } from './useImportQueries' + +export function useCancelImportMutation() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: (runId: string) => rushDBInstance.imports.cancel(runId), + onSuccess(_, runId) { + queryClient.invalidateQueries({ queryKey: importQueryKeys.run(runId) }) + queryClient.invalidateQueries({ queryKey: importQueryKeys.runs() }) + toast({ title: 'Import cancellation requested' }) + }, + onError() { + toast({ title: 'Could not cancel import', variant: 'danger' }) + } + }) +} + +export function useRetryImportMutation() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: (runId: string) => rushDBInstance.imports.retry(runId), + onSuccess(_, runId) { + queryClient.invalidateQueries({ queryKey: importQueryKeys.run(runId) }) + queryClient.invalidateQueries({ queryKey: importQueryKeys.runs() }) + toast({ title: 'Import re-queued' }) + }, + onError() { + toast({ title: 'Could not retry import', variant: 'danger' }) + } + }) +} + +export function useDeleteImportMutation() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: (runId: string) => rushDBInstance.imports.remove(runId), + onSuccess() { + queryClient.invalidateQueries({ queryKey: importQueryKeys.runs() }) + toast({ title: 'Import run deleted' }) + }, + onError() { + toast({ title: 'Could not delete import', variant: 'danger' }) + } + }) +} diff --git a/platform/dashboard/src/features/imports/hooks/useImportQueries.ts b/platform/dashboard/src/features/imports/hooks/useImportQueries.ts new file mode 100644 index 00000000..ef1a2f56 --- /dev/null +++ b/platform/dashboard/src/features/imports/hooks/useImportQueries.ts @@ -0,0 +1,29 @@ +import { useQuery } from '@tanstack/react-query' + +import { rushDBInstance } from '~/lib/sdk' + +import { isRunActive } from '../lib' + +export const importQueryKeys = { + runs: () => ['imports', 'runs'] as const, + run: (runId: string) => ['imports', 'runs', runId] as const +} + +export function useImportRunsQuery() { + return useQuery({ + queryKey: importQueryKeys.runs(), + queryFn: () => rushDBInstance.imports.list() + }) +} + +export function useImportRunQuery(runId: string | undefined) { + return useQuery({ + queryKey: importQueryKeys.run(runId ?? ''), + queryFn: () => rushDBInstance.imports.get(runId!), + enabled: Boolean(runId), + refetchInterval: (query) => { + const detail = query.state.data + return detail && isRunActive(detail.status) ? 2000 : false + } + }) +} diff --git a/platform/dashboard/src/features/imports/lib.ts b/platform/dashboard/src/features/imports/lib.ts new file mode 100644 index 00000000..318e9254 --- /dev/null +++ b/platform/dashboard/src/features/imports/lib.ts @@ -0,0 +1,106 @@ +import type { ImportFileFormat } from './types' + +const SUPPORTED_EXTENSIONS = new Set(['csv', 'jsonl', 'ndjson', 'json', 'parquet']) + +export const SUPPORTED_FORMATS: ImportFileFormat[] = ['csv', 'jsonl', 'ndjson', 'json', 'parquet'] + +export function isSupportedFileName(fileName: string): boolean { + const ext = fileName.split('.').pop()?.toLowerCase() + return Boolean(ext && SUPPORTED_EXTENSIONS.has(ext)) +} + +export function detectFormat(fileName: string): ImportFileFormat | null { + const ext = fileName.split('.').pop()?.toLowerCase() + if (!ext || !SUPPORTED_EXTENSIONS.has(ext)) return null + return ext as ImportFileFormat +} + +export function formatBytes(bytes: number): string { + if (!bytes || bytes < 0) return '0 B' + if (bytes < 1024) return `${bytes} B` + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB` + return `${(bytes / (1024 * 1024)).toFixed(2)} MB` +} + +let clientFileIdCounter = 0 +export function generateClientFileId(): string { + clientFileIdCounter += 1 + return `cf-${Date.now().toString(36)}-${clientFileIdCounter}-${Math.random().toString(36).slice(2, 8)}` +} + +/** Uppercase base filename, non-alphanumerics collapsed to underscores. */ +export function suggestLabel(fileName: string): string { + const base = fileName.replace(/\.[^.]+$/, '') + const cleaned = base + .replace(/[^a-zA-Z0-9]+/g, '_') + .replace(/^_+|_+$/g, '') + .toUpperCase() + return cleaned || 'RECORD' +} + +/** + * Parses the first data row of a file into column names, when possible. + * CSV/JSONL/NDJSON are supported; JSON and parquet return an empty list and + * callers should fall back to free-text column entry. + */ +export async function readHeaderColumns(file: File): Promise { + const ext = file.name.split('.').pop()?.toLowerCase() + + if (ext === 'csv') { + const text = await file.slice(0, 64 * 1024).text() + const firstLine = text.replace(/^\uFEFF/, '').split(/\r?\n/, 1)[0] ?? '' + return splitCsvLine(firstLine) + } + + if (ext === 'jsonl' || ext === 'ndjson') { + const text = await file.slice(0, 64 * 1024).text() + const firstLine = text.split(/\r?\n/).find((l) => l.trim().length > 0) + if (!firstLine) return [] + try { + const parsed = JSON.parse(firstLine) + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return Object.keys(parsed) + } + } catch { + return [] + } + } + + return [] +} + +/** Minimal CSV line splitter that respects double-quoted fields. */ +function splitCsvLine(line: string): string[] { + const cells: string[] = [] + let current = '' + let inQuotes = false + + for (let i = 0; i < line.length; i += 1) { + const char = line[i] + if (char === '"') { + if (inQuotes && line[i + 1] === '"') { + current += '"' + i += 1 + } else { + inQuotes = !inQuotes + } + } else if (char === ',' && !inQuotes) { + cells.push(current.trim()) + current = '' + } else { + current += char + } + } + cells.push(current.trim()) + return cells +} + +const TERMINAL_STATUSES = new Set(['completed', 'completed_with_errors', 'failed', 'canceled']) + +export function isTerminalStatus(status: string): boolean { + return TERMINAL_STATUSES.has(status) +} + +export function isRunActive(status: string): boolean { + return !isTerminalStatus(status) +} diff --git a/platform/dashboard/src/features/imports/pages/ImportsPage.tsx b/platform/dashboard/src/features/imports/pages/ImportsPage.tsx new file mode 100644 index 00000000..5becd1cc --- /dev/null +++ b/platform/dashboard/src/features/imports/pages/ImportsPage.tsx @@ -0,0 +1,53 @@ +import { ExternalLink, Plus } from 'lucide-react' + +import { Button } from '~/elements/Button' +import { PageContent, PageHeader, PageTitle } from '~/elements/PageHeader' + +import { useImportRunsQuery } from '../hooks/useImportQueries' +import { ImportsList } from '../components/ImportsList' +import { getRoutePath } from '~/lib/router' +import { useStore } from '@nanostores/react' +import { $currentProjectId } from '~/features/projects/stores/id' + +const IMPORTS_DOCS_URL = 'https://docs.rushdb.com' + +export function ImportsPage() { + const { data: runs, isPending: loading } = useImportRunsQuery() + const projectId = useStore($currentProjectId) + + return ( + <> + +
    + Imports +

    + Upload multiple CSV, JSON, JSONL, NDJSON, or Parquet files as a single import run. Each file is + parsed and committed asynchronously, with per-file progress, and you can define how record files + and relationship files map into the graph. +

    + + Read the docs + +
    + +     + +
    +
    +

    Import runs

    +

    Monitor and manage multi-file imports for this project.

    +
    + +
    +     + + ) +} diff --git a/platform/dashboard/src/features/imports/types.ts b/platform/dashboard/src/features/imports/types.ts new file mode 100644 index 00000000..646f42f9 --- /dev/null +++ b/platform/dashboard/src/features/imports/types.ts @@ -0,0 +1,124 @@ +// Local mirror of the SDK's async-import types (packages/javascript-sdk/src/api/types.ts). +// The SDK does not re-export these from its public entry point, so we keep a copy here. +// Keep in sync with the SDK + backend contract. + +export type ImportFileFormat = 'csv' | 'jsonl' | 'ndjson' | 'json' | 'parquet' + +export type ImportFileRole = 'records' | 'links' + +export interface ImportLinkEndpoint { + column: string + label: string + keyProperty: string + direction: 'source' | 'target' +} + +export interface ImportLinkSpec { + version: 1 + role: 'links' + endpoints: [ImportLinkEndpoint, ImportLinkEndpoint] + relationshipType: string + propertyColumns?: Record +} + +export type ImportFileManifest = + | { + clientFileId: string + fileName: string + size: number + format: ImportFileFormat + role?: 'records' + rootLabel: string + parseOptions?: Record + importOptions?: Record + } + | { + clientFileId: string + fileName: string + size: number + format: ImportFileFormat + role: 'links' + linkSpec: ImportLinkSpec + rootLabel?: never + parseOptions?: Record + importOptions?: Record + } + +export interface CreateImportRunResponse { + runId: string + files: Array<{ fileId: string; clientFileId: string; suggestedLabel: string | null }> +} + +export interface ImportRunFile { + id: string + runId: string + ordinal: number + clientFileId: string + fileName: string + declaredSizeBytes: number + format: ImportFileFormat + role: ImportFileRole + rootLabel: string | null + linkSpec: ImportLinkSpec | null + status: string + stage: string + parsedUnits: number + committedUnits: number + recordsCommitted: number + relationshipsCommitted: number + linksResolved: number + linksUnresolved: number + skippedUnits: number + attemptCount: number + lastErrorCode: string | null + lastErrorMessage: string | null + waitingOn?: Array<{ fileId: string; status: string }> +} + +export interface ImportRunEvent { + id: string + runId: string + fileId: string | null + type: string + code?: string | null + message?: string | null + createdAt: string +} + +export interface ImportRun { + id: string + projectId: string + name: string | null + status: + | 'draft' + | 'uploading' + | 'queued' + | 'running' + | 'blocked' + | 'canceling' + | 'finalizing' + | 'completed' + | 'completed_with_errors' + | 'failed' + | 'canceled' + failurePolicy: 'continue' | 'stop_new_files' + totalFiles: number + totalBytes: number + uploadedBytes: number + parsedUnits: number + recordsCommitted: number + relationshipsCommitted: number + skippedUnits: number + failedFiles: number + cancelRequestedAt: string | null + startedAt: string | null + finalizedAt: string | null + retentionUntil: string | null + createdAt: string + updatedAt: string +} + +export interface ImportRunDetail extends ImportRun { + files: Array + events: Array +} diff --git a/platform/dashboard/src/features/projects/components/ProjectTabs.tsx b/platform/dashboard/src/features/projects/components/ProjectTabs.tsx index fb4207e4..282f9afa 100644 --- a/platform/dashboard/src/features/projects/components/ProjectTabs.tsx +++ b/platform/dashboard/src/features/projects/components/ProjectTabs.tsx @@ -2,6 +2,7 @@ import { Book, Bookmark, Database, + FileText, FlaskConical, Key, Network, @@ -101,8 +102,13 @@ export function ProjectTabs({ const dataManagementTabs: ProjectTab[] = [ { - href: getRoutePath('projectImportData', { id: project.id }), + href: getRoutePath('projectImports', { id: project.id }), icon: , + label: 'Imports' + }, + { + href: getRoutePath('projectImportData', { id: project.id }), + icon: , label: 'Import Data', dataTour: 'project-import-data-chip' }, diff --git a/platform/dashboard/src/layout/ProjectLayout/index.tsx b/platform/dashboard/src/layout/ProjectLayout/index.tsx index 5dd40014..d8a62b9a 100644 --- a/platform/dashboard/src/layout/ProjectLayout/index.tsx +++ b/platform/dashboard/src/layout/ProjectLayout/index.tsx @@ -13,6 +13,9 @@ import { $router, getRoutePath, isProjectPage, redirectRoute } from '~/lib/route import { ProjectConnectedApps, ProjectSettings } from '~/pages/project/settings' import { ProjectTokens } from '~/pages/project/tokens' +import { ProjectImports } from '~/pages/project/imports' +import { ProjectImportsNew } from '~/pages/project/imports-new' +import { ProjectImportsRun } from '~/pages/project/imports-run' import { ProjectIndexes } from '~/pages/project/indexes' import { ProjectLiveSchema } from '~/pages/project/live-schema' import { ProjectQueryLab } from '~/pages/project/query-lab' @@ -49,6 +52,12 @@ function ProjectRoutes({ project }: { project: Project }) { return case 'projectImportData': return + case 'projectImports': + return + case 'projectImportsNew': + return + case 'projectImportsRun': + return case 'projectNewConnection': return case 'projectConnection': @@ -71,6 +80,9 @@ const PROJECT_TAB_TITLES: Record = { projectRelationships: 'Relationships', projectSuggestedRelationships: 'Suggested Relationships', projectImportData: 'Import', + projectImports: 'Imports', + projectImportsNew: 'New Import', + projectImportsRun: 'Import Run', projectLiveSchema: 'Live Schema', projectNewConnection: 'New Connection', projectConnection: 'Connection', diff --git a/platform/dashboard/src/lib/router.ts b/platform/dashboard/src/lib/router.ts index 349190ea..498ed839 100644 --- a/platform/dashboard/src/lib/router.ts +++ b/platform/dashboard/src/lib/router.ts @@ -27,6 +27,9 @@ export const projectRoutes = { projectSettings: '/projects/:id/settings', projectConnectedApps: '/projects/:id/settings/connected-apps', projectImportData: '/projects/:id/import', + projectImports: '/projects/:id/imports', + projectImportsNew: '/projects/:id/imports/new', + projectImportsRun: '/projects/:id/imports/:runId', projectLiveSchema: '/projects/:id/live-schema', projectNewConnection: '/projects/:id/connections/new/:sourceType', projectConnection: '/projects/:id/connections/:connectionId', diff --git a/platform/dashboard/src/lib/sdk.ts b/platform/dashboard/src/lib/sdk.ts index c2831a60..a67763d7 100644 --- a/platform/dashboard/src/lib/sdk.ts +++ b/platform/dashboard/src/lib/sdk.ts @@ -80,3 +80,17 @@ export const rushDBInstance = new RushDB(undefined, { url: BASE_URL || document.URL, httpClient: new CustomHttpClient() }) + +// Raw-body fetches (import source uploads bypass the JSON fetcher) need the same +// auth + project/workspace scoping headers as regular API calls. +rushDBInstance.setRawHeadersProvider(() => { + const token = $token.get() + const currentWorkspaceId = $currentWorkspaceId.get() + const currentProjectId = $currentProjectId.get() + + return { + ...(token ? { Authorization: `Bearer ${token}` } : {}), + ...(currentWorkspaceId ? { 'x-workspace-id': currentWorkspaceId } : {}), + ...(currentProjectId ? { 'x-project-id': currentProjectId } : {}) + } +}) diff --git a/platform/dashboard/src/pages/project/imports-new.tsx b/platform/dashboard/src/pages/project/imports-new.tsx new file mode 100644 index 00000000..fb3753e7 --- /dev/null +++ b/platform/dashboard/src/pages/project/imports-new.tsx @@ -0,0 +1,10 @@ +import { PageContent } from '~/elements/PageHeader' +import { NewImport } from '~/features/imports/components/NewImport' + +export function ProjectImportsNew() { + return ( + + + + ) +} diff --git a/platform/dashboard/src/pages/project/imports-run.tsx b/platform/dashboard/src/pages/project/imports-run.tsx new file mode 100644 index 00000000..54642612 --- /dev/null +++ b/platform/dashboard/src/pages/project/imports-run.tsx @@ -0,0 +1,11 @@ +import { useStore } from '@nanostores/react' + +import { $router } from '~/lib/router' +import { ImportRunDetail } from '~/features/imports/components/ImportRunDetail' + +export function ProjectImportsRun() { + const page = useStore($router) + const runId = page?.route === 'projectImportsRun' ? page.params.runId : undefined + if (!runId) return null + return +} diff --git a/platform/dashboard/src/pages/project/imports.tsx b/platform/dashboard/src/pages/project/imports.tsx new file mode 100644 index 00000000..49c6ebfa --- /dev/null +++ b/platform/dashboard/src/pages/project/imports.tsx @@ -0,0 +1,5 @@ +import { ImportsPage } from '~/features/imports/pages/ImportsPage' + +export function ProjectImports() { + return +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 64107a05..c19248ce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -162,6 +162,12 @@ importers: platform/core: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1116.0 + version: 3.1116.0 + '@aws-sdk/s3-request-presigner': + specifier: ^3.1116.0 + version: 3.1116.0 '@fastify/formbody': specifier: 8.0.2 version: 8.0.2 @@ -237,6 +243,9 @@ importers: fastify-raw-body: specifier: ^5.0.0 version: 5.0.0 + hyparquet: + specifier: ^1.29.1 + version: 1.29.1 ms: specifier: ^2.1.3 version: 2.1.3 @@ -646,6 +655,82 @@ packages: resolution: {integrity: sha512-/PZmyAlb2NGWPikRRuiWLdfHQd8Wrx6lX4HqvTcaDhlU43M3T0ud4PH2T3QDp7BzHYY92xtD8iPxX2asg67G1A==} engines: {node: ^18.19.1 || ^20.11.1 || >=22.0.0, npm: ^6.11.0 || ^7.5.6 || >=8.0.0, yarn: '>= 1.13.0'} + '@aws-sdk/checksums@3.1000.29': + resolution: {integrity: sha512-Dtu0gr4dnATZAPwEYbpCsG+MpLM7OAliy2gTepEFQwl1vZ6DL3QMH2FveMa3HLvPsOdhJsPRB3KtxVhph9T75A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-s3@3.1116.0': + resolution: {integrity: sha512-UKRl9qSVW0rZpvSOauQNpYAy8+ONBAVYnpfKVtCyOF+FZVT1tl6MunYuHvuarCrroD2/YJs+tHTALYNgAlec3Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.977.9': + resolution: {integrity: sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.70': + resolution: {integrity: sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.72': + resolution: {integrity: sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.15': + resolution: {integrity: sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.77': + resolution: {integrity: sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.81': + resolution: {integrity: sha512-Rml+WitoFvXmv6JZ18U/xGdGDGGvB/mOin0ya0lTnTrdC0Z1lrVxTYh7iNklZBcvcRMrs4DoEf6xy1KWyrLQQw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.70': + resolution: {integrity: sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.14': + resolution: {integrity: sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.76': + resolution: {integrity: sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.75': + resolution: {integrity: sha512-wMIsNumRVKaNMKhvU/s9VrdEwE8S6gSzXp4RygFG5BEMnGkkXf8cjh8zf7cKJBpUDpqTWqwbz5isEgp9rH6Lng==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.44': + resolution: {integrity: sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/s3-request-presigner@3.1116.0': + resolution: {integrity: sha512-WwaaVpvrZyML5L8SNY7sUGsYlMeCHzQ/8A/ms7dz/7sfYRvPn+qf0OasUWk+zuT8qGwjsYhILD0WVtlqUU08pQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.46': + resolution: {integrity: sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1116.0': + resolution: {integrity: sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.974.5': + resolution: {integrity: sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.40': + resolution: {integrity: sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + '@babel/code-frame@7.26.2': resolution: {integrity: sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==} engines: {node: '>=6.9.0'} @@ -3728,6 +3813,30 @@ packages: '@sinonjs/fake-timers@10.3.0': resolution: {integrity: sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==} + '@smithy/core@3.33.3': + resolution: {integrity: sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.5.2': + resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.7.2': + resolution: {integrity: sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.11.3': + resolution: {integrity: sha512-2jY1tSpERfPfWqyBV2pH+iGFaghVsIJszJNsT7hxtQYhVJpWDyc0LqOWI+nXOxOAHaEfZ4PXXtp1wW1TGpHhkA==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.7.3': + resolution: {integrity: sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.17.2': + resolution: {integrity: sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==} + engines: {node: '>=18.0.0'} + '@stripe/stripe-js@2.4.0': resolution: {integrity: sha512-WFkQx1mbs2b5+7looI9IV1BLa3bIApuN3ehp9FP58xGg7KL9hCHDECgW3BwO9l9L+xBPVAD7Yjn1EhGe6EDTeA==} @@ -4909,6 +5018,9 @@ packages: boolbase@1.0.0: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} @@ -6309,6 +6421,7 @@ packages: eslint@9.2.0: resolution: {integrity: sha512-0n/I88vZpCOzO+PQpt0lbsqmn9AsnsJAQseIqhZFI8ibQT0U1AkEKRxA3EVMos0BoHSXDQvCXY25TUjB5tr8Og==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true esm-env@1.2.2: @@ -7019,6 +7132,9 @@ packages: engines: {node: '>=14'} hasBin: true + hyparquet@1.29.1: + resolution: {integrity: sha512-Wa57A2KxGFtRTeDTJp4pT0TaqN8btb7/XTrMs17iM+xJxsbBxRyzaGaRRPQ4lOPbmso/qohDafJD7b+cFMZq8A==} + iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -11391,6 +11507,180 @@ snapshots: transitivePeerDependencies: - chokidar + '@aws-sdk/checksums@3.1000.29': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.1116.0': + dependencies: + '@aws-sdk/checksums': 3.1000.29 + '@aws-sdk/core': 3.977.9 + '@aws-sdk/credential-provider-node': 3.972.81 + '@aws-sdk/middleware-sdk-s3': 3.972.75 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/core@3.977.9': + dependencies: + '@aws-sdk/types': 3.974.5 + '@aws-sdk/xml-builder': 3.972.40 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.33.3 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.17.2 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.72': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.15': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/credential-provider-env': 3.972.70 + '@aws-sdk/credential-provider-http': 3.972.72 + '@aws-sdk/credential-provider-login': 3.972.77 + '@aws-sdk/credential-provider-process': 3.972.70 + '@aws-sdk/credential-provider-sso': 3.973.14 + '@aws-sdk/credential-provider-web-identity': 3.972.76 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.77': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.81': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.70 + '@aws-sdk/credential-provider-http': 3.972.72 + '@aws-sdk/credential-provider-ini': 3.973.15 + '@aws-sdk/credential-provider-process': 3.972.70 + '@aws-sdk/credential-provider-sso': 3.973.14 + '@aws-sdk/credential-provider-web-identity': 3.972.76 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.14': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/token-providers': 3.1116.0 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.76': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.75': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.44': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/s3-request-presigner@3.1116.0': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.46': + dependencies: + '@aws-sdk/types': 3.974.5 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1116.0': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/types@3.974.5': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.40': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.3.0': {} + '@babel/code-frame@7.26.2': dependencies: '@babel/helper-validator-identifier': 7.25.9 @@ -14566,6 +14856,39 @@ snapshots: dependencies: '@sinonjs/commons': 3.0.1 + '@smithy/core@3.33.3': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.5.2': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.7.2': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.11.3': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/signature-v4@5.7.3': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/types@4.17.2': + dependencies: + tslib: 2.8.1 + '@stripe/stripe-js@2.4.0': {} '@swc/core-darwin-arm64@1.10.1': @@ -15896,6 +16219,8 @@ snapshots: boolbase@1.0.0: {} + bowser@2.14.1: {} + brace-expansion@1.1.11: dependencies: balanced-match: 1.0.2 @@ -18386,6 +18711,8 @@ snapshots: husky@8.0.3: {} + hyparquet@1.29.1: {} + iconv-lite@0.4.24: dependencies: safer-buffer: 2.1.2