diff --git a/scripts/data-profiler.test.mjs b/scripts/data-profiler.test.mjs new file mode 100644 index 0000000..4a0893f --- /dev/null +++ b/scripts/data-profiler.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { build } from 'esbuild'; + +const { outputFiles } = await build({ entryPoints: ['src/utils/dataProfiler.ts'], bundle: true, platform: 'node', format: 'esm', write: false }); +const { profileSample, sampleCoverage } = await import(`data:text/javascript;base64,${Buffer.from(outputFiles[0].contents).toString('base64')}`); + +test('profiles nulls, distinct values, common values, and numeric ranges from sampled rows', () => { + const result = profileSample(['id', 'status', 'score'], [ + [1, 'open', 2], [2, 'open', null], [3, 'closed', 8], [4, null, 4], + ]); + assert.equal(result[1].nullCount, 1); + assert.equal(result[1].distinctCount, 2); + assert.deepEqual(result[1].commonValues, [{ value: 'open', count: 2 }, { value: 'closed', count: 1 }]); + assert.deepEqual(result[2].numericRange, { min: 2, max: 8 }); +}); + +test('never exposes common values or ranges for sensitive columns', () => { + const result = profileSample(['api_key', 'password'], [['secret-1', 123], ['secret-2', 456]]); + assert.deepEqual(result.map(column => column.commonValues), [[], []]); + assert.deepEqual(result.map(column => column.numericRange), [null, null]); + assert.equal(result[0].distinctCount, 2); +}); + +test('handles empty and structured samples without treating objects as display values', () => { + assert.deepEqual(profileSample(['id'], []), [{ name: 'id', nullCount: 0, distinctCount: 0, commonValues: [], numericRange: null }]); + const result = profileSample(['data'], [[{ a: 1 }], [{ a: 1 }], [{ a: 2 }]]); + assert.equal(result[0].distinctCount, 2); + assert.deepEqual(result[0].commonValues, []); +}); + +test('coverage labels bounded samples and full returns without implying randomness', () => { + assert.match(sampleCoverage(100, true, 100), /first 100 rows/i); + assert.match(sampleCoverage(100, true, 100), /not a full-table profile/i); + assert.match(sampleCoverage(3, false, 100), /All 3 rows returned/); +}); diff --git a/src/components/DataProfilerDialog.tsx b/src/components/DataProfilerDialog.tsx new file mode 100644 index 0000000..e9e3312 --- /dev/null +++ b/src/components/DataProfilerDialog.tsx @@ -0,0 +1,128 @@ +import { useEffect, useRef, useState } from 'react'; +import * as Dialog from '@radix-ui/react-dialog'; +import { invoke } from '@tauri-apps/api/core'; +import { BarChart3, Loader2, Search, X } from 'lucide-react'; +import { useDatabaseStore, type FilterConfig, type SortConfig } from '../store/databaseStore'; +import { isSensitiveColumn } from '../utils/sensitiveData'; +import { profileSample, sampleCoverage, type ColumnProfile } from '../utils/dataProfiler'; + +interface SampleResult { + columns: string[]; + rows: unknown[][]; + has_more?: boolean; + execution_time_ms: number; +} + +export function DataProfilerDialog({ connectionId, database, tableName, filters, sortConfig, primaryKeys, fallbackColumns, onClose }: { + connectionId: string; + database: string; + tableName: string; + filters: FilterConfig[]; + sortConfig?: SortConfig; + primaryKeys: string[]; + fallbackColumns: string[]; + onClose: () => void; +}) { + const [sampleSize, setSampleSize] = useState(100); + const [sample, setSample] = useState(null); + const [profiles, setProfiles] = useState([]); + const [search, setSearch] = useState(''); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const requestId = useRef(0); + const activeFilters = filters.filter(filter => filter.enabled); + + useEffect(() => () => { requestId.current += 1; }, []); + + const load = async (limit = sampleSize) => { + const id = ++requestId.current; + setLoading(true); + setError(null); + setSample(null); + try { + const result = await invoke('get_table_data', { + connectionId, tableName, limit, offset: 0, filters: activeFilters, + sortColumn: sortConfig?.column ?? null, + sortDirection: sortConfig?.direction ?? 'ASC', + primaryKeys, seekAfter: null, + }); + if (id !== requestId.current) return; + const state = useDatabaseStore.getState(); + if (!state.openConnectionIds.includes(connectionId) || state.activeDatabases[connectionId] !== database) { + throw new Error('The connection or database changed. Reopen the profiler for the current table.'); + } + if (!Array.isArray(result.columns) || !Array.isArray(result.rows) || result.rows.some(row => !Array.isArray(row))) { + throw new Error('The database returned an invalid sample.'); + } + setProfiles(profileSample(result.columns.length ? result.columns : fallbackColumns, result.rows)); + setSample(result); + } catch (cause) { + if (id === requestId.current) setError(String(cause)); + } finally { + if (id === requestId.current) setLoading(false); + } + }; + + const visible = profiles.filter(profile => profile.name.toLowerCase().includes(search.trim().toLowerCase())); + + return ( + { if (!open) onClose(); }}> + + + +
+
+ +
+ Data Profiler + Read-only column statistics for {tableName}. No writes or full-table count. +
+
+ +
+ +
+ + + {activeFilters.length ? `${activeFilters.length} active filter${activeFilters.length === 1 ? '' : 's'} applied` : 'No filters applied'} · {sortConfig?.column ? 'Current sort' : primaryKeys.length ? 'Primary-key order' : 'Database-return order'} +
+ + {sample &&
+ {sample.rows.length} sampled rows · {profiles.length} columns · {sample.execution_time_ms}ms +

{sampleCoverage(sample.rows.length, sample.has_more === true, sampleSize)}{!sortConfig?.column && primaryKeys.length === 0 ? ' Row order is not guaranteed without a primary key or sort.' : ''} Pending row edits are not included.

+
} + +
+ {error &&
{error}
} + {loading &&
Reading bounded sample...
} + {!loading && !sample && !error &&

Profile up to 1,000 returned rows. Statistics describe only the returned sample, not the whole table.

} + {sample && <> + + {visible.length === 0 &&

No columns match your search.

} +
+ {visible.map(profile => { + const nullPercent = sample.rows.length ? Math.round(profile.nullCount / sample.rows.length * 100) : 0; + const sensitive = isSensitiveColumn(profile.name); + return
+

{profile.name}

{sensitive && Values hidden}
+
Non-null{sample.rows.length - profile.nullCount}
Null{profile.nullCount} ({nullPercent}%)
Distinct{profile.distinctCount}
+
+ {profile.numericRange &&

Numeric range {profile.numericRange.min} to {profile.numericRange.max}

} + {profile.commonValues.length > 0 &&
Common values
{profile.commonValues.map((item, index) =>
{item.value || '(empty string)'}{item.count}×
)}
} +
; + })} +
+ } +
+
Sample statistics only · Read-only · Sensitive values are not displayed
+
+
+
+ ); +} diff --git a/src/components/TabContentTable.tsx b/src/components/TabContentTable.tsx index 04feea0..c7d948c 100644 --- a/src/components/TabContentTable.tsx +++ b/src/components/TabContentTable.tsx @@ -12,6 +12,7 @@ import { ColumnVisibilityPopover } from './ColumnVisibilityPopover'; import { TabContentStructure } from './TabContentStructure'; import { ExportModal } from './ExportModal'; import { RelationshipMapDialog } from './RelationshipMapDialog'; +import { DataProfilerDialog } from './DataProfilerDialog'; import type { RelationshipNode } from '../utils/relationshipMap'; import { createDefaultTableFilter } from '../utils/tableFilters'; import { filtersForForeignKey } from '../utils/foreignKeyNavigation'; @@ -111,6 +112,7 @@ export const TabContentTable = ({ id, tableName, connectionId }: TabContentTable const [foreignKeys, setForeignKeys] = useState([]); const [showExportModal, setShowExportModal] = useState(false); const [showRelationshipMap, setShowRelationshipMap] = useState(false); + const [showDataProfiler, setShowDataProfiler] = useState(false); const [commitError, setCommitError] = useState(null); const viewMode = activeTab?.viewMode || 'data'; const [lastExecutedSql, setLastExecutedSql] = useState(null); @@ -727,6 +729,7 @@ export const TabContentTable = ({ id, tableName, connectionId }: TabContentTable isColumnsVisible={activeTab?.isColumnsPopoverVisible} onExport={() => setShowExportModal(true)} onViewRelationships={() => setShowRelationshipMap(true)} + onProfileData={() => setShowDataProfiler(true)} /> {showExportModal && ( @@ -750,6 +753,19 @@ export const TabContentTable = ({ id, tableName, connectionId }: TabContentTable /> )} + {showDataProfiler && ( + setShowDataProfiler(false)} + /> + )} + {/* Bottom Console / Pending Changes */}
diff --git a/src/components/TableFooter.tsx b/src/components/TableFooter.tsx index 8dbe75f..6e7f1af 100644 --- a/src/components/TableFooter.tsx +++ b/src/components/TableFooter.tsx @@ -1,4 +1,4 @@ -import { ChevronLeft, ChevronRight, Columns, Filter, Plus, Download, GitFork } from 'lucide-react'; +import { ChevronLeft, ChevronRight, Columns, Filter, Plus, Download, GitFork, BarChart3 } from 'lucide-react'; import { cn } from '../utils/cn'; interface TableFooterProps { @@ -19,6 +19,7 @@ interface TableFooterProps { isColumnsVisible?: boolean; onExport?: () => void; onViewRelationships?: () => void; + onProfileData?: () => void; } export const TableFooter = ({ @@ -39,6 +40,7 @@ export const TableFooter = ({ isColumnsVisible, onExport, onViewRelationships, + onProfileData, }: TableFooterProps) => { const start = displayedRows > 0 ? offset + 1 : 0; const end = offset + displayedRows; @@ -82,6 +84,12 @@ export const TableFooter = ({ Map )} + {onProfileData && ( + + )}
diff --git a/src/utils/dataProfiler.ts b/src/utils/dataProfiler.ts new file mode 100644 index 0000000..e5f02f4 --- /dev/null +++ b/src/utils/dataProfiler.ts @@ -0,0 +1,54 @@ +import { isSensitiveColumn } from './sensitiveData'; + +export interface ColumnProfile { + name: string; + nullCount: number; + distinctCount: number; + commonValues: { value: string; count: number }[]; + numericRange: { min: number; max: number } | null; +} + +function valueKey(value: unknown): string { + return JSON.stringify(value) ?? String(value); +} + +export function profileSample(columns: string[], rows: unknown[][]): ColumnProfile[] { + return columns.map((name, index) => { + const frequencies = new Map(); + let nullCount = 0; + let min = Infinity; + let max = -Infinity; + let numericCount = 0; + let nonNullCount = 0; + for (const row of rows) { + const value = row[index]; + if (value === null || value === undefined) { nullCount++; continue; } + nonNullCount++; + const key = valueKey(value); + const existing = frequencies.get(key); + if (existing) existing.count++; + else frequencies.set(key, { value, count: 1 }); + if (typeof value === 'number' && Number.isFinite(value)) { + numericCount++; + min = Math.min(min, value); + max = Math.max(max, value); + } + } + const sensitive = isSensitiveColumn(name); + const commonValues = sensitive ? [] : [...frequencies.values()] + .filter(item => typeof item.value === 'string' || typeof item.value === 'number' || typeof item.value === 'boolean') + .sort((a, b) => b.count - a.count || String(a.value).localeCompare(String(b.value))) + .slice(0, 3) + .map(item => ({ value: String(item.value).slice(0, 72), count: item.count })); + return { + name, nullCount, distinctCount: frequencies.size, commonValues, + numericRange: !sensitive && numericCount === nonNullCount && numericCount > 0 ? { min, max } : null, + }; + }); +} + +export function sampleCoverage(rowCount: number, hasMore: boolean, limit: number): string { + return hasMore + ? `First ${limit} rows returned; more rows exist. This is not a full-table profile or a random sample.` + : `All ${rowCount} rows returned. No additional rows were found.`; +}