Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions scripts/data-profiler.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import assert from 'node:assert/strict';
import { test } from 'node:test';
import { build } from 'esbuild';

const { outputFiles } = await build({ entryPoints: ['src/utils/dataProfiler.ts'], bundle: true, platform: 'node', format: 'esm', write: false });
const { profileSample, sampleCoverage } = await import(`data:text/javascript;base64,${Buffer.from(outputFiles[0].contents).toString('base64')}`);

test('profiles nulls, distinct values, common values, and numeric ranges from sampled rows', () => {
const result = profileSample(['id', 'status', 'score'], [
[1, 'open', 2], [2, 'open', null], [3, 'closed', 8], [4, null, 4],
]);
assert.equal(result[1].nullCount, 1);
assert.equal(result[1].distinctCount, 2);
assert.deepEqual(result[1].commonValues, [{ value: 'open', count: 2 }, { value: 'closed', count: 1 }]);
assert.deepEqual(result[2].numericRange, { min: 2, max: 8 });
});

test('never exposes common values or ranges for sensitive columns', () => {
const result = profileSample(['api_key', 'password'], [['secret-1', 123], ['secret-2', 456]]);
assert.deepEqual(result.map(column => column.commonValues), [[], []]);
assert.deepEqual(result.map(column => column.numericRange), [null, null]);
assert.equal(result[0].distinctCount, 2);
});

test('handles empty and structured samples without treating objects as display values', () => {
assert.deepEqual(profileSample(['id'], []), [{ name: 'id', nullCount: 0, distinctCount: 0, commonValues: [], numericRange: null }]);
const result = profileSample(['data'], [[{ a: 1 }], [{ a: 1 }], [{ a: 2 }]]);
assert.equal(result[0].distinctCount, 2);
assert.deepEqual(result[0].commonValues, []);
});

test('coverage labels bounded samples and full returns without implying randomness', () => {
assert.match(sampleCoverage(100, true, 100), /first 100 rows/i);
assert.match(sampleCoverage(100, true, 100), /not a full-table profile/i);
assert.match(sampleCoverage(3, false, 100), /All 3 rows returned/);
});
128 changes: 128 additions & 0 deletions src/components/DataProfilerDialog.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
import { useEffect, useRef, useState } from 'react';
import * as Dialog from '@radix-ui/react-dialog';
import { invoke } from '@tauri-apps/api/core';
import { BarChart3, Loader2, Search, X } from 'lucide-react';
import { useDatabaseStore, type FilterConfig, type SortConfig } from '../store/databaseStore';
import { isSensitiveColumn } from '../utils/sensitiveData';
import { profileSample, sampleCoverage, type ColumnProfile } from '../utils/dataProfiler';

interface SampleResult {
columns: string[];
rows: unknown[][];
has_more?: boolean;
execution_time_ms: number;
}

export function DataProfilerDialog({ connectionId, database, tableName, filters, sortConfig, primaryKeys, fallbackColumns, onClose }: {
connectionId: string;
database: string;
tableName: string;
filters: FilterConfig[];
sortConfig?: SortConfig;
primaryKeys: string[];
fallbackColumns: string[];
onClose: () => void;
}) {
const [sampleSize, setSampleSize] = useState(100);
const [sample, setSample] = useState<SampleResult | null>(null);
const [profiles, setProfiles] = useState<ColumnProfile[]>([]);
const [search, setSearch] = useState('');
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const requestId = useRef(0);
const activeFilters = filters.filter(filter => filter.enabled);

useEffect(() => () => { requestId.current += 1; }, []);

const load = async (limit = sampleSize) => {
const id = ++requestId.current;
setLoading(true);
setError(null);
setSample(null);
try {
const result = await invoke<SampleResult>('get_table_data', {
connectionId, tableName, limit, offset: 0, filters: activeFilters,
sortColumn: sortConfig?.column ?? null,
sortDirection: sortConfig?.direction ?? 'ASC',
primaryKeys, seekAfter: null,
});
if (id !== requestId.current) return;
const state = useDatabaseStore.getState();
if (!state.openConnectionIds.includes(connectionId) || state.activeDatabases[connectionId] !== database) {
throw new Error('The connection or database changed. Reopen the profiler for the current table.');
}
if (!Array.isArray(result.columns) || !Array.isArray(result.rows) || result.rows.some(row => !Array.isArray(row))) {
throw new Error('The database returned an invalid sample.');
}
setProfiles(profileSample(result.columns.length ? result.columns : fallbackColumns, result.rows));
setSample(result);
} catch (cause) {
if (id === requestId.current) setError(String(cause));
} finally {
if (id === requestId.current) setLoading(false);
}
};

const visible = profiles.filter(profile => profile.name.toLowerCase().includes(search.trim().toLowerCase()));

return (
<Dialog.Root open onOpenChange={open => { if (!open) onClose(); }}>
<Dialog.Portal>
<Dialog.Overlay className="fixed inset-0 z-[80] bg-black/70 backdrop-blur-[2px]" />
<Dialog.Content className="fixed left-1/2 top-1/2 z-[81] flex h-[min(830px,calc(100vh-28px))] w-[min(1080px,calc(100vw-28px))] -translate-x-1/2 -translate-y-1/2 flex-col overflow-hidden rounded-xl border border-border-strong bg-background text-text-primary shadow-2xl">
<header className="flex shrink-0 items-start justify-between gap-4 border-b border-border bg-surface px-5 py-4">
<div className="flex min-w-0 items-start gap-3">
<span className="rounded-lg border border-accent/25 bg-accent/10 p-2 text-accent"><BarChart3 size={18} /></span>
<div className="min-w-0">
<Dialog.Title className="text-base font-semibold">Data Profiler</Dialog.Title>
<Dialog.Description className="mt-0.5 text-xs text-text-muted">Read-only column statistics for <span className="font-mono text-text-secondary">{tableName}</span>. No writes or full-table count.</Dialog.Description>
</div>
</div>
<Dialog.Close className="rounded-md p-1.5 text-text-muted hover:bg-hover hover:text-text-primary" aria-label="Close data profiler"><X size={17} /></Dialog.Close>
</header>

<div className="flex shrink-0 flex-wrap items-end gap-3 border-b border-border bg-surface px-5 py-3">
<label className="flex flex-col gap-1 text-xs text-text-secondary">Rows to sample
<select value={sampleSize} onChange={event => { requestId.current += 1; setLoading(false); setSample(null); setProfiles([]); setError(null); setSampleSize(Number(event.target.value)); }} className="h-9 rounded-md border border-border-strong bg-background px-3 text-text-primary outline-none focus:border-accent">
<option value={100}>100 rows</option><option value={500}>500 rows</option><option value={1000}>1,000 rows</option>
</select>
</label>
<button type="button" onClick={() => void load()} disabled={loading} className="flex h-9 items-center gap-2 rounded-md bg-accent px-4 text-xs font-semibold text-white hover:opacity-90 disabled:opacity-50">
{loading && <Loader2 size={14} className="animate-spin" />}{sample ? 'Refresh profile' : 'Profile sample'}
</button>
<span className="pb-1 text-xs text-text-muted">{activeFilters.length ? `${activeFilters.length} active filter${activeFilters.length === 1 ? '' : 's'} applied` : 'No filters applied'} · {sortConfig?.column ? 'Current sort' : primaryKeys.length ? 'Primary-key order' : 'Database-return order'}</span>
</div>

{sample && <div className="shrink-0 border-b border-border px-5 py-3 text-xs text-text-secondary">
<span className="font-semibold text-text-primary">{sample.rows.length} sampled rows · {profiles.length} columns · {sample.execution_time_ms}ms</span>
<p className="mt-1">{sampleCoverage(sample.rows.length, sample.has_more === true, sampleSize)}{!sortConfig?.column && primaryKeys.length === 0 ? ' Row order is not guaranteed without a primary key or sort.' : ''} Pending row edits are not included.</p>
</div>}

<div className="min-h-0 flex-1 overflow-auto px-5 py-5">
{error && <div role="alert" className="rounded-lg border border-red-400/30 bg-red-400/10 p-4 text-sm text-red-400">{error}</div>}
{loading && <div className="flex items-center gap-2 text-sm text-text-secondary"><Loader2 size={16} className="animate-spin" /> Reading bounded sample...</div>}
{!loading && !sample && !error && <div className="flex h-full flex-col items-center justify-center gap-3 text-center text-text-secondary"><BarChart3 size={32} className="text-accent" /><p className="max-w-md text-sm">Profile up to 1,000 returned rows. Statistics describe only the returned sample, not the whole table.</p></div>}
{sample && <>
<label className="mb-4 flex max-w-xs items-center gap-2 rounded-md border border-border-strong bg-surface px-3 text-text-muted focus-within:border-accent"><Search size={14} /><input value={search} onChange={event => setSearch(event.target.value)} placeholder="Find a column" aria-label="Find a column" className="h-9 w-full bg-transparent text-xs text-text-primary outline-none placeholder:text-text-muted" /></label>
{visible.length === 0 && <p className="text-sm text-text-muted">No columns match your search.</p>}
<div className="grid gap-3 md:grid-cols-2">
{visible.map(profile => {
const nullPercent = sample.rows.length ? Math.round(profile.nullCount / sample.rows.length * 100) : 0;
const sensitive = isSensitiveColumn(profile.name);
return <section key={profile.name} className="min-w-0 rounded-lg border border-border-strong bg-surface p-4">
<div className="mb-3 flex items-center justify-between gap-2"><h3 className="truncate font-mono text-sm font-semibold" title={profile.name}>{profile.name}</h3>{sensitive && <span className="shrink-0 rounded bg-amber-500/10 px-2 py-0.5 text-[10px] text-amber-500">Values hidden</span>}</div>
<div className="grid grid-cols-3 gap-2 text-xs"><div><span className="block text-text-muted">Non-null</span><strong className="font-medium">{sample.rows.length - profile.nullCount}</strong></div><div><span className="block text-text-muted">Null</span><strong className="font-medium">{profile.nullCount} ({nullPercent}%)</strong></div><div><span className="block text-text-muted">Distinct</span><strong className="font-medium">{profile.distinctCount}</strong></div></div>
<div className="mt-3 h-1.5 overflow-hidden rounded-full bg-background"><div className="h-full bg-accent" style={{ width: `${100 - nullPercent}%` }} /></div>
{profile.numericRange && <p className="mt-3 text-xs text-text-secondary">Numeric range <span className="font-mono text-text-primary">{profile.numericRange.min} to {profile.numericRange.max}</span></p>}
{profile.commonValues.length > 0 && <div className="mt-3 border-t border-border pt-3 text-xs"><span className="text-text-muted">Common values</span><div className="mt-2 space-y-1">{profile.commonValues.map((item, index) => <div key={index} className="flex min-w-0 justify-between gap-2"><span className="truncate font-mono" title={item.value}>{item.value || '(empty string)'}</span><span className="shrink-0 text-text-muted">{item.count}×</span></div>)}</div></div>}
</section>;
})}
</div>
</>}
</div>
<footer className="shrink-0 border-t border-border bg-surface px-5 py-2.5 text-[11px] text-text-muted">Sample statistics only · Read-only · Sensitive values are not displayed</footer>
</Dialog.Content>
</Dialog.Portal>
</Dialog.Root>
);
}
16 changes: 16 additions & 0 deletions src/components/TabContentTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { ColumnVisibilityPopover } from './ColumnVisibilityPopover';
import { TabContentStructure } from './TabContentStructure';
import { ExportModal } from './ExportModal';
import { RelationshipMapDialog } from './RelationshipMapDialog';
import { DataProfilerDialog } from './DataProfilerDialog';
import type { RelationshipNode } from '../utils/relationshipMap';
import { createDefaultTableFilter } from '../utils/tableFilters';
import { filtersForForeignKey } from '../utils/foreignKeyNavigation';
Expand Down Expand Up @@ -111,6 +112,7 @@ export const TabContentTable = ({ id, tableName, connectionId }: TabContentTable
const [foreignKeys, setForeignKeys] = useState<ForeignKeyStructure[]>([]);
const [showExportModal, setShowExportModal] = useState(false);
const [showRelationshipMap, setShowRelationshipMap] = useState(false);
const [showDataProfiler, setShowDataProfiler] = useState(false);
const [commitError, setCommitError] = useState<string | null>(null);
const viewMode = activeTab?.viewMode || 'data';
const [lastExecutedSql, setLastExecutedSql] = useState<string | null>(null);
Expand Down Expand Up @@ -727,6 +729,7 @@ export const TabContentTable = ({ id, tableName, connectionId }: TabContentTable
isColumnsVisible={activeTab?.isColumnsPopoverVisible}
onExport={() => setShowExportModal(true)}
onViewRelationships={() => setShowRelationshipMap(true)}
onProfileData={() => setShowDataProfiler(true)}
/>

{showExportModal && (
Expand All @@ -750,6 +753,19 @@ export const TabContentTable = ({ id, tableName, connectionId }: TabContentTable
/>
)}

{showDataProfiler && (
<DataProfilerDialog
connectionId={connectionId}
database={activeTab?.database || activeDatabase || ''}
tableName={tableName}
filters={activeTab?.filters || []}
sortConfig={activeTab?.sortConfig}
primaryKeys={pkColumns}
fallbackColumns={structureColumns}
onClose={() => setShowDataProfiler(false)}
/>
)}

{/* Bottom Console / Pending Changes */}
<div ref={consoleRef} id={`table-sql-log-${id}`} style={{ height: consoleHeight, maxHeight: 'calc(100% - 200px)' }} className="min-h-[80px] bg-sidebar border-t border-border flex flex-col shrink-0 relative">
<div className="flex-1 overflow-auto">
Expand Down
10 changes: 9 additions & 1 deletion src/components/TableFooter.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { ChevronLeft, ChevronRight, Columns, Filter, Plus, Download, GitFork } from 'lucide-react';
import { ChevronLeft, ChevronRight, Columns, Filter, Plus, Download, GitFork, BarChart3 } from 'lucide-react';
import { cn } from '../utils/cn';

interface TableFooterProps {
Expand All @@ -19,6 +19,7 @@ interface TableFooterProps {
isColumnsVisible?: boolean;
onExport?: () => void;
onViewRelationships?: () => void;
onProfileData?: () => void;
}

export const TableFooter = ({
Expand All @@ -39,6 +40,7 @@ export const TableFooter = ({
isColumnsVisible,
onExport,
onViewRelationships,
onProfileData,
}: TableFooterProps) => {
const start = displayedRows > 0 ? offset + 1 : 0;
const end = offset + displayedRows;
Expand Down Expand Up @@ -82,6 +84,12 @@ export const TableFooter = ({
<span>Map</span>
</button>
)}
{onProfileData && (
<button type="button" onClick={onProfileData} title="Profile a bounded, read-only sample of rows" className="ml-2 flex h-7 items-center gap-1.5 rounded border border-border-strong bg-surface-elevated px-2.5 text-text-primary transition-colors hover:bg-hover">
<BarChart3 size={12} className="text-accent" />
<span>Profile</span>
</button>
)}
</div>

<div className="flex shrink-0 items-center gap-3">
Expand Down
54 changes: 54 additions & 0 deletions src/utils/dataProfiler.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import { isSensitiveColumn } from './sensitiveData';

export interface ColumnProfile {
name: string;
nullCount: number;
distinctCount: number;
commonValues: { value: string; count: number }[];
numericRange: { min: number; max: number } | null;
}

function valueKey(value: unknown): string {
return JSON.stringify(value) ?? String(value);
}

export function profileSample(columns: string[], rows: unknown[][]): ColumnProfile[] {
return columns.map((name, index) => {
const frequencies = new Map<string, { value: unknown; count: number }>();
let nullCount = 0;
let min = Infinity;
let max = -Infinity;
let numericCount = 0;
let nonNullCount = 0;
for (const row of rows) {
const value = row[index];
if (value === null || value === undefined) { nullCount++; continue; }
nonNullCount++;
const key = valueKey(value);
const existing = frequencies.get(key);
if (existing) existing.count++;
else frequencies.set(key, { value, count: 1 });
if (typeof value === 'number' && Number.isFinite(value)) {
numericCount++;
min = Math.min(min, value);
max = Math.max(max, value);
}
}
const sensitive = isSensitiveColumn(name);
const commonValues = sensitive ? [] : [...frequencies.values()]
.filter(item => typeof item.value === 'string' || typeof item.value === 'number' || typeof item.value === 'boolean')
.sort((a, b) => b.count - a.count || String(a.value).localeCompare(String(b.value)))
.slice(0, 3)
.map(item => ({ value: String(item.value).slice(0, 72), count: item.count }));
return {
name, nullCount, distinctCount: frequencies.size, commonValues,
numericRange: !sensitive && numericCount === nonNullCount && numericCount > 0 ? { min, max } : null,
};
});
}

export function sampleCoverage(rowCount: number, hasMore: boolean, limit: number): string {
return hasMore
? `First ${limit} rows returned; more rows exist. This is not a full-table profile or a random sample.`
: `All ${rowCount} rows returned. No additional rows were found.`;
}
Loading