From 068a2c6bf6ab558d6954267d92958ad2830f6a2f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 02:58:17 +0000 Subject: [PATCH 1/2] Fix Chase re-import duplicating every row in IndexedDB DB.importTransactions() assigned the _2/_3 occurrence suffix whenever an ID already existed, so every re-uploaded Chase row was inserted again as _2 (165 -> 330 in testing). Suffixes are now assigned once in parseCsvFile(), counted within the file, so IDs are final and identical in memory and in IndexedDB; the import skips IDs already stored (one getAllKeys read). Each import also removes leftover copies from the bug: suffixed rows whose base is in the file but whose exact ID is not. The results card reports them. Identical Ally rows are dropped in memory too; custom schemas without a time column get suffixes by default. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AT1Bh6Sscb8ogzzuf8U3Jc --- js/core.js | 29 ++++++++++++++---- js/db.js | 76 +++++++++++++++++++++++++++--------------------- js/views-data.js | 1 + 3 files changed, 67 insertions(+), 39 deletions(-) diff --git a/js/core.js b/js/core.js index 2cf0fe9..f761429 100644 --- a/js/core.js +++ b/js/core.js @@ -328,7 +328,8 @@ function detectSchema(headers, filename) { /* ---------- GENERIC ROW NORMALIZER ---------- */ // Replaces normalizeAllyRow() and normalizeChaseRow(). -function normalizeRow(row, schema, accountId) { +// `occurrence` (1-based) distinguishes truly identical rows within one file — see parseCsvFile(). +function normalizeRow(row, schema, accountId, occurrence = 1) { const get = (col) => col && row[col] !== undefined ? row[col].toString().trim() : ''; const rawDate = get(schema.columns.date); @@ -359,7 +360,7 @@ function normalizeRow(row, schema, accountId) { const cat = (rawCat && rawCat !== '') ? rawCat.trim() : autoCategory(desc); return mkTx({ date: d, time, postDate, description: desc, amount: amt, - category: cat, account: accountId, type }); + category: cat, account: accountId, type, occurrence }); } /* ---------- DEFAULT CATEGORY RULES ---------- */ @@ -426,7 +427,9 @@ function mkTx(o) { } else { hashKey = dateStr+'|'+o.description+'|'+o.amount+'|'+o.account; } - const id = hash(hashKey); + // The 2nd, 3rd, … truly identical row in the same file gets _2, _3, … (assigned in + // parseCsvFile). The suffix is part of the ID everywhere — memory, IndexedDB, overrides. + const id = hash(hashKey) + (o.occurrence > 1 ? '_' + o.occurrence : ''); const ov = State.txOverrides[id] || {}; const cats = ov.categories && ov.categories.length ? ov.categories : (ov.category ? [ov.category] : [o.category]); return { @@ -506,11 +509,24 @@ function parseCsvFile(file, forcedKind) { State.accounts[accountId].name = State.accountLabels[accountId]; } - // Normalize all rows + // Normalize all rows. Truly identical rows (same hash inputs) are counted within + // this file: the 2nd copy gets an _2 suffix, the 3rd _3, and so on. Counting per + // file (never against the database) keeps IDs identical across re-uploads, so + // DB.importTransactions() can simply skip IDs it already holds. Schemas with a + // bank-assigned Time column (Ally) opt out: an identical row there is a true + // duplicate, so only the first copy is kept. + const allowSuffix = schema.allowOccurrenceSuffix ?? !schema.columns.time; const txs = []; + const seen = {}; for (const r of rows) { - const tx = normalizeRow(r, schema, accountId); - if (tx) txs.push(tx); + let tx = normalizeRow(r, schema, accountId); + if (!tx) continue; + const n = seen[tx.id] = (seen[tx.id] || 0) + 1; + if (n > 1) { + if (!allowSuffix) continue; + tx = normalizeRow(r, schema, accountId, n); + } + txs.push(tx); } // Persist to IndexedDB (if DB is ready) @@ -537,6 +553,7 @@ function parseCsvFile(file, forcedKind) { rowsFound: rows.length, rowsInserted: importResult.inserted, rowsSkipped: importResult.skipped, + rowsRemoved: importResult.removed, dateRangeMin: importResult.dateRangeMin, dateRangeMax: importResult.dateRangeMax, }); diff --git a/js/db.js b/js/db.js index 3f8e7de..bd1f7f3 100644 --- a/js/db.js +++ b/js/db.js @@ -146,21 +146,25 @@ const DB = { /* ── Transactions ────────────────────────────────────────────── */ /** - * Batch-insert transactions. Uses add() (not put()) — duplicate IDs are silently - * skipped (or occurrence-suffixed for Chase). Returns import stats + gap warning. + * Batch-insert transactions. IDs arrive final from parseCsvFile() — including the + * _2, _3 … suffixes for truly identical rows, counted within the file — so any ID + * already in the store is a re-imported row and is skipped. Never assign suffixes + * here: doing so on collision (≤v1.8.0) re-inserted every row of a re-uploaded + * Chase file as an "_2" copy. + * + * Also removes leftover copies from that bug: a suffixed row whose base transaction + * is in this file but whose exact ID is not (e.g. base_2 when the file has only one + * such row). Same-day identical rows always appear together in a bank export, so the + * file is authoritative for how many copies of each of its transactions exist. + * + * Returns { inserted, skipped, removed, dateRangeMin, dateRangeMax, gapWarning }. */ async importTransactions(txArray, accountId, schemaId) { if (!this._db || !txArray.length) { - return { inserted: 0, skipped: 0, dateRangeMin: null, dateRangeMax: null, gapWarning: null }; + return { inserted: 0, skipped: 0, removed: 0, dateRangeMin: null, dateRangeMax: null, gapWarning: null }; } - // Find schema config for allowOccurrenceSuffix - const allSchemas = [...(typeof BANK_SCHEMAS !== 'undefined' ? BANK_SCHEMAS : []), - ...(State.customSchemas || [])]; - const schema = allSchemas.find(s => s.id === schemaId); - const allowSuffix = schema ? !!schema.allowOccurrenceSuffix : false; - - // Sort for stable occurrence-suffix assignment + // Sort chronologically for insertion const sorted = [...txArray].sort((a, b) => { const da = a.date instanceof Date ? a.date : new Date(a.date); const db = b.date instanceof Date ? b.date : new Date(b.date); @@ -193,10 +197,18 @@ const DB = { } } catch(e) { /* account might not exist yet — first upload */ } + // IDs already stored (keys only — one read). Checking up front avoids relying on + // ConstraintError, which also logs a console error for every re-imported row. + let existingIds = new Set(); + try { + existingIds = new Set(await this._req(this._getStore('transactions').getAllKeys())); + } catch(e) { console.warn('DB.importTransactions: could not read existing IDs:', e); } + // Batch insert - let inserted = 0, skipped = 0; + let inserted = 0, skipped = 0, removed = 0; for (const tx of sorted) { + if (existingIds.has(tx.id)) { skipped++; continue; } const d = tx.date instanceof Date ? tx.date : new Date(tx.date); const storable = { id: tx.id, @@ -220,34 +232,32 @@ const DB = { await this._req(txnStore.add(storable)); inserted++; } catch (err) { - if (err && err.name === 'ConstraintError') { - if (!allowSuffix) { - skipped++; - } else { - // Occurrence suffix: try _2, _3, ... until add succeeds - let suffixNum = 2, added = false; - while (suffixNum <= 99 && !added) { - try { - const suffixStore = this._db.transaction('transactions', 'readwrite').objectStore('transactions'); - await this._req(suffixStore.add({ ...storable, id: storable.id + '_' + suffixNum })); - inserted++; - added = true; - } catch (e2) { - if (e2 && e2.name === 'ConstraintError') { suffixNum++; } - else { skipped++; break; } - } - } - if (!added && suffixNum > 99) skipped++; - } - } + // Already stored (e.g. read raced with another import): it's a duplicate. + if (err && err.name === 'ConstraintError') skipped++; // Other errors: silently skip } } + // Remove leftover copies created by the ≤v1.8.0 collision-suffix bug (see above). + const baseOf = (id) => id.replace(/_\d+$/, ''); + const batchIds = new Set(sorted.map(t => t.id)); + const batchBases = new Set(sorted.map(t => baseOf(t.id))); + const stale = [...existingIds].filter(id => + id !== baseOf(id) && !batchIds.has(id) && batchBases.has(baseOf(id))); + if (stale.length) { + removed = await new Promise((res) => { + const t = this._db.transaction('transactions', 'readwrite'); + const store = t.objectStore('transactions'); + stale.forEach(id => store.delete(id)); + t.oncomplete = () => res(stale.length); + t.onerror = t.onabort = () => res(0); + }); + } + // Update account coverage metadata - await this._updateAccountCoverage(accountId, fileMinDate, fileMaxDate, inserted); + await this._updateAccountCoverage(accountId, fileMinDate, fileMaxDate, inserted - removed); - return { inserted, skipped, dateRangeMin: fileMinDate, dateRangeMax: fileMaxDate, gapWarning }; + return { inserted, skipped, removed, dateRangeMin: fileMinDate, dateRangeMax: fileMaxDate, gapWarning }; }, async _updateAccountCoverage(accountId, fileMin, fileMax, newlyInserted) { diff --git a/js/views-data.js b/js/views-data.js index 33d8366..0fba906 100644 --- a/js/views-data.js +++ b/js/views-data.js @@ -145,6 +145,7 @@ registerView('import','Import',(root)=>{
${ir.inserted.toLocaleString()} new transaction${ir.inserted!==1?'s':''} imported · ${ir.skipped.toLocaleString()} duplicate${ir.skipped!==1?'s':''} skipped + ${ir.removed?`· ${ir.removed.toLocaleString()} leftover duplicate cop${ir.removed!==1?'ies':'y'} removed`:''}
File covered: ${fmtD(ir.dateRangeMin)} → ${fmtD(ir.dateRangeMax)}
${coverage?`
Total in DB: ${fmtD(coverage.min)} → ${fmtD(coverage.max)} (${coverage.count.toLocaleString()} transactions)
`:''} From a4d123ac6ca799072b40874bfef23bd1f65a03de Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 02:58:17 +0000 Subject: [PATCH 2/2] =?UTF-8?q?Release=20v1.8.1:=20changelog,=20version=20?= =?UTF-8?q?strings,=20CLAUDE.md=20=C2=A710?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AT1Bh6Sscb8ogzzuf8U3Jc --- CHANGELOG.md | 15 +++++++++++++++ CLAUDE.md | 11 +++++++---- README.md | 4 ++-- index.html | 2 +- js/views-settings.js | 2 +- 5 files changed, 26 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e140a4a..357f408 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). --- +## [1.8.1] — 2026-10-23 + +### Fixed (Technical) +- **Re-importing a Chase file no longer duplicates every Chase row in IndexedDB.** + - **Cause:** `DB.importTransactions()` assigned the `_2`/`_3` occurrence suffix whenever an ID already existed in the database. So every re-uploaded Chase row collided with its earlier self and was inserted again as `_2`. In testing, 165 Chase rows became 330 after one re-import. Ally was unaffected because it skips collisions. + - **Fix:** occurrence suffixes are now assigned in `parseCsvFile()`, counted within the file (`normalizeRow(..., occurrence)` → `mkTx()`). `DB.importTransactions()` reads the stored keys once and skips any ID it already holds, with no suffixing and no console error per skipped row. + - **Consistent IDs:** identical rows now have the same suffixed ID in memory and in IndexedDB. Before, both copies shared the base ID in memory, so overrides couldn't target the second copy reliably. +- **Cleanup of existing damage** — each import deletes leftover copies created by the bug: suffixed rows whose base transaction is in the file but whose exact ID is not. The import results card shows "N leftover duplicate copies removed". Re-import your Chase CSVs once to clean an affected database. +- **Identical Ally rows** (same date, bank time, description and amount) are now dropped in memory too, matching IndexedDB and §10. Custom schemas without a `time` column get occurrence suffixes by default instead of silently losing same-day identical rows. + +### Docs +- **CLAUDE.md §10** rewritten to describe where suffixes are assigned and how re-upload safety and cleanup work. + +--- + ## [1.8.0] — 2026-10-23 ### AI Analysis — Restored and Hardened diff --git a/CLAUDE.md b/CLAUDE.md index c430b38..64cbbcc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ FinLens is a **single-origin, zero-build, vanilla JavaScript personal finance dashboard** that runs entirely in the browser. There is no server, no npm, no bundler, no framework, and no TypeScript. The entry point is `index.html`; everything else is a plain `.js` file in the `js/` folder. -**Current version:** v1.8.0 (updated in `index.html` footer, `README.md`, and `CHANGELOG.md` on every release). +**Current version:** v1.8.1 (updated in `index.html` footer, `README.md`, and `CHANGELOG.md` on every release). **What it does:** Imports CSV exports from Ally Bank (Checking + Savings) and Chase credit cards, parses and normalizes them into a unified transaction model, and renders a multi-tab financial dashboard with budgeting, debt tracking, savings forecasting, investment planning, and Claude-powered AI analysis. @@ -454,9 +454,12 @@ The deterministic `id` for each transaction is computed per source to survive re | Ally Savings | `date \| time \| description \| amount \| accountId` | Same as checking | | Chase | `transactionDate \| postDate \| description \| amount \| accountId` | No timestamp; post date disambiguates most cases | -**Chase collision fallback:** Truly identical Chase rows (same transaction date, post date, description, and amount) do occur in practice — e.g. multiple TAXMANIA charges on the same day for the same amount. When detected, an occurrence suffix `_2`, `_3`, etc. is appended to the base hash. The import batch must be sorted by `(transactionDate, postDate, description, amount)` before suffix assignment to ensure the suffix is stable across re-uploads. +**Identical rows (occurrence suffix):** Truly identical Chase rows (same transaction date, post date, description, and amount) do occur in practice — e.g. multiple TAXMANIA charges on the same day for the same amount. `parseCsvFile()` counts them **within the file**: the first keeps the base hash, the 2nd gets `_2`, the 3rd `_3` (via the `occurrence` argument of `normalizeRow()` → `mkTx()`). The suffix is part of the ID everywhere — memory, IndexedDB and `txOverrides`. Because identical rows are interchangeable, file order is enough to make the suffix stable across re-uploads. -**Re-upload safety:** On re-upload of the same file with new rows appended, existing transaction IDs are identical → deduplicated silently. Only genuinely new rows are inserted. +- Suffixes are allowed when `schema.allowOccurrenceSuffix` is true, or when it's unset and the schema has no `time` column (custom schemas). Ally sets it to `false`: with a bank-assigned `Time`, an identical row is a true duplicate and only the first copy is kept. +- **Never assign suffixes on a database collision.** Up to v1.8.0, `DB.importTransactions()` did, which re-inserted every row of a re-uploaded Chase file as an `_2` copy. + +**Re-upload safety:** IDs are final when they reach `DB.importTransactions()`, which reads the stored keys once and skips any ID it already holds. Re-uploading a file — or a newer export with rows appended — inserts only genuinely new rows. On each import it also deletes leftover copies from the old bug: a suffixed row whose base transaction is in the file but whose exact ID is not. Same-day identical rows always appear together in a bank export, so the file is authoritative for how many copies exist. The import results card reports these as "leftover duplicate copies removed". --- @@ -706,7 +709,7 @@ Multiple Chase card files can be loaded simultaneously. Each gets a unique `acco --- -*Last updated: v1.8.0 — 2026-10-23 (AI Analysis restored with API-key and output hardening; `renderMarkdown()` added; `samples/` added; §1–3, §13, §14, §17 corrected to match the code)* +*Last updated: v1.8.1 — 2026-10-23 (§10 re-upload dedup rewritten for the occurrence-suffix fix; v1.8.0: AI Analysis restored with API-key and output hardening; `renderMarkdown()` added; `samples/` added; §1–3, §13, §14, §17 corrected to match the code)* --- diff --git a/README.md b/README.md index 17899b0..55a718e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > A zero-install, browser-native finance dashboard. Drop in your bank CSVs, get instant clarity on your money. -**Version:** v1.8.0  ·  **Stack:** Vanilla JS, HTML, CSS  ·  **Install:** None +**Version:** v1.8.1  ·  **Stack:** Vanilla JS, HTML, CSS  ·  **Install:** None --- @@ -227,4 +227,4 @@ See [CHANGELOG.md](CHANGELOG.md) for the full release history. --- -*FinLens v1.8.0 — Built for one person's finances, designed to be useful for anyone's.* +*FinLens v1.8.1 — Built for one person's finances, designed to be useful for anyone's.* diff --git a/index.html b/index.html index bf60425..f1e77f8 100644 --- a/index.html +++ b/index.html @@ -322,7 +322,7 @@
⚙️ Settings
diff --git a/js/views-settings.js b/js/views-settings.js index e47ad66..776ddcc 100644 --- a/js/views-settings.js +++ b/js/views-settings.js @@ -47,7 +47,7 @@ function _renderGeneralSettings(root) { -
FinLens v1.8.0 ·${State.transactions.length} transactions loaded${State.lastRefresh?' · Refreshed '+dayjs(State.lastRefresh).format('MMM D, YYYY h:mm A'):''}
`; +
FinLens v1.8.1 ·${State.transactions.length} transactions loaded${State.lastRefresh?' · Refreshed '+dayjs(State.lastRefresh).format('MMM D, YYYY h:mm A'):''}
`; root.appendChild(displayCard); /* ---- Anthropic API Key ---- */