Hardening: GC-safe lifecycle, 64-bit integers, exception-safe VFS, power-fail recovery

tests/README.md

@@ -0,0 +1,18 @@
+# usqlite regression tests
+
+Host-side tests, run against a MicroPython unix build that includes this
+module. On the Pico Computer 3 tree that is the `pc3` variant, and the heap
+size must match the board (8 MB) so memory behaviour is faithful:
+
+    cd ports/unix
+    ./build-pc3/micropython -X heapsize=8m ../../lib/usqlite/tests/test_smoke.py
+
+All scripts print `... PASSED` on success and use `/tmp` for their databases.
+
+| script            | covers |
+|-------------------|--------|
+| `test_smoke.py`   | basic CRUD, executemany error-path leak, connection.close() with live/closed cursors, GC finaliser reclaim, row/dict/named-parameter binding |
+| `test_simreset.py`| the soft-reset session cycle: `usqlite.__init__()` + `gc.collect()` simulate what a bare-metal soft reset does (roots are NOT auto-zeroed; the module `__init__` clears them on the next import), then the new session must reinitialize and close cleanly |
+| `test_fix456.py`  | 64-bit INTEGER bind/read roundtrip, `.description` with computed columns (NULL decltype) and before first fetch, VFS errors surfacing as `usqlite_Error` (not exceptions unwinding SQLite), raising trace callback swallowed |
+| `test_crash.py`   | power-fail recovery: snapshots of db+journal taken mid-transaction and post-commit are reopened; mid-transaction must roll back exactly, post-commit must keep data, a garbage journal must be ignored (`PRAGMA integrity_check` throughout) |
+| `test_churn.py`   | heavy fetchall + gc.collect churn under an open connection, then close (the historical GC-vs-SQLite corruption pattern) |

tests/test_churn.py

@@ -0,0 +1,29 @@
+import usqlite, gc, os
+
+try:
+    os.remove("/tmp/phaseb.db")
+except OSError:
+    pass
+
+db = usqlite.connect("/tmp/phaseb.db")
+db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, v REAL)")
+db.execute("BEGIN")
+for i in range(100):
+    db.execute("INSERT INTO t (name, v) VALUES (?, ?)", ("row%d" % i, i * 1.5))
+db.execute("COMMIT")
+print("count:", db.execute("SELECT COUNT(*) FROM t").fetchone()[0])
+
+# exact phase-B shape: COUNT fetchone + 20x (fetchall + churn + collect)
+row = db.execute("SELECT COUNT(*), SUM(v) FROM t").fetchone()
+assert row == (100, 7425.0), row
+for k in range(20):
+    x = ["pad%d" % i for i in range(2000)]
+    rows = db.execute("SELECT * FROM t ORDER BY v DESC").fetchall()
+    assert len(rows) == 100 and rows[0][2] == 148.5
+    gc.collect()
+print("churn done, mem:", usqlite.mem_current())
+print("closing...")
+db.close()
+print("closed ok")
+os.remove("/tmp/phaseb.db")
+print("PHASE-B REPLICA PASSED")

tests/test_crash.py

@@ -0,0 +1,92 @@
+import usqlite, os
+
+DB = "/tmp/crash.db"
+
+def rm(p):
+    try:
+        os.remove(p)
+    except OSError:
+        pass
+
+def cp(src, dst):
+    rm(dst)
+    try:
+        with open(src, "rb") as s, open(dst, "wb") as d:
+            while True:
+                b = s.read(4096)
+                if not b:
+                    break
+                d.write(b)
+    except OSError:
+        pass  # source may not exist (no journal)
+
+def snapshot(tag):
+    cp(DB, "/tmp/%s.db" % tag)
+    cp(DB + "-journal", "/tmp/%s.db-journal" % tag)
+
+def content(path):
+    db = usqlite.connect(path)
+    ic = db.execute("PRAGMA integrity_check").fetchone()[0]
+    row = db.execute("SELECT COUNT(*), SUM(v), SUM(LENGTH(pad)) FROM t").fetchone()
+    db.close()
+    return ic, row
+
+for f in os.listdir("/tmp"):
+    if f.startswith("crash") and (f.endswith(".db") or f.endswith("-journal")):
+        rm("/tmp/" + f)
+
+# --- baseline: committed state -------------------------------------------
+db = usqlite.connect(DB)
+db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v INTEGER, pad TEXT)")
+db.execute("BEGIN")
+for i in range(100):
+    db.execute("INSERT INTO t VALUES (?, ?, ?)", (i, i, "a" * 200))
+db.execute("COMMIT")
+db.close()
+baseline = content(DB)
+print("baseline:", baseline)
+assert baseline[0] == "ok" and baseline[1][0] == 100
+
+# --- big uncommitted transaction, snapshot mid-flight ---------------------
+db = usqlite.connect(DB)
+db.execute("PRAGMA cache_size=10")     # tiny cache -> spill writes db mid-txn
+db.execute("BEGIN")
+for i in range(100):
+    db.execute("UPDATE t SET v = v + 1000000, pad = ? WHERE id = ?",
+               ("z" * 2000, i))
+jsz = os.stat(DB + "-journal")[6]
+print("mid-txn journal bytes:", jsz)
+assert jsz > 0, "journal must exist mid-transaction"
+snapshot("crashA")                     # power cut mid-transaction
+
+db.execute("COMMIT")
+snapshot("crashB")                     # power cut just after commit
+db.close()
+committed = content(DB)
+print("committed:", committed)
+assert committed[0] == "ok" and committed[1][1] == sum(range(100)) + 100 * 1000000
+
+# --- recovery A: mid-transaction cut must roll back to baseline -----------
+recA = content("/tmp/crashA.db")
+print("crashA recovered:", recA)
+assert recA == baseline, (recA, baseline)
+try:
+    os.stat("/tmp/crashA.db-journal")
+    print("note: crashA journal still present after recovery")
+except OSError:
+    print("crashA journal cleaned up")
+
+# --- recovery B: post-commit cut must keep the committed data -------------
+recB = content("/tmp/crashB.db")
+print("crashB recovered:", recB)
+assert recB == committed, (recB, committed)
+
+# --- garbage journal next to a healthy db is ignored ----------------------
+cp(DB, "/tmp/crashC.db")
+with open("/tmp/crashC.db-journal", "wb") as f:
+    f.write(bytes(range(256)) * 16)
+recC = content("/tmp/crashC.db")
+print("crashC (garbage journal):", recC)
+assert recC == committed, (recC, committed)
+
+print("CRASH RECOVERY TESTS PASSED")

tests/test_fix456.py

@@ -0,0 +1,52 @@
+import usqlite, gc, os
+
+try:
+    os.remove("/tmp/t456.db")
+except OSError:
+    pass
+
+db = usqlite.connect("/tmp/t456.db")
+
+# --- fix 4: 64-bit INTEGER bind + read ------------------------------------
+db.execute("CREATE TABLE big (id INTEGER PRIMARY KEY, v INTEGER)")
+vals = [2**40, 1753305600123, -2**45, 2**62, -1, 0, 2**31, -(2**31) - 1]
+for i, v in enumerate(vals):
+    db.execute("INSERT INTO big VALUES (?, ?)", (i, v))
+back = [r[0] for r in db.execute("SELECT v FROM big ORDER BY id")]
+assert back == vals, (back, vals)
+row = db.execute("SELECT SUM(v) FROM big").fetchone()
+assert row[0] == sum(vals), row
+print("4: 64-bit roundtrip ok:", back[:3], "...")
+
+# --- fix 5: decltype NULL + description before fetch ----------------------
+cur = db.cursor()
+cur.execute("SELECT COUNT(*), v + 1, v FROM big")
+d = cur.description          # computed columns -> decltype NULL; was a crash
+assert len(d) == 3, d
+assert d[0][1] is None and d[1][1] is None and d[2][1] == "INTEGER", d
+cur.fetchall()
+cur2 = db.execute("SELECT id, v FROM big")
+d2 = cur2.description        # before any fetch consumed rows -- was empty
+assert len(d2) == 2 and d2[0][0] == "id", d2
+cur2.fetchall()
+print("5: description/decltype ok")
+
+# --- fix 6: VFS errors as codes, not exceptions ---------------------------
+try:
+    usqlite.connect("/tmp/nosuchdir/x.db")
+    raise SystemExit("expected connect to fail")
+except usqlite.usqlite_Error as e:
+    print("6: bad-path connect -> usqlite_Error:", e)
+# engine must still be fully usable afterwards
+assert db.execute("SELECT COUNT(*) FROM big").fetchone()[0] == len(vals)
+
+def bad_trace(stmt):
+    raise ValueError("boom")
+db.set_trace_callback(bad_trace)
+assert db.execute("SELECT COUNT(*) FROM big").fetchone()[0] == len(vals)
+db.set_trace_callback(None)
+print("6: raising trace callback swallowed ok")
+
+db.close()
+os.remove("/tmp/t456.db")
+print("FIX 4/5/6 TESTS PASSED")

tests/test_simreset.py

@@ -0,0 +1,43 @@
+import usqlite, gc, os
+
+try:
+    os.remove("/tmp/simreset.db")
+except OSError:
+    pass
+
+# --- session 1: mirror sqltest_a end state ---------------------------------
+db = usqlite.connect("/tmp/simreset.db")
+db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, v REAL)")
+db.execute("BEGIN")
+for i in range(100):
+    db.execute("INSERT INTO t (name, v) VALUES (?, ?)", ("row%d" % i, i * 1.5))
+db.execute("COMMIT")
+dbopen = db     # connection deliberately left open across the "reset"
+print("session 1 up, mem:", usqlite.mem_current())
+
+# --- simulated soft reset ---------------------------------------------------
+# Faithful to rp2: root pointers are NOT zeroed by the reset itself. What a
+# new session does is (a) sweep all objects (finalisers run) and (b) call the
+# module __init__ on its first import, which forgets the dead session's state.
+del db, dbopen
+gc.collect()        # ~ gc_sweep_all() at soft_reset_exit
+usqlite.__init__()  # ~ first `import usqlite` of the new session
+gc.collect()        # old pool is now unreferenced garbage
+print("simulated soft reset done")
+
+# --- session 2: mirror sqltest_b -------------------------------------------
+db = usqlite.connect("/tmp/simreset.db")
+row = db.execute("SELECT COUNT(*), SUM(v) FROM t").fetchone()
+print("B1 count/sum:", row)
+assert row == (100, 7425.0), row
+for k in range(5):
+    x = ["pad%d" % i for i in range(2000)]
+    rows = db.execute("SELECT * FROM t ORDER BY v DESC").fetchall()
+    assert len(rows) == 100 and rows[0][2] == 148.5
+    gc.collect()
+print("B2 churn ok, mem:", usqlite.mem_current())
+print("B3 closing...")
+db.close()
+print("B3 closed ok")
+os.remove("/tmp/simreset.db")
+print("SIM-RESET TEST PASSED")

tests/test_smoke.py

@@ -0,0 +1,70 @@
+import usqlite, gc, os
+
+try:
+    os.remove("/tmp/smoke.db")
+except OSError:
+    pass
+
+# --- basic sanity ---------------------------------------------------------
+db = usqlite.connect("/tmp/smoke.db")
+db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, v REAL)")
+db.executemany(
+    "INSERT INTO t (name, v) VALUES ('a', 1.5);"
+    "INSERT INTO t (name, v) VALUES ('b', 2.5)")
+cur = db.execute("SELECT * FROM t ORDER BY id")
+rows = cur.fetchall()
+assert rows == [(1, 'a', 1.5), (2, 'b', 2.5)], rows
+print("basic ok")
+
+# --- fix 9: failed executemany must not leak the error message ------------
+gc.collect()
+base = usqlite.mem_current()
+for i in range(50):
+    try:
+        db.executemany("INSERT INTO nosuch VALUES (1)")
+        raise SystemExit("expected an error")
+    except usqlite.usqlite_Error:
+        pass
+gc.collect()
+leak = usqlite.mem_current() - base
+print("executemany 50-error leak bytes:", leak)
+assert leak < 512, leak
+
+# --- fix 2: connection.close() with live / explicitly-closed cursors ------
+cur2 = db.execute("SELECT * FROM t")   # unexhausted -> still registered
+cur3 = db.execute("SELECT * FROM t")
+cur3.close()                           # explicitly closed -> must untrack
+db.close()
+gc.collect()
+for i in range(2000):                  # churn the heap: UAF would corrupt/crash
+    x = [i] * 8
+print("post-close rowcounts:", cur2.rowcount, cur3.rowcount)
+assert cur2.fetchone() is None
+assert list(cur2) == []
+print("close-with-live-cursors ok")
+
+# --- fix 8: dropped connection is closed by the finaliser -----------------
+db2 = usqlite.connect("/tmp/smoke.db")
+db2.execute("SELECT COUNT(*) FROM t").fetchone()
+mid = usqlite.mem_current()
+db2 = None
+gc.collect()
+after = usqlite.mem_current()
+print("mem with-open-conn/after-collect:", mid, after)
+assert after < mid, (mid, after)
+print("finaliser ok")
+
+# --- row_type + named params regression sweep -----------------------------
+db3 = usqlite.connect("/tmp/smoke.db")
+db3.row_type = "row"
+r = db3.execute("SELECT name, v FROM t WHERE id = :i", {"i": 2}).fetchone()
+assert tuple(r) == ('b', 2.5), r
+assert r.keys == ('name', 'v'), r.keys
+db3.row_type = "dict"
+r = db3.execute("SELECT name FROM t WHERE id = ?", (1,)).fetchone()
+assert r == {"name": "a"}, r
+db3.close()
+print("row/dict/named-params ok")
+
+os.remove("/tmp/smoke.db")
+print("ALL SMOKE TESTS PASSED")

usqlite_config.h

@@ -30,6 +30,20 @@ SOFTWARE.
 
 #undef USQLITE_DEBUG
 
+// Directory for SQLite temporary files (sort/merge spill, temp b-trees). The
+// engine asks the VFS for anonymous temp files by passing a NULL name; usqlite
+// gives them a unique name so large sorts / GROUP BY / index builds spill to
+// disk instead of failing. By default it uses the SD card (USQLITE_TEMP_SD)
+// when one is mounted -- sparing flash from the write churn of a big spill --
+// and falls back to the flash root (USQLITE_TEMP_DIR) otherwise. Either is
+// overridden at runtime by PRAGMA temp_store_directory.
+#ifndef USQLITE_TEMP_DIR
+#define USQLITE_TEMP_DIR "/"
+#endif
+#ifndef USQLITE_TEMP_SD
+#define USQLITE_TEMP_SD "/sd"
+#endif
+
 // ------------------------------------------------------------------------------
 // SQLite configuration options - https://sqlite.org/compile.html
 
@@ -50,7 +64,10 @@ SOFTWARE.
 #define SQLITE_OMIT_SHARED_CACHE 1
 #define SQLITE_OMIT_TCL_VARIABLE 1
 #define SQLITE_OMIT_UTF16 1
-// #define SQLITE_OMIT_WAL 1
+// WAL needs the xShm* shared-memory VFS methods, which this VFS does not
+// provide (they are NULL) -- PRAGMA journal_mode=WAL would call through a
+// NULL pointer. Rollback journals are the crash-safety mechanism here.
+#define SQLITE_OMIT_WAL 1
 
 #define SQLITE_UNTESTABLE 1
 
@@ -63,18 +80,46 @@ SOFTWARE.
 #undef SQLITE_ENABLE_RTREE
 
 #define SQLITE_ENABLE_MEMORY_MANAGEMENT 1
-#define SQLITE_DEFAULT_MEMSTATUS 0
+// Track heap usage so usqlite.mem_current()/mem_peak() report real figures --
+// invaluable for watching the dedicated heap fill. Cheap here: THREADSAFE is 0,
+// so it is just an unlocked counter per malloc/free.
+#define SQLITE_DEFAULT_MEMSTATUS 1
 #define SQLITE_ZERO_MALLOC 1
 
 
 // ------------------------------------------------------------------------------
 
+// Give SQLite one dedicated heap (MEMSYS5) instead of a GC-heap allocation per
+// call. The per-call allocator (gc_alloc) puts SQLite's structures on the
+// MicroPython GC heap, where the conservative collector cannot follow SQLite's
+// interior/tagged pointers and frees live memory on any gc.collect() under an
+// open connection -- corrupting the database or hanging the board. One pooled
+// block is opaque to the GC and fixes both. See usqlite_mem.c.
 #ifdef SQLITE_ZERO_MALLOC
-// #define SQLITE_ENABLE_MEMSYS5 1
+#define SQLITE_ENABLE_MEMSYS5 1
 #endif
 
 #ifdef SQLITE_ENABLE_MEMSYS5
-#define MEMSYS5_HEAP_SIZE               128 * 1024
+// Size of that pool, reserved lazily on the first connect() (see
+// usqlite_mem.c), so a program that never opens a database pays nothing. Small
+// by default so it fits constrained targets -- a plain RP2040 or ESP32 has only
+// a few hundred KB of RAM -- and raised per board where there is room:
+//   -DMEMSYS5_HEAP_SIZE=0x400000   (e.g. 4 MB on a board with PSRAM)
+#ifndef MEMSYS5_HEAP_SIZE
+#define MEMSYS5_HEAP_SIZE               (128 * 1024)
+#endif
+// Page cache: small and mostly independent of the pool size (negative = KiB).
+// A big cache (the old half-the-pool default) let the sorter hoard memory
+// before spilling, so a large ORDER BY / GROUP BY / index build climbed to the
+// edge of the pool and thrashed. Capping the cache low makes the sorter's temp
+// b-tree spill to disk early, so big sorts stay bounded (sub-MB) and the pool
+// keeps headroom -- measured at no speed cost (spilling to flash is cheap, and
+// even on SD the same query went from a 150s+ thrash to a clean 20s). ~256 KB
+// where the pool allows, never more than 1/8 of a small pool. Override to tune.
+#ifndef SQLITE_DEFAULT_CACHE_SIZE
+#define SQLITE_DEFAULT_CACHE_SIZE \
+    (MEMSYS5_HEAP_SIZE / 8 < 256 * 1024 ? -(MEMSYS5_HEAP_SIZE / 8 / 1024) : -256)
+#endif
 #endif
 
 // ------------------------------------------------------------------------------