diff --git a/Makefile b/Makefile index 26164d7d6..a0b52acae 100644 --- a/Makefile +++ b/Makefile @@ -457,8 +457,10 @@ CLOCKCACHE_SYS = $(OBJDIR)/$(SRCDIR)/clockcache.o \ $(PLATFORM_IO_SYS) BTREE_SYS = $(OBJDIR)/$(SRCDIR)/btree.o \ + $(OBJDIR)/$(SRCDIR)/btree_key.o \ $(OBJDIR)/$(SRCDIR)/blob.o \ $(OBJDIR)/$(SRCDIR)/blob_build.o \ + $(OBJDIR)/$(SRCDIR)/mini_blob.o \ $(OBJDIR)/$(SRCDIR)/data_blob_build.o \ $(OBJDIR)/$(SRCDIR)/data_internal.o \ $(OBJDIR)/$(SRCDIR)/mini_allocator.o \ diff --git a/benchmarks/blob_keys/.gitignore b/benchmarks/blob_keys/.gitignore new file mode 100644 index 000000000..01143f192 --- /dev/null +++ b/benchmarks/blob_keys/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ + +# Generated benchmark results and profiling artifacts stay local. +/*results*/ +*.csv +*.tsv +*.json +*.jsonl +*.log +*.out +*.err +perf.data* diff --git a/benchmarks/blob_keys/MUTATION_PROFILE.md b/benchmarks/blob_keys/MUTATION_PROFILE.md new file mode 100644 index 000000000..bcedc2d24 --- /dev/null +++ b/benchmarks/blob_keys/MUTATION_PROFILE.md @@ -0,0 +1,255 @@ +# Custom insertion and overwrite: continued profiling + +Latest scan follow-up: [shared-prefix Scan4096 profiling and retained fix](SHARED_SCAN4096_PROFILE.md) +makes these scans 12.8–13.4% faster than original main across all four key sizes. +The retained fix also removes the intermediate random-scan tradeoff. All 882 +runs of the final full matrix verified; custom insertion, overwrite, and hits +retain roughly 2% overhead. + + +Borrowing key and message metadata through the mutation helpers removes most of +the remaining overwrite regression. In the longer paired confirmation, +**overwrite overhead falls from 4.2% to 0.4%**, and **insertion overhead from +3.3% to 1.7%**. These are paired geometric-mean elapsed-time changes against the +same frozen main baseline used in the preceding report. + +Both improvements relative to the pre-change branch have 95% intervals below +zero. Both remaining focused intervals versus main include zero. However, the +shorter control sweep still finds **2.4% insertion overhead**, with an interval +of **+1.5% to +3.4%**. The two campaigns are consistent with a small remaining +insertion cost; the overall no-small-key-regression objective is not established. +Overwrite is close to main in both campaigns. No broader control shows a clear +new regression relative to the pre-change branch. + +## Focused uninstrumented confirmation + +| Workload | Before / main | After / main | After / main 95% interval | After / before | After / before 95% interval | +|---|---:|---:|---:|---:|---:| +| Custom insertion | +3.3% | +1.7% | -0.5% to +3.9% | -1.6% | -2.9% to -0.3% | +| Custom overwrite | +4.2% | +0.4% | -0.3% to +1.1% | -3.6% | -4.5% to -2.8% | + +Each case has 15 paired repetitions, seeds 42–56, with alternating variant order. +A run performs 16 measured batches over 65,536 keys: 1,048,576 inserts or +replacement overwrites. Keys are 64 bytes, values 32 bytes; the custom comparator +orders by the last eight bytes. Small keys remain inline in the actual +blob-capable implementation. Each batch builds a fresh mutable tree; overwrite +prefill and verification occur outside timing. The first batch is an unmeasured +warmup of the same path. + +Separate median times are **971.7 / 1012.1 / 986.6 ns per insertion**, and +**988.3 / 1028.5 / 992.9 ns per overwrite** (main / before / after). +Paired median improvements are 2.6% for insertion and 3.1% for overwrite. +Paired median after/main overhead is 1.8% for insertion and 0.5% for overwrite. +These medians need not equal ratios of the separate absolute medians. + +The table uses geometric means so its estimate and interval use the same +statistic. Intervals use Student-t on paired log ratios (14 degrees of freedom), +then transform back to percentages. The CSVs retain paired medians, extrema, +individual ratios, and every run. Negative means less elapsed time, not a +throughput percentage. No successful sample or outlier was discarded. + +## Cause and retained change + +The preceding mutation profiles concentrated in pivot search and leaf +incorporation, with additional samples around aggregate copies in the insertion +body. Disassembly showed partial metadata stores followed by wider reloads when +keys were repackaged for calls. Those sample locations motivated the experiments; +sampling skid alone cannot prove their cost. + +The existing insertion wrapper already owns by-value key and message objects. +For large keys it also keeps the encoded-key buffer alive through the entire +insertion. The inner body nevertheless copied those metadata objects into more +locals, and copied them again into by-value search/incorporation arguments. + +The retained change carries pointers to the wrapper's objects through: + +- `btree_insert_encoded`, which now reads its borrowed arguments directly; +- mutable pivot-search dispatch and its lexical/custom wrappers; +- `btree_create_leaf_incorporate_spec`, which takes borrowed key/message inputs + and reads them directly. + +The incorporation specification still stores the metadata it needs. Its inputs +must remain valid for the call and must not alias the output specification; all +production callers use the insertion wrapper's independent objects. The two +private helper-test callers are adapted to pass addresses of their own inputs. +Application comparator and merge callbacks continue to receive their original +arguments, and fallible blob comparisons still propagate errors. No public API, +B-tree format, prefix behavior, or ownership rule changes. + +The useful change is the complete path through these helpers. Merely changing a +callee to take pointers while immediately copying their contents into local +aggregates did not give a consistent improvement. Removing those intermediate +copies and retaining pointers across the next call boundaries did. + +In the selected GCC build, insertion-body stack allocation falls from 920 to +840 bytes, and leaf-incorporation stack allocation from 424 to 392 bytes. A fresh +cycle profile puts the insertion body's sample share at about 2.7% for insertion +and 2.6% for overwrite, down from about 4.8% for both. Compiler outlining changes +the exact symbol boundaries, so these shares support the diagnosis but are not +an exact cycle attribution. + +Final three-repetition hardware-counter measurements show: + +| Workload / variant | Instructions / op | Branches / op | Branch misses / op | Blocked forwarding / op | +|---|---:|---:|---:|---:| +| insert / baseline | 2843.1 | 483.2 | 11.1 | 58.7 | +| insert / before | 2958.6 | 501.6 | 11.8 | 55.5 | +| insert / final | 2884.3 | 499.8 | 11.9 | 48.5 | +| overwrite / baseline | 2818.0 | 476.8 | 9.7 | 63.4 | +| overwrite / before | 2903.7 | 493.9 | 10.3 | 56.1 | +| overwrite / final | 2816.7 | 487.9 | 10.3 | 50.7 | + +The fix removes about **74 instructions per insertion** and **87 per overwrite**. +Insertion retains about 41 additional instructions and 17 additional branches +versus main; its remaining elapsed-time cost has not been fully attributed. +Counter runs are instrumented and separate from the uninstrumented confirmation. +All events ran for 100% of enabled time without multiplexing, gated to the active +user-mode operation intervals. The final six cycle profiles lost zero samples. + +## Experiments rejected + +The artifacts retain the full progression, including negative results: + +- Pointer parameters alone, forced inlining, a single inlined insertion body, + separate checked/mutable leaf loops, a smaller cold comparison path, a compact + pivot-search return value, and callback-argument reuse or a callback helper + did not consistently improve both workloads. +- Ordinary conditional assignments of search bounds compiled back into the + original branches. Explicit mask operations did remove branches and cut + mispredictions, but increased elapsed time. For example, the combined mask + variant reduced insertion branch misses from 11.8 to 4.6 per operation while + increasing its screen time from 1011 to 1076 ns. It was rejected. +- A **benchmark-only inline-key control** bypassed stored blob/sentinel dispatch + in mutable custom searches. It removed instructions and branches but did not + consistently improve elapsed time. It cannot handle mixed/special keys and is + deliberately excluded from the working source. No node metadata or format + change was introduced based on that diagnostic. +- Further member-by-member specification copies did not improve on the retained + complete borrowing path. + +The working source contains only the retained argument-passing change. Lower +instruction counts, fewer branch misses, or a favorable short sample were not +sufficient by themselves to select a variant. + +## Broader controls + +All nine earlier small-key configurations were measured for both insertion and +overwrite, plus 64-byte random and custom hits: 20 cases, seven repetitions, +three variants, **420 verified runs**. Writes use four measured batches; reads +use one million timed lookups. These are exploratory per-case intervals with six +degrees of freedom and no multiple-comparison adjustment. + +The main baseline has the earlier lexical shortcut in **checked reads**. Its +mutable lexical pivot search still calls the application comparator, whereas +both the pre-change and final blob-key branches already use the lexical shortcut +there. Consequently, the large lexical-write advantage over this baseline +**predates this patch**. The after/before columns measure this patch's effect. +This distinction does not affect the custom-comparator target workloads. + +| Case | After / before | After / before 95% interval | After / main | +|---|---:|---:|---:| +| 24-random-insert | -2.8% | -12.6% to +8.0% | -29.4% | +| 24-random-overwrite | -0.5% | -6.4% to +5.8% | -26.8% | +| 24-common-insert | -2.0% | -3.9% to +0.0% | -29.3% | +| 24-common-overwrite | +0.6% | -3.2% to +4.5% | -26.6% | +| 64-random-insert | -2.0% | -5.9% to +1.9% | -28.4% | +| 64-random-overwrite | +0.3% | -4.5% to +5.2% | -24.4% | +| 64-random-hit | -1.8% | -5.5% to +2.0% | +0.6% | +| 64-common-insert | -6.0% | -8.0% to -3.9% | -27.3% | +| 64-common-overwrite | -2.6% | -6.6% to +1.6% | -22.6% | +| 256-random-insert | +0.7% | -0.2% to +1.6% | -23.6% | +| 256-random-overwrite | +0.4% | -0.7% to +1.5% | -25.5% | +| 256-common-insert | -5.2% | -9.3% to -1.0% | -22.0% | +| 256-common-overwrite | -1.2% | -5.4% to +3.2% | -20.4% | +| 512-random-insert | -0.1% | -0.7% to +0.6% | -18.2% | +| 512-random-overwrite | -0.1% | -1.2% to +1.1% | -20.1% | +| 512-common-insert | -1.0% | -1.4% to -0.7% | -19.3% | +| 512-common-overwrite | -0.6% | -1.2% to -0.0% | -18.5% | +| 64-nonlexical-insert | -3.5% | -5.9% to -1.0% | +2.4% | +| 64-nonlexical-overwrite | -3.0% | -4.0% to -2.1% | -0.0% | +| 64-nonlexical-hit | -2.0% | -3.4% to -0.5% | +1.5% | + +The shorter custom overwrite control is effectively at main's time (-0.02%, +95% interval -0.74% to +0.71%). Custom insertion remains +2.43% (+1.47% to +3.40%). +Custom point hits remain +1.53% (+0.89% to +2.16%) versus main in this window, +while improving relative to the pre-change binary. Random hits have no clear +remaining gap (+0.57%, -2.15% to +3.36%). No read algorithm was changed; these +controls do not attribute timing shifts to a particular compiler-layout effect. +The full 63-case matrix, packing, scans, and public API performance were not +remeasured in this round. + +## Validation and reproducibility + +- **510 uninstrumented runs, 270 counter runs, and six fresh cycle profiles** all + passed verification and residency checks, with zero measured I/O and major + faults. Work counts and checksums match within every uninstrumented pair. + Packed read layouts also match. Mutable-write layout fields in this harness + are not populated, so matching zero layout fields does not establish identical + mutable-tree geometry. +- The complete debug suite passed **295 tests, zero failures, five existing + skips**. Separate debug, release, and ASan runs each passed all 14 B-tree + layout/config tests and all three native blob-key tests. Existing native tests + exercise mixed inline/blob keys, lexical/custom ordering, updates and old + results, blob lifetimes, callback failures, and mutation blob-read failures. + ASan leak detection was disabled; these sanitizer runs did not enable UBSan. +- Formatting and whitespace checks pass. Production changes for this round are + `src/btree.c` and `src/btree_private.h`; `tests/unit/btree_test.c` adapts the two + helper calls. All 110 production source/header hashes match the frozen final + profile. Earlier unrelated working changes are preserved; no commit or push. +- `baseline` is frozen main `d9a61a9abf6feadf017c951d965e2f3224b5eb30` with the + preceding checked-read lexical shortcut. `before` is the retained + `entry_direct_leaf` implementation from [RESIDUAL_PROFILE.md](RESIDUAL_PROFILE.md). + `final` is this worktree's validated source; `borrow_through` is the selected + experiment before comment/style cleanup. Main's checkout is untouched. +- Intel Core i7-12800H, CPU 2; GCC 14.2.0. Library flags are release + `-O3 -ffast-math -flto`, with `-march=native` additionally used for the unchanged + harness. Cache is 512 MiB, configured device size 2 GiB. Governor is performance + with a 2.5 GHz cap; frequency was not fixed or changed. +- All benchmark windows held both benchmark and machine locks exclusively. + Other project completion records and the exact-content completion confirmation + for the stale async-state heading were checked. This owner had no child agents. + No builds, tests, edits, or result analysis ran during measurement. Requests, + releases, and host observations are retained; all locks are released. +- Focused timings ran October 29, 21:50–21:53 America/Los_Angeles; controls ran + 21:54–22:01; final profiling finished at 22:04 (October 30, 04:50–05:04 UTC). + These are resident B-tree CPU measurements, not disk-bound, tail-latency, + concurrent-write, compaction-policy, or end-to-end database results. + +## Evidence + +- [Insertion comparisons](mutation-results-20261029/plain-insert-comparison.csv), + [pairs](mutation-results-20261029/plain-insert-pairs.csv), and + [runs](mutation-results-20261029/plain-insert-runs.csv). +- [Overwrite comparisons](mutation-results-20261029/plain-overwrite-comparison.csv), + [pairs](mutation-results-20261029/plain-overwrite-pairs.csv), and + [runs](mutation-results-20261029/plain-overwrite-runs.csv). +- [Controls](mutation-results-20261029/plain-controls-comparison.csv), + [pairs](mutation-results-20261029/plain-controls-pairs.csv), and + [runs](mutation-results-20261029/plain-controls-runs.csv). +- [Counters](mutation-results-20261029/perf-stat.csv), + [cycle reports](mutation-results-20261029/cycle-profile-summary.txt), + [profile hashes](mutation-results-20261029/cycle-profile-index.json), + [retained patch](mutation-results-20261029/fix.patch), and + [source, validation, environment, and coordination evidence](mutation-results-20261029/evidence.json). + +Raw logs, frozen sources/binaries/libraries, disassembly, earlier profile sample +attribution, and build/report scripts are retained under +`.cache/mutation-profile-20261029/` in this topic worktree. Pre-change binaries +remain in the preceding campaign directories recorded by the profile metadata. +`report.py` regenerates the CSVs and checks identities, verification, matching +work/results, event scheduling, lost samples, and releases. Files prefixed +`experiment-` are diagnostic or intermediate patches; only `fix.patch` describes +the cumulative retained change. + +To repeat a focused workload with a fresh output directory: + +```sh +python3 .cache/mutation-profile-20261029/run.py \ + --out .cache/mutation-profile-20261029/new-overwrite \ + --mode plain --variants baseline before final --repetitions 15 \ + --cases 64-nonlexical --operation overwrite --batches 16 +``` + +The runner acquires both locks and requires matching pause acknowledgements from +any active project. Use `--operation insert` for insertion. Full control and +profiling commands are recorded in the raw manifests. diff --git a/benchmarks/blob_keys/Makefile b/benchmarks/blob_keys/Makefile new file mode 100644 index 000000000..c3a75c21e --- /dev/null +++ b/benchmarks/blob_keys/Makefile @@ -0,0 +1,6 @@ +# Invoke from the repository root: +# make -f Makefile -f benchmarks/blob_keys/Makefile blob-key-bench +.PHONY: blob-key-bench +blob-key-bench: $(BINDIR)/blob_key_bench +$(BINDIR)/blob_key_bench: $(OBJDIR)/benchmarks/blob_keys/blob_key_bench.o $(OBJDIR)/benchmarks/blob_keys/blob_key.o $(OBJDIR)/benchmarks/blob_keys/blob_key_checks.o $(LIBDIR)/libsplinterdb.a +-include $(OBJDIR)/benchmarks/blob_keys/blob_key_bench.d $(OBJDIR)/benchmarks/blob_keys/blob_key.d $(OBJDIR)/benchmarks/blob_keys/blob_key_checks.d diff --git a/benchmarks/blob_keys/OVERWRITE_PROFILE.md b/benchmarks/blob_keys/OVERWRITE_PROFILE.md new file mode 100644 index 000000000..74c80b220 --- /dev/null +++ b/benchmarks/blob_keys/OVERWRITE_PROFILE.md @@ -0,0 +1,187 @@ +# Custom-comparator overwrite profile + +Latest follow-up: [mutation profiling and fixes](MUTATION_PROFILE.md) remove +intermediate key/message metadata copies. Longer confirmation now finds +0.4% +custom overwrite and +1.7% insertion time versus main; shorter controls still +confirm roughly 2% insertion overhead. The measurements below are historical. + +Follow-up: [continued profiling](RESIDUAL_PROFILE.md) improves point lookups, +while custom overwrite still has 4.0% paired geometric-mean overhead in a new +15-pair confirmation. The historical measurements below are retained. + +The regression is reduced, but **not eliminated**. In the longer unprofiled +confirmation, the retained candidate is **5.2% slower than main** by paired median, +with a 95% paired-log-ratio interval of **+3.9% to +6.1%**. Before these fixes it +was **11.7% slower** in the same measurement window. The fixes reduce elapsed time +by **6.2%** relative to that pre-fix candidate. + +All 15 focused pairs were still slower than main; all 15 were faster than the +pre-fix candidate. The overall no-small-key-regression objective remains unmet. + +## Focused confirmation + +Each run overwrites 65,536 existing 64-byte keys in a random permutation, with +32-byte replacement values, for 16 measured batches. The application comparator +orders keys by their final eight bytes, so lexical comparison cannot substitute +for the callback. Small keys remain inline. + +| Comparison | Median elapsed-time change | 95% interval | Observed paired range | +|---|---:|---:|---:| +| Pre-fix candidate / main | +11.7% | +10.7% to +13.1% | +8.6% to +16.8% | +| Retained candidate / main | +5.2% | +3.9% to +6.1% | +0.4% to +7.4% | +| Retained / pre-fix candidate | -6.2% | -7.1% to -5.3% | -9.0% to -2.1% | + +These are 15 paired repetitions, seeds 42–56, with alternating variant order. +Negative means less elapsed time, not a throughput percentage. Median times were +1,048.5 ns/overwrite for main, 1,177.2 ns before the fix, and 1,099.7 ns after it. +Ratios are computed within each repetition before taking medians, so they need +not equal ratios of the separate median times. + +The intervals use Student-t on paired log ratios (14 degrees of freedom), then +transform back to percentages. They estimate geometric-mean ratios, not the +reported medians. No samples or outliers were discarded. + +## What profiling found and what changed + +The original counter runs reproduced approximately 11% overwrite overhead. +Cycle samples concentrated in pivot and leaf searches. The new blob-key search +path validated offsets, entry bounds, flags, and encoded lengths at each binary +search comparison, including mutations of trusted, pinned memtable nodes. + +Search implementations now distinguish mutable entries from persisted branch +entries. Mutation searches use the existing trusted entry accessors, while +persisted readers retain the same checked accessors. Both paths retain fallible +blob comparison and propagate blob-read errors. The isolated trusted-entry +variant removed about 486 instructions and 127 branches per custom overwrite, +but improved time by only about 2%; those checks were a contributor rather than +the whole cause. + +Every insertion also initialized and cleaned up a key-encoding buffer even when +the incoming key was small. The public B-tree insertion wrapper now normalizes +and validates configuration, rejects externally supplied encoded keys, and calls +a shared insertion body directly for inline keys. Only large incoming keys +initialize the encoding buffer. The shared body receives key/message pointers +to avoid another aggregate argument copy. This changes scratch-buffer setup; +it does not imply that the former small-key path allocated heap storage. + +Finally, a mutable leaf has no common-prefix compression. For custom comparators, +its search now goes directly to the callback-driven loop instead of entering +generic compressed-leaf dispatch. Lexical mutation searches keep their lexical +fast path. No user callback signatures, comparison semantics, or on-disk encoding +were changed. + +A same-window hardware-counter comparison of the retained code shows: + +| Variant | ns/overwrite | Instructions/overwrite | Branches/overwrite | Blocked forwarding/overwrite | +|---|---:|---:|---:|---:| +| Main | 1,074.7 | 2,818.0 | 476.8 | 63.2 | +| Pre-fix candidate | 1,178.0 | 3,658.6 | 649.3 | 74.0 | +| Retained candidate | 1,116.9 | 3,070.6 | 510.6 | 52.8 | + +These are medians of three **instrumented** runs and are separate from the +unprofiled results above. The counter set was cycles, instructions, branches, +branch misses, and `ld_blocks.store_forward`, restricted to active user-mode +insertion intervals. All counters ran 100% of their enabled time, without +multiplexing. Five cycle profiles had zero lost samples. + +Several experiments were rejected. Explicit callback-member assignments left +the hot pivot callback assembly unchanged. Preparing the query's callback +argument once per search reduced blocked forwarding events but did not improve +time. A smaller cleanup-based lazy-buffer implementation was slower than the +split insertion path. A final experiment bypassing merge-accumulator setup for +definitive overwrites reduced instructions but made custom overwrite slower; +it is **not in the retained implementation**. These results show why lower +instruction or forwarding-event counts alone are insufficient evidence of a fix. +The remaining approximately 5% gap has not been fully attributed or fixed. + +## Broader overwrite controls + +Before the longer focused confirmation, all nine earlier small-key overwrite +configurations plus custom-comparator insertion were remeasured: seven paired +repetitions, four measured batches per run, and three variants, for 210 runs. +This shorter sweep was noisier. Its near-zero custom-overwrite median did not +establish equivalence; the longer focused confirmation above establishes a +remaining regression. The two campaigns are kept separate. + +| Case | Before/main | After/main | After/before | After/main 95% interval | +|---|---:|---:|---:|---| +| 24-random-overwrite | +4.6% | -2.0% | -7.0% | -5.8% to +0.7% | +| 24-common-overwrite | +5.8% | -0.4% | -5.3% | -2.7% to +4.5% | +| 64-random-overwrite | +6.0% | +0.2% | -3.2% | -1.3% to +4.0% | +| 64-common-overwrite | +6.3% | +2.4% | -5.0% | +0.2% to +3.6% | +| 256-random-overwrite | +7.7% | +1.5% | -4.8% | -0.7% to +3.1% | +| 256-common-overwrite | +1.8% | -4.6% | -4.3% | -6.9% to -2.4% | +| 512-random-overwrite | -0.5% | -6.7% | -4.0% | -7.3% to -4.1% | +| 512-common-overwrite | +2.7% | -2.0% | -4.4% | -2.4% to -0.0% | +| 64-nonlexical-insert | +10.2% | +3.0% | -6.2% | +1.4% to +5.1% | +| 64-nonlexical-overwrite | +8.1% | +0.0% | -5.9% | -0.9% to +5.0% | + +The matrix intervals use six degrees of freedom and are exploratory per-case +intervals without multiple-comparison adjustment. Custom insertion retains +approximately 3.0% median overhead; the 64-byte shared-prefix overwrite case +retains approximately 2.4%. Point lookups, scans, packing, and public API +performance were not remeasured after these insertion changes. + +## Correctness and method + +- All **255 unprofiled runs** and **147 counter runs** passed verification and + residency checks. Each timed workload verifies values, ordering/checksums, + and allocator reclamation outside the active interval. All reported measured + I/O, asynchronous I/O, and major faults were zero. +- The full debug suite passed **295 tests, zero failures, five existing skips**. + The 14 B-tree layout/configuration tests and three native blob-key tests passed + in release and AddressSanitizer builds. ASan leak detection was disabled. +- The native tests now inject blob-read errors during insertion into both a root + leaf and a tree with internal pivots, for lexical, custom, and zero-prefix + configurations. Subsequent inserts and full-tree validation check that errors + leave the tree usable and release locks. Existing tests also exercise UPDATE + merging, old-result ownership, callback failure, blob lifetime, malformed + persisted entries, and rejected formats. +- Main is frozen commit `d9a61a9abf6feadf017c951d965e2f3224b5eb30`. The pre-fix + candidate is the format-4 working tree at `bb000cc3` plus compact statistics, + attachment accounting, and the retained [packing fixes](PACK_PROFILE.md). + All variants use the identical harness; hashes and incremental patches are + archived. The topic branch is `robj/blob-keys`; changes remain uncommitted. +- Intel Core i7-12800H, CPU 2; GCC 14.2.0. Libraries use release + `-O3 -ffast-math -flto`; the harness additionally uses `-march=native`. + 512 MiB cache and 2 GiB configured device. CPU frequency was not fixed; the + recorded governor was performance with a 2.5 GHz cap. No settings were changed. +- Every measurement window held both benchmark and machine locks exclusively. + No other registered project was active, and this owner had no child agents. + Completion confirmations, requests, and releases are retained. No builds, + tests, edits, or result analysis ran concurrently with measurement. +- The broad unprofiled matrix ran October 28, 21:57–22:01 America/Los_Angeles + (2026-10-29 04:57:18–05:01:18 UTC). The focused confirmation ran October 28, + 22:02–22:04 (05:02:27–05:04:44 UTC). All locks were released. +- These are resident B-tree CPU measurements, not disk-bound, tail-latency, + full-compaction, public API, or production-throughput measurements. + +## Evidence + +- [Focused comparisons and intervals](overwrite-results-20261028/focused-comparison.csv), + [all focused pairs](overwrite-results-20261028/focused-pairs.csv), and + [all 45 focused runs](overwrite-results-20261028/focused-runs.csv). +- [Broader comparisons](overwrite-results-20261028/comparison.csv), + [paired samples](overwrite-results-20261028/pairs.csv), and + [all 210 matrix runs](overwrite-results-20261028/runs.csv). +- [All hardware-counter runs](overwrite-results-20261028/perf-stat.csv), + [identity, validation and coordination records](overwrite-results-20261028/evidence.json), + and [retained source/test patch](overwrite-results-20261028/fix.patch). + +Raw frozen sources, executables, libraries, perf records, disassembly, stdout, +stderr, test logs, manifests, and build/report scripts are retained in +`.cache/overwrite-profile-20261028/` in the topic worktree. The frozen `candidate` +profile is before this work; `final` is the retained implementation. The +`definitive` profile is a rejected experiment, despite being measured later. + +Repeat the focused confirmation with a fresh output directory: + +~~~sh +python3 .cache/overwrite-profile-20261028/run.py \ + --out .cache/overwrite-profile-20261028/focused-rerun --mode plain \ + --variants baseline candidate final --cases 64-nonlexical \ + --repetitions 15 --batches 16 +~~~ + +The runner checks frozen binary/library hashes and requires current pause +acknowledgements from any newly active project before acquiring both locks. diff --git a/benchmarks/blob_keys/PACK_PROFILE.md b/benchmarks/blob_keys/PACK_PROFILE.md new file mode 100644 index 000000000..56eb93bd8 --- /dev/null +++ b/benchmarks/blob_keys/PACK_PROFILE.md @@ -0,0 +1,153 @@ +# Packing profile and fix + +The packing regression is resolved in this resident small-key sweep. The combined +fix is faster than main in all nine cases by paired median, ranging from 2.4% to +17.4% less elapsed time. Eight per-case intervals exclude zero; the 256-byte random +case remains inconclusive. This does not resolve or remeasure the point-lookup and +custom-comparator write regressions in [the earlier full matrix](SMALL_KEYS_V4.md). + +## Cause and implementation + +The dominant cost was aggregate argument copying before `btree_pack_append_leaf()`. +With GCC 14.2 and LTO, the caller assembled `key` and `message` fields using small +stack stores, then loaded larger vector-sized aggregates to pass them by value. +The candidate incurred about one additional blocked store-forwarding event per +packed tuple. Cycle samples concentrated around those aggregate loads; ordinary +cycle samples can skid, so the counter change and isolated ablation provide the +stronger evidence than assigning a precise stall to one sampled instruction. + +Changing the private helper to take `const key *` and `const message *` removes the +extra aggregate argument copies. The counter falls from about 2.2–2.4 blocked +forwarding events per tuple to about 1.1–1.2. The event count is not zero: some +forwarding stalls are common to both implementations. + +A second cost was scanning every entry in a full leaf to discover whether it +contained a blob key before finding a common prefix. As proposed during review, +`btree_pack_req` now keeps `leaf_has_blob_key`. It is initialized at pack setup, +reset on starting a leaf, and set after successfully appending a blob key. Prefix +compression caps its comparison length at P when that flag is set. It does not +need to scan the leaf. This is transient packing state; the on-disk format and +entry flags are unchanged. + +The existing append rule still starts a new leaf when a blob key encounters C>P. +Small keys remain inline. Lexical prefix shortcuts still require lexical ordering. +The native test now places inline keys both before and after blob keys and asserts +that later inline leaves regain prefixes longer than P, exercising flag reset. + +## Isolated counter experiments + +These are medians of three instrumented runs of 24-byte random lexical packing, +64 measured batches per run. Counters cover only active pack calls. Every counter +ran 100% of its enabled time; none was multiplexed. Times here are diagnostic, +not the final unprofiled comparison. + +| Variant | ns/tuple | Cycles/tuple | Instructions/tuple | Blocked forwarding/tuple | +|---|---:|---:|---:|---:| +| Main | 37.894 | 94.467 | 274.961 | 1.170 | +| Candidate before fix | 43.827 | 109.202 | 306.218 | 2.245 | +| Argument pointers only | 34.625 | 86.306 | 303.218 | 1.115 | +| Per-leaf flag only | 44.730 | 111.458 | 299.156 | 2.220 | +| Both changes | 33.504 | 83.506 | 296.093 | 1.130 | + +Passing arguments by pointer is the dominant improvement. The flag removes work, +but alone did not consistently improve elapsed time. With both changes, the +64-byte random case took 41.729 ns/tuple versus 51.952 before the fix; the 256-byte +shared-prefix case took 60.062 versus 68.447. These ablations do not establish that +the flag improves every workload: the pointer-only 256-byte shared-prefix median +was 58.270 ns/tuple. An earlier conditional-scan ablation is also retained in the +raw data; the implemented flag avoids the scan entirely. + +Four cycle profiles covered main and the pre-fix candidate on 24- and 64-byte +random keys, with zero lost samples. Sampling used `cpu_core/cycles/u`, period +1,000,003, and DWARF call chains. One earlier attempt failed before measurement +because its perf ring buffer exceeded the available mapping allowance; retrying +with 64 pages succeeded without changing machine settings. Failed setup is +retained separately and excluded from performance results. + +## Unprofiled confirmation + +All 189 runs passed: nine cases × seven repetitions × three binaries. Values below +are medians of matched elapsed-time ratios. Negative means less time, not a +throughput percentage. No outliers were discarded. Before/main and after/main +were measured in the same window. + +| Key bytes / distribution | Before/main | After/main | After/before | After/main 95% interval | +|---|---:|---:|---:|---| +| 24-random | +12.7% | -13.4% | -21.8% | -22.8% to -7.8% | +| 24-common | +5.4% | -14.8% | -19.2% | -16.8% to -11.4% | +| 64-random | +10.8% | -10.6% | -19.7% | -22.5% to -2.6% | +| 64-common | +7.9% | -12.4% | -19.1% | -14.6% to -8.3% | +| 256-random | +5.4% | -2.4% | -7.5% | -8.0% to +0.4% | +| 256-common | +9.7% | -6.1% | -13.7% | -9.3% to -3.7% | +| 512-random | +3.5% | -17.4% | -20.5% | -19.3% to -16.2% | +| 512-common | +5.3% | -5.7% | -8.9% | -8.2% to -2.4% | +| 64-nonlexical | +8.3% | -9.4% | -16.4% | -13.1% to -4.5% | + +The intervals use Student-t on seven paired log ratios and then transform back to +percentages. They estimate geometric-mean ratios, not the displayed median, and +are exploratory per-case intervals without adjustment for multiple comparisons. +The 256-byte random interval includes a small slowdown. Some baseline runs were +noisy; the per-run results and all paired ranges are retained. + +Leaf counts, compressed-leaf counts, prefix bytes, and packed allocation bytes +matched across all three variants for every seed. Every run reported zero measured +I/O, zero asynchronous I/O, and zero major faults. Values, checksums, ordering, and +allocator reclamation were verified outside the timed region. + +## Method and validation + +- Baseline: main `d9a61a9abf6feadf017c951d965e2f3224b5eb30`, the same frozen binary + used for the full format-4 matrix. Pre-fix candidate: `bb000cc3` plus the frozen + format-4 compact-statistics and attachment-accounting changes. Fixed candidate: + that same source with the two packing changes. Source/library/binary hashes + and isolated patches are preserved. +- Intel Core i7-12800H, pinned to CPU 2; GCC 14.2.0. Libraries use release + `-O3 -ffast-math -flto`; the identical harness additionally uses `-march=native`. + The host used intel_pstate's performance governor with a recorded 2.5 GHz cap; + frequency was not fixed. No machine settings were changed. +- Same nine packing configurations as the earlier matrix: 24/64/256/512-byte + lexical keys, random and shared-prefix distributions, plus a 64-byte custom + comparator; 32-byte values. 65,536 input records for 24/64-byte keys and 32,768 + for 256/512-byte keys; 512 MiB cache, 2 GiB device. Final timings use 16 batches + rather than the earlier matrix's four, with seeds 42–48 and alternating variant + order. Setup, verification, and reclamation are outside active timing. +- Both benchmark and machine locks were held exclusively for every measurement + window. No other project was active and the owner had no children. Registration + confirmations, requests, and releases are archived. No builds, edits, or result + analysis ran inside measurement windows. +- Final unprofiled window: 2026-10-29 04:22:08–04:24:10 UTC (October 28, + 21:22–21:24 America/Los_Angeles). Both locks were released. +- After the fix: 14 B-tree layout/configuration tests and three expanded native-key + tests passed in debug, release, and AddressSanitizer builds. All 66 public quick + tests passed in debug. ASan leak detection was disabled. The complete unit suite + was not rerun for this local packing change. +- This is a resident packing CPU comparison. It does not establish disk-bound, + full compaction, public API, tail-latency, or other-operation performance. + +## Evidence and reproduction + +- [All unprofiled runs](pack-results-20261028/runs.csv), + [paired samples](pack-results-20261028/pairs.csv), and + [comparisons and intervals](pack-results-20261028/comparison.csv). +- [All 99 hardware-counter runs](pack-results-20261028/perf-stat.csv). +- [Identities, coordination and test evidence](pack-results-20261028/evidence.json). +- [Final incremental source and test patch](pack-results-20261028/packing.patch). + +Raw frozen sources, binaries, stdout/stderr, perf records, disassembly, validation +logs, manifests, and build/report scripts remain in `.cache/pack-profile-20261028/` +in this topic worktree. The `stat-leafbit` campaign contains the final isolated +ablation; `plain-final` contains the unprofiled confirmation. + +With the frozen profiles in place, repeat the unprofiled sweep with a fresh output +directory from this worktree: + +~~~sh +python3 .cache/pack-profile-20261028/run.py \ + --out .cache/pack-profile-20261028/plain-rerun --mode plain \ + --variants baseline candidate combined --repetitions 7 --batches 16 \ + --cases 24-random 24-common 64-random 64-common 256-random 256-common \ + 512-random 512-common 64-nonlexical +~~~ + +The runner verifies frozen binary/library hashes and requires current pause +acknowledgements from any newly active project before measuring under both locks. diff --git a/benchmarks/blob_keys/POINT_HIT_64.md b/benchmarks/blob_keys/POINT_HIT_64.md new file mode 100644 index 000000000..bc34c6f63 --- /dev/null +++ b/benchmarks/blob_keys/POINT_HIT_64.md @@ -0,0 +1,59 @@ +# 64-byte random point-hit remeasurement + +Follow-up: [profiling and the retained lexical pivot optimization](POINT_HIT_PROFILE.md) +updates these timings. The results below describe the pre-optimization branch. + +The current blob-key B-tree remains **6.2% slower than main** by median paired +elapsed time. The 95% paired-log-ratio interval is **+3.9% to +7.0%**; 14 of +15 pairs were slower. This is similar to the earlier full matrix's +6.3%, +although the campaigns ran at different times and do not isolate the effect +of the intervening fixes. + +| Variant | Median ns/hit | +|---|---:| +| Main | 735.0 | +| Current blob-key candidate | 779.0 | + +The paired changes range from -1.4% to +9.3%. Ratios are calculated within each +pair before taking the median, so the headline need not equal the ratio of the +two median times above. The interval uses Student-t on 15 paired log ratios +(14 degrees of freedom); it estimates the geometric-mean ratio, not the median. + +## Method and validation + +- 15 paired repetitions, seeds 42–56; alternating variant order. No discarded + runs or outliers, and no profiling instrumentation. +- Same workload as the earlier small-key matrix: 65,536 random 64-byte lexical + keys, 32-byte values, one million successful point lookups per run in two + measured batches. The tree is packed before lookup timing. Small keys remain inline. +- Main is frozen commit `d9a61a9abf6feadf017c951d965e2f3224b5eb30`. + The candidate is the current `robj/blob-keys` working tree, including the + retained packing and overwrite fixes. All 110 production source hashes match + the frozen candidate; both executable and static-library hashes were verified. + Source identities and the identical harness hash are retained in the evidence. +- All 30 runs passed correctness and residency checks. Measured I/O, asynchronous + I/O, and major faults were zero. Paired checksums, leaf counts, prefix bytes, + compressed-leaf counts, and packed allocation bytes matched. +- Intel Core i7-12800H, pinned to CPU 2; GCC 14.2.0 release libraries with + `-O3 -ffast-math -flto`, identical harness with `-march=native`. + 512 MiB cache and 2 GiB configured device. CPU frequency was not fixed. +- Both coordination locks were held exclusively. Other registered projects + were complete; this owner had no children. No build, edit, or analysis ran + during measurement. Both locks were released. +- Measured October 28, 22:43:51–22:44:55 America/Los_Angeles + (2026-10-29 05:43:51–05:44:55 UTC). +- This is a resident B-tree lookup measurement, not public API throughput, + storage-bound performance, or tail latency. No source changes were made + for this remeasurement; the remaining lookup overhead has not been profiled here. + +## Evidence + +- [Comparison and interval](point-hit-results-20261028/comparison.csv) +- [All paired results](point-hit-results-20261028/pairs.csv) +- [All 30 runs](point-hit-results-20261028/runs.csv) +- [Identities, commands, host observations, and coordination](point-hit-results-20261028/evidence.json) +- [Lock release](point-hit-results-20261028/release.json) + +Raw output, the runner, and source-identity metadata are in +`.cache/point-hit-20261028/` in this worktree. Frozen executables are reused from +the preceding campaigns; their exact paths and hashes are in the evidence. diff --git a/benchmarks/blob_keys/POINT_HIT_PROFILE.md b/benchmarks/blob_keys/POINT_HIT_PROFILE.md new file mode 100644 index 000000000..0e125a214 --- /dev/null +++ b/benchmarks/blob_keys/POINT_HIT_PROFILE.md @@ -0,0 +1,175 @@ +# 64-byte random point-hit profile + +Latest scan follow-up: [shared-prefix Scan4096 profiling and retained fix](SHARED_SCAN4096_PROFILE.md) +makes these scans 12.8–13.4% faster than original main across all four key sizes. +The retained fix also removes the intermediate random-scan tradeoff. All 882 +runs of the final full matrix verified; custom insertion, overwrite, and hits +retain roughly 2% overhead. + + +Follow-up: [continued profiling](RESIDUAL_PROFILE.md) reduces the remaining +64-byte random and custom-hit overhead to about 1% against matched optimized +main; both intervals span zero. Custom-write regressions remain. The results +below describe the preceding implementation and measurement window. + +The retained change reduces 64-byte random point-hit time to **566 ns**, versus +**781 ns before this change** and **738 ns on unmodified main**. Across 15 paired +repetitions, that is **23.2% less elapsed time than main** and **27.7% less than the +pre-fix branch**. + +This is a general lexical-search optimization, not elimination of the blob-key +cost. Applying the same shortcut to main gives **538 ns/hit**. The retained branch +is still **6.0% slower than that optimized main**, with a 95% interval of +**+4.5% to +6.7%**. The added blob-key overhead remains, and the overall +no-small-key-regression objective is not yet met. + +## Final focused comparison + +| Comparison | Median elapsed-time change | 95% interval | +|---|---:|---:| +| Pre-fix branch / main | +5.9% | +4.9% to +7.0% | +| Retained branch / main | -23.2% | -24.2% to -22.5% | +| Retained / pre-fix branch | -27.7% | -28.3% to -27.1% | +| Main with lexical shortcut / main | -27.3% | -28.0% to -26.8% | +| Retained / main with lexical shortcut | +6.0% | +4.5% to +6.7% | + +Each run performs four million successful lookups in eight measured batches over +65,536 random 64-byte lexical keys, with 32-byte values. The four variants alternate +order across seeds 42–56. All 15 retained/main pairs improved; all 15 +retained/optimized-main pairs were slower. No samples were discarded. + +Ratios are calculated within each repetition before taking medians. Intervals +use Student-t on paired log ratios (14 degrees of freedom), then transform back +to percentages. They estimate geometric-mean ratios, not intervals around the +displayed median. + +## What profiling found + +The initial counters reproduced the slowdown at about 775.5 versus 735.1 ns/hit: +the branch executed roughly **372 additional instructions and 65 additional +branches per hit**. Cycle samples concentrated in pivot search: 30.8% of branch +samples and 29.5% of main samples were in that function. Leaf search accounted +for about 10.5% in both. Both profiles had zero lost samples. + +Pivot search built by-value callback arguments and invoked the application +comparator even when `key_compare_is_lexical` explicitly allowed unsigned +bytewise lexical comparison. Disassembly and sampled instruction offsets +identified callback-argument construction as a major cost in both versions. +The blob-aware path also adds entry/key validation and blob dispatch around +that existing cost. Samples can skid; the attribution to the callback path is +supported by the controlled optimization and counters, not sample addresses +alone. + +A same-window counter comparison of the final code shows: + +| Variant | Instructions/hit | Branches/hit | Blocked store-forwarding events/hit | +|---|---:|---:|---:| +| Main | 2,572.0 | 511.3 | 49.6 | +| Pre-fix branch | 2,944.4 | 576.2 | 53.4 | +| Retained branch | 2,617.5 | 507.8 | 10.9 | + +A separate matched-optimization counter window found 2,238.4 instructions, +464.7 branches, and 3.0 blocked forwarding events per hit on optimized main, +versus 2,617.5, 507.8, and 10.8 on the retained branch. Thus roughly 379 +instructions, 43 branches, and eight blocked forwarding events per hit remain. +Their exact contribution to elapsed time has not been isolated. Checked +entry/key decoding and the remaining blob-aware dispatch/argument construction +are follow-up targets; this experiment does not prove that every remaining +cycle belongs to validation. + +These counters are medians of three instrumented repetitions, separate from the +unprofiled timing comparisons. All counters ran for 100% of enabled time without +multiplexing. Events were user-mode cycles, instructions, branches, branch misses, +and `ld_blocks.store_forward`, enabled only around the measured lookup batches. + +## Retained implementation and rejected experiments + +The retained change is confined to `src/btree.c`. Pivot search dispatches once +between lexical and callback loops. The lexical loop compares two ordinary +inline keys directly with `slice_lex_cmp`, only when the existing lexical flag +is enabled. Blob and sentinel comparisons keep the fallible existing path. +Custom ordering, query-key semantics, and persisted-entry checks are preserved. +There is no on-disk format change. + +The search body remains shared source, with explicit lexical/callback and +checked/mutable specializations. Four small wrappers force separate compiled +loops. This matters: the first implementation added the lexical branch inside +the common loop and improved lexical hits, but regressed custom-comparator +insertion, overwrite, and hits by about 10–12% relative to the pre-fix branch. +Simply moving the condition or splitting only lexical/callback functions still +left GCC combining paths; those two intermediate variants were inspected but +not timed. The retained four-way specialization removes the observed custom +regression. + +A separate experiment moved blob-key materialization scratch initialization out +of ordinary inline lookups. It made little difference in the counter screen +and was not retained. Its combined version and the rejected common-loop +implementation are preserved as experiments, not current code. + +## Final controls + +Seven paired repetitions per case, with main, pre-fix, and retained variants: +126 verified runs. Reads use one million operations per run; insertion and +overwrite use four measured batches. + +| Case | Retained / main | Retained / pre-fix | Retained / pre-fix 95% interval | +|---|---:|---:|---| +| 64-byte random miss | -26.0% | -27.6% | -29.8% to -26.0% | +| 64-byte shared-prefix hit | -20.2% | -23.9% | -26.1% to -19.7% | +| 512-byte random hit | -17.1% | -18.6% | -19.4% to -18.0% | +| 64-byte custom insertion | +5.0% | -0.1% | -2.7% to +2.0% | +| 64-byte custom overwrite | +5.3% | -2.3% | -4.7% to +3.0% | +| 64-byte custom hit | +6.8% | +0.2% | -1.6% to +3.9% | + +The custom controls show no clear new regression; they do not establish +equivalence. Their previous overhead relative to main remains. These exploratory +per-case intervals use six degrees of freedom without multiple-comparison +adjustment. This is a control subset, not a rerun of the full 63-case matrix. +Packing and scans were not remeasured after this change. + +## Validation and measurement conditions + +- The retained code passed the full debug suite: **295 passed, zero failed, + five existing skips**. All 14 B-tree layout/configuration tests and three + native blob-key tests also passed in release and AddressSanitizer builds. + ASan leak detection was disabled. No new test was added: existing native + tests cover lexical/custom ordering, zero-prefix blobs, mutable and packed + lookups, corruption, and propagated blob-read errors. +- Across all experiments, **372 unprofiled runs, 54 counter runs, and two cycle + profiles** passed correctness and residency checks. Of those, 186 unprofiled + runs compare the retained code. Measured reads/writes, asynchronous I/O, and + major faults were zero. Paired checksums, allocation bytes, leaf counts, + compressed-leaf counts, and prefix bytes matched. +- Baseline main: `d9a61a9abf6feadf017c951d965e2f3224b5eb30`. Pre-fix branch: + `bb000cc3` plus the existing format-4, attachment-accounting, packing, and + overwrite changes. The retained code is still uncommitted in + `robj/blob-keys`. Optimized main is a frozen experimental copy; main itself + was not edited. Source, library, executable, and identical harness hashes + are retained, and the current production source matches the measured copy. +- Intel Core i7-12800H, pinned to CPU 2; GCC 14.2.0, release + `-O3 -ffast-math -flto`, identical harness with `-march=native`. + 512 MiB cache, 2 GiB configured device. CPU frequency was not fixed. +- Every measurement window held both benchmark and machine locks exclusively. + Other registered projects were complete; this owner had no children. + No builds, edits, tests, or result analysis ran during measurements. + All locks are released. +- Final controls ran October 28, 23:14:59–23:18:48 America/Los_Angeles; + final focused comparison ran 23:19:06–23:22:58 + (2026-10-29 06:14:59–06:18:48 and 06:19:06–06:22:58 UTC). +- Resident B-tree CPU measurements only: no storage-bound, public API, + production tail-latency, or full-compaction claim. + +## Evidence + +- [Final focused comparisons](point-hit-profile-results-20261028/plain-final-comparison.csv), + [pairs](point-hit-profile-results-20261028/plain-final-pairs.csv), and + [runs](point-hit-profile-results-20261028/plain-final-runs.csv). +- [Final controls](point-hit-profile-results-20261028/controls-final-comparison.csv), + [pairs](point-hit-profile-results-20261028/controls-final-pairs.csv), and + [runs](point-hit-profile-results-20261028/controls-final-runs.csv). +- [All hardware counters](point-hit-profile-results-20261028/perf-stat.csv), + [retained source change](point-hit-profile-results-20261028/fix.patch), and + [identities, validation, and coordination](point-hit-profile-results-20261028/evidence.json). +- Earlier candidate comparisons, rejected patches, frozen sources and binaries, + raw stdout/stderr, `perf.data`, disassembly, and analysis scripts are preserved + under `.cache/point-hit-profile-20261028/` in this worktree. diff --git a/benchmarks/blob_keys/README.md b/benchmarks/blob_keys/README.md new file mode 100644 index 000000000..d6699bb4c --- /dev/null +++ b/benchmarks/blob_keys/README.md @@ -0,0 +1,159 @@ +# Blob-backed key prototype + +Latest scan follow-up: [shared-prefix Scan4096 profiling and retained fix](SHARED_SCAN4096_PROFILE.md) +makes these scans 12.8–13.4% faster than original main across all four key sizes. +The retained fix also removes the intermediate random-scan tradeoff. All 882 +runs of the final full matrix verified; custom insertion, overwrite, and hits +retain roughly 2% overhead. + + +Native-format small-key regression results are in [SMALL_KEYS.md](SMALL_KEYS.md). +The initial native candidate regressed small-key performance. The subsequent +[Scan4096 profile](SCAN4096_PROFILE.md) identifies and fixes the large scan +slowdown. The subsequent [Scan32 remeasurement](SCAN32_RESULTS.md) confirms a +substantial improvement but still finds up to 14.1% median time overhead. +[Scan32 profiling and fixes](SCAN32_PROFILE.md) identify a tree-height threshold +and reduce the largest remaining Scan32 overhead to about 4.3%. +The [format-4 full matrix](SMALL_KEYS_V4.md) remeasures all 63 B-tree cases. +[Packing profiling and fixes](PACK_PROFILE.md) subsequently remove the measured +packing regression. [Custom-comparator overwrite profiling](OVERWRITE_PROFILE.md) +reduces its measured overhead from 11.7% to 5.2%; a regression remains. + +This experiment stores large logical keys in SplinterDB's existing blobs and +indexes compact key descriptors in the real mutable and packed B-trees. It is a +standalone experimental codec. Its measured version is commit da464513. +The subsequent [native B-tree implementation](../../docs/blob-keys.md) changes +the persistent format; this harness retains its original codec and does not +measure the native encoding. Public SplinterDB large-key writes remain disabled. + +Each encoded key contains its logical length, up to 64 actual leading key bytes, +and an existing blob descriptor bounded to 128 bytes. The maximum encoded key is +202 bytes, below the 512-byte inline-key limit for 4 KiB pages. The blob format's +existing hierarchical descriptors handle larger logical lengths. + +The comparator uses the current `data_config` callbacks. Stored keys are encoded; +`user_key.is_query_key` identifies raw query keys. This prototype defines +unsigned-byte lexical ordering: it compares cached prefixes, then streams blob +fragments when the prefixes cannot decide. The prefix is not a fingerprint, and +a prefix match never proves that two longer keys are equal. + +**Arbitrary application comparators cannot use this shortcut automatically.** +The native implementation materializes full keys for the existing custom +comparator; it does not introduce blob-aware application callbacks. The existing `key_compare_is_lexical` flag +describes stored byte representations; it is false in blob mode even though +the logical order is lexical. The inline baseline enables normal lexical +branch prefix compression. + +The logical-key hash uses a simple streaming FNV-1a implementation consistently +for raw and encoded keys, independently of fragment boundaries. This hash is +experimental, not a proposed replacement for the production hash. + +## Ownership and error scope + +Every tree owns a separate key-blob arena. Packing and repacking clone each key +into the destination arena. Full extents can be shared by the existing +`blob_clone`; tails are copied. The harness releases the source tree and arena, +flushes and evicts the cache, and then verifies destination lookups and scans. +After releasing every tree, allocator usage must return to its initial value. +The separate arenas include some overhead that a native pack implementation +could eliminate. + +The transforming iterator does work in initialization and `next()`, preserving +the observation-only contract of `curr()`. Scans reconstruct the full key, +rather than measuring descriptor traversal alone. + +Comparison and hashing callbacks currently have no error return channel. When a callback detects malformed encoding or a blob-read failure, the +prototype aborts; it never manufactures a comparison result after failure. Counters and callbacks are +single-threaded. + +The original prototype's integration checklist included fallible comparison/hash dispatch, +blob-aware user callbacks, key/pivot/trunk ownership, log reservation and key +blob durability, recovery validation, iterator/API materialization, logical +versus stored byte accounting, and concurrency/progress testing. This harness +does not exercise the full SplinterDB trunk, log, routing filters, crash recovery, +or simultaneous writers. The native implementation's current scope and +remaining work are documented separately above. This harness cannot establish +end-to-end database throughput +or latency guarantees. + +## Build and validate + +From the worktree root: + +```sh +make -j4 -f Makefile -f benchmarks/blob_keys/Makefile BUILD_MODE=debug blob-key-bench +mkdir -p .cache/blob-keys +build/debug/bin/blob_key_bench --db .cache/blob-keys/check.db \ + --mode blob --records 256 --queries 512 --key-bytes 4096 \ + --prefix 32 --common-prefix 4088 --cache-mib 32 +build/debug/bin/blob_key_bench --db .cache/blob-keys/codec-check.db \ + --mode blob --cache-mib 64 --checks +make -j4 -f Makefile -f benchmarks/blob_keys/Makefile BUILD_MODE=release blob-key-bench +``` + +Every ordinary run verifies inserts, replacements, hit/miss lookups, complete ordered +scans, cloning across source release, and final reclamation. `--variable` +varies key lengths; `--checks` instead runs focused codec boundary checks. Correctness +runs are not performance evidence. + +## Coordinated measurement + +Ordinary project work uses the shared benchmark and machine locks. Measurements +must acquire both exclusively after every other active project acknowledges +that its root and children have stopped all work. + +```sh +python3 benchmarks/blob_keys/run.py \ + --binary build/release/bin/blob_key_bench \ + --out /path/on/disk/to/new-results-directory \ + --coordination /home/rob/projects/splinterdb/.cache/projects-20261025/coordination \ + --lock-dir /home/rob/projects/splinterdb/.cache/projects-20261011/coordination \ + --repetitions 3 +python3 benchmarks/blob_keys/summarize.py /path/on/disk/to/new-results-directory +``` + +The requesting root must stop its own child agents before invoking `run.py`. +The runner records matching peer acknowledgements, source and binary hashes, +commands, raw output, and elapsed intervals. It refuses to measure without +acknowledgements and releases the lock on failure. Registered completed projects +are recorded separately. Concurrent new work must also obey the shared locks. +Only a run's successfully verified scratch database image is removed; logs remain. + +The matrix compares inline and forced blob keys at 128 and 512 bytes. At 4 KiB, +64 KiB, 1 MiB, and 2 MiB it compares zero versus 32 prefix bytes. Keys either +differ early or share all but their final eight bytes. Each case uses randomized +insertion order, repeatable 50/50 hit/miss queries prepared outside timing, a +64 MiB Splinter cache, and alternating variant order across repetitions. + +Cold-start lookup means the batch begins with an evicted Splinter cache; it does +not mean every query misses the cache. Repeated lookup is the next identical +query stream; check actual read bytes before treating it as fully memory-resident. +The corpus and query bytes are allocated separately from the Splinter cache. +The benchmark is single-threaded and does not clear the operating system's cache; +database I/O uses `O_DIRECT`. + +`read_bytes` and `write_bytes` are cache I/O counters. `allocated_bytes` +includes all live prototype arenas and trees at the end of the phase; pack +temporarily retains both source and destination. `blob_bytes` counts logical +fragment spans passed to comparisons/hashing/copies, not actual DRAM or disk +traffic. `blob_fragments` counts comparator/hash fragment visits, not unique +pages; decode uses the blob materialization API. `comparisons` counts callback +calls, so inline lexical fast paths inside B-tree code need not increment it. + +Detailed measurements and interpretation will be recorded in `RESULTS.md`. + +The [64-byte random point-hit remeasurement](POINT_HIT_64.md) found 6.2% overhead. +Subsequent [profiling and a lexical pivot shortcut](POINT_HIT_PROFILE.md) reduce +point-hit time by 23.2% relative to unmodified main. Applying the same shortcut +to main still leaves 6.0% blob-key overhead; custom-comparator overhead remains. + +[Continued profiling](RESIDUAL_PROFILE.md) streamlines ordinary pivot validation +and inline comparison. Against main with the same lexical shortcut, 64-byte +random and custom hits retain about 1% overhead, with intervals spanning zero. +Custom insertion and overwrite still regress by roughly 6% and 4%; the overall +small-key performance objective remains unmet. + +[Further mutation profiling](MUTATION_PROFILE.md) removes intermediate key/message +metadata copies through insertion and its helpers. Custom overwrite is now near +main (+0.4% in the longer paired comparison); insertion retains roughly 2% +overhead. All 786 measurements verified, with no clear new control regression. diff --git a/benchmarks/blob_keys/RESIDUAL_PROFILE.md b/benchmarks/blob_keys/RESIDUAL_PROFILE.md new file mode 100644 index 000000000..bf51ad32b --- /dev/null +++ b/benchmarks/blob_keys/RESIDUAL_PROFILE.md @@ -0,0 +1,239 @@ +# Remaining small-key regressions: profiling and fixes + +Latest scan follow-up: [shared-prefix Scan4096 profiling and retained fix](SHARED_SCAN4096_PROFILE.md) +makes these scans 12.8–13.4% faster than original main across all four key sizes. +The retained fix also removes the intermediate random-scan tradeoff. All 882 +runs of the final full matrix verified; custom insertion, overwrite, and hits +retain roughly 2% overhead. + + +Latest follow-up: [mutation profiling and fixes](MUTATION_PROFILE.md) remove +intermediate key/message metadata copies. Longer confirmation now finds +0.4% +custom overwrite and +1.7% insertion time versus main; shorter controls still +confirm roughly 2% insertion overhead. The measurements below are historical. + +The retained fix removes most of the remaining point-lookup overhead. Against +**main with the same lexical pivot shortcut**, 64-byte random hit overhead falls +from **6.0% to 1.0%**, and custom-comparator hit overhead from **5.4% to 1.0%**. +Both final 95% intervals include zero. This supports no clear residual regression +in these two cases; it does not establish equivalence. + +The overall no-small-key-regression objective remains unmet. Custom insertion +retains about **5.9%** overhead in the shorter control sweep, custom overwrite +retains **4.0%** in its longer confirmation, and 256-byte random hits retain +**2.5%**. The fix and all experiments stay in the `robj/blob-keys` worktree; +production changes in this round are confined to `src/btree.c`, with a boundary +validation case added to `tests/unit/btree_stress_test.c`. + +## Focused uninstrumented comparisons + +| Case | Before / matched main | After / matched main | After / main 95% interval | After / before | +|---|---:|---:|---:|---:| +| 64-byte random hit | +6.0% | +1.0% | -0.9% to +2.9% | -4.7% | +| 64-byte custom hit | +5.4% | +1.0% | -0.4% to +2.5% | -4.1% | +| 64-byte custom overwrite | +5.3% | +4.0% | +2.4% to +5.6% | -1.2% | + +All percentages in this report's timing tables are **paired geometric-mean +elapsed-time changes**, so the displayed estimate and interval use the same +statistic. Negative means less time, not a throughput percentage. Each interval +uses Student-t on paired log ratios, transformed back to percentages. The CSVs +also retain paired medians, extrema, and every sample. Earlier reports primarily +displayed paired medians; do not compare those estimates as though they were the +same statistic or the same measurement window. + +There are 15 repetitions per case, alternating variant order across seeds 42–56. +Each read run makes four million measured lookups in eight batches over 65,536 +keys. Each overwrite run makes 16 random-permutation passes over 65,536 existing +keys, for 1,048,576 measured overwrites. Values are 32 bytes. The custom comparator +orders by the last eight key bytes; it must use the application callback. + +Separate median absolute times are 512.1 / 551.4 / 522.5 ns for random hits, +787.9 / 824.6 / 791.5 ns for custom hits, and 995.4 / 1049.9 / 1031.1 ns for +custom overwrites (matched main / before / after). Ratios of these separate +medians need not equal the paired estimates above. After/before 95% intervals +are -5.8% to -3.7% for random hits, -5.4% to -2.8% for custom hits, and -2.6% to ++0.2% for overwrite. The overwrite improvement is therefore inconclusive. + +## What changed and why + +The first cycle profiles put approximately 25–29% of lookup samples in pivot +search and 13–14% in leaf search. The blob-capable implementation decoded keys +and selected general validation paths at each pivot comparison. Disassembly +also showed repeated by-value callback argument construction. Controlled variants +isolated three useful changes: + +1. **Validate ordinary pivots directly.** After checking the entry offset and + fixed header bounds, an entry with zero flags and an inline key length uses + one bounds check for header, key, and three contiguous 32-bit counters. Blob, + wide-counter, sentinel, unknown-flag, and oversized-key cases use the existing + complete validator in a separate function. The common case avoids constructing + a general internal key and selecting several encoded layouts. +2. **Compare lexical pivots from their stored header.** An ordinary inline pivot + supplies its length and bytes directly to `slice_lex_cmp`. The existing + lexical/callback specializations remain separate. This shortcut still requires + the application's exact unsigned-byte lexical-order contract. +3. **Prepare callback arguments once per search.** For ordinary inline targets, + pivot search prepares the target `user_key` outside the loop and builds the + stored argument directly from the entry. Callback leaf search does likewise + when the leaf has no common prefix. Query-key flags are preserved; stored keys + carry a false query flag. Blobs, sentinels, and prefix-compressed leaves retain + their existing comparison and materialization paths. + +The entry encoding, format version, callbacks, comparator ordering, and blob +ownership are unchanged. Small keys still remain inline. The ordinary persisted +entry path checks that the counters fit in the page before returning the entry; +it does not merely validate the key bytes. + +A same-window three-repetition counter screen shows the reduction in work: + +| Workload / variant | Instructions / operation | Branches / operation | +|---|---:|---:| +| Random hit: matched main | 2238.4 | 464.7 | +| Random hit: before | 2617.5 | 507.8 | +| Random hit: after | 2344.5 | 476.1 | +| Custom hit: matched main | 2783.9 | 542.4 | +| Custom hit: before | 3162.6 | 609.1 | +| Custom hit: after | 2849.7 | 560.2 | + +These counters are from `stat-direct`; they are instrumented measurements, +separate from the final timing comparisons. The changes reduce about 273 +instructions per random hit and 313 per custom hit. Counter reductions alone +were not sufficient to select a fix: several other variants reduced work but +failed to improve time consistently. + +## Remaining mutation overhead and rejected experiments + +The custom overwrite screen still executes about 2904 instructions per operation +against 2818 on matched main. A separate cache-event screen found no increase in +L1/L2/L3 miss counts after the retained fix relative to main, and only a small +increase in split loads (11.9 versus 11.1 per overwrite). These measurements do +not attribute the remaining 4% elapsed-time gap to a specific memory event. + +The final mutation profiles use 64 measured batches each, covering matched main, +the pre-change branch, and the retained code for insertion and overwrite. Pivot +search remains roughly 20–22% of retained samples and leaf incorporation roughly +15%. The insertion body and libc routines also account for substantial samples. +Function boundaries differ between variants, so sample shares are not a direct +measurement of the added cost. These profiles narrow follow-up work to mutation +search, argument passing, and incorporation; they do not establish a single +remaining cause. + +Experiments retained only as evidence include simpler general validation, +pointer or forced-inline traversal, a different internal key flag layout, +scoped materialization scratch, global callback-argument zeroing, combined leaf +flag checks, and forced inlining of the insertion body. Changing key flag layout +or globally zeroing callback arguments substantially hurt custom lookups. Forced +inlining of insertion made the overwrite counter screen slower (1092 versus +1073 ns for the retained code in that window). None of those experiments is in +the selected source. Normal configuration validation and blob-capable fallbacks +remain necessary even when the incoming key itself is small. + +## Broader uninstrumented controls + +Seven paired repetitions per case and three variants: 273 runs. Reads use one +million timed operations; insertion uses four batches, and Scan 32 uses two +batches of 131,072 scan starts. These are exploratory per-case intervals, without +multiple-comparison adjustment. The broader 512-byte intervals are noisy. + +| Case | Before / matched main | After / matched main | After / main 95% interval | After / before | +|---|---:|---:|---:|---:| +| 24-random-hit | +3.0% | -0.7% | -4.1% to +2.9% | -3.6% | +| 24-common-hit | +3.7% | -1.2% | -3.6% to +1.3% | -4.7% | +| 64-random-miss | +4.6% | +0.5% | -0.7% to +1.7% | -3.9% | +| 64-random-scan32 | -5.8% | -6.8% | -14.6% to +1.7% | -1.1% | +| 64-common-hit | +6.2% | +1.7% | -0.2% to +3.7% | -4.2% | +| 64-common-miss | +3.2% | -1.9% | -4.2% to +0.4% | -4.9% | +| 256-random-hit | +8.6% | +2.5% | +0.9% to +4.1% | -5.6% | +| 256-common-hit | +5.2% | +1.5% | +0.1% to +2.9% | -3.6% | +| 512-random-hit | +7.1% | +3.1% | -0.4% to +6.6% | -3.8% | +| 512-common-hit | +2.2% | +2.3% | -2.6% to +7.4% | +0.1% | +| 64-nonlexical-insert | +5.6% | +5.9% | +3.1% to +8.7% | +0.3% | +| 64-nonlexical-miss | +4.1% | +1.0% | -0.4% to +2.3% | -3.0% | +| 64-nonlexical-scan32 | -0.2% | +0.3% | -1.0% to +1.6% | +0.5% | + +No control has a clear new regression relative to the pre-change branch. This +subset is not a rerun of all 63 cases: packing, long scans, and the full lexical +write matrix were not remeasured in this round. The remaining custom insertion +regression is not materially improved. Both 256-byte hit cases retain positive +intervals versus matched main, though the common-prefix result is marginal. + +## Validation and measurement conditions + +- All **408 uninstrumented runs, 192 counter runs, and 10 cycle profiles** passed + verification and residency checks. All measured I/O and major faults were zero. + Paired uninstrumented runs match operation counts, checksums, packed bytes, + leaf counts, compressed-leaf counts, and prefix bytes. No successful sample or + outlier was discarded. All cycle profiles report zero lost samples. +- The complete debug suite passed **295 tests, zero failures, five existing + skips**. The three native blob-key tests also passed separately in debug. + Release and AddressSanitizer builds each passed all 14 B-tree layout/config + tests and all three native tests. Leak detection was disabled; these current + sanitizer runs used ASan, not UBSan. +- The added malformed-index case enlarges the first key so its narrow counters + extend past the page. The custom-order 400-to-401-byte case exercises the new + ordinary-entry path; the lexical 512-to-513-byte case exercises maximum-length + rejection. Synchronous and asynchronous lookups reject both, and the test + restores the entry before proceeding. Existing tests cover wide counters, + invalid flags, rejected formats, mixed/blob keys, prefix compression, failures, + and ownership. +- Matched main is frozen `d9a61a9abf6feadf017c951d965e2f3224b5eb30` with the + [previous lexical shortcut](POINT_HIT_PROFILE.md) applied in an isolated + experiment. The main checkout is untouched. The pre-change branch is that + report's retained `split` profile. This round's retained profile is + `entry_direct_leaf`, also named `final`. All 110 production source/header hashes + match the topic worktree. Branch HEAD is `bb000cc3`; working changes are uncommitted. +- Intel Core i7-12800H, pinned to CPU 2; GCC 14.2.0. Libraries use release + `-O3 -ffast-math -flto`; the common harness adds `-march=native`. Cache is 512 MiB, + configured device size 2 GiB. The performance governor had a 2.5 GHz maximum; + frequency was not fixed, and no frequency settings were changed. +- Counter events were enabled only around timed user-mode operations and ran + 100% of enabled time without multiplexing. Event lists, commands, source and + binary hashes, per-run outputs, and host observations are retained. +- Every measurement window held both coordination locks exclusively. Other + registrations were complete, with a recorded exact-content completion + confirmation for the stale async-state heading. This root had no child agents. + No builds, tests, edits, or analysis ran during measurement. Each request has a + matching release; all locks are released. +- Final timing windows ran October 29, 16:16–16:32 America/Los_Angeles + (23:16–23:32 UTC); mutation profiles finished at 16:34. These are resident + B-tree CPU measurements, not disk-bound, tail-latency, full-compaction, or + public API performance results. + +## Evidence and reproduction + +- [Focused read comparisons](residual-results-20261029/plain-reads-comparison.csv), + [pairs](residual-results-20261029/plain-reads-pairs.csv), and + [runs](residual-results-20261029/plain-reads-runs.csv). +- [Overwrite comparisons](residual-results-20261029/plain-overwrite-comparison.csv), + [pairs](residual-results-20261029/plain-overwrite-pairs.csv), and + [runs](residual-results-20261029/plain-overwrite-runs.csv). +- [Control comparisons](residual-results-20261029/plain-controls-comparison.csv), + [pairs](residual-results-20261029/plain-controls-pairs.csv), and + [runs](residual-results-20261029/plain-controls-runs.csv). +- [Hardware counters](residual-results-20261029/perf-stat.csv), + [cycle reports](residual-results-20261029/cycle-profile-summary.txt), + [profile hashes](residual-results-20261029/cycle-profile-index.json), + [source/test patch](residual-results-20261029/fix.patch), and + [identity, validation, and coordination evidence](residual-results-20261029/evidence.json). + +Frozen sources, binaries, libraries, raw perf data, stdout/stderr, validation +logs, and all build/report scripts are under `.cache/residual-profile-20261029/` +in this worktree; baseline and pre-change binaries remain in the preceding +campaign's `.cache/point-hit-profile-20261028/`. The evidence JSON records exact +paths and hashes. `report.py` regenerates the tables and checks source identity, +run validation, matching paired layouts, counter scheduling, and lost samples. +Individual experimental patches in the results directory are rejected or +intermediate variants; only `fix.patch` describes the cumulative retained change. + +After coordinating a fresh exclusive window, repeat the focused read set with: + +```sh +python3 .cache/residual-profile-20261029/run.py \ + --out .cache/residual-profile-20261029/new-plain-reads \ + --mode plain --variants baseline candidate final --repetitions 15 \ + --cases 64-random 64-nonlexical --batches 8 +``` + +The runner acquires both locks and requires matching pause acknowledgements from +any active peer. Use a new output directory; the complete control and overwrite +commands are recorded in their raw manifests. diff --git a/benchmarks/blob_keys/RESULTS.md b/benchmarks/blob_keys/RESULTS.md new file mode 100644 index 000000000..ecaab8e9a --- /dev/null +++ b/benchmarks/blob_keys/RESULTS.md @@ -0,0 +1,146 @@ +# Blob-key prototype results + +Blob-backed keys are feasible with the existing blob storage and B-tree +machinery. This prototype verified logical keys through 2 MiB, including +hierarchical descriptors, source-tree reclamation, repacking, and cold-cache +reads. It does **not** enable large keys through the public SplinterDB API. + +The useful performance result is conditional: an inline logical prefix avoids +most blob comparisons when keys differ early. Long common prefixes still +require blob reads, and forcing already-small keys into blobs can lose the +substantial benefit of current branch prefix compression. + +The subsequent [native format-3 implementation](../../docs/blob-keys.md) is +validated separately. These measurements do not describe its encoding or cost. + +## Measurement + +Measured implementation: local commit `da464513`, based on `6e550e1e`, branch +`robj/blob-keys`. Release binary SHA256: +`b6310c06fbc2d9ca1adeb3a9df227d6b2dd84238b4071c8cea533a65bce84036`. + +There were 72 successful runs: six key lengths, early/late differences, two +representations or prefix lengths, and three repetitions. Variant order reverses +in the middle repetition. All runs verify results and reclamation. Single +foreground thread, Intel Core i7-12800H, ext4 on encrypted local block storage, +`O_DIRECT`, 4 KiB pages, 128 KiB extents, 64 MiB Splinter cache. Key/query +generation happens outside timers; queries are 50% hits and 50% misses. The +corpus occupies separate heap memory, so 64 MiB is a cache limit, not a total +process memory cap. Small-key and large-key cases use different record counts; +compare variants within the same row. + +All three other active projects explicitly acknowledged that their roots and +children had stopped work. Two review-only projects were already complete. +Both machine and benchmark locks were held exclusively for the measurement +window, which lasted about 104 seconds. All blob-key child agents were idle. +This excludes registered agents' work, not arbitrary operating-system activity. + +## Prefix effect + +Repeated-lookup batch mean time, in microseconds per operation. Parentheses +show the minimum and maximum batch means across three runs, not latency +percentiles. Neither 4 KiB case issued disk reads during the repeated pass. + +| Logical key | Difference | No prefix | 32-byte prefix | +|---|---|---:|---:| +| 4 KiB | Early | 1.413 (1.386–1.414) | 0.681 (0.659–0.805) | +| 4 KiB | Last eight bytes | 2.596 (2.501–2.603) | 2.636 (2.473–2.728) | +| 64 KiB | Early | 3.852 (3.412–4.605) | 2.899 (2.752–3.437) | +| 1 MiB | Early | 43.388 (40.759–46.479) | 39.800 (37.061–46.037) | +| 2 MiB | Early | 82.001 (72.025–91.647) | 92.028 (84.977–99.330) | + +For early-difference 4 KiB keys, the prefix gave about 2.07 times the lookup +throughput. It reduced blob-comparison fallbacks from 439,721 to 16,743 per +32,768-query pass. Hits still compare the complete logical key for equality. +The 2 MiB case did not improve: only 16 records and 64 queries were used, and +full-key equality work dominates. The prefix is not a universal large-key +speedup. + +Late-difference 64 KiB and 1 MiB keys exposed cache pressure. With a 32-byte +prefix, repeated lookup took 71.777 and 824.875 microseconds per operation, +respectively, and read another 14.402 and 31.984 MiB per batch. Calling those +passes entirely memory-resident would be incorrect. A cold-start batch means +the cache was evicted before the batch; it does not mean every lookup was cold. + +## Inline versus forced blobs + +Both variants use the same logical order and hash. The inline variant retains +normal lexical branch prefix compression. All repeated lookup passes below +had zero measured disk-read bytes. + +| Key size | Difference | Inline lookup, us/op | Blob/32 lookup, us/op | Inline allocated MiB | Blob allocated MiB | +|---|---|---:|---:|---:|---:| +| 128 B | Early | 0.461 | 0.568 | 5.500 | 7.750 | +| 128 B | Last eight bytes | 0.382 | 1.175 | 1.750 | 7.750 | +| 512 B | Early | 0.596 | 0.534 | 11.625 | 10.250 | +| 512 B | Last eight bytes | 0.441 | 1.405 | 1.375 | 10.250 | + +Allocation is total live allocator footprint after source release, including +prototype arena/tree metadata. It is not process RSS or key payload alone. +The 512-byte late-difference blob case was about 3.19 times slower and used +7.45 times the allocator footprint. Inline compression is particularly valuable +for that workload. + +At 512 bytes with early differences, blob insertion including key construction +was 0.915 versus 1.117 microseconds per operation for inline keys. Packing +including clone and logical hashing was slower: 1.025 versus 0.809. For +128-byte early-difference keys, blob insertion was 0.766 versus 0.609 and packing +0.459 versus 0.216. These short samples support keeping an inline path, not a +universal crossover threshold. + +Packing must process the logical key for hash construction even if prefix +comparison avoids its bytes. The prototype uses streaming FNV-1a for both +variants, an experimental hash rather than the production hash. Native +performance will depend on the actual hash and callback implementation. + +The existing clone policy also matters: 64 KiB keys have tails to copy, whereas +1 MiB keys can share full extents. During packing, live allocation for the +64 KiB/prefix32 early-difference case reached 140.125 MiB before source release; +the 1 MiB case, with the same 64 MiB logical key payload, reached 75.875 MiB. +Both settled to 64.750 MiB after source release. This is measured allocation +behavior of these prototype arenas, not a full compaction-space bound. + +## Correctness and integration boundary + +Eleven correctness cases passed across debug, release, and GCC +AddressSanitizer/UndefinedBehaviorSanitizer builds. Coverage includes empty and +proper-prefix keys, embedded zero bytes, first/last-byte differences, variable +lengths, seeded hash equivalence, page/extent/hierarchy boundaries, malformed +descriptors, prefix inconsistency, clone lifetime, cache eviction, and exact +allocator-baseline restoration. LeakSanitizer was disabled; allocator checks +do not substitute for a general heap-leak test. + +Each stored key contains its logical length, a prefix, and a descriptor bounded +to 128 bytes. The configured maximum encoded length is 202 bytes. The measured +32-byte-prefix representations were 76 bytes except the 1 MiB flat descriptor, +which was 132 bytes. Hierarchical encoding keeps larger descriptors compact. + +The prefix shortcut lives inside this prototype's lexical comparator. +**An arbitrary application comparator must retain control over ordering.** +Native support should use its blob-aware callback, or materialize keys for its +existing callback. The current `key_compare_is_lexical` flag remains false +for the encoded blob representation; logical and stored lexical properties +are different. + +This is a real-cache, real-B-tree experiment with explicit tree/key-arena +ownership pairs. Native integration still needs fallible comparison/hash +contracts, key-aware log/durability/recovery handling, pivot/trunk ownership, +public iterator and callback semantics, and concurrent progress validation. +Comparison failures currently abort the harness. No production crash recovery, +routing-filter lookup, full trunk compaction, concurrent insertion, tail-latency +distribution, or end-to-end database throughput claim follows from these runs. + +## Reproduce and inspect + +Build and run instructions are in [README.md](README.md). Generated evidence is +retained locally, outside version control: + +- [Complete measurement manifest and pause acknowledgements](../../.cache/blob-key-results-20261025/manifest.json). +- [All phase medians and ranges](../../.cache/blob-key-results-20261025/summary.csv). +- [Readable phase tables](../../.cache/blob-key-results-20261025/summary.md). +- [Validation commands, results, compiler, and source/binary hashes](../../.cache/blob-key-validation/validation.json). + +The measurement directory also contains every raw stdout/stderr file and the +request/release records. Successful scratch benchmark databases were removed; +raw evidence was retained. No additional blob-key performance experiment was run after +this window. Source and measured-binary hashes were independently audited. diff --git a/benchmarks/blob_keys/SCAN32_PROFILE.md b/benchmarks/blob_keys/SCAN32_PROFILE.md new file mode 100644 index 000000000..0981fb288 --- /dev/null +++ b/benchmarks/blob_keys/SCAN32_PROFILE.md @@ -0,0 +1,193 @@ +# Scan32 profiling and fixes + +The subsequent [full format-4 matrix](SMALL_KEYS_V4.md) remeasures every small-key +B-tree operation after restoring compact contiguous statistics. The profiling +and measurements below describe the preceding fixed-64-bit candidate. + +The largest Scan32 regressions came from an extra internal tree level and +avoidable work in iterator setup and tuple validation. The implemented changes +reduce the remaining median Scan32 overhead to 4.3% for 64-byte random lexical +keys and 4.2% for the 64-byte custom comparator. The 256-byte random case is +2.3% faster than main in the final sweep. + +The no-small-key-regression objective is not yet established: the two 64-byte +cases retain overhead, and some paired ranges are broad. Small keys stay inline, +and the requested fixed-width uint64 statistics remain in place. + +## Tree height explains the size-dependent slowdowns + +Index-entry headers grew from 23 bytes on main to 35 bytes with three uint64 +statistics. Leaves have the same occupancy, but internal-node fanout falls. +Direct inspection of the packed trees found: + +| Case | Main levels, including leaf | Native levels, including leaf | +|---|---:|---:| +| 65,536 64-byte random keys | 3 | 4 | +| 65,536 64-byte custom-comparator keys | 3 | 4 | +| 32,768 256-byte random keys | 4 | 5 | + +A controlled threshold comparison uses the **before-fix native candidate** and +main with 62,000 versus 63,000 random 64-byte keys. Both have three levels at +62,000 keys. At 63,000, only the native candidate grows to four levels. + +| Records | Main ns/range | Before-fix native ns/range | Paired median overhead | +|---:|---:|---:|---:| +| 62,000 | 1,451.3 | 1,537.7 | +6.0% | +| 63,000 | 1,450.5 | 1,658.7 | +14.0% | + +These are three paired repetitions per size. Main stays nearly flat while the +native tree adds about 121 ns/range. This supports the additional level as a +major contributor; matching leaf counts alone did not reveal it. The CSV +retains the paired ranges, including the noisier 62,000-key case. + +## Implemented changes + +1. **Defer the unbounded upper-endpoint walk.** Initialization used to traverse + the tree again to find the rightmost page even when the caller requested + only a short range. For pinned immutable iterators, that lookup now waits + until a visited page claims to be last in its level. It still reconciles + that page with the root and propagates lookup failures. A false early + last-page marker returns an error instead of silently truncating the scan. + Forward movement, reverse movement and seeks prepare bounds before returning. + Finite upper bounds and mutable/copied-page semantics retain their paths. +2. **Combine full-entry bounds validation.** Reading a complete leaf tuple had + started doing both key-only bounds checks and the full key-plus-message + check. The complete span now bounds both, as on main. The full-key size limit + rejects infinity encodings; the static assertion documents that constraint. + Key-only searches retain their checks, and key flags, message flags/types, + blob descriptors and prefix constraints are still validated. +3. **Keep blob parsing/comparison out of ordinary search code.** Explicit + noinline attributes prevent link-time optimization from pulling descriptor + scratch and blob comparison code into each pivot-search function. This had + a modest benefit in isolation. + +There is no persistent-format change or statistics-width rollback in this fix. + +## Hardware-counter evidence + +The following are per-range medians from two paired runs of the 64-byte random +case, with counters enabled only during the measured Scan32 loop. The final +unprofiled timing sweep below uses five pairs. + +| Metric per 32-key range | Main | Before fix | Final | +|---|---:|---:|---:| +| Time, ns | 1,469.8 | 1,670.1 | 1,516.5 | +| User cycles | 3,661.3 | 4,160.8 | 3,778.0 | +| Retired instructions | 10,716.4 | 12,367.1 | 10,967.4 | +| Branches | 2,402.8 | 2,680.7 | 2,448.0 | +| Branch misses | 9.32 | 9.25 | 9.13 | + +The final path removes about 1,400 instructions per range. The custom-comparator +case shows the same reduction, from 12,575.8 to 11,176.1 instructions/range. +The full-entry validation change alone removes about 12 instructions per +returned tuple in the long-scan counter experiment. + +This was not another large store-forwarding regression: before-fix branch-miss +counts were similar to main, and blocked-forwarding counts were lower. +Cycle samples highlighted pivot search and cache-page acquisition. All accepted +cycle profiles report zero lost samples. + +An intermediate endpoint-only change improved Scan32 but worsened long scans. +Its hot iterator instructions matched the old version after relocation, while +cycles increased: code layout was affecting performance. The exact processor +mechanism was not isolated. That version was not retained; the final version +also shortens validation and was remeasured across the complete matrix. +Intermediate binaries, counters and failed trial results remain in the raw +artifacts. + +## Final unprofiled Scan32 results + +Positive changes mean slower than main. Each value is the median of five +paired time ratios; the range is the smallest and largest paired change, +**not a confidence interval**. + +| Key bytes | Distribution / comparator | Time change vs main | Paired range | +|---:|---|---:|---:| +| 24 | Random lexical | -2.9% | -5.0% to +0.5% | +| 24 | Shared prefix | -3.7% | -4.9% to -1.0% | +| 64 | Random lexical | +4.3% | -0.8% to +7.2% | +| 64 | Shared prefix | -3.3% | -4.7% to -1.4% | +| 64 | Custom comparator | +4.2% | +0.3% to +6.6% | +| 256 | Random lexical | -2.3% | -6.0% to +0.0% | +| 256 | Shared prefix | -4.6% | -5.4% to -2.3% | +| 512 | Random lexical | -10.7% | -11.5% to -8.8% | +| 512 | Shared prefix | -6.6% | -7.9% to -5.5% | + +The [earlier remeasurement](SCAN32_RESULTS.md) found +14.1%, +12.0% and +9.4% +for 64-byte random, 64-byte custom and 256-byte random, respectively. Those +earlier values came from a different measurement window; the three-way counter +runs above provide a same-window before/final comparison for the 64-byte cases. + +## Scan4096 confirmation + +Three paired repetitions per case. This screen checks long scans after the +additional hot-path change; it does not prove universal performance neutrality. + +| Key bytes | Distribution / comparator | Time change vs main | Paired range | +|---:|---|---:|---:| +| 24 | Random lexical | -2.3% | -2.8% to -2.1% | +| 24 | Shared prefix | -7.8% | -7.9% to -7.6% | +| 64 | Random lexical | -2.8% | -2.8% to -1.9% | +| 64 | Shared prefix | -6.2% | -6.5% to -4.5% | +| 64 | Custom comparator | +0.6% | -3.5% to +1.7% | +| 256 | Random lexical | +2.7% | +2.5% to +3.9% | +| 256 | Shared prefix | -7.4% | -7.4% to -6.9% | +| 512 | Random lexical | +1.8% | +0.1% to +2.4% | +| 512 | Shared prefix | -8.1% | -8.3% to -7.6% | + +The 256/512-byte random cases retain small median long-scan overhead. The +intermediate version's roughly 8–12% regression in smaller random keys is gone. + +## Reproducibility and validation + +Main is frozen at `d9a61a9a`; the before-fix branch is `6689e29a` (production +code identical to `737b6161`). The final raw profile is named `combined`. +The candidate base commit, exact patch, all 110 source/header hashes, binary and +library hashes, and identical harness hash are recorded with the evidence. + +Intel Core i7-12800H, pinned to CPU 2; GCC 14.2.0 with matching +`-O3 -ffast-math -flto -march=native` link/build configuration. Values are +32 bytes, cache 512 MiB, configured device 2 GiB. There are 65,536 records for +24/64-byte cases and 32,768 for 256/512-byte cases. + +Final Scan32 uses four batches of 131,072 starts: 524,288 ranges and 16,777,216 +returned tuples per run. Scan4096 uses two batches of 4,096 starts: +8,192 ranges and 33,554,432 tuples. Main/candidate order alternates, with matching +seeds within each pair. Setup, warmup, full replay verification and reclamation +are outside timing. Pinning and alternating order do not eliminate clock, +thermal or layout sensitivity. + +- All 144 final timing runs verified, were cache resident, and had zero + measured synchronous/asynchronous I/O, process I/O and major faults. +- Leaf counts, compressed-leaf counts, prefix bytes, returned counts and + checksums match across every final pair. +- Full debug suite on the final source: 300 tests, 295 passed, zero failed, + five skipped. +- Release native blob-key tests: 3/3; format/layout tests: 14/14. +- The new unbounded-iterator regression fails on the previous code (two root + acquisitions instead of one) and passes on the final code. It also covers + pinned/copied pages, both initialization APIs, forward/reverse movement, + seeks, and false last-page markers. +- Formatting and whitespace checks passed. + +The user confirmed no other agents were running; the live agent list contained +only this root. Both benchmark and machine locks were held exclusively for each +measurement window. No compilation, testing or analysis overlapped measurements. +This remains a resident direct-B-tree investigation; other operations and the +public API were not rebenchmarked. + +## Evidence + +- [Final paired comparisons and absolute times](scan32-profile-results-20261028/comparison.csv) +- [All final timing/validation rows](scan32-profile-results-20261028/timings.csv) +- [Per-run hardware counters](scan32-profile-results-20261028/counters.csv) +- [Height-threshold comparison](scan32-profile-results-20261028/height-threshold.csv) +- [Hashes and validation results](scan32-profile-results-20261028/evidence.json) +- [Raw artifacts](../../.cache/scan32-profile-20261028/): source patch, frozen + profiles/binaries, layout diagnostics, perf data/reports, commands, manifests, + stdout/stderr and correctness logs. + +`campaign_combined.py` reproduces the final windows and +`summarize_combined.py` regenerates the checked CSVs and metadata. Run them from +the worktree root. The session-specific runner records this campaign's +no-other-agents confirmation; re-establish coordination before any future reuse. diff --git a/benchmarks/blob_keys/SCAN32_RESULTS.md b/benchmarks/blob_keys/SCAN32_RESULTS.md new file mode 100644 index 000000000..d293db1a4 --- /dev/null +++ b/benchmarks/blob_keys/SCAN32_RESULTS.md @@ -0,0 +1,94 @@ +# Scan32 remeasurement after the Scan4096 fix + +The iterator assignment fix substantially improves direct B-tree Scan32, but +the branch still misses the no-small-key-regression objective. The largest +remaining median time increases are 14.1% for 64-byte random lexical keys, +12.0% for the 64-byte custom comparator, and 9.4% for 256-byte random keys. +Shared-prefix cases show 1.8–2.6% median overhead. + +These are fresh October 28 measurements of main `d9a61a9a`, the rebased native +candidate before the fix (`fdbffde0`), and the fixed candidate (`737b6161`). +They reuse the frozen binaries and identical harness from the +[Scan4096 investigation](SCAN4096_PROFILE.md). Small keys remain inline. +There are no new source changes in this measurement. + +This report records the pre-profiling candidate. The subsequent +[Scan32 profile and fixes](SCAN32_PROFILE.md) supersede its current-performance +conclusion; the original measurements below remain unchanged. + +## Results + +Each percentage is the median of five paired elapsed-time changes. Positive +means slower than main. The last column is the smallest and largest of the +five fixed/main changes, not a confidence interval. + +| Key bytes | Distribution / comparator | Before fix vs main | Fixed vs main | Fixed paired range | +|---:|---|---:|---:|---:| +| 24 | Random lexical | +33.8% | +7.1% | +0.1% to +14.0% | +| 24 | Shared prefix | +21.3% | +1.8% | -0.9% to +6.4% | +| 64 | Random lexical | +36.3% | +14.1% | +10.4% to +15.3% | +| 64 | Shared prefix | +18.0% | +2.6% | +1.7% to +2.6% | +| 64 | Custom comparator | +31.1% | +12.0% | +8.9% to +18.8% | +| 256 | Random lexical | +23.8% | +9.4% | +8.0% to +11.4% | +| 256 | Shared prefix | +18.6% | +2.6% | +2.0% to +2.6% | +| 512 | Random lexical | +10.9% | +1.0% | -2.0% to +2.7% | +| 512 | Shared prefix | +17.2% | +2.5% | +2.1% to +2.9% | + +In throughput terms, the largest remaining median losses are 12.4% (64 random), +10.7% (64 custom), and 8.6% (256 random). Time increases and throughput losses +use different denominators; the CSV records both. + +The fix reduces Scan32 time by 9.5–18.9% versus the before-fix candidate across +these nine cases. Including that control in the same windows isolates the +benefit from the change of main baseline since the +[October 26 report](SMALL_KEYS.md). Its historical percentages should not be +treated as a paired before/after comparison with this campaign. + +The residual 64-byte random and custom-comparator slowdowns occur in all five +pairs. The 24-byte cases are noisier, and the 512-byte random range includes +zero. This sweep does not diagnose the residual overhead. Iterator setup and +search are candidates for follow-up profiling because short scans repeat those +steps more often per returned tuple, but no new CPU profiles were collected here. + +## Method and validation + +- Intel Core i7-12800H, process pinned to CPU 2; GCC 14.2.0, matching + `-O3 -ffast-math -flto -march=native` builds. +- Nine cases, three variants, five paired repetitions: 135 runs. Seeds 42–46; + main/before-fix/fixed order alternates with its reverse. +- 65,536 keys for 24/64-byte cases; 32,768 for 256/512-byte cases. + Values are 32 bytes, cache 512 MiB, configured device 2 GiB. +- Each run executes two measured batches of 131,072 range starts, returning + exactly 32 tuples per range: 262,144 ranges and 8,388,608 tuples. +- Full replay warms the exact access sequence and verifies keys/values outside + timing. Measured checksums, post-measurement replay, full-tree verification, + and allocator reclamation all passed. +- All 135 runs were resident. Measured process and hooked read/write I/O, + including asynchronous I/O, and major faults were zero. +- Leaf counts, compressed-leaf counts, prefix bytes, returned counts and + checksums matched across all three variants for every case/seed. +- Binary hashes were checked before every window. All 110 recorded fixed + source/header hashes also match the current checkout. +- The user confirmed no other agents were active; the live agent list contained + only the root. Both benchmark and machine locks were held exclusively during + each measurement window. No other work ran concurrently from this agent. + +The samples measure resident single-thread B-tree scans. They do not establish +cold-cache, long-running, or other-hardware performance. The paired ranges retain +observed variation; CPU pinning and alternating order do not eliminate clock +or thermal drift. The existing correctness suite results are recorded in the +Scan4096 report; this task changed only measurement artifacts and documentation. + +## Evidence + +- [All 135 timing and validation rows](scan32-results-20261028/runs.csv) +- [Paired comparisons, absolute times and throughput](scan32-results-20261028/comparison.csv) +- [Commits, hashes, coordination windows and validation](scan32-results-20261028/evidence.json) +- [Raw stdout/stderr, commands, manifests, runner and summarizer](../../.cache/scan32-bench-20261028/) +- [Frozen binaries, source hashes and identical harness](../../.cache/scan4096-profile-20261028/) + +The raw runner records the user's no-other-agents confirmation for this +campaign. Re-establish coordination before reusing it; that recorded assumption +does not authorize a future uncoordinated run. Each window's manifest contains +its exact invocation. The summarizer can be rerun from the worktree root with +`python3 .cache/scan32-bench-20261028/summarize.py`. diff --git a/benchmarks/blob_keys/SCAN4096_PROFILE.md b/benchmarks/blob_keys/SCAN4096_PROFILE.md new file mode 100644 index 000000000..9e5d8b2f6 --- /dev/null +++ b/benchmarks/blob_keys/SCAN4096_PROFILE.md @@ -0,0 +1,157 @@ +# Scan4096 profiling after rebasing onto main + +Latest scan follow-up: [shared-prefix Scan4096 profiling and retained fix](SHARED_SCAN4096_PROFILE.md) +makes these scans 12.8–13.4% faster than original main across all four key sizes. +The retained fix also removes the intermediate random-scan tradeoff. All 882 +runs of the final full matrix verified; custom insertion, overwrite, and hits +retain roughly 2% overhead. + +The subsequent [full format-4 matrix](SMALL_KEYS_V4.md) includes a fresh Scan4096 +sweep after the additional iterator fixes and compact contiguous statistics. +The diagnosis and measurements below remain the history of this scan fix. + +The large inline-key scan regression was caused by a store-to-load forwarding +stall in the code GCC 14 generated for `btree_iterator_curr()`. A small source +change removes the stalled aggregate copy. It preserves the blob path, +entry validation, prefix compression, and the native format. + +The branch was first rebased onto main `d9a61a9a`. The uncommitted native work +was saved in a local commit; `backup/blob-keys-before-main-20261028` retains the +pre-rebase state. Rebase integration preserves main's smaller async state, +fallible reference ownership, and new message API. The rebased native candidate +before the performance fix is `fdbffde0`. No changes were pushed. + +## Diagnosis + +Cycle sampling during the measured scan puts 36.9% of the rebased candidate's +CPU samples in `btree_iterator_curr()`, versus 14.9% on main. After the fix, +that function returns to 14.9%. The confirmation profile uses one sample per +1,000,003 user cycles and reports no lost records. An earlier, higher-rate +profile lost a few ring-buffer chunks and is retained only as exploratory +evidence. The installed `perf annotate` crashed, so instruction locations were +obtained with `perf script` and matched against `objdump`. + +For an ordinary inline key, this source expression: + +```c +*curr_key = key_create(FALSE, length, data); +``` + +generated a temporary with this sequence (other instructions omitted): + +```asm +movq $0, 0x20(%rsp) # first 8 bytes of the key +mov %rax, 0x28(%rsp) # next 8 bytes +movdqa 0x20(%rsp), %xmm0 # immediately reload both stores as 16 bytes +movups %xmm0, (%r12) # copy to the output key +``` + +On the tested Intel processor, the wide load cannot be forwarded from those +two narrower stores without a stall. The load and its dependent instruction +are the dominant sampled instruction region. This happens on every tuple, +so a 4096-key range pays it roughly 4096 times. + +The fix assigns `kind`, `is_query_key`, `is_blob`, and `user_slice` directly +through the output pointer. GCC then writes the result directly, eliminating +the temporary and wide reload. This is a code-generation problem observed +with this compiler and CPU, not evidence that the logical aggregate assignment +is incorrect C. + +## Causal check + +The following are medians of three paired runs of 24-byte random keys. +Counters cover only the scan interval; setup and verification are excluded. + +| Metric per tuple | Main | Rebased native | Assignment fix | +|---|---:|---:|---:| +| Time, ns | 16.15 | 28.40 | 16.60 | +| User cycles | 40.22 | 70.70 | 41.34 | +| Retired instructions | 221.07 | 239.33 | 236.33 | +| Blocked store forwarding | 0.014 | 1.008 | 0.008 | +| Branch misses | 0.019 | 0.019 | 0.019 | + +The relevant hardware event is `cpu_core/ld_blocks.store_forward/u`. The single +source change removes about one blocked forwarding event and 29 cycles per +tuple while retiring only three fewer instructions. It removes about 96% of +the excess scan time in this case. Shared-prefix 24-byte keys show the same +counter change: 1.008 to 0.008 blocked events per tuple, with time falling from +30.86 to 19.02 ns (main: 19.88 ns). + +These are actual blob-capable binaries with small keys kept inline. No checks +were disabled for the experiment, and no hypothetical blob I/O was removed. + +## Unprofiled confirmation + +Five paired repetitions per case, alternating main/fixed run order and using +the same seed within each pair. Positive time change means slower than main. +The range shows the smallest and largest paired change, not a confidence +interval. Small remaining differences should not be described as neutrality. + +| Key bytes | Distribution / comparator | Median time change | Paired range | +|---:|---|---:|---:| +| 24 | Random lexical | +2.8% | +2.7% to +3.8% | +| 24 | Shared prefix | -4.8% | -5.1% to -4.4% | +| 64 | Random lexical | +3.0% | +2.9% to +3.6% | +| 64 | Shared prefix | -3.2% | -3.6% to -3.0% | +| 64 | Custom comparator | +2.8% | +2.8% to +3.1% | +| 256 | Random lexical | +2.9% | -3.8% to +4.6% | +| 256 | Shared prefix | -3.8% | -5.6% to -2.9% | +| 512 | Random lexical | +1.9% | -8.7% to +14.2% | +| 512 | Shared prefix | +1.7% | -2.9% to +6.4% | + +All 90 unprofiled runs passed value, order, checksum, residency and reclamation +checks. Measured read/write I/O and major faults were zero. Leaf counts, +compressed-leaf counts and prefix bytes matched main for every paired seed. +The 512-byte cases were noisier; their full ranges are retained above. + +Configuration: Intel Core i7-12800H, pinned to CPU 2; GCC 14.2.0 with matching +`-O3 -ffast-math -flto -march=native` builds and identical harness sources. +Each run uses a 512 MiB cache, a 2 GiB configured device, 32-byte values, +4096 scan starts per batch, and scans 4096 keys at each start. The final timing +runs use four measured batches (67,108,864 returned tuples), with full warmup +and verification outside timing. There are 65,536 keys for 24/64-byte cases +and 32,768 keys for 256/512-byte cases. + +The hardware-counter experiment used eight measured batches. The three +counter variants were main, the rebased candidate, and the single assignment +change. The last differs from the final source only by an explanatory comment. + +## Validation and scope + +- Full debug unit suite: 299 total, 294 passed, zero failed, five skipped. +- Release native blob-key tests: 3/3 passed. +- Release B-tree format/configuration tests: 14/14 passed. +- Formatting and whitespace checks passed. +- All 90 final timing runs and all three confirmation profiles verified. + +The large Scan4096 slowdown is resolved. Random-key scans still show roughly +2–3% median overhead in this sweep. This investigation does not establish that +point lookups, packing, overwrites or the public API have recovered: those +workloads were not rebenchmarked after this fix. The earlier +[small-key report](SMALL_KEYS.md) remains historical evidence for those paths. +The subsequent [Scan32 remeasurement](SCAN32_RESULTS.md) confirms that this fix +also helps short scans, but leaves up to 14.1% median time overhead versus main. +The later [Scan32 profiling report](SCAN32_PROFILE.md) records additional fixes +and a fresh Scan4096 confirmation sweep. + +The user confirmed no other agents were running. Each measurement window still +held the shared benchmark and machine locks exclusively. No compilation, +correctness tests or analysis ran concurrently with measurement. + +## Evidence + +- [Per-run hardware counters](scan4096-results-20261028/counters.csv) +- [Unprofiled paired comparison](scan4096-results-20261028/scan-comparison.csv) +- [Binary hashes and validation](scan4096-results-20261028/evidence.json) +- [Raw profiling artifacts](../../.cache/scan4096-profile-20261028/): + manifests, commands, identical harness, frozen main source, binaries, + build logs, perf recordings, per-symbol reports, disassemblies, isolated + fix patch, stdout/stderr, and correctness logs. + +The raw `run.py` is a session-specific runner recording the user's explicit +no-other-agents confirmation. Before reusing it, re-establish quiescence; +its recorded assumption must not replace coordination with future agents. +Its `--mode stat|record|plain`, `--variants`, `--patterns`, +`--key-bytes`, and `--repetitions` arguments reproduce each recorded window. +`PREFIX_PERF_CONTROL` and `PREFIX_PERF_ACK` FIFOs enable perf only inside +the harness's measured intervals. diff --git a/benchmarks/blob_keys/SHARED_SCAN4096_PROFILE.md b/benchmarks/blob_keys/SHARED_SCAN4096_PROFILE.md new file mode 100644 index 000000000..83fec0e29 --- /dev/null +++ b/benchmarks/blob_keys/SHARED_SCAN4096_PROFILE.md @@ -0,0 +1,260 @@ +# Shared-prefix Scan4096 profiling and fix + +The shared-prefix Scan4096 regression is resolved in the tested workloads. +The retained implementation is **12.8–13.4% faster than original main** by +paired median across 24-, 64-, 256-, and 512-byte shared-prefix keys. A longer +15-pair confirmation finds **18.1% less time than the pre-fix branch** for +64-byte shared-prefix scans. + +An initial inlining-only fix introduced about 1.2% overhead on 24-byte random +scans. A second change isolates blob/index result construction from ordinary +leaf results and removes that tradeoff. The retained version is **4.5% faster +than the pre-fix branch** and **2.0% faster than main** for 24-byte random scans +in the longer confirmation. + +The retained source received a fresh complete 63-case matrix: **882/882 runs +verified**, seven pairs per case, with zero measured I/O or major faults. +No long-scan case has a clearly positive slowdown interval. The overall +no-small-key-regression objective remains open: custom insertion, overwrite, +and point hits retain roughly 2% overhead. + +## Focused confirmation + +These are 15 paired repetitions, seeds 42–56, alternating variant order. +Each run has eight measured batches of 4096 starts scanning 4096 tuples: +134,217,728 returned tuples. Negative time changes mean faster execution. + +| Case / comparison | Paired median | Geometric mean | Geometric-mean 95% interval | +|---|---:|---:|---:| +| 64-common / before versus baseline | +6.4% | +6.0% | +5.0% to +7.1% | +| 64-common / retained versus baseline | -12.7% | -12.9% | -14.0% to -11.8% | +| 64-common / retained versus before | -18.1% | -17.9% | -18.7% to -17.0% | +| 24-random / before versus baseline | +2.7% | +2.6% | +1.9% to +3.3% | +| 24-random / retained versus baseline | -2.0% | -1.9% | -2.8% to -1.0% | +| 24-random / retained versus before | -4.5% | -4.4% | -5.1% to -3.6% | + +Baseline is unmodified frozen main; before is the exact binary from the preceding +full report; retained is the final working source. The experimental profile +named final is the intermediate inlining-only version, not the retained source. +Intervals use Student-t on paired log ratios (14 degrees of freedom), transformed +back to percentages. They estimate geometric-mean ratios, not medians. + +## Diagnosis and retained change + +The initial same-window comparison reproduced the slowdown: about 19.9 ns/tuple +on original main, 20.1 ns on the earlier format-4 candidate, and 21.1 ns on the +pre-fix branch. The latter two execute essentially the same number of +instructions and branches; the slower binary has fewer blocked store-forwarding +events. This is not the earlier one-stall-per-tuple aggregate-copy problem. + +Samples concentrate in iterator advance, current-result construction, and +compressed-key decoding. An alignment-only diagnostic recovered part of the +difference without reducing instructions, showing code-layout sensitivity. +This investigation did not isolate the later regression to one prior change +or identify a precise microarchitectural bottleneck. + +The retained change reduces recurring work: + +1. Explicitly inline btree_iterator_decode_current(), reconstructing the suffix + in the positioning path. Allocation, growth, and prefix replacement stay + in the existing out-of-line prepare_decode() helper. +2. Explicitly inline btree_iterator_check_current() too. Inlining only the + decoder made GCC move the enlarged entry checker out of next(), introducing + a call per tuple on uncompressed leaves. Keeping both inline avoids that. +3. Move aggregate construction into btree_iterator_curr_blob_key() and + btree_iterator_curr_index(). In this GCC build, the ordinary curr() function + drops from 241 to 79 assembly lines. Local stack allocation falls from + 64 to 8 bytes; one additional saved register makes the total stack-use + reduction 48 bytes. + +Every validation, bounds check, prefix preparation, blob materialization, +callback, and error path remains. Positioning still finishes fallible work +before curr() exposes an entry. Ordinary leaf keys retain the direct member +assignments from the earlier store-forwarding fix. No public API, ownership +rule, persistent format, iterator structure, or compression policy changes. + +Only src/btree.c changes in this round. No bounds rewrite or explicit function +alignment is retained. + +## Hardware evidence + +Medians of three counter runs, counting only active user-mode scans. All events +ran for 100% of enabled time without multiplexing. + +| Case / variant | ns/tuple | Cycles/tuple | Instructions/tuple | Branches/tuple | +|---|---:|---:|---:|---:| +| 64-common / baseline | 19.76 | 49.24 | 261.54 | 62.37 | +| 64-common / before | 21.14 | 52.66 | 262.67 | 61.38 | +| 64-common / retained | 17.29 | 43.09 | 247.67 | 56.38 | +| 24-random / baseline | 16.08 | 40.04 | 221.07 | 52.23 | +| 24-random / before | 16.54 | 41.20 | 224.20 | 53.23 | +| 24-random / retained | 15.76 | 39.26 | 223.21 | 50.25 | +| 64-random / baseline | 16.95 | 42.23 | 224.23 | 52.87 | +| 64-random / before | 17.22 | 42.91 | 227.53 | 53.89 | +| 64-random / retained | 16.59 | 41.34 | 226.55 | 50.91 | + +For shared-prefix keys, the retained source removes about **15 instructions, +five branches, and 9.6 cycles per tuple** versus the pre-fix branch. For 24-byte +random keys it removes about one instruction and three branches. Store-forwarding +event counts remain very small. + +The decoder no longer appears as a separate sampled function because its work +is inlined. All nine cycle profiles lost zero samples. Sample shares change +when functions are inlined or outlined; they are not an exact cycle attribution. + +## Full matrix on retained source + +Median elapsed-time changes against original main. Positive means slower, +negative means faster. All small keys remain inline. + +| Key bytes | Distribution / comparator | Insert | Overwrite | Hit | Miss | Scan32 | Scan4096 | Pack | +|---:|---|---:|---:|---:|---:|---:|---:|---:| +| 24 | Random lexical | -30.0% | -28.9% | -26.4% | -26.8% | -14.6% | -2.4% | -12.1% | +| 24 | Shared prefix | -29.4% | -28.7% | -26.7% | -27.2% | -20.7% | -12.9% | -11.6% | +| 64 | Random lexical | -29.0% | -25.7% | -26.6% | -26.4% | -15.3% | -2.4% | -14.0% | +| 64 | Shared prefix | -27.0% | -24.5% | -24.2% | -25.4% | -18.8% | -12.8% | -12.8% | +| 256 | Random lexical | -24.2% | -23.2% | -22.6% | -25.1% | -17.2% | +0.2% | -4.7% | +| 256 | Shared prefix | -21.8% | -22.1% | -20.2% | -19.3% | -19.0% | -13.4% | -8.5% | +| 512 | Random lexical | -18.5% | -20.0% | -19.0% | -24.0% | -18.9% | -1.7% | -16.4% | +| 512 | Shared prefix | -18.5% | -18.4% | -16.2% | -17.3% | -18.9% | -13.3% | -4.2% | +| 64 | Custom comparator | +2.4% | +2.1% | +1.8% | +0.1% | -0.8% | -2.0% | -11.6% | + +The per-case geometric-mean intervals classify **55 cases as faster, +3 as slower, and 5 as overlapping zero**. +These are exploratory intervals without correction across 63 comparisons. + +All shared-prefix Scan4096 cases have intervals below zero. So do 24-, 64-, +and 512-byte random long scans and custom long scans. The 256-byte random case +is +0.2% by median, with a geometric-mean interval of -0.1% to +0.7%: inconclusive, +not proof of equivalence. + +The remaining clear regressions are: + +| Case | Paired median | Geometric mean | Geometric-mean 95% interval | +|---|---:|---:|---:| +| 64-nonlexical-insert | +2.4% | +2.3% | +1.5% to +3.1% | +| 64-nonlexical-overwrite | +2.1% | +2.0% | +1.2% to +2.9% | +| 64-nonlexical-hit | +1.8% | +1.7% | +0.9% to +2.5% | + +All packing medians remain faster than main, but intervals for 256- and 512-byte +shared-prefix packing overlap zero in this window. Outliers were retained. +Large lexical-write and lookup gains include the earlier lexical shortcut; +this baseline does not have the modified-main shortcut used in focused profiles. + +Historical long-scan comparison against the same main binary: + +| Case | Previous full-suite median | Retained full-suite median | Retained geometric-mean 95% interval | +|---|---:|---:|---:| +| 24-random-scan4096 | +2.9% | -2.4% | -7.8% to -0.2% | +| 24-common-scan4096 | +4.3% | -12.9% | -13.1% to -12.4% | +| 64-random-scan4096 | +1.4% | -2.4% | -2.8% to -2.0% | +| 64-common-scan4096 | +6.7% | -12.8% | -13.2% to -11.7% | +| 256-random-scan4096 | +1.4% | +0.2% | -0.1% to +0.7% | +| 256-common-scan4096 | +5.2% | -13.4% | -15.4% to -12.4% | +| 512-random-scan4096 | +1.0% | -1.7% | -2.6% to -0.2% | +| 512-common-scan4096 | +5.2% | -13.3% | -13.6% to -12.9% | +| 64-nonlexical-scan4096 | +1.6% | -2.0% | -2.2% to -1.8% | + +Those full-suite columns come from different windows. The 15-pair focused +comparison above provides direct same-window evidence for the change. + +## Rejected and intermediate experiments + +- Inlining only the decoder helped compressed scans, but outlining the entry + checker added overhead to uncompressed scans. +- A separate helper wrapping decode/materialize/error handling increased call + overhead. Combining that helper with an inline decoder helped less than + inlining the positioning path. +- Inlining only the entry checker did not help. +- Explicit alignment changed timing with unchanged instruction counts; it was + diagnostic and is not retained as a production setting. +- The two-function inline version passed a full matrix but introduced a + confirmed +1.24% geometric-mean cost on 24-byte random scans (95% interval + +0.88% to +1.59%). The retained implementation improves that case relative + to the pre-fix branch. +- A compact rewrite of the existing bounds predicates was slower and rejected. + +The intermediate full matrix remains in the artifacts. The current full matrix +is full-retained; fix.patch is the cumulative retained change. + +## Validation and method + +- Retained source: complete debug suite **295 passed, zero failed, five existing + skips**. Separate debug, release, and ASan runs each passed 14 B-tree + configuration/layout tests and all three native blob-key tests. Existing + coverage includes prefix reconstruction, mixed inline/blob keys, custom + ordering, forward/backward and copied iterators, endpoint errors, and blob-key + read failures. +- ASan leak detection was disabled; UBSan was not enabled. Early standalone + native-test invocations printed usage but returned zero; invalid logs are + retained and not counted as tests. Valid invocations ran all three cases. + The runner now checks the reported test summary. +- **2,085 measurements verified: 1,941 plain runs, 135 counter runs, and nine + cycle profiles.** This includes both full matrices, intermediate screens, + the tradeoff check, and focused confirmations. +- Paired plain runs match work counts, checksums, verified values, and reported + packed layout counters. Active I/O and major faults are zero. Mutable layout + counters are unpopulated; matching zeros do not establish identical mutable + geometry. Tree height was not instrumented. +- Original main: d9a61a9abf6feadf017c951d965e2f3224b5eb30. Before: the exact prior + full-suite candidate. Retained: robj/blob-keys working source on bb000cc3. + All 110 production source/header hashes match the frozen retained source; + only btree.c differs from the start of this round. +- GCC 14.2.0, release -O3 -ffast-math -flto; identical harness additionally uses + -march=native. Intel Core i7-12800H, CPU 2, performance governor and unchanged + 2.5 GHz cap. Frequency was not fixed; between-run observations are not + active-operation clock measurements. +- Full matrix: 24/64/256/512-byte random/shared-prefix lexical keys and 64-byte + custom ordering by the last eight bytes; 32-byte values. 65,536 records for + 24/64-byte keys and 32,768 for larger keys. Cache 512 MiB, device 2 GiB. + Four measured batches for insert/overwrite/pack, two for reads; one million + lookups, about 8.4 million Scan32 or 33.6 million Scan4096 tuples per run. + Seeds 42–48, seven alternating pairs. Setup, warmup, verification, and + reclamation are outside timing. +- Full-matrix intervals use paired-log Student-t with six degrees of freedom. + No successful sample or outlier was discarded. +- Every window held both benchmark and machine locks exclusively after checking + project completion/current acknowledgements. The exact-content completion + confirmation for the unchanged async-state registration is retained. No child + agents; no builds, tests, edits, or analysis during measurement. Requests and + matching releases are preserved; all locks are released. +- Scope: resident single-threaded B-tree CPU performance, not disk-bound, + public API, large-key/blob throughput, concurrency, compaction-policy, or + tail-latency measurements. No commit or push. + +## Evidence + +Retained full matrix: 2026-10-30 00:26:30–00:49:18 PDT (2026-10-30 07:26:30–07:49:18 UTC). + +- [Retained full matrix](shared-scan4096-results-20261030/full-retained-comparison.csv), + [441 pairs](shared-scan4096-results-20261030/full-retained-pairs.csv), + [882 runs](shared-scan4096-results-20261030/full-retained-runs.csv). +- [Focused comparisons](shared-scan4096-results-20261030/plain-retained-comparison.csv), + [pairs](shared-scan4096-results-20261030/plain-retained-pairs.csv), + [runs](shared-scan4096-results-20261030/plain-retained-runs.csv). +- [Counters](shared-scan4096-results-20261030/perf-stat.csv), + [cycle reports](shared-scan4096-results-20261030/cycle-profile-summary.txt), + [raw profile hashes](shared-scan4096-results-20261030/cycle-profiles.json). +- [Cumulative patch](shared-scan4096-results-20261030/fix.patch) and + [source, validation, samples, coordination](shared-scan4096-results-20261030/evidence.json). + +Raw binaries/libraries, frozen sources, logs, disassembly, manifests, counter +and record data, and scripts remain under .cache/shared-scan4096-profile-20261029/. +Earlier source/binary locations are recorded in profiles.json. + +Repeat the full matrix with a fresh output directory: + +~~~sh +python3 .cache/shared-scan4096-profile-20261029/run_full.py \ + --profiles .cache/shared-scan4096-profile-20261029/retained-full-profiles.json \ + --out .cache/shared-scan4096-profile-20261029/new-retained-matrix \ + --coordination /home/rob/projects/splinterdb/.cache/projects-20261025/coordination \ + --lock-dir /home/rob/projects/splinterdb/.cache/projects-20261011/coordination \ + --variants baseline candidate --repetitions 7 --expected-minutes 25 +~~~ + +The runner requires matching pause acknowledgements from active projects. +The focused run.py accepts --variants baseline before retained, --mode +plain/stat/record, --cases 64-common 24-random, --batches 8, and --repetitions 15. +Analyze completed windows with analyze.py, then generate this narrative with +write_report.py; the narrative uses the original full-retained campaign. diff --git a/benchmarks/blob_keys/SMALL_KEYS.md b/benchmarks/blob_keys/SMALL_KEYS.md new file mode 100644 index 000000000..dfa072f39 --- /dev/null +++ b/benchmarks/blob_keys/SMALL_KEYS.md @@ -0,0 +1,160 @@ +# Small-key regression benchmarks — native blob-key format + +The [complete format-4 rerun](SMALL_KEYS_V4.md) supersedes these historical +B-tree results. It covers all 63 cases after the scan fixes and contiguous +32/64-bit statistics, with seven paired repetitions per case. + +The candidate measured on October 26 **did not meet the no-small-key-regression objective**. +The public API loses 7.3–9.5% throughput on successful point lookups and +2.7–9.3% on the measured ranges. Direct B-tree long scans lose 24.9–39.2%. +Small keys remained inline throughout these experiments. + +These measurements compare the native B-tree changes, including the requested +uint64 statistics and format version changes, against main at `6e550e1e`. +They are separate from the older forced-blob codec experiments in RESULTS.md. +The working branch is `robj/blob-keys`; native changes and performance changes +were uncommitted at measurement time. Public large-key writes are still outside this implementation. + +The subsequent [Scan4096 profiling and fix](SCAN4096_PROFILE.md) rebases onto +newer main and identifies the large scan slowdown. It supersedes the Scan4096 +conclusion for that newer candidate. A fresh [Scan32 remeasurement](SCAN32_RESULTS.md) +also shows substantial recovery, with remaining regressions. The historical +data below remain unchanged. + +## Public API results + +Each cell is the median of three paired candidate/baseline throughput changes. +Negative numbers mean fewer operations per second. Each pair uses the same seed; +run order alternates. These are bounded throughput samples, not tail-latency or +long-term stability claims. + +| Operation | Log off, 1 thread | Log off, 4 threads | Log on, 1 thread | Log on, 4 threads | +|---|---:|---:|---:|---:| +| Random load | -1.0% | -7.3% | -3.8% | -4.1% | +| Point hit | -8.3% | -9.5% | -8.0% | -7.3% | +| Point miss | -2.6% | -1.5% | -2.5% | +5.9% | +| Scan 32 | -7.9% | -9.3% | -6.3% | -5.9% | +| Scan 4096 | -5.2% | -7.3% | -5.4% | -2.7% | +| Overwrite | -0.6% | -5.4% | -0.5% | -1.5% | + +Load throughput above excludes final close/drain. Including that drain, median +elapsed time increases were 2.7%, 11.9%, 4.2%, and 2.7%, respectively, in the +same column order. Do not substitute operation throughput for completed-load +cost. + +Public setup: 1,000,000 keys, 24-byte keys, 100-byte values, 512 MiB cache, +4 GiB configured device, 24 MiB memtable, one memtable worker, four normal +background workers, process affinity CPUs 0–7. Each timed lookup/range/overwrite +phase has one second of warmup and three seconds of measurement. Load has a +fixed key count, closes/drains, and reopens before the read phases. A full +verified scan precedes reads; point results, range order/values, and overwrite +samples are checked. The `warm` phase is retained in the CSV but is a warmup +and verification pass, not an independently stabilized scan benchmark. + +## Direct B-tree results + +Representative cases below use the same throughput convention. Full results for +all 63 cases, including paired timing ranges, are in +[final-v3.csv](small-key-results-20261026/final-v3.csv). + +| Keys / comparator | Insert | Overwrite | Hit | Miss | Scan 32 | Scan 4096 | Pack | +|---|---:|---:|---:|---:|---:|---:|---:| +| 24 random lexical | +2.5% | -3.9% | -3.6% | -4.0% | -19.9% | -39.2% | -8.3% | +| 24 common prefix | +2.2% | -3.3% | -1.6% | -1.1% | -17.0% | -34.5% | -12.4% | +| 64 random lexical | -0.1% | -5.4% | -13.2% | -14.9% | -24.4% | -37.7% | -11.2% | +| 64 common prefix | +1.2% | -5.9% | -0.7% | -1.9% | -15.4% | -34.4% | -6.2% | +| 256 random lexical | -0.7% | -3.1% | -12.8% | -12.2% | -20.0% | -30.2% | -1.3% | +| 512 random lexical | -1.1% | +0.1% | -1.7% | +1.2% | -10.2% | -24.9% | -3.3% | +| 512 common prefix | -0.1% | +0.1% | -2.7% | -1.7% | -14.0% | -33.8% | -3.8% | +| 64 custom comparator | -8.9% | -11.1% | -14.6% | -13.5% | -22.1% | -37.6% | -7.3% | + +The matrix covers 24/64/256/512-byte lexical keys with random and shared-prefix +distributions, plus a 64-byte nonlexical comparator. Values are 32 bytes. +There are 65,536 keys at 24/64 bytes and 32,768 keys at 256/512 bytes. Cache size +is 512 MiB; configured device size is 2 GiB; the process is pinned to CPU 2. +Lookups execute one million measured operations. Short/long scans process +approximately 8.4/33.6 million tuples. Insert, overwrite, and pack use four +batches; reads use two. Setup, warmup, full verification and teardown are outside +the active timing interval. + +All measured direct B-tree operations were cache resident, with zero measured +I/O according to the harness hooks and process I/O counters. Every run checks +values, checksums, and allocator reclamation. This isolates CPU overhead; it +does not establish cold-cache or storage-bound performance. + +Leaf counts and compressed-prefix bytes match baseline. Some packed allocations +increase due to the larger fixed-width index entries. There was no ablation +separating the required uint64 statistics changes from blob dispatch, so the +remaining regressions cannot all be attributed to one feature. + +## Iterations and validation + +Three complete campaigns produced 1,158 valid process runs: + +- `screen-v1b`: 378 runs of the initial native implementation. For example, + 24-byte random long scans took 98.0% more time and packing took 73.0% more time. +- `screen-v2`: 378 runs after preserving the original 24-byte in-memory key + size, inlining ordinary comparisons, separating descriptor parsing from + ordinary leaf reads, and reusing the pack key buffer. +- `final-v3`: 378 B-tree runs plus 24 public API runs, after simplifying logical + key construction in iterator `curr()` and moving index-only materialization + out of leaf validation. All 402 final runs passed their checks. This last + change did not eliminate the scan regression; do not describe it as a proven + scan performance fix. + +The combined changes reduced the 24-byte random packing time regression from +73.0% to 9.1%; long-scan time remained 64.5% above baseline. The residual CPU +cost still needs profiling and isolation before performance sign-off. + +Correctness validation: the v2 full debug suite passed 246 tests, with four +existing skips and zero failures. After the v3 changes, all three native +blob-key tests, all 14 B-tree layout/configuration tests, and all 62 public API +quick tests passed. The final full 250-test suite was not repeated after v3. +The queued-owner lock protocol was separately checked with an existing owner, +a replacement request ID, and lock release/acquisition. + +## Evidence and reproduction + +Machine: Intel Core i7-12800H, GCC 14.2.0, release `-O3 -ffast-math -flto +-march=native` on both sides. Both sides use byte-identical benchmark harness +sources. The final campaign ran 2026-10-26 23:58:14 through 2026-10-27 00:13:51 UTC. + +Every measured window held both benchmark and machine locks exclusively and +saved fresh acknowledgements that all other active project roots and children +were paused. No compilation, source work, or result analysis ran during these +windows. Other projects' windows were also respected. + +Raw artifacts are under +[`.cache/small-key-bench-20261026`](../../.cache/small-key-bench-20261026/): +source snapshots, source/binary/library/harness hashes, compiler commands, +patches, per-run stdout/stderr and CSV, full manifests, lock requests, pause +acknowledgements and release records. The initial `screen-v1` attempt was +aborted by scratch-file cleanup after one run; its data are excluded. The +cleanup was corrected before `screen-v1b`. + +Maintained summary evidence: + +- [Profile hashes and validation](small-key-results-20261026/evidence.json) +- [Initial candidate](small-key-results-20261026/screen-v1b.csv) +- [First optimization](small-key-results-20261026/screen-v2.csv) +- [Final candidate and public API](small-key-results-20261026/final-v3.csv) + +To rerun the frozen final profiles from the topic worktree: + +```sh +python3 benchmarks/blob_keys/small_keys.py \ + --profiles .cache/small-key-bench-20261026/profiles.json \ + --public-profiles .cache/small-key-bench-20261026/public-profiles.json \ + --out .cache/small-key-bench-20261026/new-run \ + --coordination /home/rob/projects/splinterdb/.cache/projects-20261025/coordination \ + --lock-dir /home/rob/projects/splinterdb/.cache/projects-20261011/coordination \ + --variants baseline candidate-v3 --repetitions 3 --expected-minutes 20 + +python3 benchmarks/blob_keys/summarize_small_keys.py \ + .cache/small-key-bench-20261026/new-run/manifest.json --candidate candidate-v3 +``` + +Only invoke the runner after all local builds, tests and child-agent work stop. +The runner now polls an occupied reservation and acknowledges its current owner +while queued; this avoids the handoff stalls encountered in the recorded +campaigns. The manifest snapshots the runner actually used for each campaign. diff --git a/benchmarks/blob_keys/SMALL_KEYS_FINAL.md b/benchmarks/blob_keys/SMALL_KEYS_FINAL.md new file mode 100644 index 000000000..db97bb2c8 --- /dev/null +++ b/benchmarks/blob_keys/SMALL_KEYS_FINAL.md @@ -0,0 +1,185 @@ +# Full small-key B-tree rerun after profiling fixes + +Latest scan follow-up: [shared-prefix Scan4096 profiling and retained fix](SHARED_SCAN4096_PROFILE.md) +makes these scans 12.8–13.4% faster than original main across all four key sizes. +The retained fix also removes the intermediate random-scan tradeoff. All 882 +runs of the final full matrix verified; custom insertion, overwrite, and hits +retain roughly 2% overhead. + +The combined fixes improve packing and lexical mutation/lookup performance, but +**the no-small-key-regression objective is still not met**. The clearest remaining +regression is shared-prefix Scan4096: 64-, 256-, and 512-byte keys take about +**5–7% longer** by paired median. Custom overwrite retains about **2% overhead**. + +All **882 runs** passed verification and residency checks, with zero measured +I/O or major faults. This is the complete original 63-case matrix, seven paired +repetitions, on the exact source validated in the latest mutation profile. +No production source changes were made for this rerun. + +## Full matrix + +Values are medians of seven paired **elapsed-time changes**, current/main. +Positive means slower; negative means faster. These are not throughput changes. +The baseline is the original frozen main binary from SMALL_KEYS_V4.md. + +| Key bytes | Distribution / comparator | Insert | Overwrite | Hit | Miss | Scan 32 | Scan 4096 | Pack | +|---:|---|---:|---:|---:|---:|---:|---:|---:| +| 24 | Random lexical | -29.7% | -26.6% | -26.5% | -26.4% | -11.4% | +2.9% | -12.5% | +| 24 | Shared prefix | -29.4% | -28.4% | -26.4% | -26.3% | -14.0% | +4.3% | -11.7% | +| 64 | Random lexical | -28.2% | -25.0% | -27.4% | -28.0% | -12.5% | +1.4% | -12.0% | +| 64 | Shared prefix | -27.7% | -24.7% | -23.7% | -25.2% | -13.3% | +6.7% | -11.5% | +| 256 | Random lexical | -23.5% | -23.0% | -23.2% | -26.6% | -15.3% | +1.4% | -5.0% | +| 256 | Shared prefix | -22.0% | -20.8% | -19.8% | -20.2% | -12.8% | +5.2% | -10.1% | +| 512 | Random lexical | -17.5% | -20.5% | -19.5% | -24.3% | -17.3% | +1.0% | -16.9% | +| 512 | Shared prefix | -16.9% | -18.7% | -16.1% | -16.4% | -13.2% | +5.2% | -4.9% | +| 64 | Custom comparator | +1.0% | +1.9% | +1.8% | -0.6% | +1.5% | +1.6% | -14.5% | + +All nine packing configurations are faster: paired median reductions range from +4.9% to 16.9%, and every packing case has a geometric-mean interval below zero. +Lexical hits are 16.1–27.4% faster by paired median; lexical Scan32 is 11.4–17.3% +faster. These are the combined branch's gains against original main, including +the lexical comparison shortcut added during profiling. They do not isolate the +cost of blob-key support. + +The exploratory per-case intervals classify **7 cases as slower, +49 as faster, and 7 as overlapping zero**. +These are not simultaneous confidence guarantees across all 63 comparisons. + +## Remaining regressions + +Every row below has a positive lower bound in its paired-log 95% interval. +The interval and geometric mean use the same statistic; the median is reported +separately. All seven pairs were slower in every row except 256-random-scan4096, +where six were slower. + +| Case | Paired median | Geometric mean | Geometric-mean 95% interval | Slower pairs | +|---|---:|---:|---:|---:| +| 64-common-scan4096 | +6.7% | +6.8% | +6.3% to +7.2% | 7/7 | +| 256-common-scan4096 | +5.2% | +6.7% | +3.1% to +10.4% | 7/7 | +| 512-common-scan4096 | +5.2% | +5.8% | +4.4% to +7.3% | 7/7 | +| 24-random-scan4096 | +2.9% | +3.1% | +2.4% to +3.8% | 7/7 | +| 64-nonlexical-overwrite | +1.9% | +2.2% | +0.7% to +3.7% | 7/7 | +| 64-random-scan4096 | +1.4% | +1.6% | +0.9% to +2.3% | 7/7 | +| 256-random-scan4096 | +1.4% | +1.2% | +0.2% to +2.2% | 6/7 | + +**64-byte shared-prefix Scan4096 is the clearest next profiling target**: its ++6.7% median and +6.3% to +7.2% geometric-mean interval are both substantial and +consistent. The 256- and 512-byte shared-prefix cases confirm that the issue +extends beyond one key size. These measurements establish the slowdown; they do +not identify its cause. + +Custom insertion is +1.0% by median, +0.8% by geometric mean, with an interval +of -0.8% to +2.3%. Custom hits are +1.8% by median, +2.9% by geometric mean, +with an interval of -0.6% to +6.6%. Neither is a clear regression in this +seven-pair window, but neither demonstrates equivalence or overturns the +earlier evidence of a small insertion cost. Custom overwrite is +1.9% by median, ++2.2% by geometric mean, with an interval of +0.7% to +3.7%. + +## Relation to earlier measurements + +The following comparison uses the same original-main binary and workload as the +previous full suite, but different measurement windows. It summarizes progress +and does not attribute differences to any single fix. + +| Case | Previous full-suite median | Current full-suite median | +|---|---:|---:| +| 64-random-pack | +12.1% | -12.0% | +| 64-nonlexical-insert | +9.9% | +1.0% | +| 64-nonlexical-overwrite | +10.6% | +1.9% | +| 64-random-hit | +6.3% | -27.4% | +| 64-nonlexical-hit | +5.0% | +1.8% | +| 64-common-scan4096 | -0.5% | +6.7% | +| 256-common-scan4096 | +2.8% | +5.2% | +| 512-common-scan4096 | +1.5% | +5.2% | + +The most recent focused [mutation profile](MUTATION_PROFILE.md) used main with +the diagnostic **checked-read lexical shortcut** added; this full matrix uses +**unmodified frozen main**, as did the previous full matrix. The focused runs +also used 16 write batches, whereas this matrix uses four. Even custom workloads +can respond to the compiler/layout changes between those baseline binaries. +Therefore, the earlier +0.4% overwrite estimate and this +2.2% estimate are not +a same-baseline, same-workload comparison. The candidate executable is +byte-identical in both campaigns. This rerun does not show that the latest fix +was undone; it does show residual overwrite overhead against original main. + +The increased Scan4096 overhead relative to the earlier full suite merits a +same-window comparison of intermediate candidates. This campaign did not run +those intermediate candidates, so it cannot locate when the slowdown arose. +The original large scan losses remain historical context, not evidence that the +current long-scan path is fully fixed. + +## Verification and method + +- All 441 pairs have equal operation/tuple counts, checksums, verified values, + and workload parameters. Packed leaf counts, compressed-leaf counts, prefix + bytes, and allocated bytes match in every pair. The harness does not populate + mutable-write layout counters; their matching zeros establish no mutable-tree + geometry claim. Tree height was not instrumented. +- All 882 runs passed, with zero active read/write bytes, synchronous/asynchronous + cache I/O, and major faults. No successful sample or outlier was discarded. +- Baseline: main d9a61a9abf6feadf017c951d965e2f3224b5eb30, exact binary from the + previous full matrix. Candidate: robj/blob-keys at bb000cc3 plus the current + retained working changes, B-tree format 4. All 110 current production + source/header hashes match the frozen final mutation-profile source. +- The unchanged candidate already passed the full debug suite (295 passed, + zero failures, five existing skips) and targeted debug/release/ASan B-tree and + native blob-key tests. Those logs were hash-verified; correctness suites were + not rerun during this measurement-only task. ASan leak detection was disabled + in that prior validation, and UBSan was not enabled. +- Both libraries use GCC 14.2.0 release flags -O3 -ffast-math -flto. The identical + harness additionally uses -march=native. No perf instrumentation was enabled. + CPU is an Intel Core i7-12800H, pinned to CPU 2. The performance governor and + 2.5 GHz cap were unchanged; frequency was not fixed. Frequency observations + occur between runs and are not active-operation clock measurements. +- Four key sizes (24/64/256/512 bytes), random and shared-prefix lexical ordering, + plus 64-byte custom ordering by the last eight bytes. Values are 32 bytes. + There are 65,536 records for 24/64-byte keys and 32,768 for 256/512-byte keys; + all keys in this small-key workload remain inline. +- Cache is 512 MiB; configured device size is 2 GiB. Insert, overwrite, and pack + use four measured batches; reads use two. Lookups perform one million + operations. Scan32 returns about 8.4 million tuples and Scan4096 about + 33.6 million per run. Setup, warmup, verification, and reclamation are outside + active timing. +- Seeds 42–48 match within each pair; baseline/candidate order alternates. + Intervals use Student-t on seven paired log ratios (six degrees of freedom), + transformed back to percentages. They estimate geometric-mean ratios, not + medians, and are exploratory per-case intervals without multiplicity correction. +- Both benchmark and machine locks were held exclusively throughout measurement. + Other projects were explicitly complete; the unchanged async-state registration + has an archived exact-content completion confirmation superseding its stale + PREPARING heading. There were no child agents. No builds, edits, tests, or + result analysis ran during measurement. Request, registrations, confirmations, + and matching release are retained. Both locks are released. +- Host observations show no cgroup CPU throttling. The scope is resident, + single-threaded B-tree CPU cost, not disk-bound performance, public API + throughput, concurrency, compaction policy, or tail latency. + +## Evidence + +Measured 2026-10-29 22:20:32–22:43:34 PDT (2026-10-30 05:20:32–05:43:34 UTC), 23.0 minutes. + +- [All 63 comparisons and intervals](small-key-final-results-20261029/comparison.csv). +- [All 882 runs](small-key-final-results-20261029/runs.csv) and + [441 paired samples](small-key-final-results-20261029/pairs.csv). +- [Comparison with the earlier full suite](small-key-final-results-20261029/historical-comparison.csv). +- [Source, validation, sample hashes, and coordination evidence](small-key-final-results-20261029/evidence.json). + +Raw stdout/stderr, frozen binaries/libraries, harness, runner, metadata, and report +generator are retained under .cache/small-key-final-20261029/. Source archives and +frozen source trees are retained at the paths recorded by the profiles. No +production code was edited, committed, or pushed for this rerun. + +To reproduce with a fresh output directory: + +~~~sh +python3 .cache/small-key-final-20261029/run.py \ + --profiles .cache/small-key-final-20261029/profiles.json \ + --out .cache/small-key-final-20261029/new-matrix \ + --coordination /home/rob/projects/splinterdb/.cache/projects-20261025/coordination \ + --lock-dir /home/rob/projects/splinterdb/.cache/projects-20261011/coordination \ + --variants baseline candidate --repetitions 7 --expected-minutes 25 +~~~ + +The runner requires fresh matching pause acknowledgements from any active project. +Run report.py --matrix .cache/small-key-final-20261029/new-matrix after completion +to validate and regenerate the report and CSVs. diff --git a/benchmarks/blob_keys/SMALL_KEYS_V4.md b/benchmarks/blob_keys/SMALL_KEYS_V4.md new file mode 100644 index 000000000..d066554bd --- /dev/null +++ b/benchmarks/blob_keys/SMALL_KEYS_V4.md @@ -0,0 +1,162 @@ +# Small-key B-tree rerun after compact statistics + +Latest scan follow-up: [shared-prefix Scan4096 profiling and retained fix](SHARED_SCAN4096_PROFILE.md) +makes these scans 12.8–13.4% faster than original main across all four key sizes. +The retained fix also removes the intermediate random-scan tradeoff. All 882 +runs of the final full matrix verified; custom insertion, overwrite, and hits +retain roughly 2% overhead. + + +The no-small-key-regression objective is **not yet met**. The large scan regressions +have largely cleared, but packing is 2.7–12.1% slower by paired median, and the +64-byte custom comparator retains 9.9% insertion and 10.6% overwrite overhead. +Successful point lookups are 1.2–6.3% slower by paired median. + +Follow-up: [packing profiling and fixes](PACK_PROFILE.md) remove the packing +regressions in a same-window comparison of all nine packing cases. The combined +fix is 2.4–17.4% faster than main by paired median; the 256-byte random case remains +inconclusive. The historical full matrix below predates that fix; other operations +have not been remeasured as a full matrix after it. + +The later [overwrite profile](OVERWRITE_PROFILE.md) reduces custom-comparator +overwrite overhead from 11.7% to 5.2% in its longer focused comparison. The +remaining regression is confirmed; the fixes do not establish overall parity. + +A [later 64-byte random point-hit remeasurement](POINT_HIT_64.md) finds ++6.2% median elapsed time versus main (15 pairs; 95% interval +3.9% to +7.0%). +The lookup regression remains after the packing and overwrite fixes. +Subsequent [point-hit profiling](POINT_HIT_PROFILE.md) adds a lexical pivot +shortcut: the retained branch takes 23.2% less time than unmodified main, but +remains 6.0% slower than main with the same shortcut. Custom controls show no +clear new regression relative to the pre-fix branch. + +[Continued profiling](RESIDUAL_PROFILE.md) reduces ordinary key decoding and +comparison overhead: 64-byte random and custom hits now have about 1% overhead +against matched optimized main, with intervals spanning zero. Custom writes +and some larger-key lookups still regress. That follow-up is a targeted subset. + +The latest [mutation follow-up](MUTATION_PROFILE.md) removes intermediate +metadata copies: custom overwrite is near main, while insertion retains roughly +2% overhead. Its full write controls show no clear new regression. + +All **882 runs** passed verification and residency checks: 63 cases, seven paired +repetitions, main versus the actual blob-capable format-4 candidate. Small keys +remain inline. This campaign measures B-trees only; no public API timings were run. + +## Full matrix + +Values are the median of seven paired **elapsed-time changes**, candidate/main. +Positive means slower; negative means faster. These are time changes, not +throughput changes. No samples or outliers were discarded. + +| Key bytes | Distribution / comparator | Insert | Overwrite | Hit | Miss | Scan 32 | Scan 4096 | Pack | +|---:|---|---:|---:|---:|---:|---:|---:|---:| +| 24 | Random lexical | -2.5% | +2.9% | +4.3% | +3.5% | +0.7% | -0.4% | +11.1% | +| 24 | Shared prefix | -1.5% | +4.3% | +3.5% | +3.7% | +2.7% | -1.0% | +6.4% | +| 64 | Random lexical | -0.6% | +5.4% | +6.3% | +4.4% | +0.0% | +0.5% | +12.1% | +| 64 | Shared prefix | +0.3% | +4.2% | +2.2% | +1.0% | +0.3% | -0.5% | +8.2% | +| 256 | Random lexical | +3.0% | +3.0% | +5.8% | +3.0% | -5.3% | +2.3% | +5.4% | +| 256 | Shared prefix | +1.2% | +1.6% | +2.7% | +1.0% | -1.9% | +2.8% | +9.5% | +| 512 | Random lexical | +1.9% | -0.9% | +1.6% | +0.5% | -9.1% | +1.1% | +4.4% | +| 512 | Shared prefix | +1.3% | +1.0% | +1.2% | +2.1% | -3.1% | +1.5% | +2.7% | +| 64 | Custom comparator | +9.9% | +10.6% | +5.0% | +3.2% | -0.8% | +0.7% | +7.1% | + +## Largest remaining regressions + +Each range below is the smallest/largest observed paired change, **not a +confidence interval**. All seven pairs were slower in these cases. + +| Case | Median time change | Paired range | +|---|---:|---:| +| 64-random-pack | +12.1% | +6.6% to +16.3% | +| 24-random-pack | +11.1% | +5.8% to +18.5% | +| 64-nonlexical-overwrite | +10.6% | +6.9% to +13.6% | +| 64-nonlexical-insert | +9.9% | +5.3% to +14.3% | +| 256-common-pack | +9.5% | +6.7% to +16.5% | +| 64-common-pack | +8.2% | +6.0% to +13.6% | +| 64-random-hit | +6.3% | +4.0% to +8.5% | + +The CSV also includes 95% Student-t intervals computed on the seven paired log +time ratios (six degrees of freedom), then transformed back to percentage changes. +Those intervals estimate the geometric-mean ratio; they are not intervals around +the displayed median. They are per-case exploratory intervals without correction +for the 63 comparisons. Some cases have wide ranges; seven pairs do not establish +performance equivalence. + +## Scan comparison and storage layout + +The earlier [Scan32 profile](SCAN32_PROFILE.md), against the same main commit, +found +4.3% for 64-byte random lexical keys and +4.2% for the custom comparator. +Those cases now have paired medians of +0.0% and -0.8%, with ranges of -4.3% to ++1.4% and -1.9% to +1.7%. This is consistent with removing the remaining +statistics-width penalty; it is not a same-window ablation of that change alone. + +Scan32 medians now range from -9.1% to +2.7%. Scan4096 medians range from -1.0% +to +2.8%, far smaller than the original 25–39% throughput losses. Small residual +scan overhead remains: 256-byte shared-prefix Scan4096 is slower in all seven +pairs (+1.4% to +6.3%). The original [small-key report](SMALL_KEYS.md) used an +older baseline, so its percentages are historical context, not direct ablation data. + +Leaf counts, compressed-leaf counts, prefix bytes, and total packed allocation +bytes matched main in **every pair**. This campaign did not instrument tree +height. Index entries use 23 bytes plus the key for these small statistics, +versus 35 in the previous fixed-64-bit candidate. The matched allocation counts +do not explain the remaining CPU regressions; packing and custom-comparator +mutable operations are the clearest next profiling targets. + +## Method and limitations + +- Baseline: main d9a61a9abf6feadf017c951d965e2f3224b5eb30, using the identical + frozen baseline binary from the Scan4096/Scan32 investigations. +- Candidate: bb000cc3 plus the frozen working-tree patch implementing contiguous + 32/64-bit counters (B-tree format 4) and attachment accounting. Source, patch, + static library, harness and executable hashes are preserved. +- CPU: Intel Core i7-12800H, pinned to CPU 2; GCC 14.2.0. Both libraries use + release -O3 -ffast-math -flto; the identical harness uses those flags plus + -march=native at compilation/linking. No perf instrumentation was enabled. +- Same original 63-case workload: 24/64/256/512-byte lexical keys, random and + shared-prefix distributions; 64-byte nonlexical comparator; 32-byte values. + 65,536 records for 24/64-byte keys and 32,768 for 256/512-byte keys. +- 512 MiB cache and 2 GiB configured device. Insert/overwrite/pack use four + measured batches; reads use two. Lookups execute 1,000,000 operations; + Scan32 returns about 8.4 million tuples and Scan4096 about 33.6 million. +- Seeds 42–48, matched within each pair; alternating baseline/candidate order. + Setup, warmup, full verification and reclamation are outside active timing. +- Every run checks values, checksums, ordering where applicable, and reclamation. + All measured read/write I/O, asynchronous I/O and major faults were zero. +- Both coordination locks were held exclusively throughout the measurement window. + No other project was active; the owner had no children. Registrations, completion + confirmations, request and release records are archived. No build, edit or + result analysis ran during measurement. +- This is a resident CPU-cost comparison. It does not measure storage-bound + behavior, public API throughput, tail latency, or prolonged production workloads. + CPU frequency was not fixed. Host load/frequency observations are retained; + no cgroup CPU throttling was reported. Outlying samples remain in the data. + +## Evidence + +Measured 2026-10-29T00:15:55.125080+00:00 through 2026-10-29T00:40:18.548267+00:00 UTC +(October 28, 17:15–17:40 America/Los_Angeles). Both locks were released successfully. + +- [All 63 comparisons and intervals](small-key-results-20261028-v4/comparison.csv) +- [All 882 per-run results](small-key-results-20261028-v4/runs.csv) +- [All 441 paired samples](small-key-results-20261028-v4/pairs.csv) +- [Identity, validation and coordination evidence](small-key-results-20261028-v4/evidence.json) + +Raw frozen sources, libraries, executables, per-run stdout/stderr, runner and +manifest are in .cache/small-key-btree-v4-20261028/ in the topic worktree. + +Reproduce from that worktree, with a fresh output directory: + +~~~sh +python3 .cache/small-key-btree-v4-20261028/run.py \ + --profiles .cache/small-key-btree-v4-20261028/profiles.json \ + --out .cache/small-key-btree-v4-20261028/new-matrix \ + --coordination /home/rob/projects/splinterdb/.cache/projects-20261025/coordination \ + --lock-dir /home/rob/projects/splinterdb/.cache/projects-20261011/coordination \ + --variants baseline candidate --repetitions 7 --expected-minutes 25 +~~~ + +The runner requires current acknowledgements from any newly active project. +The archived completion confirmation is accepted only while that project record +remains byte-identical; a changed registration requires fresh coordination. diff --git a/benchmarks/blob_keys/blob_key.c b/benchmarks/blob_keys/blob_key.c new file mode 100644 index 000000000..a5ef59682 --- /dev/null +++ b/benchmarks/blob_keys/blob_key.c @@ -0,0 +1,424 @@ +// Copyright 2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "blob_key.h" +#include "splinterdb/default_data_config.h" + +#define BLOB_KEY_FORMAT 1 + +typedef struct ONDISK blob_key_header { + uint64 length; + uint8 prefix_bytes; + uint8 format; +} blob_key_header; + +typedef struct blob_key_view { + uint64 length; + slice prefix; + slice descriptor; + bool32 is_blob; +} blob_key_view; + +/* Bounds checks on every comparison, including a prefix-only comparison. */ +static platform_status +blob_key_view_init(blob_key_config *cfg, user_key input, blob_key_view *view) +{ + ZERO_CONTENTS(view); + if (!cfg->blob_keys || input.is_query_key) { + view->length = slice_length(input.key); + view->prefix = input.key; + return STATUS_OK; + } + + uint64 size = slice_length(input.key); + if (size < sizeof(blob_key_header)) { + return STATUS_INVALID_STATE; + } + const blob_key_header *hdr = slice_data(input.key); + uint64 prefix = hdr->prefix_bytes; + if (hdr->format != BLOB_KEY_FORMAT || prefix > BLOB_KEY_MAX_PREFIX_BYTES + || prefix > hdr->length || prefix > size - sizeof(*hdr) + || size - sizeof(*hdr) - prefix < sizeof(blob) + || size - sizeof(*hdr) - prefix > BLOB_KEY_DESCRIPTOR_BYTES) + { + return STATUS_INVALID_STATE; + } + const char *bytes = slice_data(input.key); + view->length = hdr->length; + view->prefix = slice_create(prefix, bytes + sizeof(*hdr)); + view->descriptor = + slice_create(size - sizeof(*hdr) - prefix, bytes + sizeof(*hdr) + prefix); + const blob *descriptor = slice_data(view->descriptor); + if (descriptor->length != hdr->length || descriptor->format != BLOB_FORMAT + || (slice_length(view->descriptor) - sizeof(blob)) % sizeof(uint64) != 0) + { + return STATUS_INVALID_STATE; + } + view->is_blob = TRUE; + return STATUS_OK; +} + +/* + * Existing data_config callbacks cannot return an I/O status. This harness + * therefore stops on malformed keys or read failures. Never synthesize an + * ordering after an error: that could direct a mutation into the wrong leaf. + */ +static void +blob_key_callback_check(platform_status rc) +{ + platform_assert(SUCCESS(rc), + "blob-key prototype callback failed: %s; production " + "integration requires fallible comparison and hashing", + platform_status_to_string(rc)); +} + +platform_status +blob_key_validate(blob_key_config *cfg, slice encoded) +{ + blob_key_view view; + platform_status rc = blob_key_view_init( + cfg, (user_key){.key = encoded, .is_query_key = FALSE}, &view); + if (!SUCCESS(rc) || !view.is_blob) { + return rc; + } + parsed_blob parsed; + return blob_parse_descriptor(cfg->cc, view.descriptor, &parsed); +} + +typedef struct blob_key_cursor { + blob_page_iterator iter; + const blob_key_view *view; + uint64 offset; + bool32 initialized; +} blob_key_cursor; + +static platform_status +blob_key_cursor_init(blob_key_config *cfg, + blob_key_cursor *cursor, + const blob_key_view *view, + uint64 offset) +{ + cursor->view = view; + cursor->offset = offset; + cursor->initialized = FALSE; + if (!view->is_blob || offset == view->length) { + return STATUS_OK; + } + platform_status rc = + blob_page_iterator_init(cfg->cc, + &cursor->iter, + view->descriptor, + offset, + BLOB_PAGE_ITERATOR_MODE_NO_PREFETCH); + cursor->initialized = SUCCESS(rc); + return rc; +} + +static void +blob_key_cursor_deinit(blob_key_cursor *cursor) +{ + if (cursor->initialized) { + blob_page_iterator_deinit(&cursor->iter); + } +} + +static platform_status +blob_key_cursor_get(blob_key_config *cfg, blob_key_cursor *cursor, slice *part) +{ + if (!cursor->view->is_blob) { + *part = slice_create(cursor->view->length - cursor->offset, + (const char *)slice_data(cursor->view->prefix) + + cursor->offset); + return STATUS_OK; + } + uint64 offset; + platform_status rc = + blob_page_iterator_get_curr(&cursor->iter, &offset, part); + if (SUCCESS(rc)) { + cfg->counters.blob_fragments++; + platform_assert(offset == cursor->offset); + } + return rc; +} + +static void +blob_key_cursor_advance(blob_key_cursor *cursor, uint64 bytes) +{ + cursor->offset += bytes; + if (cursor->initialized) { + blob_page_iterator_advance_bytes(&cursor->iter, bytes); + } +} + +static int +blob_key_compare_lengths(uint64 a, uint64 b) +{ + return (a > b) - (a < b); +} + +/* Keep two large blob iterator frames off the inline-prefix fast path. */ +static __attribute__((noinline)) int +blob_key_compare_suffix(blob_key_config *cfg, + const blob_key_view *a, + const blob_key_view *b, + uint64 offset) +{ + cfg->counters.blob_fallbacks++; + blob_key_cursor left; + blob_key_cursor right; + blob_key_callback_check(blob_key_cursor_init(cfg, &left, a, offset)); + blob_key_callback_check(blob_key_cursor_init(cfg, &right, b, offset)); + uint64 end = MIN(a->length, b->length); + int result = 0; + while (offset < end) { + slice lpart; + slice rpart; + blob_key_callback_check(blob_key_cursor_get(cfg, &left, &lpart)); + blob_key_callback_check(blob_key_cursor_get(cfg, &right, &rpart)); + uint64 bytes = MIN(slice_length(lpart), slice_length(rpart)); + bytes = MIN(bytes, end - offset); + platform_assert(bytes != 0); + cfg->counters.blob_bytes += bytes * (a->is_blob + b->is_blob); + result = memcmp(slice_data(lpart), slice_data(rpart), bytes); + if (result != 0) { + break; + } + offset += bytes; + if (offset < end) { + blob_key_cursor_advance(&left, bytes); + blob_key_cursor_advance(&right, bytes); + } + } + blob_key_cursor_deinit(&right); + blob_key_cursor_deinit(&left); + return result == 0 ? blob_key_compare_lengths(a->length, b->length) : result; +} + +static int +blob_key_compare(const data_config *data_cfg, user_key left, user_key right) +{ + blob_key_config *cfg = (blob_key_config *)data_cfg; + cfg->counters.comparisons++; + if (!cfg->blob_keys) { + return slice_lex_cmp(left.key, right.key); + } + blob_key_view a; + blob_key_view b; + blob_key_callback_check(blob_key_view_init(cfg, left, &a)); + blob_key_callback_check(blob_key_view_init(cfg, right, &b)); + uint64 prefix = MIN(slice_length(a.prefix), slice_length(b.prefix)); + int result = + prefix ? memcmp(slice_data(a.prefix), slice_data(b.prefix), prefix) : 0; + if (result != 0 || prefix == MIN(a.length, b.length)) { + cfg->counters.prefix_resolved++; + return result == 0 ? blob_key_compare_lengths(a.length, b.length) + : result; + } + return blob_key_compare_suffix(cfg, &a, &b, prefix); +} + +/* FNV-1a is deliberately simple and independent of fragment boundaries. Both + * inline and blob benchmark variants use the same nonproduction hash choice. */ +static uint32 +blob_key_hash_bytes(uint32 hash, slice bytes) +{ + const uint8 *data = slice_data(bytes); + for (uint64 i = 0; i < slice_length(bytes); i++) { + hash = (hash ^ data[i]) * UINT32_C(16777619); + } + return hash; +} + +static __attribute__((noinline)) uint32 +blob_key_hash_blob(blob_key_config *cfg, const blob_key_view *view, uint32 hash) +{ + blob_key_cursor cursor; + blob_key_callback_check(blob_key_cursor_init(cfg, &cursor, view, 0)); + while (cursor.offset < view->length) { + slice part; + blob_key_callback_check(blob_key_cursor_get(cfg, &cursor, &part)); + cfg->counters.blob_bytes += slice_length(part); + hash = blob_key_hash_bytes(hash, part); + blob_key_cursor_advance(&cursor, slice_length(part)); + } + blob_key_cursor_deinit(&cursor); + return hash; +} + +static uint32 +blob_key_hash(const data_config *data_cfg, user_key input, uint32 seed) +{ + blob_key_config *cfg = (blob_key_config *)data_cfg; + cfg->counters.hash_calls++; + blob_key_view view; + blob_key_callback_check(blob_key_view_init(cfg, input, &view)); + uint32 hash = UINT32_C(2166136261) ^ seed; + return view.is_blob ? blob_key_hash_blob(cfg, &view, hash) + : blob_key_hash_bytes(hash, view.prefix); +} + +static void +blob_key_to_string(const data_config *data_cfg, + user_key input, + char *str, + uint64 max_len) +{ + blob_key_config *cfg = (blob_key_config *)data_cfg; + blob_key_view view; + platform_status rc = blob_key_view_init(cfg, input, &view); + if (!SUCCESS(rc)) { + if (max_len != 0) { + str[0] = '\0'; + } + return; + } + debug_hex_encode( + str, max_len, slice_data(view.prefix), slice_length(view.prefix)); +} + +platform_status +blob_key_config_init(blob_key_config *cfg, + cache *cc, + bool32 blob_keys, + uint64 prefix_bytes) +{ + if (cfg == NULL || prefix_bytes > BLOB_KEY_MAX_PREFIX_BYTES + || (blob_keys && cc == NULL)) + { + return STATUS_BAD_PARAM; + } + ZERO_CONTENTS(cfg); + default_data_config_init(&cfg->data_cfg); + cfg->cc = cc; + cfg->blob_keys = blob_keys; + cfg->prefix_bytes = prefix_bytes; + cfg->data_cfg.key_compare = blob_key_compare; + cfg->data_cfg.key_hash = blob_key_hash; + cfg->data_cfg.key_to_string = blob_key_to_string; + /* Bytewise prefix compression is valid only for the raw inline variant. */ + cfg->data_cfg.key_compare_is_lexical = !blob_keys; + return STATUS_OK; +} + +void +blob_key_reset_counters(blob_key_config *cfg) +{ + ZERO_CONTENTS(&cfg->counters); +} + +static platform_status +blob_key_assemble(uint64 length, + slice prefix, + slice descriptor, + writable_buffer *encoded) +{ + if (slice_length(descriptor) > BLOB_KEY_DESCRIPTOR_BYTES) { + return STATUS_LIMIT_EXCEEDED; + } + uint64 bytes = + sizeof(blob_key_header) + slice_length(prefix) + slice_length(descriptor); + platform_status rc = writable_buffer_resize(encoded, bytes); + if (!SUCCESS(rc)) { + return rc; + } + blob_key_header *header = writable_buffer_data(encoded); + *header = (blob_key_header){ + .length = length, + .prefix_bytes = slice_length(prefix), + .format = BLOB_KEY_FORMAT, + }; + char *dst = (char *)header + sizeof(*header); + if (slice_length(prefix) != 0) { + memcpy(dst, slice_data(prefix), slice_length(prefix)); + } + memcpy(dst + slice_length(prefix), + slice_data(descriptor), + slice_length(descriptor)); + return STATUS_OK; +} + +platform_status +blob_key_encode(blob_key_config *cfg, + const blob_build_config *build_cfg, + mini_allocator *mini, + slice logical, + writable_buffer *encoded) +{ + if (!cfg->blob_keys) { + return writable_buffer_copy_slice(encoded, logical); + } + blob_build_config compact = *build_cfg; + compact.blob_max_descriptor_size = BLOB_KEY_DESCRIPTOR_BYTES; + auto_writable_buffer descriptor; + writable_buffer_init(&descriptor, encoded->heap_id); + platform_status rc = + blob_build(&compact, cfg->cc, mini, logical, &descriptor); + if (!SUCCESS(rc)) { + return rc; + } + slice prefix = slice_create(MIN(cfg->prefix_bytes, slice_length(logical)), + slice_data(logical)); + return blob_key_assemble(slice_length(logical), + prefix, + writable_buffer_to_slice(&descriptor), + encoded); +} + +platform_status +blob_key_clone(blob_key_config *cfg, + const blob_build_config *build_cfg, + mini_allocator *mini, + slice encoded, + writable_buffer *cloned) +{ + if (!cfg->blob_keys) { + return writable_buffer_copy_slice(cloned, encoded); + } + blob_key_view view; + platform_status rc = blob_key_view_init( + cfg, (user_key){.key = encoded, .is_query_key = FALSE}, &view); + if (!SUCCESS(rc)) { + return rc; + } + blob_build_config compact = *build_cfg; + compact.blob_max_descriptor_size = BLOB_KEY_DESCRIPTOR_BYTES; + auto_writable_buffer descriptor; + writable_buffer_init(&descriptor, cloned->heap_id); + rc = blob_clone(&compact, cfg->cc, mini, view.descriptor, &descriptor); + if (!SUCCESS(rc)) { + return rc; + } + return blob_key_assemble( + view.length, view.prefix, writable_buffer_to_slice(&descriptor), cloned); +} + +platform_status +blob_key_decode(blob_key_config *cfg, user_key input, writable_buffer *logical) +{ + blob_key_view view; + platform_status rc = blob_key_view_init(cfg, input, &view); + if (!SUCCESS(rc)) { + return rc; + } + if (!view.is_blob) { + return writable_buffer_copy_slice(logical, input.key); + } + /* blob_materialize_full validates descriptor shape before page reads. */ + rc = blob_key_validate(cfg, input.key); + if (!SUCCESS(rc)) { + return rc; + } + rc = blob_materialize_full(cfg->cc, view.descriptor, logical); + if (SUCCESS(rc)) { + cfg->counters.blob_bytes += view.length; + if (slice_length(view.prefix) != 0 + && memcmp(writable_buffer_data(logical), + slice_data(view.prefix), + slice_length(view.prefix)) + != 0) + { + return STATUS_INVALID_STATE; + } + } + return rc; +} diff --git a/benchmarks/blob_keys/blob_key.h b/benchmarks/blob_keys/blob_key.h new file mode 100644 index 000000000..90cd3863a --- /dev/null +++ b/benchmarks/blob_keys/blob_key.h @@ -0,0 +1,69 @@ +// Copyright 2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "blob_build.h" +#include "data_internal.h" + +/* + * Experimental codec for the real-cache/B-tree benchmark. Not a public API + * or persistent database format. Counters and callbacks are single-threaded. + * This codec implements unsigned bytewise lexical ordering only. Its prefix + * shortcut belongs inside that comparator; arbitrary application comparators + * must receive logical bytes and cannot use this shortcut without a contract. + */ +#define BLOB_KEY_MAX_PREFIX_BYTES 64 +#define BLOB_KEY_DESCRIPTOR_BYTES 128 + +typedef struct blob_key_counters { + uint64 comparisons; + uint64 prefix_resolved; + uint64 blob_fallbacks; + /* Comparator/hash fragment visits; decode uses blob_materialize_full. */ + uint64 blob_fragments; + /* Logical bytes passed to memcmp/hash/copy from blob fragments, not I/O. */ + uint64 blob_bytes; + uint64 hash_calls; +} blob_key_counters; + +typedef struct blob_key_config { + data_config data_cfg; /* Must remain first: callback downcast. */ + cache *cc; + bool32 blob_keys; + uint64 prefix_bytes; + blob_key_counters counters; +} blob_key_config; + +platform_status +blob_key_config_init(blob_key_config *cfg, + cache *cc, + bool32 blob_keys, + uint64 prefix_bytes); + +void +blob_key_reset_counters(blob_key_config *cfg); + +/* Output buffers must not alias input storage. Blob batches belong to mini. */ +platform_status +blob_key_encode(blob_key_config *cfg, + const blob_build_config *build_cfg, + mini_allocator *mini, + slice logical, + writable_buffer *encoded); + +platform_status +blob_key_clone(blob_key_config *cfg, + const blob_build_config *build_cfg, + mini_allocator *mini, + slice encoded, + writable_buffer *cloned); + +platform_status +blob_key_decode(blob_key_config *cfg, + user_key encoded_or_query, + writable_buffer *logical); + +/* Validate the encoded envelope and descriptor without reading blob pages. */ +platform_status +blob_key_validate(blob_key_config *cfg, slice encoded); diff --git a/benchmarks/blob_keys/blob_key_bench.c b/benchmarks/blob_keys/blob_key_bench.c new file mode 100644 index 000000000..dd5020c30 --- /dev/null +++ b/benchmarks/blob_keys/blob_key_bench.c @@ -0,0 +1,590 @@ +// Copyright 2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 +/* Experimental ordered blob keys over the real mutable and packed B-trees. + * Each tree has a separately owned key arena. This is not a persisted format. + */ +#include +#include +#include +#include +#include +#include +#include "blob_key.h" +#include "blob_key_checks.h" +#include "btree.h" +#include "clockcache.h" +#include "platform_threads.h" +#include "rc_allocator.h" +#include "task.h" + +#define MIB (UINT64_C(1) << 20) +#define GIB (UINT64_C(1) << 30) +#define CHECK(x) \ + do { \ + platform_status check_rc = (x); \ + if (!SUCCESS(check_rc)) { \ + fprintf(stderr, \ + "%s:%d: %s failed: %d\n", \ + __FILE__, \ + __LINE__, \ + #x, \ + check_rc.r); \ + exit(2); \ + } \ + } while (0) +#define REQUIRE(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "%s:%d: %s failed\n", __FILE__, __LINE__, #x); \ + exit(2); \ + } \ + } while (0) + +typedef struct { + const char *db; + bool32 blob, variable, checks; + uint64 n, queries, bytes, prefix, common, cache_mib; +} options; + +typedef struct { + options opt; + platform_heap_id hid; + io_config io_cfg; + allocator_config al_cfg; + clockcache_config cc_cfg; + task_system_config task_cfg; + task_system tasks; + io_handle *io; + rc_allocator al; + clockcache cc; + blob_key_config codec; + btree_config tree_cfg; + btree_scratch *scratch; + unsigned char *corpus; + uint64 *lengths, *order; + uint64 baseline_allocated; + uint64 encoded_bytes; +} bench; + +typedef struct { + uint64 root, arena_root; + page_type type; + mini_allocator mini, arena; + bool32 mini_active, arena_active; +} owned_tree; + +static uint64 +now_ns(void) +{ + struct timespec ts; + REQUIRE(clock_gettime(CLOCK_MONOTONIC, &ts) == 0); + return (uint64)ts.tv_sec * 1000000000 + ts.tv_nsec; +} +static uint64 +random_next(uint64 *s) +{ + uint64 x = (*s += UINT64_C(0x9e3779b97f4a7c15)); + x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); + x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb); + return x ^ (x >> 31); +} +static slice +raw_key(bench *b, uint64 id, bool32 missing) +{ + return slice_create(b->lengths[id], + b->corpus + (2 * id + missing) * b->opt.bytes); +} +static uint64 +expected_value(uint64 id) +{ + return id % 8 == 0 ? id + UINT64_C(0x100000000) : id; +} + +static void +generate(bench *b) +{ + const options *o = &b->opt; + REQUIRE(o->n <= SIZE_MAX / (2 * o->bytes)); + b->corpus = malloc(2 * o->n * o->bytes); + b->lengths = malloc(o->n * sizeof(*b->lengths)); + b->order = malloc(o->n * sizeof(*b->order)); + REQUIRE(b->corpus && b->lengths && b->order); + for (uint64 i = 0; i < o->n; i++) { + uint64 minlen = o->common + 8; + b->lengths[i] = + o->variable ? minlen + i % (o->bytes - minlen + 1) : o->bytes; + b->order[i] = i; + for (uint64 missing = 0; missing < 2; missing++) { + unsigned char *p = b->corpus + (2 * i + missing) * o->bytes; + memset(p, 0, o->bytes); + uint64 number = 2 * i + missing; + for (unsigned j = 0; j < 8; j++) + p[o->common + j] = number >> (56 - 8 * j); + uint64 state = i + 1; + for (uint64 j = minlen; j < b->lengths[i]; j++) + p[j] = random_next(&state); + } + } + uint64 state = 42; + for (uint64 i = o->n - 1; i > 0; i--) { + uint64 j = random_next(&state) % (i + 1), tmp = b->order[i]; + b->order[i] = b->order[j]; + b->order[j] = tmp; + } +} + +static void +setup(bench *b) +{ + platform_register_thread(); + CHECK(platform_heap_create(platform_get_module_id(), GIB, FALSE, &b->hid)); + io_config_init(&b->io_cfg, + 4096, + 128 * 1024, + O_RDWR | O_CREAT | O_TRUNC | O_DIRECT, + 0600, + 256, + b->opt.db); + uint64 capacity = 4 * GIB; + while (capacity < 8 * b->opt.n * b->opt.bytes) + capacity *= 2; + allocator_config_init(&b->al_cfg, &b->io_cfg, capacity); + clockcache_config_init( + &b->cc_cfg, &b->io_cfg, b->opt.cache_mib * MIB, "/dev/null", TRUE); + uint64 workers[NUM_TASK_TYPES] = {0}; + CHECK(task_system_config_init(&b->task_cfg, TRUE, workers)); + b->io = io_handle_create(&b->io_cfg, b->hid); + REQUIRE(b->io != NULL); + CHECK(task_system_init(&b->tasks, b->hid, &b->task_cfg)); + CHECK(rc_allocator_init( + &b->al, &b->al_cfg, b->io, b->hid, platform_get_module_id())); + CHECK(clockcache_init(&b->cc, + &b->cc_cfg, + b->io, + (allocator *)&b->al, + "blob-keys", + b->hid, + platform_get_module_id())); + CHECK(blob_key_config_init( + &b->codec, (cache *)&b->cc, b->opt.blob, b->opt.prefix)); + btree_config_init(&b->tree_cfg, &b->cc_cfg.super, &b->codec.data_cfg); + /* Bound key metadata plus prefix to the existing 512-byte inline limit. */ + b->tree_cfg.blob_cfg.blob_max_descriptor_size = 128; + b->scratch = calloc(1, btree_scratch_size(&b->tree_cfg)); + REQUIRE(b->scratch); + b->baseline_allocated = allocator_in_use((allocator *)&b->al); + generate(b); +} + +static void +tree_create_arena(bench *b, owned_tree *t) +{ + CHECK(btree_create((cache *)&b->cc, + &b->tree_cfg, + &t->arena, + PAGE_TYPE_BRANCH, + &t->arena_root)); + t->arena_active = TRUE; +} +static void +tree_release(bench *b, owned_tree *t) +{ + if (t->mini_active) + mini_release(&t->mini); + if (t->root) + REQUIRE(btree_dec_ref((cache *)&b->cc, &b->tree_cfg, t->root, t->type)); + if (t->arena_active) + mini_release(&t->arena); + if (t->arena_root) + REQUIRE(btree_dec_ref( + (cache *)&b->cc, &b->tree_cfg, t->arena_root, PAGE_TYPE_BRANCH)); + memset(t, 0, sizeof(*t)); +} +static uint64 +phase_begin(bench *b) +{ + blob_key_reset_counters(&b->codec); + cache_reset_stats((cache *)&b->cc); + return now_ns(); +} +static void +phase_end(bench *b, const char *stage, uint64 operations, uint64 start) +{ + double seconds = (now_ns() - start) / 1e9; + uint64 read_bytes, write_bytes; + cache_io_stats((cache *)&b->cc, &read_bytes, &write_bytes); + blob_key_counters *c = &b->codec.counters; + printf("{\"stage\":\"%s\",\"mode\":\"%s\",\"key_bytes\":%lu,\"prefix_" + "bytes\":%lu,\"common_prefix\":%lu,\"records\":%lu,\"operations\":%" + "lu,\"seconds\":%.9f,\"ops_per_sec\":%.3f,\"read_bytes\":%lu,\"write_" + "bytes\":%lu,\"allocated_bytes\":%lu,\"corpus_bytes\":%lu," + "\"comparisons\":%lu,\"prefix_resolved\":%lu,\"blob_fallbacks\":%lu," + "\"blob_fragments\":%lu,\"blob_bytes\":%lu,\"hash_calls\":%lu}\n", + stage, + b->opt.blob ? "blob" : "inline", + b->opt.bytes, + b->opt.prefix, + b->opt.common, + b->opt.n, + operations, + seconds, + operations / seconds, + read_bytes, + write_bytes, + allocator_in_use((allocator *)&b->al) * b->io_cfg.extent_size, + 2 * b->opt.n * b->opt.bytes, + c->comparisons, + c->prefix_resolved, + c->blob_fallbacks, + c->blob_fragments, + c->blob_bytes, + c->hash_calls); + fflush(stdout); +} + +static void +insert(bench *b, owned_tree *t, bool32 overwrite) +{ + writable_buffer encoded; + writable_buffer_init(&encoded, b->hid); + btree_insert_results results; + btree_insert_results_init(&results, NULL); + uint64 operations = 0, start = phase_begin(b); + for (uint64 j = 0; j < b->opt.n; j++) { + uint64 i = b->order[j]; + if (overwrite && i % 8 != 0) + continue; + CHECK(blob_key_encode(&b->codec, + &b->tree_cfg.blob_cfg, + &t->arena, + raw_key(b, i, FALSE), + &encoded)); + if (!overwrite) + b->encoded_bytes += writable_buffer_length(&encoded); + key k = key_create_from_slice(FALSE, writable_buffer_to_slice(&encoded)); + uint64 value = overwrite ? expected_value(i) : i; + message msg = message_create_internal( + MESSAGE_TYPE_INSERT, NULL, slice_create(sizeof(value), &value)); + CHECK(btree_insert((cache *)&b->cc, + &b->tree_cfg, + b->hid, + b->scratch, + t->root, + &t->mini, + k, + msg, + &results)); + operations++; + } + phase_end(b, overwrite ? "overwrite" : "insert", operations, start); + btree_insert_results_deinit(&results); + writable_buffer_deinit(&encoded); +} + +static void +lookups(bench *b, owned_tree *t, const char *stage) +{ + lookup_result result; + lookup_result_init( + &result, &b->codec.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + uint64 rng = 0x12345, start = phase_begin(b); + for (uint64 j = 0; j < b->opt.queries; j++) { + uint64 i = random_next(&rng) % b->opt.n; + bool32 missing = j & 1; + lookup_result_reset(&result); + CHECK(btree_lookup((cache *)&b->cc, + &b->tree_cfg, + t->root, + BTREE_DEFAULT_FORMAT, + t->type, + key_create_from_slice(TRUE, raw_key(b, i, missing)), + &result)); + REQUIRE(lookup_result_found(&result) == !missing); + if (!missing) { + slice value = + merge_accumulator_to_value(lookup_result_accumulator(&result)); + REQUIRE(slice_length(value) == sizeof(uint64)); + uint64 got; + memcpy(&got, slice_data(value), sizeof(got)); + REQUIRE(got == expected_value(i)); + } + } + phase_end(b, stage, b->opt.queries, start); + lookup_result_deinit(&result); +} + +/* A private owner-pair adapter. Cloning happens during initialization/next, + * preserving the observation-only iterator_curr contract. */ +typedef struct { + iterator super; + btree_iterator source; + bench *b; + owned_tree *destination; + writable_buffer encoded; + message msg; +} clone_iterator; +static platform_status +clone_prepare(clone_iterator *it) +{ + if (!iterator_can_next(&it->source.super)) + return STATUS_OK; + key k; + iterator_curr(&it->source.super, &k, &it->msg); + return blob_key_clone(&it->b->codec, + &it->b->tree_cfg.blob_cfg, + &it->destination->arena, + key_slice(k), + &it->encoded); +} +static void +clone_curr(iterator *base, key *k, message *m) +{ + clone_iterator *it = (clone_iterator *)base; + *k = key_create_from_slice(FALSE, writable_buffer_to_slice(&it->encoded)); + *m = it->msg; +} +static bool32 +clone_next_available(iterator *base) +{ + return iterator_can_next(&((clone_iterator *)base)->source.super); +} +static bool32 +clone_prev_available(iterator *base) +{ + (void)base; + return FALSE; +} +static platform_status +clone_next(iterator *base) +{ + clone_iterator *it = (clone_iterator *)base; + platform_status rc = iterator_next(&it->source.super); + return SUCCESS(rc) ? clone_prepare(it) : rc; +} +static const iterator_ops clone_ops = {.curr = clone_curr, + .can_next = clone_next_available, + .can_prev = clone_prev_available, + .next = clone_next}; + +static void +pack(bench *b, owned_tree *source, owned_tree *destination, const char *stage) +{ + uint64 start = phase_begin(b); + tree_create_arena(b, destination); + destination->type = PAGE_TYPE_BRANCH; + clone_iterator it = { + .super = {.ops = &clone_ops}, .b = b, .destination = destination}; + writable_buffer_init(&it.encoded, b->hid); + CHECK(btree_iterator_init((cache *)&b->cc, + &b->tree_cfg, + &it.source, + source->root, + BTREE_DEFAULT_FORMAT, + source->type, + &(btree_iterator_init_options){0})); + CHECK(clone_prepare(&it)); + btree_pack_req req; + CHECK(btree_pack_req_init(&req, + (cache *)&b->cc, + &b->tree_cfg, + &it.super, + b->opt.n, + b->hid, + .collect_fingerprints = TRUE)); + CHECK(btree_pack(&req)); + destination->root = req.root_addr; + REQUIRE(req.num_tuples == b->opt.n); + btree_iterator_deinit(&it.source); + writable_buffer_deinit(&it.encoded); + mini_release(&destination->arena); + destination->arena_active = FALSE; + phase_end(b, stage, b->opt.n, start); + for (uint64 i = 0; i < b->opt.n; i++) { + user_key raw = {.key = raw_key(b, i, FALSE), .is_query_key = TRUE}; + REQUIRE(req.fingerprint_arr[i] + == b->codec.data_cfg.key_hash(&b->codec.data_cfg, raw, 0)); + } + btree_pack_req_deinit(&req, b->hid); +} + +static void +scan(bench *b, owned_tree *t, const char *stage) +{ + writable_buffer decoded; + writable_buffer_init(&decoded, b->hid); + btree_iterator it; + uint64 start = phase_begin(b); + CHECK(btree_iterator_init( + (cache *)&b->cc, + &b->tree_cfg, + &it, + t->root, + BTREE_DEFAULT_FORMAT, + t->type, + &(btree_iterator_init_options){.prefetch_lookahead = 2})); + uint64 i = 0; + while (iterator_can_next(&it.super)) { + key k; + message msg; + iterator_curr(&it.super, &k, &msg); + REQUIRE(i < b->opt.n); + CHECK( + blob_key_decode(&b->codec, + (user_key){.key = key_slice(k), .is_query_key = FALSE}, + &decoded)); + if (slice_lex_cmp(writable_buffer_to_slice(&decoded), + raw_key(b, i, FALSE)) + != 0) + { + uint64 got_id = 0; + memcpy(&got_id, message_data(msg), sizeof(got_id)); + fprintf(stderr, + "scan mismatch index=%lu value=%lu decoded_length=%lu " + "expected_length=%lu\n", + i, + got_id, + writable_buffer_length(&decoded), + b->lengths[i]); + exit(2); + } + uint64 got; + REQUIRE(message_length(msg) == sizeof(got)); + memcpy(&got, message_data(msg), sizeof(got)); + REQUIRE(got == expected_value(i)); + i++; + CHECK(iterator_next(&it.super)); + } + REQUIRE(i == b->opt.n); + btree_iterator_deinit(&it); + phase_end(b, stage, i, start); + writable_buffer_deinit(&decoded); +} + +static options +parse_options(int argc, char **argv) +{ + options o = {.db = NULL, + .blob = TRUE, + .n = 1024, + .queries = 2048, + .bytes = 4096, + .prefix = 32, + .cache_mib = 128}; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--checks")) { + o.checks = TRUE; + continue; + } + if (!strcmp(argv[i], "--variable")) { + o.variable = TRUE; + continue; + } + REQUIRE(i + 1 < argc); + const char *name = argv[i++], *value = argv[i]; + if (!strcmp(name, "--db")) { + o.db = value; + continue; + } + if (!strcmp(name, "--mode")) { + REQUIRE(!strcmp(value, "inline") || !strcmp(value, "blob")); + o.blob = !strcmp(value, "blob"); + continue; + } + char *end; + errno = 0; + uint64 number = strtoull(value, &end, 10); + REQUIRE(errno == 0 && *end == 0 && *value != 0); + if (!strcmp(name, "--records")) + o.n = number; + else if (!strcmp(name, "--queries")) + o.queries = number; + else if (!strcmp(name, "--key-bytes")) + o.bytes = number; + else if (!strcmp(name, "--prefix")) + o.prefix = number; + else if (!strcmp(name, "--common-prefix")) + o.common = number; + else if (!strcmp(name, "--cache-mib")) + o.cache_mib = number; + else { + fprintf(stderr, "Unknown argument: %s\n", name); + exit(2); + } + } + REQUIRE(o.db && o.n > 0 && o.queries > 0 && o.bytes >= 8 + && o.bytes <= 16 * MIB && o.common <= o.bytes - 8 + && o.prefix <= BLOB_KEY_MAX_PREFIX_BYTES && o.cache_mib >= 16); + REQUIRE(o.blob || o.bytes <= 512); + return o; +} + +int +main(int argc, char **argv) +{ + bench b = {.opt = parse_options(argc, argv)}; + setup(&b); + printf("{\"configuration\":true,\"cache_mib\":%lu,\"variable_keys\":%s," + "\"page_bytes\":4096,\"extent_bytes\":131072,\"fingerprints\":true," + "\"direct_io\":true}\n", + b.opt.cache_mib, + b.opt.variable ? "true" : "false"); + if (b.opt.checks) { + owned_tree checks_arena = {0}; + tree_create_arena(&b, &checks_arena); + CHECK(blob_key_checks( + &b.codec, &b.tree_cfg.blob_cfg, &checks_arena.arena, b.hid)); + tree_release(&b, &checks_arena); + goto cleanup; + } + owned_tree mutable = {.type = PAGE_TYPE_MEMTABLE}, branch = {0}, + repacked = {0}; + tree_create_arena(&b, &mutable); + CHECK(btree_create( + (cache *)&b.cc, &b.tree_cfg, &mutable.mini, mutable.type, &mutable.root)); + mutable.mini_active = TRUE; + insert(&b, &mutable, FALSE); + printf( + "{\"encoded_key_bytes_total\":%lu,\"encoded_key_bytes_average\":%.3f}\n", + b.encoded_bytes, + (double)b.encoded_bytes / b.opt.n); + insert(&b, &mutable, TRUE); + lookups(&b, &mutable, "memtable_lookup"); + scan(&b, &mutable, "memtable_scan"); + pack(&b, &mutable, &branch, "pack"); + tree_release(&b, &mutable); + /* The source arena is gone; branch key clones must independently survive. */ + CHECK(cache_writeback_dirty((cache *)&b.cc)); + CHECK(cache_durable_barrier((cache *)&b.cc)); + CHECK(cache_evict((cache *)&b.cc, FALSE)); + lookups(&b, &branch, "branch_cold_lookup"); + lookups(&b, &branch, "branch_warm_lookup"); + scan(&b, &branch, "branch_scan"); + pack(&b, &branch, &repacked, "repack"); + tree_release(&b, &branch); + CHECK(cache_writeback_dirty((cache *)&b.cc)); + CHECK(cache_durable_barrier((cache *)&b.cc)); + CHECK(cache_evict((cache *)&b.cc, FALSE)); + lookups(&b, &repacked, "repacked_lookup"); + scan(&b, &repacked, "repacked_scan"); + tree_release(&b, &repacked); +cleanup: + REQUIRE(allocator_in_use((allocator *)&b.al) == b.baseline_allocated); + free(b.scratch); + free(b.corpus); + free(b.lengths); + free(b.order); + clockcache_deinit(&b.cc); + rc_allocator_deinit(&b.al); + task_system_deinit(&b.tasks); + io_handle_destroy(b.io); + platform_heap_destroy(&b.hid); + platform_deregister_thread(); + if (b.opt.checks) { + puts("{\"validation\":\"passed\",\"codec_checks\":true,\"allocation_" + "baseline_restored\":true}"); + return 0; + } + puts("{\"validation\":\"passed\",\"source_release\":true,\"repack_source_" + "release\":true,\"allocation_baseline_restored\":true}"); + return 0; +} diff --git a/benchmarks/blob_keys/blob_key_checks.c b/benchmarks/blob_keys/blob_key_checks.c new file mode 100644 index 000000000..a221bc0c0 --- /dev/null +++ b/benchmarks/blob_keys/blob_key_checks.c @@ -0,0 +1,239 @@ +// Copyright 2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "blob_key_checks.h" + +#define CHECK(expr) \ + do { \ + if (!(expr)) { \ + platform_error_log("blob-key check failed at %s:%d: %s\n", \ + __FILE__, \ + __LINE__, \ + #expr); \ + return STATUS_TEST_FAILED; \ + } \ + } while (0) +#define CHECK_OK(expr) CHECK(SUCCESS(expr)) + +static int +sign_of(int value) +{ + return (value > 0) - (value < 0); +} + +static user_key +stored_key(writable_buffer *buffer) +{ + return (user_key){.key = writable_buffer_to_slice(buffer), + .is_query_key = FALSE}; +} + +static platform_status +check_length(blob_key_config *cfg, + const blob_build_config *build_cfg, + mini_allocator *mini, + platform_heap_id hid, + uint64 length) +{ + auto_writable_buffer logical; + auto_writable_buffer first; + auto_writable_buffer second; + auto_writable_buffer decoded; + writable_buffer_init(&logical, hid); + writable_buffer_init(&first, hid); + writable_buffer_init(&second, hid); + writable_buffer_init(&decoded, hid); + /* Keep a non-NULL source pointer even for the empty logical key. */ + CHECK_OK(writable_buffer_resize(&logical, length + 1)); + uint8 *bytes = writable_buffer_data(&logical); + for (uint64 i = 0; i < length; i++) { + bytes[i] = (uint8)((i * UINT64_C(157) + (i >> 8)) & 255); + } + user_key query = {.key = slice_create(length, bytes), .is_query_key = TRUE}; + CHECK_OK(blob_key_encode(cfg, build_cfg, mini, query.key, &first)); + CHECK_OK(blob_key_encode(cfg, build_cfg, mini, query.key, &second)); + CHECK_OK(blob_key_validate(cfg, writable_buffer_to_slice(&first))); + CHECK_OK(blob_key_decode(cfg, stored_key(&first), &decoded)); + CHECK(writable_buffer_length(&decoded) == length); + CHECK(length == 0 + || memcmp(writable_buffer_data(&decoded), bytes, length) == 0); + + data_config *dc = &cfg->data_cfg; + CHECK(dc->key_compare(dc, stored_key(&first), stored_key(&second)) == 0); + CHECK(dc->key_compare(dc, stored_key(&first), query) == 0); + CHECK(dc->key_compare(dc, query, stored_key(&first)) == 0); + CHECK(dc->key_compare(dc, query, query) == 0); + blob_key_config inline_cfg; + CHECK_OK(blob_key_config_init(&inline_cfg, cfg->cc, FALSE, 0)); + const uint32 seeds[] = {0, 1, UINT32_MAX}; + for (uint64 i = 0; i < ARRAY_SIZE(seeds); i++) { + uint32 expected = + inline_cfg.data_cfg.key_hash(&inline_cfg.data_cfg, query, seeds[i]); + CHECK(dc->key_hash(dc, query, seeds[i]) == expected); + CHECK(dc->key_hash(dc, stored_key(&first), seeds[i]) == expected); + CHECK(dc->key_hash(dc, stored_key(&second), seeds[i]) == expected); + } + CHECK_OK(blob_key_decode(cfg, query, &decoded)); + CHECK(writable_buffer_length(&decoded) == slice_length(query.key)); + CHECK(slice_length(query.key) == 0 + || memcmp(writable_buffer_data(&decoded), + slice_data(query.key), + slice_length(query.key)) + == 0); + + if (length != 0) { + /* A key that is a proper prefix sorts first, including an empty prefix. + */ + user_key shorter = {.key = slice_create(length / 2, bytes), + .is_query_key = TRUE}; + CHECK_OK(blob_key_encode(cfg, build_cfg, mini, shorter.key, &second)); + CHECK(dc->key_compare(dc, stored_key(&second), stored_key(&first)) < 0); + CHECK(dc->key_compare(dc, stored_key(&first), stored_key(&second)) > 0); + CHECK(dc->key_compare(dc, shorter, stored_key(&first)) < 0); + CHECK(dc->key_compare(dc, stored_key(&first), shorter) > 0); + + /* Different first/last bytes exercise prefix and streamed suffix paths. + */ + const uint64 positions[] = {0, length - 1}; + for (uint64 p = 0; p < ARRAY_SIZE(positions); p++) { + uint64 at = positions[p]; + uint8 before = bytes[at]; + bytes[at] ^= 0x80; + int expected = (before > bytes[at]) - (before < bytes[at]); + CHECK_OK(blob_key_encode(cfg, build_cfg, mini, query.key, &second)); + CHECK(sign_of( + dc->key_compare(dc, stored_key(&first), stored_key(&second))) + == expected); + CHECK(sign_of( + dc->key_compare(dc, stored_key(&second), stored_key(&first))) + == -expected); + CHECK(sign_of(dc->key_compare(dc, stored_key(&first), query)) + == expected); + CHECK(sign_of(dc->key_compare(dc, query, stored_key(&first))) + == -expected); + bytes[at] = before; + } + } + return STATUS_OK; +} + +static platform_status +check_malformed(blob_key_config *cfg, + const blob_build_config *build_cfg, + mini_allocator *mini, + platform_heap_id hid) +{ + uint8 logical[65]; + for (uint64 i = 0; i < sizeof(logical); i++) { + logical[i] = (uint8)i; + } + auto_writable_buffer original; + auto_writable_buffer changed; + auto_writable_buffer decoded; + writable_buffer_init(&original, hid); + writable_buffer_init(&changed, hid); + writable_buffer_init(&decoded, hid); + CHECK_OK(blob_key_encode( + cfg, build_cfg, mini, slice_create(sizeof(logical), logical), &original)); + slice valid = writable_buffer_to_slice(&original); + /* Codec v1 envelope: uint64 length, uint8 prefix length, uint8 format. */ + const uint64 header_bytes = 10; + for (uint64 length = 0; length < header_bytes; length++) { + slice short_key = slice_create(length, slice_data(valid)); + CHECK(!SUCCESS(blob_key_validate(cfg, short_key))); + CHECK(!SUCCESS( + blob_key_decode(cfg, (user_key){.key = short_key}, &decoded))); + } + for (uint64 variant = 0; variant < 6; variant++) { + CHECK_OK(writable_buffer_copy_slice(&changed, valid)); + uint8 *bytes = writable_buffer_data(&changed); + uint64 descriptor = header_bytes + bytes[8]; + switch (variant) { + case 0: + bytes[8] = BLOB_KEY_MAX_PREFIX_BYTES + 1; + break; + case 1: + bytes[9] = 0; + break; + case 2: + { + uint64 wrong_length = sizeof(logical) + 1; + memcpy(bytes, &wrong_length, sizeof(wrong_length)); + break; + } + case 3: + { + uint16 wrong_format = 0; + memcpy(bytes + descriptor + offsetof(blob, format), + &wrong_format, + sizeof(wrong_format)); + break; + } + case 4: + CHECK_OK(writable_buffer_resize( + &changed, writable_buffer_length(&changed) - 1)); + break; + case 5: + { + uint64 invalid_addr = 0; + memcpy(bytes + descriptor + sizeof(blob), + &invalid_addr, + sizeof(invalid_addr)); + break; + } + } + CHECK( + !SUCCESS(blob_key_validate(cfg, writable_buffer_to_slice(&changed)))); + CHECK(!SUCCESS(blob_key_decode(cfg, stored_key(&changed), &decoded))); + } + if (cfg->prefix_bytes != 0) { + CHECK_OK(writable_buffer_copy_slice(&changed, valid)); + ((uint8 *)writable_buffer_data(&changed))[header_bytes] ^= 0x80; + /* Shape-only validation cannot detect this; materialization must. */ + CHECK_OK(blob_key_validate(cfg, writable_buffer_to_slice(&changed))); + CHECK(!SUCCESS(blob_key_decode(cfg, stored_key(&changed), &decoded))); + } + return STATUS_OK; +} + +platform_status +blob_key_checks(blob_key_config *cfg, + const blob_build_config *build_cfg, + mini_allocator *mini, + platform_heap_id hid) +{ + blob_key_config invalid; + CHECK(!SUCCESS(blob_key_config_init( + &invalid, cfg->cc, TRUE, BLOB_KEY_MAX_PREFIX_BYTES + 1))); + CHECK(!SUCCESS(blob_key_config_init(&invalid, NULL, TRUE, 0))); + uint64 page = cache_page_size(cfg->cc); + uint64 extent = cache_extent_size(cfg->cc); + const uint64 lengths[] = { + 0, + 1, + 7, + 16, + 63, + 64, + 65, + page - 1, + page, + page + 1, + extent - 1, + extent, + extent + 1, + /* With 128 descriptor bytes, thirteen extents require an indirect tree. + */ + 13 * extent, + }; + const uint64 prefixes[] = {0, 16, BLOB_KEY_MAX_PREFIX_BYTES}; + for (uint64 p = 0; p < ARRAY_SIZE(prefixes); p++) { + blob_key_config check_cfg; + CHECK_OK(blob_key_config_init(&check_cfg, cfg->cc, TRUE, prefixes[p])); + for (uint64 i = 0; i < ARRAY_SIZE(lengths); i++) { + CHECK_OK(check_length(&check_cfg, build_cfg, mini, hid, lengths[i])); + } + CHECK_OK(check_malformed(&check_cfg, build_cfg, mini, hid)); + } + return STATUS_OK; +} diff --git a/benchmarks/blob_keys/blob_key_checks.h b/benchmarks/blob_keys/blob_key_checks.h new file mode 100644 index 000000000..97aa8ac2b --- /dev/null +++ b/benchmarks/blob_keys/blob_key_checks.h @@ -0,0 +1,12 @@ +// Copyright 2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "blob_key.h" + +/* Untimed codec checks. The caller owns and later releases the supplied arena. + */ +platform_status +blob_key_checks(blob_key_config *cfg, + const blob_build_config *build_cfg, + mini_allocator *mini, + platform_heap_id hid); diff --git a/benchmarks/blob_keys/run.py b/benchmarks/blob_keys/run.py new file mode 100644 index 000000000..2e4dabb95 --- /dev/null +++ b/benchmarks/blob_keys/run.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Run real blob-key/B-tree experiments with cross-project quiescence.""" +import argparse +import datetime +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import time +import uuid + +def write_json(path, value): + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n") + temporary.replace(path) + +def completed(path): + text = path.read_text() + status = re.search(r"^Status:\s*(.*)$", text, re.MULTILINE | re.IGNORECASE) + return bool(status and re.match(r"(completed|complete|done|inactive)\b", + status[1], re.IGNORECASE)) + +def active_projects(coord): + return {p.stem for p in coord.glob("*.md") + if p.stem != "blob-keys" and not completed(p)} + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--coordination", type=Path, required=True) + parser.add_argument("--lock-dir", type=Path, required=True) + parser.add_argument("--repetitions", type=int, default=3) + parser.add_argument("--timeout", type=int, default=900, + help="seconds to wait for every explicit pause acknowledgement") + args = parser.parse_args() + if args.repetitions < 1: + parser.error("repetitions must be positive") + args.binary = args.binary.resolve() + args.out = args.out.resolve() + args.out.mkdir(parents=True, exist_ok=False) + coord = args.coordination.resolve() + request_id = "blob-keys-" + uuid.uuid4().hex + request = dict(request_id=request_id, owner="blob-keys", + purpose="real B-tree inline/blob-key comparison", + requested_at=datetime.datetime.now(datetime.timezone.utc).isoformat()) + root = Path(__file__).resolve().parents[2] + fs = subprocess.check_output( + ["findmnt", "-n", "-o", "FSTYPE,SOURCE", "-T", str(args.out)], text=True).strip() + if fs.split()[0] in {"tmpfs", "ramfs"}: + raise RuntimeError("Use disk-backed storage for this experiment") + manifest = dict(request=request, binary=str(args.binary), + binary_sha256=sha256(args.binary), storage=fs, + revision=subprocess.check_output(["git", "rev-parse", "HEAD"], + cwd=root, text=True).strip(), + source_sha256={p.name: sha256(p) for p in Path(__file__).parent.iterdir() + if p.suffix in {".c", ".h", ".py"}}, + cpu=Path("/proc/cpuinfo").read_text().split("\n\n")[0], + command_line=os.sys.argv, runs=[]) + write_json(args.out / "manifest.json", manifest) + benchmark_lock = open(args.lock_dir / "benchmark.lock", "a+") + print("Waiting for benchmark lock", flush=True) + fcntl.flock(benchmark_lock, fcntl.LOCK_EX) + write_json(coord / "benchmark-request.json", request) + write_json(args.out / "request.json", request) + try: + deadline = time.monotonic() + args.timeout + while True: + missing, acknowledged = [], {} + for project in sorted(active_projects(coord)): + path = coord / (project + "-paused.json") + try: + ack = json.loads(path.read_text()) + except (FileNotFoundError, json.JSONDecodeError): + ack = {} + if ack.get("request_id") == request_id and ack.get("participants_paused") is True: + acknowledged[project] = ack + else: + missing.append(project) + if not missing: + break + if time.monotonic() >= deadline: + raise TimeoutError("No quiet-machine acknowledgement: " + ", ".join(missing)) + print("Waiting for paused acknowledgements: " + ", ".join(missing), flush=True) + time.sleep(10) + manifest["acknowledgements"] = acknowledged + manifest["completed_projects"] = [p.stem for p in coord.glob("*.md") if completed(p)] + write_json(args.out / "manifest.json", manifest) + with open(args.lock_dir / "machine.lock", "a+") as machine: + fcntl.flock(machine, fcntl.LOCK_EX) + print("All active projects paused; exclusive measurements starting", flush=True) + cases = [ + (128, 32768, 131072), + (512, 16384, 65536), + (4096, 8192, 32768), + (65536, 1024, 4096), + (1048576, 64, 256), + (2097152, 16, 64), + ] + for repetition in range(args.repetitions): + for size, records, queries in cases: + for pattern in ("early", "late"): + variants = [("inline", 0), ("blob", 32)] if size <= 512 else [ + ("blob", 0), ("blob", 32)] + if repetition % 2: + variants.reverse() + for mode, prefix in variants: + name = f"r{repetition}-{size}-{pattern}-{mode}-p{prefix}" + command = [str(args.binary), "--db", str(args.out / (name + ".db")), + "--mode", mode, "--records", str(records), + "--queries", str(queries), "--key-bytes", str(size), + "--prefix", str(prefix), "--common-prefix", + str(0 if pattern == "early" else size - 8), + "--cache-mib", "64"] + print(name, flush=True) + start = time.time() + with open(args.out / (name + ".jsonl"), "w") as stdout, \ + open(args.out / (name + ".stderr"), "w") as stderr: + result = subprocess.run(command, stdout=stdout, stderr=stderr) + run = dict(name=name, repetition=repetition, pattern=pattern, + command=command, returncode=result.returncode, + started=start, finished=time.time()) + manifest["runs"].append(run) + write_json(args.out / "manifest.json", manifest) + if result.returncode != 0: + raise RuntimeError("Benchmark failed: " + name) + lines = [json.loads(line) for line in + (args.out / (name + ".jsonl")).read_text().splitlines() + if line.startswith("{")] + if not any(row.get("validation") == "passed" for row in lines): + raise RuntimeError("Missing successful validation: " + name) + # Only delete the scratch image produced by this completed run. + (args.out / (name + ".db")).unlink(missing_ok=True) + manifest["completed"] = True + write_json(args.out / "manifest.json", manifest) + finally: + release = dict(request_id=request_id, owner="blob-keys", + released_at=datetime.datetime.now(datetime.timezone.utc).isoformat(), + complete=manifest.get("completed", False)) + write_json(coord / "benchmark-release.json", release) + write_json(args.out / "release.json", release) + fcntl.flock(benchmark_lock, fcntl.LOCK_UN) + benchmark_lock.close() + print("Benchmark lock released", flush=True) + +if __name__ == "__main__": + main() diff --git a/benchmarks/blob_keys/small_keys.py b/benchmarks/blob_keys/small_keys.py new file mode 100644 index 000000000..d1842cbb5 --- /dev/null +++ b/benchmarks/blob_keys/small_keys.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""Coordinated small-key regression measurements of frozen source profiles. + +Build/profile metadata is prepared before acquiring the benchmark reservation. +No compilation, source edits, or analysis runs inside the measurement window. +""" +import argparse +import csv +import datetime +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import time +import uuid + + +def now(): + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def write_json(path, value): + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(value, indent=2) + "\n") + temp.replace(path) + + +def registrations(coord): + result = {} + for path in coord.glob("*.md"): + text = path.read_text() + status = re.search(r"^Status:\s*(.*)$", text, re.M | re.I) + # Auxiliary notes without Status are not independent projects. + if status: + result[path.stem] = { + "status": status[1], + "complete": bool(re.match(r"(complete|completed|done|inactive)\b", + status[1], re.I)), + "text": text, + } + return result + + +def acquire_reservation(reservation, coord): + """Queued owners are idle and must acknowledge the current owner.""" + while True: + try: + fcntl.flock(reservation, fcntl.LOCK_EX | fcntl.LOCK_NB) + return + except BlockingIOError: + try: + request = json.loads((coord / "benchmark-request.json").read_text()) + except (FileNotFoundError, json.JSONDecodeError): + request = {} + try: + release = json.loads((coord / "benchmark-release.json").read_text()) + except (FileNotFoundError, json.JSONDecodeError): + release = {} + if (request.get("request_id") + and request.get("owner") != "blob-keys" + and request["request_id"] != release.get("request_id")): + write_json(coord / "blob-keys-paused.json", { + "request_id": request["request_id"], + "participants_paused": True, + "participants": ["blob-keys"], + "owner_children_paused": True, + "paused_at": now(), + "note": "Queued benchmark owner idle; all preparation completed.", + }) + time.sleep(1) + + +def micro_cases(): + cases = [] + for size in (24, 64, 256, 512): + for pattern in ("random", "common"): + for op in ("insert", "overwrite", "hit", "miss", "scan32", "scan4096", "pack"): + cases.append((size, pattern, op)) + for op in ("insert", "overwrite", "hit", "miss", "scan32", "scan4096", "pack"): + cases.append((64, "nonlexical", op)) + return cases + + +def run_micro(args, out, manifest): + cases = micro_cases() + if args.cases: + wanted = set(args.cases.split(",")) + cases = [c for c in cases if f"{c[0]}-{c[1]}-{c[2]}" in wanted] + if len(cases) != len(wanted): + raise ValueError("unknown or duplicate case") + profiles = json.loads(args.profiles.read_text()) + for name in args.variants: + p = profiles[name] + if sha(p["binary"]) != p["binary_sha256"]: + raise RuntimeError("binary changed: " + name) + manifest["profiles"] = {name: profiles[name] for name in args.variants} + write_json(out / "manifest.json", manifest) + for repetition in range(args.repetitions): + for case_no, (size, pattern, op) in enumerate(cases): + variants = args.variants[:] + if (repetition + case_no) % 2: + variants.reverse() + for variant in variants: + name = f"r{repetition}-{size}-{pattern}-{op}-{variant}" + db = out / (name + ".db") + keys = 65536 if size <= 64 else 32768 + operation = "lookup" if op in ("hit", "miss") else "scan" if op.startswith("scan") else op + batches = 4 if operation in ("insert", "overwrite", "pack") else 2 + ops = 500000 if operation == "lookup" else 4096 if op == "scan4096" else 131072 + command = [profiles[variant]["binary"], "--db", str(db), + "--operation", operation, "--pattern", pattern, + "--keys", str(keys), "--key-bytes", str(size), + "--value-bytes", "32", "--cache-mib", "512", + "--disk-mib", "2048", "--cpu", str(args.cpu), + "--batches", str(batches), "--ops", str(ops), + "--seed", str(42 + repetition)] + if op == "miss": + command += ["--miss"] + if op.startswith("scan"): + command += ["--scan-length", op[4:]] + print(name, flush=True) + started = now() + with (out / (name + ".stdout")).open("x") as stdout, \ + (out / (name + ".stderr")).open("x") as stderr: + result = subprocess.run(command, stdout=stdout, stderr=stderr, + timeout=180) + rows = [json.loads(line) for line in (out / (name + ".stdout")).read_text().splitlines() + if line.startswith("{")] + run = dict(name=name, variant=variant, repetition=repetition, + case=f"{size}-{pattern}-{op}", command=command, + started=started, finished=now(), returncode=result.returncode, + rows=rows) + manifest["runs"].append(run) + write_json(out / "manifest.json", manifest) + if result.returncode or len(rows) != 1 or not rows[0].get("verified") or not rows[0].get("resident"): + raise RuntimeError("invalid benchmark run: " + name) + if rows[0]["active_wall_ns"] < 1000000: + raise RuntimeError("measurement too short: " + name) + # Only our freshly created, successfully verified scratch image. + db.unlink(missing_ok=True) + + +def run_public(args, out, manifest): + profiles = json.loads(args.public_profiles.read_text()) + manifest["public_profiles"] = {v: profiles[v] for v in args.variants} + for variant in args.variants: + if sha(profiles[variant]["binary"]) != profiles[variant]["binary_sha256"]: + raise RuntimeError("public binary changed: " + variant) + for rep in range(args.repetitions): + for log in (0, 1): + for threads in (1, 4): + variants = args.variants[:] + if (rep + log + threads) % 2: + variants.reverse() + for variant in variants: + name = f"public-r{rep}-log{log}-t{threads}-{variant}" + db, csv_path = out / (name + ".db"), out / (name + ".csv") + command = ["taskset", "-c", "0-7", profiles[variant]["binary"], + "--db", str(db), "--create", "--n", "1000000", + "--cache-mib", "512", "--disk-gib", "4", + "--log", str(log), "--threads", str(threads), + "--mt-bg", "1", "--normal-bg", "4", + "--memtable-mib", "24", "--secs", "3", + "--warmup", "1", "--csv", str(csv_path), + "--hang-dir", str(out), "--vtag", variant, + "--stag", "1m-24byte-keys", "--otag", "random", + "--rep", str(rep), "--seed", str(42 + rep), + "load:random", "warm", "point:uniform", "point:neg", + "range:uniform:32", "range:uniform:4096", "ow:uniform"] + print(name, flush=True) + started = now() + with (out / (name + ".stdout")).open("x") as stdout, \ + (out / (name + ".stderr")).open("x") as stderr: + result = subprocess.run(command, stdout=stdout, stderr=stderr, + timeout=300) + rows = list(csv.DictReader(csv_path.open())) if csv_path.exists() else [] + manifest["runs"].append(dict(name=name, variant=variant, + repetition=rep, case=f"public-log{log}-t{threads}", + command=command, started=started, finished=now(), + returncode=result.returncode, rows=rows)) + write_json(out / "manifest.json", manifest) + if result.returncode or len(rows) != 7 or any( + int(r["violations"]) or int(r["errors"]) for r in rows): + raise RuntimeError("invalid public run: " + name) + db.unlink(missing_ok=True) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--profiles", type=Path, required=True) + p.add_argument("--out", type=Path, required=True) + p.add_argument("--coordination", type=Path, required=True) + p.add_argument("--lock-dir", type=Path, required=True) + p.add_argument("--variants", nargs="+", default=["baseline", "candidate-v1"]) + p.add_argument("--repetitions", type=int, default=3) + p.add_argument("--cpu", type=int, default=2) + p.add_argument("--cases") + p.add_argument("--public-profiles", type=Path) + p.add_argument("--public-only", action="store_true") + p.add_argument("--ack-timeout", type=int, default=1800) + p.add_argument("--expected-minutes", type=int, default=12) + a = p.parse_args() + a.out = a.out.resolve() + a.profiles = a.profiles.resolve() + a.out.mkdir(parents=True, exist_ok=False) + request = dict(request_id="blob-keys-small-" + uuid.uuid4().hex, + owner="blob-keys", purpose="small-key B-tree and public API regression comparison", + requested_at=now(), expected_duration_minutes=a.expected_minutes, + participants=["blob-keys"], owner_children_paused=True) + manifest = dict(request=request, command_line=os.sys.argv, runs=[], + runner_sha256=sha(__file__), started=now(), + cpu=Path("/proc/cpuinfo").read_text().split("\n\n")[0], + affinity=sorted(os.sched_getaffinity(0)), + storage=subprocess.check_output( + ["findmnt", "-n", "-o", "FSTYPE,SOURCE", "-T", str(a.out)], + text=True).strip()) + write_json(a.out / "manifest.json", manifest) + shutil_source = a.out / "runner.py" + shutil_source.write_bytes(Path(__file__).read_bytes()) + with (a.lock_dir / "benchmark.lock").open("r") as reservation: + print("Waiting for benchmark reservation", flush=True) + acquire_reservation(reservation, a.coordination) + request["requested_at"] = now() + request["participants"] += sorted(k for k, v in registrations(a.coordination).items() + if k != "blob-keys" and not v["complete"]) + write_json(a.coordination / "benchmark-request.json", request) + write_json(a.out / "request.json", request) + try: + deadline = time.monotonic() + a.ack_timeout + while True: + peers = registrations(a.coordination) + missing, acknowledgements = [], {} + for name, info in peers.items(): + if name == "blob-keys" or info["complete"]: + continue + try: + ack = json.loads((a.coordination / (name + "-paused.json")).read_text()) + except (FileNotFoundError, json.JSONDecodeError): + ack = {} + if ack.get("request_id") == request["request_id"] and ack.get("participants_paused") is True: + acknowledgements[name] = ack + else: + missing.append(name) + if not missing: + manifest["acknowledgements"] = acknowledgements + manifest["registrations"] = peers + break + if time.monotonic() >= deadline: + raise TimeoutError("Missing explicit acknowledgements: " + ", ".join(missing)) + print("Awaiting paused roots and children: " + ", ".join(missing), flush=True) + time.sleep(10) + with (a.lock_dir / "machine.lock").open("r") as machine: + fcntl.flock(machine, fcntl.LOCK_EX) + manifest["measurement_start"] = now() + write_json(a.out / "manifest.json", manifest) + print("All peers paused; exclusive measurement window starts", flush=True) + if not a.public_only: + run_micro(a, a.out, manifest) + if a.public_profiles: + run_public(a, a.out, manifest) + manifest["complete"] = True + manifest["measurement_end"] = now() + write_json(a.out / "manifest.json", manifest) + finally: + release = dict(request_id=request["request_id"], owner="blob-keys", + released_at=now(), complete=manifest.get("complete", False), + measurements_completed=len(manifest["runs"])) + write_json(a.coordination / "benchmark-release.json", release) + write_json(a.out / "release.json", release) + print("Exclusive benchmark window released", flush=True) + + +if __name__ == "__main__": + def interrupt(signum, frame): + raise KeyboardInterrupt(f"signal {signum}") + signal.signal(signal.SIGTERM, interrupt) + main() diff --git a/benchmarks/blob_keys/summarize.py b/benchmarks/blob_keys/summarize.py new file mode 100644 index 000000000..e56980981 --- /dev/null +++ b/benchmarks/blob_keys/summarize.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Summarize complete run.py output without hiding repetitions or cache I/O.""" +import argparse +import csv +import json +from pathlib import Path +from statistics import median + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("results", type=Path) + args = parser.parse_args() + manifest = json.loads((args.results / "manifest.json").read_text()) + if not manifest.get("completed"): + raise SystemExit("Refusing to summarize an incomplete campaign") + groups = {} + for run in manifest["runs"]: + if run["returncode"]: + raise SystemExit("Failed run: " + run["name"]) + rows = [json.loads(line) for line in + (args.results / (run["name"] + ".jsonl")).read_text().splitlines() + if line.startswith("{")] + if not any(r.get("validation") == "passed" for r in rows): + raise SystemExit("Missing validation: " + run["name"]) + for row in rows: + if "stage" not in row: + continue + key = (row["key_bytes"], run["pattern"], row["mode"], + row["prefix_bytes"], row["stage"]) + groups.setdefault(key, []).append(row) + summary = [] + for key, rows in sorted(groups.items()): + size, pattern, mode, prefix, stage = key + row = dict(key_bytes=size, pattern=pattern, mode=mode, + prefix_bytes=prefix, stage=stage, repetitions=len(rows), + records=rows[0]["records"], operations=rows[0]["operations"]) + for metric in ("seconds", "ops_per_sec", "read_bytes", "write_bytes", + "allocated_bytes", "comparisons", "prefix_resolved", + "blob_fallbacks", "blob_fragments", "blob_bytes", "hash_calls"): + row["median_" + metric] = median(r[metric] for r in rows) + row["min_seconds"] = min(r["seconds"] for r in rows) + row["max_seconds"] = max(r["seconds"] for r in rows) + summary.append(row) + with open(args.results / "summary.csv", "w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(summary[0])) + writer.writeheader() + writer.writerows(summary) + selected = ("insert", "pack", "branch_cold_lookup", + "branch_warm_lookup", "branch_scan") + lines = [ + "# Blob-key measurements", + "", + "Medians across repetitions. Lookup batches contain 50% hits and 50% " + "misses. Cold-start begins with an empty Splinter cache; repeated is the " + "next identical pass, whose measured I/O is retained below. Scans decode " + "and verify every logical key. These are B-tree prototype measurements.", + "", + ] + titles = {"insert": "Insert including key construction", + "pack": "Pack including clone and logical-key hashing", + "branch_cold_lookup": "Lookup from a cold cache", + "branch_warm_lookup": "Repeated lookup", + "branch_scan": "Scan including full key reconstruction"} + for stage in selected: + lines += ["## " + titles[stage], "", + "| Key bytes | Difference | Mode/prefix | us/op | Read MiB | Allocated MiB |", + "|---:|:---|:---|---:|---:|---:|"] + for row in summary: + if row["stage"] != stage: + continue + us = row["median_seconds"] * 1e6 / row["operations"] + lines.append(f'| {row["key_bytes"]} | {row["pattern"]} | ' + f'{row["mode"]}/{row["prefix_bytes"]} | {us:.3f} | ' + f'{row["median_read_bytes"] / 2**20:.3f} | ' + f'{row["median_allocated_bytes"] / 2**20:.3f} |') + lines.append("") + (args.results / "summary.md").write_text("\n".join(lines)) + print(args.results / "summary.csv") + print(args.results / "summary.md") + +if __name__ == "__main__": + main() diff --git a/benchmarks/blob_keys/summarize_small_keys.py b/benchmarks/blob_keys/summarize_small_keys.py new file mode 100644 index 000000000..e5d471ba2 --- /dev/null +++ b/benchmarks/blob_keys/summarize_small_keys.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Paired small-key timings. Positive time changes mean slower.""" +import argparse +import csv +import json +import statistics +from pathlib import Path + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument("manifest", type=Path) +p.add_argument("--candidate", required=True) +a = p.parse_args() +m = json.loads(a.manifest.read_text()) +groups = {} +for run in m["runs"]: + if run["case"].startswith("public-"): + for row in run["rows"]: + case = run["case"] + "-" + row["phase"] + groups.setdefault(case, {}).setdefault(run["variant"], {})[ + run["repetition"]] = 1 / float(row["ops_s"]) + else: + groups.setdefault(run["case"], {}).setdefault(run["variant"], {})[ + run["repetition"]] = run["rows"][0]["active_wall_ns"] +out = a.manifest.with_name("comparison.csv") +with out.open("w") as f: + w = csv.writer(f) + w.writerow(["case", "paired_repetitions", "median_time_change_pct", + "minimum_time_change_pct", "maximum_time_change_pct", + "median_throughput_change_pct", "baseline_median_time", + "candidate_median_time"]) + for case, variants in groups.items(): + b, c = variants["baseline"], variants[a.candidate] + pairs = sorted(b.keys() & c.keys()) + ratios = [c[i] / b[i] for i in pairs] + med = statistics.median(ratios) + w.writerow([case, len(pairs), 100*(med-1), 100*(min(ratios)-1), + 100*(max(ratios)-1), 100*(1/med-1), + statistics.median(b.values()), statistics.median(c.values())]) + print(f"{case:38s} time {100*(med-1):+6.1f}% " + f"range {100*(min(ratios)-1):+6.1f}..{100*(max(ratios)-1):+6.1f}%") +print(out) diff --git a/benchmarks/btree_prefix_bench.c b/benchmarks/btree_prefix_bench.c index 5e6f9dc5f..4d96a8b1d 100644 --- a/benchmarks/btree_prefix_bench.c +++ b/benchmarks/btree_prefix_bench.c @@ -42,6 +42,11 @@ # define BTREE_BENCH_PREFIX_SUPPORT 0 #endif +/* The identical workload also builds against the pre-blob-key B-tree API. */ +#ifndef BTREE_DEFAULT_FORMAT +# define BTREE_DEFAULT_FORMAT BTREE_BRANCH_FORMAT_VERSION +#endif + #define MIB (UINT64_C(1) << 20) #define PAGE_BYTES 4096 #define EXTENT_BYTES (32 * PAGE_BYTES) @@ -282,6 +287,7 @@ measured_write(io_handle *io, void *buf, uint64 bytes, uint64 addr) static platform_status measured_async_init(io_async_state *state, + uint64 state_size, io_handle *io, io_async_cmd cmd, uint64 addr, @@ -296,7 +302,7 @@ measured_async_init(io_async_state *state, } } return underlying_io_ops->async_state_init( - state, io, cmd, addr, callback, arg); + state, state_size, io, cmd, addr, callback, arg); } static void @@ -436,8 +442,8 @@ make_inputs(benchmark *b) >> (8 * (j % 8))); } if (strcmp(o->pattern, "common") == 0) { - memset(k, 'p', 48); - put_be(k + 48, mix(i + o->seed)); + memset(k, 'p', o->key_bytes - 16); + put_be(k + o->key_bytes - 16, mix(i + o->seed)); } else { /* mix is a permutation, so first words are random and unique. */ put_be(k, mix(i + o->seed)); @@ -513,7 +519,7 @@ init_iterator(benchmark *b, &b->btree_cfg, it, root, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, &(const btree_iterator_init_options){ .start_position = key_start_at_or_after(start)}), @@ -722,7 +728,7 @@ run_lookups(benchmark *b, uint64 root, bool verify) check(btree_lookup((cache *)&b->cc, &b->btree_cfg, root, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, target, &result), @@ -976,7 +982,7 @@ parse_options(int argc, char **argv) o.read_lexical = false; #endif if (o.keys < 2 || o.keys > UINT64_MAX / 2 || o.ops == 0 || o.batches == 0 - || o.key_bytes < 64 || o.key_bytes > MAX_INLINE_KEY_SIZE(PAGE_BYTES) + || o.key_bytes < 16 || o.key_bytes > MAX_INLINE_KEY_SIZE(PAGE_BYTES) || o.value_bytes < 8 || o.value_bytes > MAX_INLINE_MESSAGE_SIZE(PAGE_BYTES) || o.cache_mib < 32 || o.disk_mib < 64 || o.cache_mib > UINT64_MAX / MIB diff --git a/docs/blob-keys.md b/docs/blob-keys.md new file mode 100644 index 000000000..8f77428e1 --- /dev/null +++ b/docs/blob-keys.md @@ -0,0 +1,417 @@ +# Native blob keys (B-tree 6, trunk 5, shard log 4) + +Large keys are supported through the public SplinterDB API, including B-tree +keys, trunk pivots, and log records. Keys at or below page_size / 8 remain inline. +Applications continue to pass ordinary key slices and receive logical key bytes +in comparison, hashing, merge callbacks, and iterators. No blob-aware key +callback is required. + +## Encoding + +Keys of at most MAX_INLINE_KEY_SIZE(page_size) remain inline. Larger keys use +bit 1 (ONDISK_KEY_IS_BLOB) of the entry's existing key-flags byte. Index pivots +use the same encoding as leaf keys. There is no extra key version, logical-length +field, prefix-length field, or descriptor-length field in an individual key. + +For lexical keys, the uncompressed entry key is: + + [ P bytes of logical key prefix | existing blob descriptor ] + +For nonlexical keys, P is zero and the key contains only the descriptor. The +descriptor describes the entire logical key, including the prefix. Its existing +logical length and checksum remain unchanged. Its encoded length follows from +the entry's key length and P. + +A branch reference carries { root address, B-tree version, P }. P is a uint16 +in that record, with the configuration constraint below enforced before +construction. Readers use the saved P, independently of the current construction +setting. Memtables use their fixed creation configuration. + +The descriptor budget defaults to 256 bytes. P defaults to 256 for lexical keys +and is always zero when constructing nonlexical trees. Construction requires: + + P + blob_max_descriptor_size <= MAX_INLINE_KEY_SIZE(page_size) + +The descriptor budget must also satisfy the blob builder's minimum (114 bytes). +At the public configuration boundary, zero selects the default P and +UINT64_MAX selects P=0. Internal B-tree configuration and the test CLI +--blob-key-prefix-size use literal lengths, including zero. + +## Leaf common prefixes + +Let C be the leaf's common-prefix length. Compression removes C bytes from the +logical inline prefix; a blob entry then stores: + + [ P-C remaining prefix bytes | unchanged blob descriptor ] + +Compression never includes descriptor bytes. When a leaf contains a blob key, +C is capped at P. If an inline-only leaf already has C>P, appending a blob key +starts a new leaf. The existing leaf retains its encoding and prefix. + +Small keys keep their usual leaf compression and carry no blob flag. Mutable +nodes and index pivots do not use leaf common-prefix compression. + +## Comparison, iteration, ownership + +Only a comparator opting into unsigned bytewise lexical ordering may use the +prefix shortcut. Ties stream the remaining bytes from the blobs, preserving +exact comparison and length ordering. Custom comparators receive materialized +full keys through the existing callback API. Descriptor bytes never determine +logical order. Packing hashes the logical source keys using the existing hash +callback and records logical key-byte totals. + +B-tree searches propagate blob-read failures through their status return. +Iterator positioning materializes the current logical key before returning; +curr() performs no allocation or I/O. Uncompressed inline keys still borrow +their page storage. Blob comparisons currently use synchronous blob reads even +inside the asynchronous B-tree APIs. + +Key blobs belong to the tree's existing mini allocator, alongside value blobs. +The mini allocator records compact blob-root references and owns packed-tail +storage; immutable pointer extents own their children. Pivots reference the same +tree-owned key blobs. Repacking materializes and builds +new key blobs; it does not yet share source key extents. Repeated large-key +updates can retain obsolete key storage until the owning tree is released. +These are performance limitations of this scoped implementation. + +Memtable fullness charges both newly allocated extents and every successfully +attached extent reference. Shared storage is charged again for each attachment +so reusing blobs cannot bypass the memtable's compaction-input size budget. +This counter includes allocation reserves and metadata; it is not a count of +unique physical disk extents. Failed attachment does not charge the payload. +Trunk byte statistics continue to measure full logical key and value lengths. + +## Format compatibility + +B-tree version 6 stores each index entry as: + + [ child address | key length and flags | key bytes | counters ] + +The counters are num_kvs, key_bytes, and message_bytes, stored contiguously as +three uint32 fields or three uint64 fields. Bit 0 of the index key-flags byte +selects the wide representation; bit 1 independently marks a blob key. The key +has the same offset in both representations. Counter access uses memcpy because +the key can leave the counters unaligned. Leaf entries cannot set bit 0. + +In memory, counters remain uint64 and UINT64_MAX means unknown. Narrow entries +encode unknown as UINT32_MAX and exact counts through UINT32_MAX-1. If any known +counter exceeds that limit, all three counters use the wide representation. +Mutable trees, whose counters are unknown, remain narrow. Narrow entries use +23 bytes plus their key, and wide entries use 35 bytes plus their key. There is +no split-word encoding or per-statistics version. + +B-tree versions 1 through 5 are rejected before page access by the versioned query, +iterator, metadata, and allocation-recovery interfaces. Trunk version 5 and +superblock version 10 identify the new branch and allocation ownership scheme. +Existing database files need rebuilding; no compatibility reader or migration +is included. + +## Validation and performance evidence + +The native regressions cover mixed inline/blob keys, the 512/513-byte boundary +on 4 KiB pages, lexical and custom ordering, P=0, mutable splits and updates, +full-key merge callbacks, synchronous/asynchronous lookup and iterator setup, +forward/reverse scans, seek, logical ranks, C>P leaf transitions, malformed +entries, blob-read failures, and repacking with a different P after source +release. Public reopen coverage checks that a saved branch P survives a change +in construction configuration. Direct layout tests cover the narrow/wide +boundary, contiguous unaligned counters, unknown sentinels, flag preservation, +and growth/shrinkage during entry updates and defragmentation. Old/future +versions are tested with a NULL cache to prove rejection before I/O. + +The earlier [prototype results](../benchmarks/blob_keys/RESULTS.md) measured +commit da464513, which used a separate experimental codec. They are not +performance measurements of the native format. The initial native reports +measure format 3. The [complete format-4 rerun](../benchmarks/blob_keys/SMALL_KEYS_V4.md) +measures all 63 small-key B-tree cases after restoring compact counters. +Benchmark campaigns must stop all other registered projects and hold both +shared coordination locks exclusively. + +Format-4 validation: the complete debug unit suite passed 295 tests with five +existing skips before the attachment-accounting correction. The new attachment +regression failed on the old code (expected 13 extent charges, observed 12) and +passed after correction. With that fix, the separate Splinter suite passed all +54 tests, including the sparse height-three blob / large-statistics test; the +public blob and quick suites passed 11 and 66 tests respectively. The 14 B-tree +layout/configuration tests and three native blob-key regressions passed in +debug, release, and GCC ASan+UBSan builds (ASan leak detection disabled). +Formatting and whitespace checks passed. Logs and hashes are retained under +.cache/compact-stats-validation.json and .cache/compact-stats-*.log in this +worktree. The separate format-4 benchmark report preserves its own raw evidence. + +## Small-key performance + +The initial native implementation had measured small-key regressions. +[Scan4096 profiling](../benchmarks/blob_keys/SCAN4096_PROFILE.md) identified and +fixed the large scan slowdown after rebasing onto main. The complete format-4 +rerun passed all 882 checks but still found packing, point-lookup, and +custom-comparator write regressions. Subsequent [packing profiling and fixes](../benchmarks/blob_keys/PACK_PROFILE.md) +remove aggregate argument copies and track whether the current leaf contains a +blob key, avoiding the prefix-compression entry scan. All nine packing cases now +have lower median elapsed time than main (189 verified runs); the 256-byte random +case remains statistically inconclusive. Subsequent +[overwrite profiling](../benchmarks/blob_keys/OVERWRITE_PROFILE.md) removes +unnecessary persisted-entry validation and encoding-buffer setup during +mutation, and simplifies custom-comparator leaf search. Fifteen longer paired +runs reduce custom overwrite overhead from 11.7% to 5.2%, with a remaining +95% interval of +3.9% to +6.1%. The full debug suite passes 295 tests with five +existing skips after those changes. The overall no-small-key-regression +objective is not yet met. The [earlier report](../benchmarks/blob_keys/SMALL_KEYS.md) preserves +the original public API and B-tree results. + +[Point-hit profiling](../benchmarks/blob_keys/POINT_HIT_PROFILE.md) adds a lexical +pivot shortcut with separate lexical/callback and checked/mutable search loops. +The retained branch takes 23.2% less time than unmodified main for 64-byte random +hits; it remains 6.0% slower than main with the same shortcut. Custom controls +show no clear new regression, but their existing overhead remains. The retained +code passed the full debug suite (295 passed, five skips), release and ASan +B-tree checks; all 428 profiling/benchmark runs verified. + +[Continued profiling](../benchmarks/blob_keys/RESIDUAL_PROFILE.md) validates ordinary +narrow-counter pivots directly and prepares inline comparison arguments once per +search. Against main with the same lexical shortcut, 64-byte random and custom +hit overhead falls to about 1%, with intervals spanning zero. Custom insertion +and overwrite retain roughly 6% and 4% overhead, and 256-byte random hits retain +2.5%; the overall objective remains unmet. The current code passes the full +debug suite, release and ASan targeted tests, and all 610 profiling/benchmark +runs. Boundary validation for narrow-counter entries now has an additional +synchronous/asynchronous rejection case. + +[Further mutation profiling](../benchmarks/blob_keys/MUTATION_PROFILE.md) carries +borrowed key/message objects through the insertion body, mutable pivot search, +and leaf incorporation, removing intermediate metadata copies. Longer paired +confirmation finds +0.4% custom overwrite and +1.7% insertion time versus main; +shorter controls retain a measurable insertion gap of about 2%. All 786 +measurements verified, the full debug suite passed 295 tests with five skips, +and debug/release/ASan targeted tests passed. Public APIs and the B-tree format +are unchanged by this follow-up; the overall performance objective remains open. + +The [full small-key rerun after these fixes](../benchmarks/blob_keys/SMALL_KEYS_FINAL.md) +completed all 63 cases and 882 measurements against the original frozen main +binary. Packing is faster in every configuration, but 64/256/512-byte shared-prefix +Scan4096 remains about 5–7% slower and custom overwrite about 2% slower. All work, +results, packed layouts, and residency checks passed, with zero measured I/O. +This measurement-only follow-up changed no production code; the small-key +performance objective remains unmet. + +[Shared-prefix Scan4096 profiling](../benchmarks/blob_keys/SHARED_SCAN4096_PROFILE.md) +keeps suffix decoding and entry checks inline and moves blob/index result +construction out of the ordinary leaf result path. The retained source is +12.8–13.4% faster than original main for shared-prefix Scan4096 across all four +key sizes, while also improving random long scans. Its fresh 63-case matrix +verified all 882 runs; custom insertion, overwrite, and hits retain roughly 2% +overhead. Full debug and targeted release/ASan tests passed. No persistent +format, public API, or ownership rule changed in this follow-up. + +## Container-owned blob references + +The ownership refactor separates blob construction from container allocation. +`blob_build()` and `blob_clone()` allocate whole extents directly and accept a +callback only for packed-tail storage. Successful results transfer whole-extent +references to their destination container; descriptors themselves are borrowed +views. Retain the container to extend a descriptor's lifetime, or clone into +another container. There is no descriptor-level `blob_incref()`. + +`mini_blob_build()` and `mini_blob_clone()` provide the mini-allocator adapter. +Failed construction releases unadopted roots, while tails and roots already +adopted by a mini allocator remain tracked for bulk cleanup. Mini admission and +space accounting still charge every referenced descendant, even when shared. +Efficient append/truncation topology is deferred. Trunk pivots use the same +container ownership contract, with their own tail allocation callback. + +Validation of the initial ownership refactor passed the full debug unit suite (295 +passed, five skipped) and standalone Splinter suite (56 passed). All 28 targeted +checks passed in debug, release, and AddressSanitizer builds, covering native +blob keys, public blob reuse, crash recovery, failure cleanup, and mini metadata +cursors. AddressSanitizer ran with leak detection disabled; allocation-map and +failure-injection tests separately verify extent reclamation. + +New regressions verify that 600 clones retain only root references without +reading pointer extents, that recovery reproduces the entire reference-count +map, and that a custom tail allocator works without a mini allocator, including +source and destination tails in the same page. They also inject failures at +every hierarchical construction allocation and at tail allocation. Commands, +logs, hashes, and a patch isolating this refactor from earlier uncommitted work +are retained in `.cache/blob-ownership-refactor/` in this worktree. + +Earlier performance reports describe format 4 before this ownership refactor; +they are not measurements of the refactored implementation. + +## Ordered mini metadata and terminated pointer extents + +Blob roots now use the same eight-byte entries as ordinary mini allocations. +The entry's page-type byte is split into four type bits and four pointer-depth +bits. The bidirectional cursor returns every entry in recording order, including +blob roots and packed-tail extents, with optional type/height outputs. The +special cursor blob-prefetch API and the second record region are removed. +B-tree scan prefetching consumes this ordered stream and prefetches blob roots +and tails; descendants still use the blob iterator's read-ahead. + +Bulk operations use a single fallible extent callback with a height parameter. +Reclamation and recovery delegate blob entries to the blob module. References +contain only an address and height in memory. Each metadata page persists a +64-bit sum of the charges for its entries, preserving full descendant accounting +without blob reads during adoption or space queries. + +Blob format 2 requires each pointer extent to end with a zero pointer unless +completely full. The terminator is initialized and persisted even at an exact +page boundary, and validation checks it before log recovery accepts a blob. +This is the only blob-format change; append/truncate topology remains deferred. +Mini format 3 and the enclosing versions reject earlier interpretations. + + +Validation passed the full debug unit suite (295 passed, five skipped), the +standalone Splinter suite (57 passed), and all 29 targeted checks in debug, +release and AddressSanitizer builds. Leak detection was disabled in ASan. +Routing-filter metadata corruption tests and the functional log recovery suite +also passed, including replay of all 500,000 records in its final case. + +Regressions check forward/reverse cursor order over 600 shared roots across +metadata pages, preservation of the current entry on cache misses and end of +stream, accurate persisted charges without pointer-tree reads, terminators +before/at/after page boundaries and at full extent boundaries, malformed and +unreadable terminators, recovery after eviction, and complete reclamation. +The functional B-tree corruption helper was updated to use the current child +address field so the functional driver builds. + +Commands, raw logs, hashes, and a patch against the previous worktree state are +retained in `.cache/blob-ordered-metadata/`. No new performance measurements +were made. + + +## Trunk pivots: copy all tails + +Trunk format 5 stores the fixed blob-key prefix length P in each node header. +Pivot keys use the same flag and `[prefix | descriptor]` representation as +B-tree keys; P is zero for custom ordering. Small pivots remain inline. +In-memory pivots and child-reference keys use the full-width `key` length, +so materialized separators are not limited by the on-disk uint16 encoded length. + +Every serialized node owns its pivot blob references. An unchanged pivot is +cloned from its source descriptor: whole extents are shared, and every tail is +copied. New pivots are built from their logical bytes. Source descriptors borrow +the same protected source-root lifetime as the node's child and branch references; +cloning acquires the new references before the old root can be released. + +A layout pass reserves the metadata body, including a bound on descriptor sizes +because splitting a tail across an extent boundary can add an address. Tails +start at the next page boundary in the node's own extent. The header records +that boundary so readers reject metadata/tail overlap. Metadata pages remain +PAGE_TYPE_TRUNK; tail pages are PAGE_TYPE_BLOB. This avoids using two cache page +types for the same physical page. The allocation callback packs further tails +into overflow extents only when the current extent is exhausted. It normalizes +zero alignment to one, matching the mini allocator's callback semantics. + +There is no per-node mini allocator. Normal reclamation already walks the node; +it releases each descriptor's whole-extent roots and each distinct overflow tail +extent once. Tails in the node's own extent require no additional allocator +reference. Recovery reads descriptors before accessing blob contents, rebuilds +root and tail references, and then materializes keys to validate ordering. +Failed serialization rolls back completed blob clones and all newly allocated +tail extents without publishing the node. + +The first version deliberately materializes keys for in-memory node operations. +Point comparisons reuse B-tree lexical streaming or the standard custom-comparator +materialization fallback. Blob reads in the async lookup path are currently +synchronous, as in the B-tree prototype; errors still propagate to the caller. +Selective tail retention, subextent reclamation, and optimizing huge-key mutation +cost are outside this implementation. + +Tests in `tests/unit/trunk_split_test.c` cover small colocated tails, overflow +tails and whole-extent sharing, keys above 64 KiB, custom comparators, cold sync +and async lookup, range bounds, allocation/retain failure rollback, read errors, +and exact allocator-reference reconstruction followed by recovery mount. These +are direct B-tree-to-trunk tests. Public large-key coverage is described below. + + +Validation of trunk format 5 passed 301 debug unit tests (five existing skips) +and all 57 standalone Splinter tests. The 12 trunk split/blob tests, two trunk +GC tests, and two async-lifetime tests passed in debug, release, and ASan builds +(ASan leak detection disabled). All 12 trunk split/blob tests also passed with +shared memory enabled. These tests preceded log/public API integration; the +end-to-end small-key measurements below include the trunk implementation. + + +## Log and public API integration + +Shard-log format 4 uses the tuple key flag to distinguish inline keys from +descriptor-only blob keys. The log needs no inline prefix because replay orders +records by memtable and leaf generation. Keys above the shared inline threshold +are built with the log's mini allocator; its page-aligned tail policy applies to +key and value blobs alike. Their logical page counts both contribute to the +group-size estimate. + +Preparation bounds the entire encoded record before allocating either blob, +builds key and value blobs, and enrolls both in the group's writeback set before +reserving publication. An aborted preparation leaves unused allocations tracked +by the log's mini allocator for bulk reclamation. Small keys need no additional +allocation. Memtables independently encode the logical key; sharing the log's +key extents with a memtable is a possible later optimization. + +Allocation reconstruction discovers both kinds of blob in needed generations. +An unreadable blob cuts off its entire group and all subsequent groups. +Checkpointed generations do not require their blob payloads to remain readable. +The iterator validates blob checksums before accepting a group, and materializes +its current key during initialization/advance, where errors can be returned. +The infallible current-record accessor therefore always supplies logical keys. +Existing log versions are rejected, with no new blob-descriptor version. + +Public tests cover the inline boundary, 4 KiB, 64 KiB, and 256 KiB keys, +lexical and custom ordering, logging on/off, insert/lookup/miss/update/delete, +fetching old values, forward/reverse/bounded iteration, flush, close/reopen, +and abrupt-exit recovery with simultaneous key and value blobs. A damaged key +tail verifies whole-group truncation. The log's indirect-blob recovery tests +also cover key blobs, missing first/later pointer pages, generation cutoff, and +reclaiming conservative allocations in a subsequent root-only rebuild. + + +The integrated final source passes the full debug unit suite: 303 passes and +five existing skips. Seventeen focused trunk/public checks each pass release +and ASan+UBSan (leak detection disabled), including a custom-comparator P=0 +layout check and reopen with a changed configured prefix. The functional log +suite passes, including 500,000 records and indirect key-blob corruption tests. + +The [public API small-key report](../benchmarks/blob_keys/PUBLIC_SMALL_KEYS.md) +records the 200-run full end-to-end comparison, focused profiling, and final +source confirmation. Most write throughput improves, but lookup misses and +custom-comparator hits retain repeatable regressions; logged shared-prefix +write latency also needs investigation. The no-small-key-regression objective +is not met. Raw measurements remain local and untracked. + + +[Public lookup profiling](../benchmarks/blob_keys/PUBLIC_LOOKUP_PROFILE.md) +borrows query descriptors through checked B-tree lookup and pivot search. A +seven-pair confirmation improves custom hits 3.4% and random misses 2.3% over +the pre-change branch; remaining single-thread gaps are small. The maintained +public benchmark now stores per-worker latency samples in separate segments. +Four-thread misses still regress in the tested workloads, so the overall +small-key performance objective remains open. The retained source passes 303 +debug unit tests (five existing skips), and 31 focused checks each in release +and ASan+UBSan with leak detection disabled. + + +[Four-thread miss profiling](../benchmarks/blob_keys/PUBLIC_LOOKUP4_PROFILE.md) +identifies false sharing between adjacent batch-lock reader rows. Giving each +thread a full cache line costs 6 KiB per database and removes almost all +measured modified-line RFO traffic. The earlier final confirmation mistakenly +linked a stale static archive without this fix; the source and shared-library +correctness tests did contain it. + +The [artifact audit and fresh comparison](../benchmarks/blob_keys/PUBLIC_LOOKUP_RESIDUAL.md) +explicitly rebuild both libraries and verify the linked reader stride. +Sustained four-thread misses improve 35% for random keys and 6.8% for shared +prefixes versus frozen main; custom misses improve 16% with a wide interval. +The original public workload has inconclusive miss changes of +2.8%, -1.5%, +and +1.3%, respectively for random, custom, and shared-prefix keys. All 30 runs +and 810 phase rows verify. A separate key-flags experiment was reverted after +clean controls showed no compelling incremental benefit. Other operation +regressions remain; the overall small-key performance objective is still open. + + +The [final full public matrix](../benchmarks/blob_keys/PUBLIC_PR_CHECK.md) +uses fresh builds of current main and this unchanged source. All 200 runs and +5,400 phase checks pass. Across 300 ordinary CRUD/scan comparisons, none has a +95% interval wholly below a -3% throughput change, but 104 remain uncertain and +11 median losses exceed 3%. Durability, optimize, and post-delete iteration +still have outliers; insertion stall bins rise from 84 to 106. This supports +progress toward a PR but does not establish the 3% limit across every operation. diff --git a/docs/branch-prefix-experiment.md b/docs/branch-prefix-experiment.md index cbaf80ffa..789c6c22a 100644 --- a/docs/branch-prefix-experiment.md +++ b/docs/branch-prefix-experiment.md @@ -1,6 +1,6 @@ # Branch leaf prefix experiment -Persisted branch version 1 is rejected before branch-page interpretation. +Persisted branch versions 1 and 2 are rejected before branch-page interpretation. A clean mount remains lazy and rejects when an affected branch is accessed; recovery rejects during mount when rebuilding allocation state. No format migration or eager tree walk is added. @@ -42,9 +42,9 @@ can use direct lexical comparison during checked binary search when the lexical contract permits it. Nonlexical comparisons, infinity targets on uncompressed leaves, boundary comparisons and index pivots use the configured comparator. -This prototype now uses branch format version `2` within main's superblock -format `8`. Branch readers reject format `1` before interpreting its different -page header. Earlier standalone prefix images used superblock `0x8007` and are +The prefix experiment originally used branch format version 2 within +superblock format 8. [Native blob keys](blob-keys.md) now use B-tree version +3 and trunk version 2, rejecting both earlier B-tree layouts. Earlier standalone prefix images used superblock `0x8007` and are rejected. Page encoding is determined from the page itself; existing compressed leaves remain readable after changing the construction opt-in on reopen. @@ -78,3 +78,9 @@ buffer. Entry validation and comparison directions are preserved. Regression coverage includes nearby and cross-leaf seeks, an aliased current-key seek target, prefix mismatches, and finite inclusive/exclusive bounds with present and absent endpoints in both directions. + +For blob keys, common-prefix compression is limited to the logical inline +prefix and excludes descriptors. Iterator positioning materializes blob keys +even on an uncompressed leaf; the direct-page and no-scratch observations above +apply to inline keys. The historical measurements in this document predate the +native blob-key and uint64-statistics format. diff --git a/docs/on-disk-format.md b/docs/on-disk-format.md index 9b033ad85..9dd938b57 100644 --- a/docs/on-disk-format.md +++ b/docs/on-disk-format.md @@ -1,6 +1,6 @@ # On-disk versions and read configuration -Format 8 rejects files written by earlier formats. There is no migration reader. +Format 10 rejects files written by earlier formats. There is no migration reader. Creation writes both superblock slots. Mount and bootstrap select the newest checksummed record before checking its structure versions, so an unsupported newest version cannot silently select an older checkpoint. A damaged slot still @@ -10,13 +10,13 @@ is limited to the current checksum layout. | Structure | Version location | Persisted configuration required to read | | --- | --- | --- | | Superblock | Superblock header | Device size, page size, extent size | -| Trunk | Superblock | None beyond device geometry and application callbacks | -| Branch B-tree | Each branch reference in its trunk bundle | None beyond device geometry and application callbacks | -| Mini allocator | Each metadata page header | Entries already carry extent number, page type and batch; geometry is global | +| Trunk | Superblock | Fixed inline blob-key prefix length P in each node header | +| Branch B-tree | Each branch reference in its trunk bundle | Fixed inline blob-key prefix length P | +| Mini allocator | Each metadata page header | Entries carry extent number, page type, batch and blob pointer depth; each metadata page stores its aggregate extent charge | | Routing filter | Descriptor in its trunk bundle | Fingerprint width, index width, hash seed, value width, counts and metadata address | | Shard log | Log head in the superblock | Checksum seed and stream nonce | | Reference-count allocator | Superblock | Device geometry; map address is already persisted | -| Blob | Existing blob descriptor version | Length and checksum already persisted; descriptor length and geometry determine the extent list | +| Blob | Existing blob descriptor version | Length and checksum persisted; descriptor geometry locates data; pointer extents are zero-terminated unless full | Memtables are reconstructed and have no persistent configuration record. Fanout, incorporation thresholds, scheduling, cache size, prefetching and @@ -98,3 +98,76 @@ Direct B-tree regressions cover old/future versions and null roots before any cache access, including safe iterator cleanup. The core corruption regression covers synchronous/asynchronous point hits and ranges, and verifies that routing-filter misses succeed without inspecting the irrelevant branch. + +## B-tree format 6, trunk format 4, and mini-allocator format 3 + +Native blob keys add a fixed prefix length to each branch reference. Index +statistics are three contiguous uint32 or uint64 fields after the key, selected +by an entry flag. B-tree versions 1 through 5 are rejected, including the old +split-word encoding and earlier blob ownership encodings. Superblock format 10, +trunk format 4, and shard-log format 3 identify the new ownership and pointer +termination contracts. + +Mini-allocator format 3 uses one ordered stream of 8-byte entries: + +| Bits | Meaning | +| --- | --- | +| 0–7 | Batch | +| 8–11 | Page type | +| 12–15 | Blob pointer depth; zero for all non-blob entries | +| 16–63 | Extent number | + +Blob depth zero denotes a data extent, including container-owned tail storage. +Positive depth denotes a pointer extent whose children have depth one less. +Roots are recorded when adopted, interleaved with ordinary extent entries. +Descendants are not separate mini entries. There is no second record region. +The cursor returns the address, batch, type and height; bulk traversal uses one +fallible callback with those extent semantics. + +Each metadata page stores a uint64 total of charged extents for its entries. +An ordinary entry charges one; a blob root charges itself and all descendants, +including shared extents again. The builder supplies this charge using the +descriptor's geometry, without traversing pointer extents. Appending the entry +and updating its page total use the same metadata lock. Space queries sum page +totals and metadata bytes without reading blobs. + +Blob format 2 keeps the descriptor fields and pointer-array layout, but requires +a zero immediately after the valid pointers unless an entire pointer extent is +full. The builder initializes the terminator even when it falls on a new page, +and maintains a terminated prefix during failed construction. Descriptor-based +validation checks the terminator, including its disk readability during log +validation. Old blob format 1 is rejected. + +A container holds references to whole-extent roots; each immutable pointer +extent owns its children, and the container owns packed-tail storage. Sharing a +root does not increment descendants. Recovery counts incoming edges and +discovers outgoing edges once. Reclamation follows zero-terminated child sets +only when the last reference disappears. The blob module's in-memory reference +contains only an address and height, with no persistent count or separate +reference-record encoding. Efficient append/truncate topology remains separate +work. See [Native blob keys](blob-keys.md). + + +## Trunk format 5 + +The node header adds a uint16 fixed blob-key prefix length before the inflight +bundle offset, and a uint32 page-aligned tail offset after that offset. The +tail offset bounds metadata traversal and prevents blob reads from addressing +metadata pages in the same extent. Pivot key bit `ONDISK_KEY_IS_BLOB` identifies a bounded encoded +key containing that prefix followed by a blob descriptor. The descriptor covers +the complete logical key; inline pivots retain their existing encoding. +Versions before 5 are rejected by the superblock's trunk-format validation. + +The metadata body still fits in one extent. Blob tails occupy separate pages in +that extent, then overflow into additional extents owned by the node. Each node +copy clones blob roots and copies all tails. Overflow tail extents are enumerated +from the descriptors and deduplicated per node for reclamation and recovery; +the node's own extent is excluded from that extra ownership. No per-node mini +allocator or new blob descriptor format is introduced. + + +Shard-log format 4 permits blob keys using the existing tuple key flag. Their +encoded key bytes contain only a blob descriptor; no lexical prefix is stored. +Key and value blobs share the log mini allocator and durability group. Recovery +validates both before accepting a group, and skips payloads in generations +already included in the durable root. Log formats through 3 are rejected. diff --git a/include/splinterdb/splinterdb.h b/include/splinterdb/splinterdb.h index 2d074c56d..b89792ca9 100644 --- a/include/splinterdb/splinterdb.h +++ b/include/splinterdb/splinterdb.h @@ -92,10 +92,15 @@ typedef struct splinterdb_config { uint64 num_normal_bg_threads; // Maximum encoded blob descriptor bytes, including its header. Zero selects - // 512. This is a runtime encoding policy; existing descriptors are decoded + // 256. This is a runtime encoding policy; existing descriptors are decoded // independently of this setting. Must be at least 114 and no greater than - // the B-tree inline message limit (35% of page_size). + // page_size / 8 minus the effective blob key prefix length. uint64 blob_max_descriptor_size; + // Fixed logical prefix retained with lexical blob keys in new B-trees. + // Zero selects 256; UINT64_MAX selects no prefix. Non-lexical keys have no + // prefix. Prefix + descriptor budget must not exceed page_size / 8. + // The effective prefix is persisted with each immutable branch pointer. + uint64 blob_key_prefix_size; // btree uint64 btree_rough_count_height; @@ -451,6 +456,11 @@ splinterdb_lookup(splinterdb *kvs, // IN // incorporated before catch-up can increment its reference count. In that // case the update has not been applied. +// Keys are ordinary byte slices, including keys larger than a page. The engine +// stores large keys in blob storage automatically; callbacks and iterators +// continue to receive their logical bytes. Small keys remain inline. +// Large-key operations can require additional allocation and I/O. +// // Insert a key and value. Overwrites any previous value associated with key. // The message may hold direct bytes or a blob from this open database instance. // Its type field is ignored: this operation always inserts/overwrites. diff --git a/src/batch_rwlock.h b/src/batch_rwlock.h index d69d2baff..4fc80167c 100644 --- a/src/batch_rwlock.h +++ b/src/batch_rwlock.h @@ -28,11 +28,13 @@ typedef struct { typedef struct { batch_claimlock write_lock[PLATFORM_CACHELINE_SIZE / 2]; - volatile uint8 read_counter[MAX_THREADS][PLATFORM_CACHELINE_SIZE / 2]; + // Each thread writes its own cache line. Half-line rows make adjacent + // threads invalidate one another even when all callers only hold reads. + volatile uint8 read_counter[MAX_THREADS][PLATFORM_CACHELINE_SIZE]; } PLATFORM_CACHELINE_ALIGNED batch_rwlock; _Static_assert(sizeof(batch_rwlock) - == PLATFORM_CACHELINE_SIZE * (MAX_THREADS / 2 + 1), + == PLATFORM_CACHELINE_SIZE * (MAX_THREADS + 1), "Missized batch_rwlock\n"); diff --git a/src/blob.c b/src/blob.c index 8070583d6..d1b6bd417 100644 --- a/src/blob.c +++ b/src/blob.c @@ -321,13 +321,35 @@ blob_fragment_for_offset(blob_page_iterator *iter) static void maybe_do_prefetch(blob_page_iterator *iter) { - uint64 curr_extent_num = iter->offset / iter->extent_size; - if (iter->pblob.height == 1 && iter->mode == BLOB_PAGE_ITERATOR_MODE_PREFETCH - && curr_extent_num + 1 < iter->pblob.num_extents) + if (iter->mode != BLOB_PAGE_ITERATOR_MODE_PREFETCH) { + return; + } + uint64 next = iter->offset / iter->extent_size + 1; + if (iter->pblob.height == 1) { + if (next < iter->pblob.num_extents) { + cache_prefetch( + iter->cc, iter->pblob.base->addrs[next], PAGE_TYPE_BLOB); + } + return; + } + uint64 count = iter->pblob.base->length / iter->extent_size + + (iter->pblob.base->length % iter->extent_size != 0); + uint64 per_page = iter->page_size / sizeof(uint64); + /* The current lowest-level pointer page already contains the next address, + * except at page boundaries. Read-ahead never blocks on pointer metadata. */ + if (next < count && next % per_page != 0 + && !page_handle_is_null(&iter->index_pages[1])) { - cache_prefetch(iter->cc, - iter->pblob.base->addrs[curr_extent_num + 1], - PAGE_TYPE_BLOB); + uint64 addr; + memcpy(&addr, + page_handle_data(&iter->index_pages[1]) + + (next % per_page) * sizeof(uint64), + sizeof(addr)); + if (blob_extent_is_valid(iter->cc, addr) + && allocator_page_valid(cache_get_allocator(iter->cc), addr)) + { + cache_prefetch(iter->cc, addr, PAGE_TYPE_BLOB); + } } } @@ -375,25 +397,15 @@ blob_page_iterator_init(cache *cc, return STATUS_OK; } -/* - * This function must be kept in sync with the code that decides - * whether to call cache_alloc in blob_build.c. - * - * The current policy is: The blob_build thread that gets the first - * byte of any page is reponsible for calling cache_alloc on that - * page. If the thread gets only part of the page, then it calls - * cache_alloc before calling mini_alloc_bytes_finish - * (i.e. immediately after getting the page from the mini_allocator). - * - * Thus we need to call cache_alloc now only if we got the entire - * page. Any partial page that we got will have already been - * cache_alloced by us (in blob_build) or by some other thread. - */ +/* Whole-extent pages are initialized by the builder's iterator. Tail pages + * are initialized by the container's allocation callback before publication. */ static bool should_alloc(blob_page_iterator *iter) { - return iter->mode == BLOB_PAGE_ITERATOR_MODE_ALLOC + bool32 tail = iter->pblob.height == 1 + && iter->offset / iter->extent_size >= iter->pblob.num_extents; + return !tail && iter->mode == BLOB_PAGE_ITERATOR_MODE_ALLOC && iter->fragment.offset == 0 && (iter->page_size <= iter->fragment.length || can_round_up(iter->page_size, iter->pblob.base->length)); @@ -580,7 +592,9 @@ blob_validate(cache *cc, slice sblob) out: blob_page_iterator_deinit(&iter); XXH3_freeState(checksum_state); - return rc; + /* Check the terminator pages before log recovery admits this descriptor. + * They are ownership metadata even though the checksum covers only data. */ + return SUCCESS(rc) ? blob_walk_extents(cc, sblob, TRUE, NULL, NULL) : rc; } platform_status @@ -656,7 +670,7 @@ blob_walk_extent(cache *cc, if (!blob_extent_is_valid(cc, addr)) { return STATUS_INVALID_STATE; } - platform_status rc = visit(cc, addr, arg); + platform_status rc = visit != NULL ? visit(cc, addr, arg) : STATUS_OK; if (!SUCCESS(rc) || height == 0) { return rc; } @@ -691,6 +705,27 @@ blob_walk_extent(cache *cc, } data_extents -= count; } + /* Format 2 requires the physical child set to match the descriptor, even + * when its terminator occupies a page with no child pointers. */ + if (SUCCESS(rc) && children < fanout) { + if (children % per_page == 0) { + if (!page_handle_is_null(page)) { + cache_unget(cc, page); + *page = (page_handle){0}; + } + rc = blob_get_pointer_page( + cc, addr + children * sizeof(uint64), disk_only, page); + } + if (SUCCESS(rc)) { + uint64 terminator; + memcpy(&terminator, + page_handle_data(page) + (children % per_page) * sizeof(uint64), + sizeof(terminator)); + if (terminator != 0) { + rc = STATUS_INVALID_STATE; + } + } + } if (!page_handle_is_null(page)) { cache_unget(cc, page); } @@ -730,6 +765,150 @@ blob_walk_extents(cache *cc, return STATUS_OK; } + +blob_ref +blob_ref_create(uint64 addr, uint64 height) +{ + platform_assert(height < BLOB_MAX_HEIGHT); + return (blob_ref){addr, height}; +} + +blob_ref +blob_get_ref(cache *cc, const parsed_blob *parsed, uint64 index) +{ + platform_assert(index < parsed->num_extents); + return blob_ref_create(parsed->base->addrs[index], parsed->height - 1); +} + +platform_status +blob_ref_validate(cache *cc, blob_ref ref) +{ + return blob_extent_is_valid(cc, ref.addr) && ref.height < BLOB_MAX_HEIGHT + ? STATUS_OK + : STATUS_INVALID_STATE; +} + +uint64 +blob_root_num_extents(cache *cc, const parsed_blob *parsed, uint64 index) +{ + platform_assert(index < parsed->num_extents); + uint64 extent = cache_extent_size(cc); + uint64 count = + parsed->base->length / extent + (parsed->base->length % extent != 0); + count = MIN(count - index * parsed->extent_span, parsed->extent_span); + uint64 total = count; + uint64 fanout = extent / sizeof(uint64); + for (uint64 level = 1; level < parsed->height; level++) { + count = count / fanout + (count % fanout != 0); + total += count; + } + return total; +} + +/* Each pointer extent owns its outgoing edges once, irrespective of the number + * of references to that extent. Its zero terminator defines its child set; + * blob edits must rebuild any pointer extent whose child set changes. */ +static platform_status +blob_ref_children(cache *cc, blob_ref ref, bool32 recovery) +{ + uint64 height = ref.height; + if (height == 0) { + return STATUS_OK; + } + uint64 fanout = cache_extent_size(cc) / sizeof(uint64); + uint64 per_page = cache_page_size(cc) / sizeof(uint64); + page_handle page = {0}; + platform_status rc = STATUS_OK; + for (uint64 i = 0; i < fanout; i++) { + if (i % per_page == 0) { + if (!page_handle_is_null(&page)) { + cache_unget(cc, &page); + page = (page_handle){0}; + } + rc = blob_get_pointer_page( + cc, ref.addr + i * sizeof(uint64), recovery, &page); + if (!SUCCESS(rc)) { + break; + } + } + uint64 addr; + memcpy(&addr, + page_handle_data(&page) + (i % per_page) * sizeof(uint64), + sizeof(addr)); + if (addr == 0) { + break; + } + blob_ref child = blob_ref_create(addr, height - 1); + if (addr == ref.addr) { + rc = STATUS_INVALID_STATE; + break; + } + rc = recovery ? blob_ref_recover(cc, child) : blob_ref_release(cc, child); + if (!SUCCESS(rc)) { + break; + } + } + if (!page_handle_is_null(&page)) { + cache_unget(cc, &page); + } + return rc; +} + +platform_status +blob_ref_release(cache *cc, blob_ref ref) +{ + allocator *al = cache_get_allocator(cc); + platform_status rc = blob_ref_validate(cc, ref); + if (SUCCESS(rc) && allocator_get_refcount(al, ref.addr) <= AL_NO_REFS) { + rc = STATUS_INVALID_STATE; + } + if (SUCCESS(rc) + && allocator_dec_ref(al, ref.addr, PAGE_TYPE_BLOB) == AL_NO_REFS) + { + rc = blob_ref_children(cc, ref, FALSE); + if (SUCCESS(rc)) { + cache_extent_discard(cc, ref.addr, PAGE_TYPE_BLOB); + refcount count = allocator_dec_ref(al, ref.addr, PAGE_TYPE_BLOB); + platform_assert(count == AL_FREE); + } + } + if (!SUCCESS(rc)) { + allocator_record_reclamation_failure(al, rc); + } + return rc; +} + +platform_status +blob_ref_recover(cache *cc, blob_ref ref) +{ + platform_status rc = blob_ref_validate(cc, ref); + if (!SUCCESS(rc)) { + return rc; + } + allocator *al = cache_get_allocator(cc); + bool32 first = allocator_get_refcount(al, ref.addr) == AL_FREE; + rc = allocator_recovery_record_reference(al, ref.addr, PAGE_TYPE_BLOB); + return SUCCESS(rc) && first ? blob_ref_children(cc, ref, TRUE) : rc; +} + +platform_status +blob_release_refs(cache *cc, slice descriptor) +{ + parsed_blob parsed; + platform_status rc = blob_parse_descriptor(cc, descriptor, &parsed); + if (!SUCCESS(rc)) { + return rc; + } + for (uint64 i = 0; i < parsed.num_extents; i++) { + platform_status release_rc = + blob_ref_release(cc, blob_get_ref(cc, &parsed, i)); + if (SUCCESS(rc)) { + rc = release_rc; + } + } + return rc; +} + /* * Record one reference for the extent containing addr, unless something already * has. See blob_recover_allocations(). @@ -758,7 +937,12 @@ blob_recover_allocations(cache *cc, slice sblob) return rc; } - rc = blob_walk_extents(cc, sblob, TRUE, blob_recover_extent, NULL); + for (uint64 i = 0; i < pblob.num_extents; i++) { + rc = blob_ref_recover(cc, blob_get_ref(cc, &pblob, i)); + if (!SUCCESS(rc)) { + return rc; + } + } if (!SUCCESS(rc)) { return rc; } diff --git a/src/blob.h b/src/blob.h index 8341c2b39..2303ae74e 100644 --- a/src/blob.h +++ b/src/blob.h @@ -10,9 +10,9 @@ #include "util.h" #include "writeback_set.h" -#define BLOB_FORMAT UINT16_C(1) +#define BLOB_FORMAT UINT16_C(2) #define BLOB_CHECKSUM_SEED UINT64_C(0x424C4F424353554D) -#define BLOB_MAX_HEIGHT 64 +#define BLOB_MAX_HEIGHT 16 typedef struct ONDISK blob { uint64 length; @@ -45,6 +45,44 @@ typedef struct parsed_blob { parsed_blob_entry leftovers[3]; } parsed_blob; +/* In-memory ownership reference to an immutable whole-extent subtree. + * Height zero is data; higher extents contain child pointers terminated by + * zero, unless completely full. Packed tails belong to their container. + * Keep the owning container alive, or clone into another container. */ +typedef struct blob_ref { + uint64 addr; + uint64 height; +} blob_ref; + +blob_ref +blob_ref_create(uint64 addr, uint64 height); + +blob_ref +blob_get_ref(cache *cc, const parsed_blob *parsed, uint64 index); + +platform_status +blob_ref_validate(cache *cc, blob_ref ref); + +/* Whole extents charged to this descriptor root, including pointer extents. + * Counts shared storage again and uses descriptor geometry without I/O. */ +uint64 +blob_root_num_extents(cache *cc, const parsed_blob *parsed, uint64 index); + +/* Consumes the reference, including on failure. Failed reclamation is recorded + * with the allocator and must not be retried. Never touches packed tails. */ +platform_status +blob_ref_release(cache *cc, blob_ref ref); + +/* Count this incoming edge, and discover outgoing edges only on first visit. + * Only valid during an allocator recovery rebuild. */ +platform_status +blob_ref_recover(cache *cc, blob_ref ref); + +/* Release the whole-extent references of a successfully built, unadopted + * descriptor. Tail allocations remain owned by the destination container. */ +platform_status +blob_release_refs(cache *cc, slice descriptor); + typedef struct page_fragment { uint64 addr; uint64 offset; @@ -147,12 +185,14 @@ blob_writeback(cache *cc, slice sblob, writeback_set *set); /* Preorder walk of whole extents, including intermediate pointer extents. * The callback runs before an extent is read, permitting recovery to mark it. * Packed tails are excluded. disk_only checks pointer-page readability before - * strict reads. Child counts and depth come only from the descriptor's shape. + * strict reads. Descriptor geometry bounds traversal; each partial pointer + * extent must terminate at the expected zero slot. */ typedef platform_status (*blob_extent_callback)(cache *cc, uint64 addr, void *arg); +/* A NULL visitor validates pointer topology and terminators only. */ platform_status blob_walk_extents(cache *cc, slice sblob, @@ -160,20 +200,9 @@ blob_walk_extents(cache *cc, blob_extent_callback visit, void *arg); -/* - * Record the allocator references this blob's storage holds, for a crash- - * recovery rebuild. Call between allocator_recovery_begin() and - * allocator_recovery_finish(). - * - * Marks each intermediate extent before reading it, then follows its children. - * Stops at the first bad address or absent pointer page. References recorded - * before failure remain conservative reservations until the root-only rebuild. - * - * Records at most one reference per extent, skipping any that already has one. - * That is the correct count and not merely deduplication: an extent gets a - * single reference when its mini allocator hands it out, and blobs share - * extents -- one holds the tails of many -- so counting per blob would leave - * extents referenced several times over and never freed. - */ +/* Recover one container-owned descriptor from a validated log record. Whole + * extent roots count incoming edges; pointer descendants are enumerated once. + * Tail extents are reserved once because packed tails share container storage. + * Call only during an allocator recovery rebuild. */ platform_status blob_recover_allocations(cache *cc, slice sblob); diff --git a/src/blob_build.c b/src/blob_build.c index 3722a16c9..ca0311545 100644 --- a/src/blob_build.c +++ b/src/blob_build.c @@ -12,13 +12,16 @@ checksum_blob_data(slice data) } static platform_status -allocate_leftover_entries(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - uint64 data_len, - uint64 remainder, - writable_buffer *result) +allocate_leftover_entries(const blob_build_config *cfg, + cache *cc, + const blob_tail_allocator *tails, + uint64 data_len, + uint64 remainder, + writable_buffer *result) { + if (remainder != 0 && (tails == NULL || tails->alloc == NULL)) { + return STATUS_BAD_PARAM; + } uint64 page_size = cache_page_size(cc); uint64 extent_size = cache_extent_size(cc); @@ -32,17 +35,14 @@ allocate_leftover_entries(const blob_build_config *cfg, uint64 alloced_pages[2]; platform_status rc; if (num_pages != 0) { - rc = mini_alloc_bytes(mini, - cfg->page_batch, - num_pages * page_size, - MAX(page_size, cfg->alignment), - 0, - alloced_pages, - NULL); + rc = tails->alloc(tails->arg, + num_pages * page_size, + MAX(page_size, cfg->alignment), + 0, + alloced_pages); if (!SUCCESS(rc)) { return rc; } - mini_alloc_bytes_finish(mini, cfg->page_batch); if (alloced_pages[1] != 0) { rc = writable_buffer_append( @@ -63,39 +63,12 @@ allocate_leftover_entries(const blob_build_config *cfg, } if (remainder != 0) { - rc = mini_alloc_bytes(mini, - cfg->subpage_batch, - remainder, - cfg->alignment, - extent_size, - alloced_pages, - NULL); + rc = tails->alloc( + tails->arg, remainder, cfg->alignment, extent_size, alloced_pages); if (!SUCCESS(rc)) { return rc; } platform_assert(alloced_pages[1] == 0); - - if (alloced_pages[0] % page_size == 0) { - page_handle storage; - page_handle *page = - cache_alloc(cc, alloced_pages[0], PAGE_TYPE_BLOB, &storage); - cache_unlock(cc, page); - cache_unclaim(cc, page); - cache_unget(cc, page); - } else if (alloced_pages[0] / page_size - != (alloced_pages[0] + remainder - 1) / page_size) - { - uint64 next_page_addr = - (alloced_pages[0] + remainder - 1) / page_size * page_size; - page_handle storage; - page_handle *page = - cache_alloc(cc, next_page_addr, PAGE_TYPE_BLOB, &storage); - cache_unlock(cc, page); - cache_unclaim(cc, page); - cache_unget(cc, page); - } - - mini_alloc_bytes_finish(mini, cfg->subpage_batch); rc = writable_buffer_append( result, sizeof(alloced_pages[0]), &alloced_pages[0]); if (!SUCCESS(rc)) { @@ -166,25 +139,28 @@ blob_descriptor_size_bound(const blob_build_config *cfg, return STATUS_OK; } -/* Construct private, immutable pointer extents. Only pages containing used - * slots are initialized. source is non-NULL when re-encoding a large clone; - * in that case share data extents while constructing a smaller descriptor. */ +/* Construct private, immutable pointer extents. Pages containing used slots + * and the zero terminator (unless full) are initialized. source is non-NULL + * when re-encoding a large clone; in that case share data extents while + * constructing a smaller descriptor. */ static platform_status -build_blob_extent(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - uint64 height, - uint64 span, - uint64 count, - blob_page_iterator *source, - uint64 *addr) +build_blob_extent(const blob_build_config *cfg, + cache *cc, + const blob_tail_allocator *tails, + uint64 height, + uint64 span, + uint64 count, + blob_page_iterator *source, + uint64 *addr) { + *addr = 0; if (height == 0 && source != NULL) { if (!SUCCESS(source->status)) { return source->status; } - *addr = source->fragment.addr; - platform_status rc = mini_attach_extent(mini, cfg->extent_batch, *addr); + *addr = source->fragment.addr; + platform_status rc = + allocator_inc_ref(cache_get_allocator(cc), *addr, NULL); if (SUCCESS(rc)) { uint64 bytes = MIN(cache_extent_size(cc), source->pblob.base->length - source->offset); @@ -192,7 +168,8 @@ build_blob_extent(const blob_build_config *cfg, } return rc; } - platform_status rc = mini_alloc_extent(mini, cfg->extent_batch, addr, NULL); + platform_status rc = + allocator_alloc(cache_get_allocator(cc), addr, PAGE_TYPE_BLOB); if (!SUCCESS(rc) || height == 0) { return rc; } @@ -201,7 +178,8 @@ build_blob_extent(const blob_build_config *cfg, uint64 per_page = cache_page_size(cc) / sizeof(uint64); page_handle page_storage = {0}; page_handle *page = &page_storage; - for (uint64 i = 0; i < children; i++) { + uint64 fanout = cache_extent_size(cc) / sizeof(uint64); + for (uint64 i = 0; i <= children && i < fanout; i++) { if (i % per_page == 0) { if (!page_handle_is_null(page)) { cache_unlock(cc, page); @@ -211,10 +189,14 @@ build_blob_extent(const blob_build_config *cfg, cache_alloc(cc, *addr + i * sizeof(uint64), PAGE_TYPE_BLOB, page); memset(page_handle_data(page), 0, cache_page_size(cc)); } + /* Initialize the terminator even when it is on the next page. */ + if (i == children) { + break; + } uint64 child; uint64 child_count = MIN(count, child_span); rc = build_blob_extent( - cfg, cc, mini, height - 1, child_span, child_count, source, &child); + cfg, cc, tails, height - 1, child_span, child_count, source, &child); if (!SUCCESS(rc)) { break; } @@ -228,16 +210,31 @@ build_blob_extent(const blob_build_config *cfg, cache_unclaim(cc, page); cache_unget(cc, page); } + if (!SUCCESS(rc)) { + /* The first unused slot is zero even after a failed child build. */ + (void)blob_ref_release(cc, blob_ref_create(*addr, height)); + *addr = 0; + } return rc; } +/* Undo only acquired roots, including partially constructed descriptors. */ +static void +release_roots(cache *cc, writable_buffer *result, uint64 roots, uint64 height) +{ + blob *descriptor = writable_buffer_data(result); + for (uint64 i = 0; i < roots; i++) { + (void)blob_ref_release(cc, blob_ref_create(descriptor->addrs[i], height)); + } +} + static platform_status -build_blob_table(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - uint64 data_len, - blob_page_iterator *source, - writable_buffer *result) +build_blob_table(const blob_build_config *cfg, + cache *cc, + const blob_tail_allocator *tails, + uint64 data_len, + blob_page_iterator *source, + writable_buffer *result) { uint64 extent_size = cache_extent_size(cc); if (data_len > allocator_get_config(cache_get_allocator(cc))->capacity) { @@ -271,26 +268,31 @@ build_blob_table(const blob_build_config *cfg, uint64 alloced_page; uint64 count = MIN(remaining, span); rc = build_blob_extent( - cfg, cc, mini, height - 1, span, count, source, &alloced_page); + cfg, cc, tails, height - 1, span, count, source, &alloced_page); if (!SUCCESS(rc)) { + release_roots(cc, result, i, height - 1); return rc; } blobby->addrs[i] = alloced_page; remaining -= count; } - return allocate_leftover_entries(cfg, cc, mini, data_len, remainder, result); + rc = allocate_leftover_entries(cfg, cc, tails, data_len, remainder, result); + if (!SUCCESS(rc)) { + release_roots(cc, result, num_extents, height - 1); + } + return rc; } platform_status -blob_build(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - slice data, - writable_buffer *result) +blob_build(const blob_build_config *cfg, + cache *cc, + const blob_tail_allocator *tails, + slice data, + writable_buffer *result) { platform_status rc = - build_blob_table(cfg, cc, mini, slice_length(data), NULL, result); + build_blob_table(cfg, cc, tails, slice_length(data), NULL, result); if (!SUCCESS(rc)) { return rc; } @@ -302,6 +304,7 @@ blob_build(const blob_build_config *cfg, 0, BLOB_PAGE_ITERATOR_MODE_ALLOC); if (!SUCCESS(rc)) { + (void)blob_release_refs(cc, writable_buffer_to_slice(result)); return rc; } @@ -327,30 +330,19 @@ blob_build(const blob_build_config *cfg, if (SUCCESS(rc)) { blob *blobby = writable_buffer_data(result); blobby->checksum = checksum_blob_data(data); + } else { + (void)blob_release_refs(cc, writable_buffer_to_slice(result)); } return rc; } -typedef struct blob_clone_extents { - mini_allocator *mini; - uint64 batch; -} blob_clone_extents; - -static platform_status -blob_attach_extent(cache *cc, uint64 addr, void *arg) -{ - (void)cc; - blob_clone_extents *state = arg; - return mini_attach_extent(state->mini, state->batch, addr); -} - static platform_status -clone_blob_table(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - parsed_blob *pblobby, - slice sblob, - writable_buffer *result) +clone_blob_table(const blob_build_config *cfg, + cache *cc, + const blob_tail_allocator *tails, + parsed_blob *pblobby, + slice sblob, + writable_buffer *result) { platform_status rc = writable_buffer_resize( result, sizeof(blob) + pblobby->num_extents * sizeof(uint64)); @@ -364,9 +356,18 @@ clone_blob_table(const blob_build_config *cfg, memcpy(blobby->addrs, pblobby->base->addrs, pblobby->num_extents * sizeof(uint64)); - blob_clone_extents state = {.mini = mini, .batch = cfg->extent_batch}; - rc = blob_walk_extents(cc, sblob, FALSE, blob_attach_extent, &state); + uint64 acquired = 0; + for (; acquired < pblobby->num_extents; acquired++) { + rc = allocator_inc_ref( + cache_get_allocator(cc), blobby->addrs[acquired], NULL); + if (!SUCCESS(rc)) { + break; + } + } if (!SUCCESS(rc)) { + for (uint64 i = 0; i < acquired; i++) { + (void)blob_ref_release(cc, blob_get_ref(cc, pblobby, i)); + } return rc; } @@ -377,18 +378,23 @@ clone_blob_table(const blob_build_config *cfg, uint64 remainder = pblobby->base->length - pblobby->num_extents * extent_size; rc = allocate_leftover_entries( - cfg, cc, mini, pblobby->base->length, remainder, result); + cfg, cc, tails, pblobby->base->length, remainder, result); } + if (!SUCCESS(rc)) { + for (uint64 i = 0; i < pblobby->num_extents; i++) { + (void)blob_ref_release(cc, blob_get_ref(cc, pblobby, i)); + } + } return rc; } platform_status -blob_clone(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - slice sblob, - writable_buffer *result) +blob_clone(const blob_build_config *cfg, + cache *cc, + const blob_tail_allocator *tails, + slice sblob, + writable_buffer *result) { if (slice_length(sblob) < sizeof(blob)) { return STATUS_INVALID_STATE; @@ -415,8 +421,8 @@ blob_clone(const blob_build_config *cfg, if (!SUCCESS(rc)) { return rc; } - rc = - build_blob_table(cfg, cc, mini, pblobby.base->length, &source, result); + rc = build_blob_table( + cfg, cc, tails, pblobby.base->length, &source, result); blob_page_iterator_deinit(&source); if (SUCCESS(rc)) { ((blob *)writable_buffer_data(result))->checksum = @@ -425,7 +431,7 @@ blob_clone(const blob_build_config *cfg, return rc; } - rc = clone_blob_table(cfg, cc, mini, &pblobby, sblob, result); + rc = clone_blob_table(cfg, cc, tails, &pblobby, sblob, result); if (!SUCCESS(rc)) { return rc; } @@ -435,57 +441,54 @@ blob_clone(const blob_build_config *cfg, { uint64 start = pblobby.num_extents * extent_size; - blob_page_iterator src_iter; + auto_writable_buffer staging; + writable_buffer_init(&staging, result->heap_id); + uint64 length = pblobby.base->length; blob_page_iterator dst_iter; - rc = blob_page_iterator_init( - cc, &src_iter, sblob, start, BLOB_PAGE_ITERATOR_MODE_PREFETCH); - if (!SUCCESS(rc)) { - return rc; - } rc = blob_page_iterator_init(cc, &dst_iter, writable_buffer_to_slice(result), start, BLOB_PAGE_ITERATOR_MODE_ALLOC); if (!SUCCESS(rc)) { - blob_page_iterator_deinit(&src_iter); + (void)blob_release_refs(cc, writable_buffer_to_slice(result)); return rc; } - - while (!blob_page_iterator_at_end(&src_iter) - && !blob_page_iterator_at_end(&dst_iter)) - { - uint64 src_offset; - slice src_data; - uint64 dst_offset; - slice dst_data; - rc = blob_page_iterator_get_curr(&src_iter, &src_offset, &src_data); + while (start < length) { + uint64 amount = MIN(dst_iter.fragment.length, length - start); + rc = blob_materialize(cc, sblob, start, start + amount, &staging); if (!SUCCESS(rc)) { - goto out; + break; } - rc = blob_page_iterator_get_curr(&dst_iter, &dst_offset, &dst_data); + uint64 offset; + slice dst; + rc = blob_page_iterator_get_curr(&dst_iter, &offset, &dst); if (!SUCCESS(rc)) { - goto out; + break; } - - platform_assert(src_offset == dst_offset); - uint64 amount_to_copy = - MIN(slice_length(src_data), slice_length(dst_data)); memcpy(page_handle_data(&dst_iter.page) + dst_iter.fragment.offset, - slice_data(src_data), - amount_to_copy); - - blob_page_iterator_advance_bytes(&src_iter, amount_to_copy); - blob_page_iterator_advance_bytes(&dst_iter, amount_to_copy); + writable_buffer_data(&staging), + amount); + blob_page_iterator_advance_bytes(&dst_iter, amount); + /* Release the destination write handle before the next source read, + * including when a partial page still has space remaining. */ + blob_page_iterator_deinit(&dst_iter); + start += amount; + if (start < length) { + rc = blob_page_iterator_init(cc, + &dst_iter, + writable_buffer_to_slice(result), + start, + BLOB_PAGE_ITERATOR_MODE_ALLOC); + if (!SUCCESS(rc)) { + break; + } + } } - - debug_assert(blob_page_iterator_at_end(&src_iter)); - debug_assert(blob_page_iterator_at_end(&dst_iter)); - - out: - blob_page_iterator_deinit(&src_iter); blob_page_iterator_deinit(&dst_iter); + if (!SUCCESS(rc)) { + (void)blob_release_refs(cc, writable_buffer_to_slice(result)); + } } - return rc; } diff --git a/src/blob_build.h b/src/blob_build.h index c47b74829..4e5a614ec 100644 --- a/src/blob_build.h +++ b/src/blob_build.h @@ -4,23 +4,37 @@ #pragma once #include "blob.h" -#include "mini_allocator.h" -#define NUM_BLOB_BATCHES (3) -#define BLOB_DEFAULT_MAX_DESCRIPTOR_SIZE UINT64_C(512) +#define BLOB_DEFAULT_MAX_DESCRIPTOR_SIZE UINT64_C(256) /* Enough for every legacy packed-tail descriptor (at most eight full extents * plus three tail addresses). Smaller budgets would make tail/tree decoding * ambiguous. Zero in blob_build_config selects the default. */ #define BLOB_MIN_MAX_DESCRIPTOR_SIZE (sizeof(blob) + 11 * sizeof(uint64)) typedef struct blob_build_config { - uint64 extent_batch; - uint64 page_batch; - uint64 subpage_batch; uint64 alignment; uint64 blob_max_descriptor_size; } blob_build_config; +/* The container owns all returned tail storage. Allocate at most one extent + * of bytes, in at most two extent-bounded fragments, and initialize newly + * allocated cache pages before returning. Existing partial pages may be reused. + * alignment/boundary have mini_alloc_bytes semantics; boundary is zero for + * whole pages and extent_size for a final subpage fragment. */ +typedef platform_status (*blob_tail_alloc_fn)(void *arg, + uint64 bytes, + uint64 alignment, + uint64 boundary, + uint64 addrs[2]); +typedef struct blob_tail_allocator { + blob_tail_alloc_fn alloc; + void *arg; +} blob_tail_allocator; + +/* On success the caller owns the whole-extent references until a container + * adopts them. On failure those references are released; allocated tails stay + * with the destination container. Source containers must remain alive through + * clone. Descriptors and pointer extents retain their existing disk format. */ /* Allocation- and I/O-free bound; exact for extent-only trees. */ platform_status blob_descriptor_size_bound(const blob_build_config *cfg, @@ -29,15 +43,15 @@ blob_descriptor_size_bound(const blob_build_config *cfg, uint64 *size); platform_status -blob_build(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - slice data, - writable_buffer *result); +blob_build(const blob_build_config *cfg, + cache *cc, + const blob_tail_allocator *tails, + slice data, + writable_buffer *result); platform_status -blob_clone(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - slice sblob, - writable_buffer *result); +blob_clone(const blob_build_config *cfg, + cache *cc, + const blob_tail_allocator *tails, + slice sblob, + writable_buffer *result); diff --git a/src/btree.c b/src/btree.c index 708c0aea7..e367ee724 100644 --- a/src/btree.c +++ b/src/btree.c @@ -139,8 +139,7 @@ btree_reset_node_entries(const btree_config *cfg, btree_hdr *hdr) static inline uint64 index_entry_required_capacity(key pivot, btree_pivot_stats stats) { - return sizeof(index_entry) + key_length(pivot) - + (btree_stats_need_wide(stats) ? BTREE_WIDE_STATS_BYTES : 0); + return sizeof(index_entry) + key_length(pivot) + btree_stats_size(stats); } static inline uint64 @@ -218,7 +217,7 @@ btree_fill_index_entry(const btree_config *cfg, + index_entry_required_capacity(new_pivot_key, stats) <= btree_page_size(cfg)); copy_key_to_ondisk_key(&entry->pivot, new_pivot_key); - entry->pivot_data.child_addr = new_addr; + entry->child_addr = new_addr; index_entry_set_stats(entry, stats); } @@ -570,13 +569,57 @@ btree_find_tuple(const btree_config *cfg, * - incorporate_tuple() is a convenience wrapper. *----------------------------------------------------------------------------- */ -static inline int +static platform_status +btree_find_pivot_checked(const btree_config *, + const btree_hdr *, + const key *, + bool32 *, + int64 *, + const index_entry **); + +static platform_status +btree_find_pivot_mutable(const btree_config *, + const btree_hdr *, + const key *, + bool32 *, + int64 *, + const index_entry **); +static platform_status +btree_find_tuple_checked(const btree_config *, + const btree_hdr *, + key, + bool32 *, + int64 *, + const leaf_entry **); + +static platform_status +btree_find_tuple_mutable(const btree_config *, + const btree_hdr *, + key, + bool32 *, + int64 *, + const leaf_entry **); + +static inline platform_status btree_merge_tuples(const btree_config *cfg, key tuple_key, message old_data, merge_accumulator *new_data) { - return data_merge_tuples(cfg->data_cfg, tuple_key, old_data, new_data); + if (merge_accumulator_is_definitive(new_data)) { + return STATUS_OK; + } + DECLARE_AUTO_KEY_BUFFER(logical, PROCESS_PRIVATE_HEAP_ID); + if (tuple_key.is_blob) { + platform_status rc = btree_key_materialize(cfg, tuple_key, &logical); + if (!SUCCESS(rc)) { + return rc; + } + tuple_key = key_buffer_key(&logical); + } + return data_merge_tuples(cfg->data_cfg, tuple_key, old_data, new_data) + ? STATUS_NO_MEMORY + : STATUS_OK; } static message @@ -612,8 +655,16 @@ btree_record_old_result(const btree_config *cfg, return rc; } } - platform_status rc = lookup_result_update( - old_result, leaf_entry_key(entry), old_msg, &blob_ref); + DECLARE_AUTO_KEY_BUFFER(logical, PROCESS_PRIVATE_HEAP_ID); + key stored = leaf_entry_key(entry); + platform_status rc = STATUS_OK; + if (stored.is_blob) { + rc = btree_key_materialize(cfg, stored, &logical); + stored = key_buffer_key(&logical); + } + if (SUCCESS(rc)) { + rc = lookup_result_update(old_result, stored, old_msg, &blob_ref); + } ondisk_ref_deinit(&blob_ref); return rc; } @@ -639,33 +690,37 @@ btree_create_leaf_incorporate_spec(const btree_config *cfg, mini_allocator *mini, platform_heap_id heap_id, btree_hdr *hdr, - key tuple_key, - message msg, + const key *tuple_key_ptr, + const message *msg_ptr, leaf_incorporate_spec *spec) { platform_status rc = STATUS_OK; - spec->tuple_key = tuple_key; + spec->tuple_key = *tuple_key_ptr; bool32 found; - spec->idx = btree_find_tuple(cfg, hdr, tuple_key, &found); + rc = btree_find_tuple_mutable( + cfg, hdr, *tuple_key_ptr, &found, &spec->idx, NULL); + if (!SUCCESS(rc)) { + return rc; + } spec->old_entry_state = found ? ENTRY_STILL_EXISTS : ENTRY_DID_NOT_EXIST; if (!found) { spec->idx++; - if (message_is_blob(msg)) { + if (message_is_blob(*msg_ptr)) { spec->use_new_message = FALSE; merge_accumulator_init(&spec->msg.modified_message, heap_id); rc = message_clone( - &cfg->blob_cfg, cc, mini, msg, &spec->msg.modified_message); + &cfg->blob_cfg, cc, mini, *msg_ptr, &spec->msg.modified_message); } else if (MAX_INLINE_MESSAGE_SIZE(btree_page_size(cfg)) - < message_length(msg)) + < message_length(*msg_ptr)) { spec->use_new_message = FALSE; merge_accumulator_init(&spec->msg.modified_message, heap_id); rc = message_to_blob( - &cfg->blob_cfg, cc, mini, msg, &spec->msg.modified_message); + &cfg->blob_cfg, cc, mini, *msg_ptr, &spec->msg.modified_message); } else { spec->use_new_message = TRUE; - spec->msg.new_message = msg; + spec->msg.new_message = *msg_ptr; } } else { leaf_entry *entry = btree_get_leaf_entry(cfg, hdr, spec->idx); @@ -673,15 +728,15 @@ btree_create_leaf_incorporate_spec(const btree_config *cfg, bool32 success; spec->use_new_message = FALSE; success = merge_accumulator_init_from_message( - &spec->msg.modified_message, heap_id, msg); + &spec->msg.modified_message, heap_id, *msg_ptr); if (!success) { return STATUS_NO_MEMORY; } - if (btree_merge_tuples( - cfg, tuple_key, oldmessage, &spec->msg.modified_message)) - { + rc = btree_merge_tuples( + cfg, *tuple_key_ptr, oldmessage, &spec->msg.modified_message); + if (!SUCCESS(rc)) { merge_accumulator_deinit(&spec->msg.modified_message); - return STATUS_NO_MEMORY; + return rc; } else if (merge_accumulator_is_blob(&spec->msg.modified_message)) { /* * A definitive incoming blob survives the merge unchanged. Take the @@ -1064,8 +1119,10 @@ static inline bool32 btree_index_is_full(const btree_config *cfg, // IN const btree_hdr *hdr) // IN { + // Mutable trees use narrow unknown counters in every index entry. return hdr->next_entry < diff_ptr(hdr, &hdr->offsets[hdr->num_entries + 2]) + sizeof(index_entry) + + sizeof(btree_pivot_stats32) + MAX_INLINE_KEY_SIZE(btree_page_size(cfg)); } @@ -1321,6 +1378,36 @@ btree_read_entry_bounds(const btree_config *cfg, return STATUS_OK; } +/* The ordinary inline/narrow-stat case is validated in the caller. */ +static __attribute__((noinline)) platform_status +btree_validate_index_key_special(const btree_config *cfg, + const index_entry *entry, + uint64 remaining) +{ + platform_status rc; + if ((entry->pivot.flags & ~(ONDISK_KEY_IS_BLOB | BTREE_INDEX_WIDE_STATS)) + != 0 + || sizeof_index_entry(entry) > remaining + || ((entry->pivot.flags & ONDISK_KEY_IS_BLOB) + && (entry->pivot.length == ONDISK_KEY_NEGATIVE_INFINITY + || entry->pivot.length == ONDISK_KEY_POSITIVE_INFINITY))) + { + return STATUS_INVALID_STATE; + } + key pivot = index_entry_key(entry); + if (pivot.is_blob) { + rc = btree_key_validate(cfg, pivot); + if (!SUCCESS(rc)) { + return rc; + } + } else if (key_is_user_key(pivot) + && key_length(pivot) > MAX_INLINE_KEY_SIZE(cfg->page_size)) + { + return STATUS_INVALID_STATE; + } + return STATUS_OK; +} + static inline platform_status btree_read_index_key(const btree_config *cfg, const btree_hdr *hdr, @@ -1335,10 +1422,19 @@ btree_read_index_key(const btree_config *cfg, return rc; } const index_entry *entry = (const index_entry *)((const char *)hdr + offset); - if ((entry->pivot.flags & ~BTREE_INDEX_WIDE_STATS) != 0 - || sizeof_index_entry(entry) > remaining) + if (entry->pivot.flags == 0 + && entry->pivot.length <= MAX_INLINE_KEY_SIZE(cfg->page_size)) { - return STATUS_INVALID_STATE; + if (sizeof(*entry) + sizeof(btree_pivot_stats32) + entry->pivot.length + > remaining) + { + return STATUS_INVALID_STATE; + } + } else { + rc = btree_validate_index_key_special(cfg, entry, remaining); + if (!SUCCESS(rc)) { + return rc; + } } *entry_out = entry; return STATUS_OK; @@ -1356,6 +1452,34 @@ btree_read_index_entry(cache *cc, return rc; } +/* Blob descriptor parsing must not add a stack frame to every inline read. */ +static __attribute__((noinline)) platform_status +btree_read_leaf_blob_key(const btree_config *cfg, + const btree_hdr *hdr, + const leaf_entry *entry) +{ + if (hdr->prefix_length > cfg->blob_key_prefix_size) { + return STATUS_INVALID_STATE; + } + uint64 p = cfg->blob_key_prefix_size - hdr->prefix_length; + if (p > entry->key_length || entry->key_length - p < sizeof(blob)) { + return STATUS_INVALID_STATE; + } + slice descriptor = + slice_create(entry->key_length - p, entry->key_and_message + p); + parsed_blob parsed; + platform_status rc = blob_parse_descriptor(cfg->cc, descriptor, &parsed); + if (!SUCCESS(rc)) { + return rc; + } + if (blob_length(descriptor) <= MAX_INLINE_KEY_SIZE(cfg->page_size) + || blob_length(descriptor) <= cfg->blob_key_prefix_size) + { + return STATUS_INVALID_STATE; + } + return STATUS_OK; +} + static inline platform_status btree_read_leaf_key(const btree_config *cfg, const btree_hdr *hdr, @@ -1370,7 +1494,7 @@ btree_read_leaf_key(const btree_config *cfg, return rc; } const leaf_entry *entry = (const leaf_entry *)((const char *)hdr + offset); - if (entry->key_flags != ONDISK_KEY_DEFAULT_FLAGS + if ((entry->key_flags & ~ONDISK_KEY_IS_BLOB) != 0 || entry->key_length == ONDISK_KEY_NEGATIVE_INFINITY || entry->key_length == ONDISK_KEY_POSITIVE_INFINITY || entry->key_length > remaining - sizeof(*entry) @@ -1379,6 +1503,12 @@ btree_read_leaf_key(const btree_config *cfg, { return STATUS_INVALID_STATE; } + if (entry->key_flags & ONDISK_KEY_IS_BLOB) { + rc = btree_read_leaf_blob_key(cfg, hdr, entry); + if (!SUCCESS(rc)) { + return rc; + } + } *entry_out = entry; return STATUS_OK; } @@ -1447,43 +1577,116 @@ btree_read_leaf_entry(cache *cc, return rc; } const leaf_entry *entry = (const leaf_entry *)((const char *)hdr + offset); - /* Checking the full key plus message below also checks the key's byte - * range. Keep key-only reads separate because they do not inspect messages. + /* The full key+message bound below also bounds the key. The full-key size + * limit rejects both infinity encodings, as asserted above. Keep the + * key-only checks in btree_read_leaf_key() for searches that skip messages. */ - /* Supported pages are at most MAX_PAGE_SIZE bytes. The widened full-key - * bound therefore rejects both infinity length encodings as well. */ - if (entry->key_flags != ONDISK_KEY_DEFAULT_FLAGS + if ((entry->key_flags & ~ONDISK_KEY_IS_BLOB) != 0 || (uint64)entry->key_length + hdr->prefix_length > MAX_INLINE_KEY_SIZE(btree_page_size(cfg))) { return STATUS_INVALID_STATE; } rc = btree_read_leaf_message(cc, cfg, hdr, entry); - if (SUCCESS(rc)) { - *entry_out = entry; + if (!SUCCESS(rc)) { + return rc; } - return rc; + if (entry->key_flags & ONDISK_KEY_IS_BLOB) { + rc = btree_read_leaf_blob_key(cfg, hdr, entry); + if (!SUCCESS(rc)) { + return rc; + } + } + *entry_out = entry; + return STATUS_OK; +} + +static __attribute__((noinline)) platform_status +btree_compare_blob_leaf_key(const btree_config *cfg, + const btree_hdr *hdr, + key target, + const leaf_entry *entry, + int *cmp) +{ + DECLARE_AUTO_KEY_BUFFER(buffer, PROCESS_PRIVATE_HEAP_ID); + key stored; + platform_status rc = btree_leaf_full_key(cfg, hdr, entry, &buffer, &stored); + if (!SUCCESS(rc)) { + return rc; + } + return btree_key_compare_checked(cfg, target, stored, cmp); } static platform_status -btree_find_pivot_checked(const btree_config *cfg, - const btree_hdr *hdr, - key target, - bool32 *found, - int64 *idx_out, - const index_entry **entry_out) +btree_compare_leaf_key_checked(const btree_config *cfg, + const btree_hdr *hdr, + key target, + const leaf_entry *entry, + int *cmp) +{ + if ((entry->key_flags & ONDISK_KEY_IS_BLOB) || target.is_blob) { + return btree_compare_blob_leaf_key(cfg, hdr, target, entry, cmp); + } + *cmp = btree_compare_leaf_key(cfg, hdr, target, entry); + return STATUS_OK; +} + +static inline __attribute__((always_inline)) platform_status +btree_find_pivot_impl(const btree_config *cfg, + const btree_hdr *hdr, + key target, + bool32 *found, + int64 *idx_out, + const index_entry **entry_out, + bool32 validate_entries, + bool32 lexical) { int64 lo = 0, hi = btree_num_entries(hdr); - const index_entry *best = NULL; - *found = FALSE; + const index_entry *best = NULL; + *found = FALSE; + const bool32 inline_target = !target.is_blob && key_is_user_key(target); + user_key user_target; + memset(&user_target, 0, sizeof(user_target)); + user_target.key = target.user_slice; + user_target.is_query_key = target.is_query_key; while (lo < hi) { int64 mid = (lo + hi) / 2; const index_entry *entry; - platform_status rc = btree_read_index_key(cfg, hdr, mid, &entry); - if (!SUCCESS(rc)) { - return rc; + platform_status rc; + if (validate_entries) { + rc = btree_read_index_key(cfg, hdr, mid, &entry); + if (!SUCCESS(rc)) { + return rc; + } + } else { + entry = btree_get_index_entry(cfg, hdr, mid); + } + int cmp; + if (lexical && !(entry->pivot.flags & ONDISK_KEY_IS_BLOB) + && entry->pivot.length != ONDISK_KEY_NEGATIVE_INFINITY + && entry->pivot.length != ONDISK_KEY_POSITIVE_INFINITY) + { + cmp = slice_lex_cmp( + key_slice(target), + slice_create(entry->pivot.length, entry->pivot.bytes)); + } else if (!lexical && inline_target + && !(entry->pivot.flags & ONDISK_KEY_IS_BLOB) + && entry->pivot.length != ONDISK_KEY_NEGATIVE_INFINITY + && entry->pivot.length != ONDISK_KEY_POSITIVE_INFINITY) + { + user_key user_pivot; + memset(&user_pivot, 0, sizeof(user_pivot)); + user_pivot.key = slice_create(entry->pivot.length, entry->pivot.bytes); + user_pivot.is_query_key = FALSE; + cmp = + cfg->data_cfg->key_compare(cfg->data_cfg, user_target, user_pivot); + } else { + rc = btree_key_compare_checked( + cfg, target, index_entry_key(entry), &cmp); + if (!SUCCESS(rc)) { + return rc; + } } - int cmp = btree_key_compare(cfg, target, index_entry_key(entry)); if (cmp == 0) { *found = TRUE; *idx_out = mid; @@ -1507,25 +1710,50 @@ btree_find_pivot_checked(const btree_config *cfg, } /* The caller initializes *found and selects callback semantics once for the - * leaf. Keep all checked-key reads before invoking the user's comparator. */ + * leaf. Persisted entries are validated before invoking the user comparator. */ static inline platform_status btree_find_tuple_callback_checked(const btree_config *cfg, const btree_hdr *hdr, key target, bool32 *found, int64 *idx_out, - const leaf_entry **entry_out) -{ - int64 lo = 0, hi = btree_num_entries(hdr); + const leaf_entry **entry_out, + bool32 validate_entries) +{ + int64 lo = 0, hi = btree_num_entries(hdr); + const bool32 inline_target = + !target.is_blob && key_is_user_key(target) && hdr->prefix_length == 0; + user_key user_target; + memset(&user_target, 0, sizeof(user_target)); + user_target.key = target.user_slice; + user_target.is_query_key = target.is_query_key; while (lo < hi) { int64 mid = (lo + hi) / 2; const leaf_entry *entry; - platform_status rc = btree_read_leaf_key(cfg, hdr, mid, &entry); - if (!SUCCESS(rc)) { - return rc; + platform_status rc; + if (validate_entries) { + rc = btree_read_leaf_key(cfg, hdr, mid, &entry); + if (!SUCCESS(rc)) { + return rc; + } + } else { + entry = btree_get_leaf_entry(cfg, hdr, mid); + } + int cmp; + if (inline_target && !(entry->key_flags & ONDISK_KEY_IS_BLOB)) { + user_key user_stored; + memset(&user_stored, 0, sizeof(user_stored)); + user_stored.key = + slice_create(entry->key_length, entry->key_and_message); + user_stored.is_query_key = FALSE; + cmp = + cfg->data_cfg->key_compare(cfg->data_cfg, user_target, user_stored); + } else { + rc = btree_compare_leaf_key_checked(cfg, hdr, target, entry, &cmp); + if (!SUCCESS(rc)) { + return rc; + } } - int cmp = - btree_key_compare(cfg, target, leaf_entry_key((leaf_entry *)entry)); if (cmp == 0) { *found = TRUE; *idx_out = mid; @@ -1548,13 +1776,20 @@ btree_find_tuple_callback_checked(const btree_config *cfg, } static platform_status -btree_find_tuple_checked(const btree_config *cfg, - const btree_hdr *hdr, - key target, - bool32 *found, - int64 *idx_out, - const leaf_entry **entry_out) +btree_find_tuple_impl(const btree_config *cfg, + const btree_hdr *hdr, + key target, + bool32 *found, + int64 *idx_out, + const leaf_entry **entry_out, + bool32 validate_entries) { + *found = FALSE; + if (target.is_blob) { + return btree_find_tuple_callback_checked( + cfg, hdr, target, found, idx_out, entry_out, validate_entries); + } + key full_target = target; int64 lo = 0, hi = btree_num_entries(hdr); *found = FALSE; bool32 compressed = hdr->prefix_length != 0; @@ -1577,17 +1812,31 @@ btree_find_tuple_checked(const btree_config *cfg, && !key_is_positive_infinity(target)); if (!lexical) { return btree_find_tuple_callback_checked( - cfg, hdr, target, found, idx_out, entry_out); + cfg, hdr, target, found, idx_out, entry_out, validate_entries); } while (lo < hi) { int64 mid = (lo + hi) / 2; const leaf_entry *entry; - platform_status rc = btree_read_leaf_key(cfg, hdr, mid, &entry); - if (!SUCCESS(rc)) { - return rc; + platform_status rc; + if (validate_entries) { + rc = btree_read_leaf_key(cfg, hdr, mid, &entry); + if (!SUCCESS(rc)) { + return rc; + } + } else { + entry = btree_get_leaf_entry(cfg, hdr, mid); + } + int cmp; + if (entry->key_flags & ONDISK_KEY_IS_BLOB) { + rc = + btree_compare_leaf_key_checked(cfg, hdr, full_target, entry, &cmp); + if (!SUCCESS(rc)) { + return rc; + } + } else { + cmp = slice_lex_cmp(key_slice(target), + key_slice(leaf_entry_key((leaf_entry *)entry))); } - int cmp = slice_lex_cmp(key_slice(target), - key_slice(leaf_entry_key((leaf_entry *)entry))); if (cmp == 0) { *found = TRUE; *idx_out = mid; @@ -1609,6 +1858,125 @@ btree_find_tuple_checked(const btree_config *cfg, return STATUS_OK; } +/* Compile separate lexical/callback and checked/mutable search loops. + * Mixing their temporaries penalizes the ordinary callback path. */ +static __attribute__((noinline)) platform_status +btree_find_pivot_lexical_checked(const btree_config *cfg, + const btree_hdr *hdr, + const key *target, + bool32 *found, + int64 *idx_out, + const index_entry **entry_out) +{ + return btree_find_pivot_impl( + cfg, hdr, *target, found, idx_out, entry_out, TRUE, TRUE); +} + +static __attribute__((noinline)) platform_status +btree_find_pivot_callback_checked(const btree_config *cfg, + const btree_hdr *hdr, + const key *target, + bool32 *found, + int64 *idx_out, + const index_entry **entry_out) +{ + return btree_find_pivot_impl( + cfg, hdr, *target, found, idx_out, entry_out, TRUE, FALSE); +} + +static __attribute__((noinline)) platform_status +btree_find_pivot_lexical_mutable(const btree_config *cfg, + const btree_hdr *hdr, + const key *target_ptr, + bool32 *found, + int64 *idx_out, + const index_entry **entry_out) +{ + return btree_find_pivot_impl( + cfg, hdr, *target_ptr, found, idx_out, entry_out, FALSE, TRUE); +} + +static __attribute__((noinline)) platform_status +btree_find_pivot_callback_mutable(const btree_config *cfg, + const btree_hdr *hdr, + const key *target_ptr, + bool32 *found, + int64 *idx_out, + const index_entry **entry_out) +{ + return btree_find_pivot_impl( + cfg, hdr, *target_ptr, found, idx_out, entry_out, FALSE, FALSE); +} + +/* Mutable nodes are trusted in-memory state, so searches need not repeat disk + * encoding checks. Blob comparisons remain fallible on both paths. */ +static inline platform_status +btree_find_pivot_checked(const btree_config *cfg, + const btree_hdr *hdr, + const key *target, + bool32 *found, + int64 *idx_out, + const index_entry **entry_out) +{ + if (cfg->data_cfg->key_compare_is_lexical && !target->is_blob + && key_is_user_key(*target)) + { + return btree_find_pivot_lexical_checked( + cfg, hdr, target, found, idx_out, entry_out); + } + return btree_find_pivot_callback_checked( + cfg, hdr, target, found, idx_out, entry_out); +} + +static inline platform_status +btree_find_pivot_mutable(const btree_config *cfg, + const btree_hdr *hdr, + const key *target_ptr, + bool32 *found, + int64 *idx_out, + const index_entry **entry_out) +{ + if (cfg->data_cfg->key_compare_is_lexical && !target_ptr->is_blob + && key_is_user_key(*target_ptr)) + { + return btree_find_pivot_lexical_mutable( + cfg, hdr, target_ptr, found, idx_out, entry_out); + } + return btree_find_pivot_callback_mutable( + cfg, hdr, target_ptr, found, idx_out, entry_out); +} + +static inline platform_status +btree_find_tuple_checked(const btree_config *cfg, + const btree_hdr *hdr, + key target, + bool32 *found, + int64 *idx_out, + const leaf_entry **entry_out) +{ + return btree_find_tuple_impl( + cfg, hdr, target, found, idx_out, entry_out, TRUE); +} + +static inline platform_status +btree_find_tuple_mutable(const btree_config *cfg, + const btree_hdr *hdr, + key target, + bool32 *found, + int64 *idx_out, + const leaf_entry **entry_out) +{ + debug_assert(hdr->prefix_length == 0); + if (!cfg->data_cfg->key_compare_is_lexical) { + *found = FALSE; + return btree_find_tuple_callback_checked( + cfg, hdr, target, found, idx_out, entry_out, FALSE); + } + return btree_find_tuple_impl( + cfg, hdr, target, found, idx_out, entry_out, FALSE); +} + + static platform_status btree_node_get_checked(cache *cc, const btree_config *cfg, @@ -1794,19 +2162,23 @@ btree_create(cache *cc, /* The B-tree owns branch format compatibility, independent of its container. */ platform_status -btree_validate_root(uint64 root_addr, uint16 format_version, page_type type) +btree_validate_root(uint64 root_addr, btree_format format, page_type type) { if (type == PAGE_TYPE_BRANCH - && format_version != BTREE_BRANCH_FORMAT_VERSION) + && format.version != BTREE_BRANCH_FORMAT_VERSION) { platform_error_log("Unsupported branch format version %u at %lu; " "this build requires version %u.\n", - format_version, + format.version, root_addr, BTREE_BRANCH_FORMAT_VERSION); return STATUS_INVALID_STATE; } - if (root_addr == 0) { + if (root_addr == 0 + || (type == PAGE_TYPE_BRANCH + && format.key_prefix_size + > MAX_INLINE_KEY_SIZE(MAX_PAGE_SIZE) - sizeof(blob))) + { return STATUS_INVALID_STATE; } return STATUS_OK; @@ -1829,11 +2201,10 @@ platform_status btree_recover_allocations(cache *cc, const btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, page_type type) { - platform_status version_rc = - btree_validate_root(root_addr, format_version, type); + platform_status version_rc = btree_validate_root(root_addr, format, type); if (!SUCCESS(version_rc)) { return version_rc; } @@ -1857,12 +2228,11 @@ platform_status btree_get_root_height(cache *cc, const btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, page_type type, uint32 *height) { - platform_status version_rc = - btree_validate_root(root_addr, format_version, type); + platform_status version_rc = btree_validate_root(root_addr, format, type); if (!SUCCESS(version_rc)) { return version_rc; } @@ -1886,13 +2256,13 @@ btree_get_root_height(cache *cc, static platform_status btree_blob_ref_inc(const ondisk_ref *ref) { - return btree_inc_ref(ref->cc, ref->arg, ref->addr); + return mini_inc_ref(ref->cc, ref->addr); } static void btree_blob_ref_dec(const ondisk_ref *ref) { - btree_dec_ref(ref->cc, ref->arg, ref->addr, ref->type); + mini_dec_ref(ref->cc, ref->addr, ref->type); } static platform_status @@ -1902,8 +2272,13 @@ btree_blob_ref_init(ondisk_ref *ref, uint64 root_addr, page_type type) { - return ondisk_ref_init( - ref, cc, root_addr, type, cfg, btree_blob_ref_inc, btree_blob_ref_dec); + return ondisk_ref_init(ref, + cc, + btree_root_to_meta_addr(cfg, root_addr, 0), + type, + NULL, + btree_blob_ref_inc, + btree_blob_ref_dec); } /* @@ -2341,8 +2716,17 @@ accumulate_node_ranks(cache *cc, } else { entry = btree_get_leaf_entry(cfg, hdr, i); } - stats->key_bytes = add_unknown( - stats->key_bytes, leaf_entry_key_size(entry) + hdr->prefix_length); + uint64 logical_length = + leaf_entry_key_size(entry) + hdr->prefix_length; + if (entry->key_flags & ONDISK_KEY_IS_BLOB) { + uint64 prefix_suffix = + cfg->blob_key_prefix_size - hdr->prefix_length; + slice descriptor = + slice_create(entry->key_length - prefix_suffix, + entry->key_and_message + prefix_suffix); + logical_length = blob_length(descriptor); + } + stats->key_bytes = add_unknown(stats->key_bytes, logical_length); stats->message_bytes = add_unknown(stats->message_bytes, leaf_entry_message_size(entry)); } @@ -2419,32 +2803,31 @@ btree_grow_root(cache *cc, // IN * Inserts the tuple into the dynamic btree. *----------------------------------------------------------------------------- */ -platform_status -btree_insert(cache *cc, // IN - const btree_config *cfg, // IN - platform_heap_id heap_id, // IN - btree_scratch *scratch, // IN - uint64 root_addr, // IN - mini_allocator *mini, // IN - key tuple_key, // IN - message msg, // IN - btree_insert_results *results) // IN/OUT +/* Borrow the wrapper's key/message objects through mutation search and leaf + * incorporation. The wrapper owns them (and any encoded-key buffer) until this + * call returns; intermediate aggregate copies are unnecessary. */ +static platform_status +btree_insert_encoded(cache *cc, // IN + const btree_config *cfg, // IN + platform_heap_id heap_id, // IN + btree_scratch *scratch, // IN + uint64 root_addr, // IN + mini_allocator *mini, // IN + const key *tuple_key_ptr, // IN + const message *msg_ptr, // IN + btree_insert_results *results) // IN/OUT { platform_status rc; leaf_incorporate_spec spec; - if (MAX_INLINE_KEY_SIZE(btree_page_size(cfg)) < key_length(tuple_key)) { - return STATUS_BAD_PARAM; - } - - if (message_is_invalid_user_type(msg)) { + if (message_is_invalid_user_type(*msg_ptr)) { return STATUS_BAD_PARAM; } btree_node root_node; root_node.addr = root_addr; - log_trace_key(tuple_key, "btree_insert"); + log_trace_key(*tuple_key_ptr, "btree_insert"); if (results->old_result_buffer != NULL) { lookup_result_reset(results->old_result_buffer); @@ -2459,7 +2842,7 @@ btree_insert(cache *cc, // IN if (btree_height(root_node.hdr) == 0) { rc = btree_create_leaf_incorporate_spec( - cfg, cc, mini, heap_id, root_node.hdr, tuple_key, msg, &spec); + cfg, cc, mini, heap_id, root_node.hdr, tuple_key_ptr, msg_ptr, &spec); if (!SUCCESS(rc)) { btree_node_unget(cc, cfg, &root_node); return rc; @@ -2504,7 +2887,13 @@ btree_insert(cache *cc, // IN /* read lock on root_node, root_node is an index. */ bool32 found; - int64 child_idx = btree_find_pivot(cfg, root_node.hdr, tuple_key, &found); + int64 child_idx; + rc = btree_find_pivot_mutable( + cfg, root_node.hdr, tuple_key_ptr, &found, &child_idx, NULL); + if (!SUCCESS(rc)) { + btree_node_unget(cc, cfg, &root_node); + return rc; + } index_entry *parent_entry; if (child_idx < 0 || btree_index_is_full(cfg, root_node.hdr)) { @@ -2521,7 +2910,7 @@ btree_insert(cache *cc, // IN !btree_set_index_entry(cfg, root_node.hdr, 0, - tuple_key, + *tuple_key_ptr, index_entry_child_addr(parent_entry), index_entry_stats(parent_entry)); } @@ -2539,7 +2928,7 @@ btree_insert(cache *cc, // IN btree_set_index_entry(cfg, root_node.hdr, 0, - tuple_key, + *tuple_key_ptr, index_entry_child_addr(parent_entry), index_entry_stats(parent_entry)); platform_assert(success); @@ -2590,8 +2979,14 @@ btree_insert(cache *cc, // IN * - read lock on child_node * - height >= 1 */ - int64 next_child_idx = - btree_find_pivot(cfg, child_node.hdr, tuple_key, &found); + int64 next_child_idx; + rc = btree_find_pivot_mutable( + cfg, child_node.hdr, tuple_key_ptr, &found, &next_child_idx, NULL); + if (!SUCCESS(rc)) { + btree_node_unget(cc, cfg, &parent_node); + btree_node_unget(cc, cfg, &child_node); + return rc; + } if (next_child_idx < 0 || btree_index_is_full(cfg, child_node.hdr)) { if (!btree_node_claim(cc, cfg, &parent_node)) { btree_node_unget(cc, cfg, &parent_node); @@ -2617,7 +3012,7 @@ btree_insert(cache *cc, // IN !btree_set_index_entry(cfg, child_node.hdr, 0, - tuple_key, + *tuple_key_ptr, index_entry_child_addr(child_entry), index_entry_stats(child_entry)); } @@ -2631,7 +3026,7 @@ btree_insert(cache *cc, // IN &parent_node, child_idx, &child_node, - tuple_key, + *tuple_key_ptr, &new_child, &next_child_idx); if (!SUCCESS(rc)) { @@ -2651,7 +3046,7 @@ btree_insert(cache *cc, // IN btree_set_index_entry(cfg, parent_node.hdr, 0, - tuple_key, + *tuple_key_ptr, index_entry_child_addr(child_entry), index_entry_stats(child_entry)); platform_assert(success); @@ -2691,7 +3086,7 @@ btree_insert(cache *cc, // IN * - height of parent == 1 */ rc = btree_create_leaf_incorporate_spec( - cfg, cc, mini, heap_id, child_node.hdr, tuple_key, msg, &spec); + cfg, cc, mini, heap_id, child_node.hdr, tuple_key_ptr, msg_ptr, &spec); if (!SUCCESS(rc)) { btree_node_unget(cc, cfg, &parent_node); btree_node_unget(cc, cfg, &child_node); @@ -2758,7 +3153,7 @@ btree_insert(cache *cc, // IN */ destroy_leaf_incorporate_spec(&spec); rc = btree_create_leaf_incorporate_spec( - cfg, cc, mini, heap_id, child_node.hdr, tuple_key, msg, &spec); + cfg, cc, mini, heap_id, child_node.hdr, tuple_key_ptr, msg_ptr, &spec); if (!SUCCESS(rc)) { btree_node_unclaim(cc, cfg, &parent_node); btree_node_unget(cc, cfg, &parent_node); @@ -2787,6 +3182,45 @@ btree_insert(cache *cc, // IN return STATUS_OK; } +/* Allocate encoding scratch only for keys that require a blob. */ +platform_status +btree_insert(cache *cc, // IN + const btree_config *cfg, // IN + platform_heap_id heap_id, // IN + btree_scratch *scratch, // IN + uint64 root_addr, // IN + mini_allocator *mini, // IN + key tuple_key, // IN + message msg, // IN + btree_insert_results *results) // IN/OUT +{ + platform_status rc; + btree_config local_cfg = *cfg; + local_cfg.cc = cc; + local_cfg.blob_key_prefix_size = + cfg->data_cfg->key_compare_is_lexical ? cfg->blob_key_prefix_size : 0; + cfg = &local_cfg; + rc = btree_config_validate(cfg); + if (!SUCCESS(rc)) { + return rc; + } + if (tuple_key.is_blob) { + return STATUS_BAD_PARAM; + } + if (key_length(tuple_key) <= MAX_INLINE_KEY_SIZE(btree_page_size(cfg))) { + return btree_insert_encoded( + cc, cfg, heap_id, scratch, root_addr, mini, &tuple_key, &msg, results); + } + DECLARE_AUTO_KEY_BUFFER(encoded, heap_id); + rc = btree_key_encode(cfg, mini, tuple_key, &encoded); + if (!SUCCESS(rc)) { + return rc; + } + tuple_key = key_buffer_key(&encoded); + return btree_insert_encoded( + cc, cfg, heap_id, scratch, root_addr, mini, &tuple_key, &msg, results); +} + /* *----------------------------------------------------------------------------- @@ -2808,7 +3242,7 @@ static platform_status btree_lookup_node_internal(cache *cc, // IN const btree_config *cfg, // IN uint64 root_addr, // IN - key target, // IN + const key *target, // IN uint16 stop_at_height, // IN page_type type, // IN bool32 get_node, // IN @@ -2839,17 +3273,15 @@ btree_lookup_node_internal(cache *cc, // IN for (h = btree_height(node.hdr); h > stop_at_height; h--) { bool32 found; const index_entry *entry = NULL; - if (key_is_positive_infinity(target)) { + if (key_is_positive_infinity(*target)) { child_idx = btree_num_entries(node.hdr) - 1; - } else if (validate_on_demand) { + } else { rc = btree_find_pivot_checked( cfg, node.hdr, target, &found, &child_idx, &entry); if (!SUCCESS(rc)) { btree_node_unget(cc, cfg, &node); return rc; } - } else { - child_idx = btree_find_pivot(cfg, node.hdr, target, &found); } if (child_idx < 0) { child_idx = 0; @@ -2922,10 +3354,12 @@ btree_lookup_node(cache *cc, // IN btree_node *out_node, // OUT btree_pivot_stats *stats) // OUT { + btree_config local_cfg = *cfg; + local_cfg.cc = cc; return btree_lookup_node_internal(cc, - cfg, + &local_cfg, root_addr, - target, + &target, stop_at_height, type, get_node, @@ -3002,10 +3436,10 @@ btree_lookup_node_async(btree_lookup_async_state *state, uint64 depth) const index_entry *entry = NULL; if (key_is_positive_infinity(state->target)) { child_idx = btree_num_entries(state->node.hdr) - 1; - } else if (state->type == PAGE_TYPE_BRANCH) { + } else { rc = btree_find_pivot_checked(state->cfg, state->node.hdr, - state->target, + &state->target, &found, &child_idx, &entry); @@ -3013,9 +3447,6 @@ btree_lookup_node_async(btree_lookup_async_state *state, uint64 depth) btree_node_unget(state->cc, state->cfg, &state->node); async_return(state, rc); } - } else { - child_idx = btree_find_pivot( - state->cfg, state->node.hdr, state->target, &found); } if (child_idx < 0) { child_idx = 0; @@ -3141,16 +3572,13 @@ btree_lookup_with_ref_async(btree_lookup_async_state *state, uint64 depth) int64 idx; const leaf_entry *entry = NULL; platform_status rc = STATUS_OK; - if (state->type == PAGE_TYPE_BRANCH) { + { rc = btree_find_tuple_checked( state->cfg, state->node.hdr, state->target, &found, &idx, &entry); if (!SUCCESS(rc)) { btree_node_unget(state->cc, state->cfg, &state->node); async_return(state, rc); } - } else { - idx = - btree_find_tuple(state->cfg, state->node.hdr, state->target, &found); } if (found) { if (state->type == PAGE_TYPE_BRANCH) { @@ -3163,11 +3591,13 @@ btree_lookup_with_ref_async(btree_lookup_async_state *state, uint64 depth) } else { entry = btree_get_leaf_entry(state->cfg, state->node.hdr, idx); } - state->work.found_key = state->node.hdr->prefix_length == 0 - ? leaf_entry_key((leaf_entry *)entry) - : key_create(FALSE, - key_length(state->target), - key_data(state->target)); + state->work.found_key = + state->node.hdr->prefix_length == 0 + && (!(entry->key_flags & ONDISK_KEY_IS_BLOB) + || !state->cfg->data_cfg->key_compare_is_lexical) + ? leaf_entry_key((leaf_entry *)entry) + : key_create( + FALSE, key_length(state->target), key_data(state->target)); state->work.msg = leaf_entry_message(state->cc, (leaf_entry *)entry); } else { btree_node_unget(state->cc, state->cfg, &state->node); @@ -3181,7 +3611,7 @@ btree_lookup_with_ref(cache *cc, // IN const btree_config *cfg, // IN uint64 root_addr, // IN page_type type, // IN - key target, // IN + const key *target, // IN btree_node *node, // OUT key *found_key, // OUT message *msg) // OUT @@ -3198,15 +3628,13 @@ btree_lookup_with_ref(cache *cc, // IN } int64 idx; const leaf_entry *entry = NULL; - if (validate_on_demand) { - rc = - btree_find_tuple_checked(cfg, node->hdr, target, &found, &idx, &entry); + { + rc = btree_find_tuple_checked( + cfg, node->hdr, *target, &found, &idx, &entry); if (!SUCCESS(rc)) { btree_node_unget(cc, cfg, node); return rc; } - } else { - idx = btree_find_tuple(cfg, node->hdr, target, &found); } if (found) { if (validate_on_demand) { @@ -3218,10 +3646,13 @@ btree_lookup_with_ref(cache *cc, // IN } else { entry = btree_get_leaf_entry(cfg, node->hdr, idx); } - *found_key = node->hdr->prefix_length == 0 - ? leaf_entry_key((leaf_entry *)entry) - : key_create(FALSE, key_length(target), key_data(target)); - *msg = leaf_entry_message(cc, (leaf_entry *)entry); + *found_key = + node->hdr->prefix_length == 0 + && (!(entry->key_flags & ONDISK_KEY_IS_BLOB) + || !cfg->data_cfg->key_compare_is_lexical) + ? leaf_entry_key((leaf_entry *)entry) + : key_create(FALSE, key_length(*target), key_data(*target)); + *msg = leaf_entry_message(cc, (leaf_entry *)entry); } else { btree_node_unget(cc, cfg, node); } @@ -3229,45 +3660,51 @@ btree_lookup_with_ref(cache *cc, // IN } platform_status -btree_lookup(cache *cc, // IN - btree_config *cfg, // IN - uint64 root_addr, // IN - uint16 format_version, // IN - page_type type, // IN - key target, // IN - lookup_result *result) // OUT +btree_lookup(cache *cc, // IN + btree_config *cfg, // IN + uint64 root_addr, // IN + btree_format format, // IN + page_type type, // IN + key target, // IN + lookup_result *result) // OUT { lookup_result_reset(result); return btree_lookup_and_merge( - cc, cfg, root_addr, format_version, type, target, result, NULL); + cc, cfg, root_addr, format, type, &target, result, NULL); } platform_status -btree_lookup_and_merge(cache *cc, // IN - const btree_config *cfg, // IN - uint64 root_addr, // IN - uint16 format_version, // IN - page_type type, // IN - key target, // IN - lookup_result *result, // IN/OUT - bool32 *local_found) // OUT +btree_lookup_and_merge(cache *cc, // IN + const btree_config *cfg, // IN + uint64 root_addr, // IN + btree_format format, // IN + page_type type, // IN + const key *target, // IN + lookup_result *result, // IN/OUT + bool32 *local_found) // OUT { if (local_found != NULL) { *local_found = FALSE; } - platform_status version_rc = - btree_validate_root(root_addr, format_version, type); + platform_status version_rc = btree_validate_root(root_addr, format, type); if (!SUCCESS(version_rc)) { return version_rc; } + btree_config local_cfg = *cfg; + local_cfg.cc = cc; + local_cfg.blob_key_prefix_size = + type == PAGE_TYPE_BRANCH ? format.key_prefix_size + : btree_config_format(cfg).key_prefix_size; + cfg = &local_cfg; + btree_node node; key found_key; message local_data; platform_status rc = STATUS_OK; - log_trace_key(target, "btree_lookup"); + log_trace_key(*target, "btree_lookup"); rc = btree_lookup_with_ref( cc, cfg, root_addr, type, target, &node, &found_key, &local_data); @@ -3282,6 +3719,11 @@ btree_lookup_and_merge(cache *cc, // IN if (message_is_blob(local_data)) { rc = btree_blob_ref_init(&blob_ref, cc, cfg, root_addr, type); } + DECLARE_AUTO_KEY_BUFFER(logical, PROCESS_PRIVATE_HEAP_ID); + if (SUCCESS(rc) && found_key.is_blob) { + rc = btree_key_materialize(cfg, found_key, &logical); + found_key = key_buffer_key(&logical); + } if (SUCCESS(rc)) { rc = lookup_result_update(result, found_key, local_data, &blob_ref); } @@ -3316,11 +3758,19 @@ btree_lookup_and_merge_async(btree_lookup_async_state *state) async_begin(state, 0); platform_status version_rc = - btree_validate_root(state->root_addr, state->format_version, state->type); + btree_validate_root(state->root_addr, state->format, state->type); if (!SUCCESS(version_rc)) { async_return(state, version_rc); } + state->local_cfg = *state->cfg; + state->local_cfg.cc = state->cc; + state->local_cfg.blob_key_prefix_size = + state->type == PAGE_TYPE_BRANCH + ? state->format.key_prefix_size + : btree_config_format(state->cfg).key_prefix_size; + state->cfg = &state->local_cfg; + async_await_subroutine(state, btree_lookup_with_ref_async); if (!SUCCESS(state->__async_result)) { @@ -3334,9 +3784,15 @@ btree_lookup_and_merge_async(btree_lookup_async_state *state) rc = btree_blob_ref_init( &blob_ref, state->cc, state->cfg, state->root_addr, state->type); } + DECLARE_AUTO_KEY_BUFFER(logical, PROCESS_PRIVATE_HEAP_ID); + key found_key = state->work.found_key; + if (SUCCESS(rc) && found_key.is_blob) { + rc = btree_key_materialize(state->cfg, found_key, &logical); + found_key = key_buffer_key(&logical); + } if (SUCCESS(rc)) { rc = lookup_result_update(state->result, - state->work.found_key, + found_key, state->work.msg, ondisk_ref_is_null(&blob_ref) ? NULL : &blob_ref); @@ -3410,12 +3866,7 @@ find_key_in_node(btree_iterator *itor, int64 tmp; platform_status rc = STATUS_OK; if (itor->height == 0) { - if (itor->page_type == PAGE_TYPE_BRANCH) { - rc = - btree_find_tuple_checked(itor->cfg, hdr, target, found, &tmp, NULL); - } else { - tmp = btree_find_tuple(itor->cfg, hdr, target, found); - } + rc = btree_find_tuple_checked(itor->cfg, hdr, target, found, &tmp, NULL); } else if (itor->height > hdr->height) { // so we will always exceed height in future lookups itor->height = (uint32)-1; @@ -3423,12 +3874,7 @@ find_key_in_node(btree_iterator *itor, *idx_out = 0; return STATUS_OK; } else { - if (itor->page_type == PAGE_TYPE_BRANCH) { - rc = - btree_find_pivot_checked(itor->cfg, hdr, target, found, &tmp, NULL); - } else { - tmp = btree_find_pivot(itor->cfg, hdr, target, found); - } + rc = btree_find_pivot_checked(itor->cfg, hdr, &target, found, &tmp, NULL); } if (!SUCCESS(rc)) { itor->validation_status = rc; @@ -3461,6 +3907,7 @@ find_key_in_node(btree_iterator *itor, * iterators small and avoid initializing buffers they will never use. */ typedef struct btree_iterator_prefix_state { key_buffer decoded_key; + key_buffer materialized_key; uint64 prefix_page_addr; // Cache the buffer's logical high-water length, not its allocation size. // Keeping that length above the prefix preserves it when the buffer grows. @@ -3481,6 +3928,8 @@ btree_iterator_ensure_prefix_state(btree_iterator *itor) } key_buffer_init(&itor->prefix_state->decoded_key, PROCESS_PRIVATE_HEAP_ID); + key_buffer_init(&itor->prefix_state->materialized_key, + PROCESS_PRIVATE_HEAP_ID); } return STATUS_OK; } @@ -3525,15 +3974,15 @@ btree_iterator_compare_node_key(btree_iterator *itor, if (itor->page_type != PAGE_TYPE_BRANCH) { key node_key = itor->height ? btree_get_pivot(itor->cfg, hdr, idx) : btree_get_tuple_key(itor->cfg, hdr, idx); - *result = btree_key_compare(itor->cfg, target, node_key); - return STATUS_OK; + return btree_key_compare_checked(itor->cfg, target, node_key, result); } if (itor->height) { const index_entry *entry; platform_status rc = btree_read_index_key(itor->cfg, hdr, idx, &entry); if (SUCCESS(rc)) { - *result = btree_key_compare(itor->cfg, target, index_entry_key(entry)); + rc = btree_key_compare_checked( + itor->cfg, target, index_entry_key(entry), result); } return rc; } @@ -3541,7 +3990,8 @@ btree_iterator_compare_node_key(btree_iterator *itor, const leaf_entry *entry; platform_status rc = btree_read_leaf_key(itor->cfg, hdr, idx, &entry); if (SUCCESS(rc)) { - *result = btree_compare_leaf_key(itor->cfg, hdr, target, entry); + rc = + btree_compare_leaf_key_checked(itor->cfg, hdr, target, entry, result); } return rc; } @@ -3706,9 +4156,46 @@ btree_iterator_end_key_beyond_curr(btree_iterator *itor, bool32 *result) return STATUS_OK; } +/* Defer an unbounded immutable range's endpoint lookup until reaching a page + * marked last in its level. Still look up the endpoint there: a corrupt early + * LAST_IN_LEVEL flag must not silently truncate a pinned scan. Copied iterators + * already derive bounds from their own snapshot in set_copy_bounds(). */ +static inline bool32 +btree_iterator_defer_unbounded_branch_end(btree_iterator *itor) +{ + if (itor->copy_nodes || itor->page_type != PAGE_TYPE_BRANCH + || !key_is_positive_infinity(itor->max_key) + || btree_hdr_is_last_in_level(itor->curr.hdr)) + { + return FALSE; + } + itor->end_addr = 0; + itor->end_idx_valid = FALSE; + return TRUE; +} + +static platform_status +btree_iterator_set_end_addr(btree_iterator *itor, uint64 addr) +{ + if (itor->page_type == PAGE_TYPE_BRANCH + && key_is_positive_infinity(itor->max_key) + && btree_hdr_is_last_in_level(itor->curr.hdr) && itor->curr.addr != addr) + { + itor->validation_status = STATUS_INVALID_STATE; + return itor->validation_status; + } + itor->end_addr = addr; + itor->end_idx_valid = FALSE; + return STATUS_OK; +} + static platform_status btree_iterator_find_end_addr(btree_iterator *itor) { + if (btree_iterator_defer_unbounded_branch_end(itor)) { + return STATUS_OK; + } + if (itor->curr.hdr != NULL && itor->curr.addr == itor->root_addr) { itor->end_addr = itor->root_addr; itor->end_idx_valid = FALSE; @@ -3729,9 +4216,7 @@ btree_iterator_find_end_addr(btree_iterator *itor) itor->validation_status = rc; return rc; } - itor->end_addr = end.addr; - itor->end_idx_valid = FALSE; - return STATUS_OK; + return btree_iterator_set_end_addr(itor, end.addr); } static inline platform_status @@ -3769,6 +4254,17 @@ btree_iterator_set_end_idx(btree_iterator *itor) static platform_status btree_iterator_prepare_end_idx(btree_iterator *itor) { + if (btree_iterator_defer_unbounded_branch_end(itor)) { + return STATUS_OK; + } + + if (!itor->copy_nodes && itor->end_addr == 0) { + platform_status rc = btree_iterator_find_end_addr(itor); + if (!SUCCESS(rc)) { + return rc; + } + } + while (itor->curr.addr == itor->end_addr && !itor->end_idx_valid) { debug_assert(!itor->copy_nodes); bool32 beyond_curr; @@ -3824,10 +4320,10 @@ btree_iterator_can_next(iterator *base_itor) || (itor->idx < itor->end_idx && itor->curr_min_idx != itor->end_idx); } -/* Keep the uncompressed entry-validation path small. Compressed leaves pay - * this call after validation; their decode algorithm and buffer ownership are - * unchanged. */ -static __attribute__((noinline)) platform_status +/* Reconstruct the suffix in the positioning path without a per-key call. + * Allocation, buffer growth, and prefix replacement remain in the out-of-line + * prepare_decode() helper. check_current() must stay inline as well. */ +static inline __attribute__((always_inline)) platform_status btree_iterator_decode_current(btree_iterator *itor, const leaf_entry *entry) { uint64 prefix_length = itor->curr.hdr->prefix_length; @@ -3848,16 +4344,57 @@ btree_iterator_decode_current(btree_iterator *itor, const leaf_entry *entry) return rc; } +static __attribute__((noinline)) platform_status +btree_iterator_materialize_key(btree_iterator *itor, const leaf_entry *entry) +{ + platform_status rc = btree_iterator_ensure_prefix_state(itor); + if (!SUCCESS(rc)) { + return rc; + } + key stored = leaf_entry_key((leaf_entry *)entry); + if (itor->curr.hdr->prefix_length != 0) { + stored = key_create(FALSE, + itor->curr.hdr->prefix_length + entry->key_length, + itor->prefix_state->decoded_data); + stored.is_blob = TRUE; + } + return btree_key_materialize( + itor->cfg, stored, &itor->prefix_state->materialized_key); +} + +static __attribute__((noinline)) platform_status +btree_iterator_check_index_current(btree_iterator *itor) +{ + const index_entry *entry; + platform_status rc = btree_read_index_entry( + itor->cc, itor->cfg, itor->curr.hdr, itor->idx, &entry); + if (SUCCESS(rc)) { + itor->validated_entry = entry; + if (entry->pivot.flags & ONDISK_KEY_IS_BLOB) { + rc = btree_iterator_ensure_prefix_state(itor); + if (SUCCESS(rc)) { + rc = btree_key_materialize(itor->cfg, + index_entry_key((index_entry *)entry), + &itor->prefix_state->materialized_key); + } + } + } + return rc; +} + /* * curr() cannot report errors. Every positioning operation therefore reads * and checks the entry it is about to expose before returning success. + * + * Keep this inline even with suffix reconstruction in its leaf path. Otherwise + * GCC can move the entire check out of next()/prev(), adding a call per tuple + * on uncompressed leaves as well. */ -static platform_status +static inline __attribute__((always_inline)) platform_status btree_iterator_check_current(btree_iterator *itor) { itor->validated_entry = NULL; - if (itor->page_type != PAGE_TYPE_BRANCH - || !btree_iterator_can_next(&itor->super) + if (!btree_iterator_can_next(&itor->super) || !btree_iterator_can_prev(&itor->super)) { return STATUS_OK; @@ -3873,14 +4410,12 @@ btree_iterator_check_current(btree_iterator *itor) if (itor->curr.hdr->prefix_length != 0) { rc = btree_iterator_decode_current(itor, entry); } + if (SUCCESS(rc) && (entry->key_flags & ONDISK_KEY_IS_BLOB)) { + rc = btree_iterator_materialize_key(itor, entry); + } } } else { - const index_entry *entry; - rc = btree_read_index_entry( - itor->cc, itor->cfg, itor->curr.hdr, itor->idx, &entry); - if (SUCCESS(rc)) { - itor->validated_entry = entry; - } + rc = btree_iterator_check_index_current(itor); } if (!SUCCESS(rc)) { itor->validated_entry = NULL; @@ -3889,6 +4424,38 @@ btree_iterator_check_current(btree_iterator *itor) return rc; } +/* Keep aggregate construction for blob keys and index pivots off the ordinary + * leaf path. Their temporary keys and pivot data otherwise enlarge curr()'s + * frame and increase its per-tuple scan cost. */ +static __attribute__((noinline)) void +btree_iterator_curr_blob_key(btree_iterator *itor, key *curr_key) +{ + *curr_key = key_buffer_key(&itor->prefix_state->materialized_key); +} + +static __attribute__((noinline)) void +btree_iterator_curr_index(btree_iterator *itor, key *curr_key, message *data) +{ + index_entry *entry; + if (itor->page_type == PAGE_TYPE_BRANCH) { + debug_assert(itor->validated_entry != NULL); + entry = (index_entry *)itor->validated_entry; + } else { + entry = btree_get_index_entry(itor->cfg, itor->curr.hdr, itor->idx); + } + *curr_key = (entry->pivot.flags & ONDISK_KEY_IS_BLOB) + ? key_buffer_key(&itor->prefix_state->materialized_key) + : index_entry_key(entry); + itor->pivot_data = (btree_pivot_data){ + .child_addr = entry->child_addr, + .stats = index_entry_stats(entry), + }; + *data = message_create_internal( + MESSAGE_TYPE_PIVOT_DATA, + NULL, + slice_create(sizeof(itor->pivot_data), &itor->pivot_data)); +} + void btree_iterator_curr(iterator *base_itor, key *curr_key, message *data) { @@ -3918,31 +4485,26 @@ btree_iterator_curr(iterator *base_itor, key *curr_key, message *data) } else { entry = btree_get_leaf_entry(itor->cfg, itor->curr.hdr, itor->idx); } - *curr_key = itor->curr.hdr->prefix_length == 0 - ? leaf_entry_key(entry) - : key_create(FALSE, - (uint64)itor->curr.hdr->prefix_length - + entry->key_length, - itor->prefix_state->decoded_data); - *data = leaf_entry_message(itor->cc, entry); + if (entry->key_flags & ONDISK_KEY_IS_BLOB) { + btree_iterator_curr_blob_key(itor, curr_key); + } else { + /* Write the logical key directly. With GCC 14, assigning the whole + * key_create() aggregate here creates two 8-byte stack stores followed + * by a 16-byte reload. That blocks store forwarding once per tuple + * on the hot scan path. Member assignments avoid the temporary. */ + curr_key->kind = USER_KEY; + curr_key->is_query_key = FALSE; + curr_key->is_blob = FALSE; + curr_key->user_slice = slice_create( + (uint64)itor->curr.hdr->prefix_length + entry->key_length, + itor->curr.hdr->prefix_length == 0 + ? entry->key_and_message + : itor->prefix_state->decoded_data); + } + *data = leaf_entry_message(itor->cc, entry); log_trace_key(*curr_key, "btree_iterator_get_curr"); } else { - index_entry *entry; - if (itor->page_type == PAGE_TYPE_BRANCH) { - debug_assert(itor->validated_entry != NULL); - entry = (index_entry *)itor->validated_entry; - } else { - entry = btree_get_index_entry(itor->cfg, itor->curr.hdr, itor->idx); - } - *curr_key = index_entry_key(entry); - itor->pivot_data = (btree_pivot_data){ - .child_addr = entry->pivot_data.child_addr, - .stats = index_entry_stats(entry), - }; - *data = message_create_internal( - MESSAGE_TYPE_PIVOT_DATA, - NULL, - slice_create(sizeof(itor->pivot_data), &itor->pivot_data)); + btree_iterator_curr_index(itor, curr_key, data); } } @@ -3952,6 +4514,10 @@ btree_iterator_find_end_addr_async(btree_iterator_async_state *state, { async_begin(state, depth); + if (btree_iterator_defer_unbounded_branch_end(state->itor)) { + async_return(state, STATUS_OK); + } + if (state->itor->curr.hdr != NULL && state->itor->curr.addr == state->itor->root_addr) { @@ -3964,7 +4530,7 @@ btree_iterator_find_end_addr_async(btree_iterator_async_state *state, state->itor->cc, state->itor->cfg, state->itor->root_addr, - state->format_version, + state->format, state->itor->page_type, state->itor->max_key, NULL, @@ -3980,10 +4546,9 @@ btree_iterator_find_end_addr_async(btree_iterator_async_state *state, state->itor->validation_status = state->lookup_state.__async_result; async_return(state, state->lookup_state.__async_result); } - state->itor->end_addr = state->lookup_state.node.addr; - state->itor->end_idx_valid = FALSE; - - async_return(state); + async_return( + state, + btree_iterator_set_end_addr(state->itor, state->lookup_state.node.addr)); } static async_status @@ -3992,6 +4557,17 @@ btree_iterator_prepare_end_idx_async(btree_iterator_async_state *state, { async_begin(state, depth); + if (btree_iterator_defer_unbounded_branch_end(state->itor)) { + async_return(state, STATUS_OK); + } + + if (!state->itor->copy_nodes && state->itor->end_addr == 0) { + async_await_subroutine(state, btree_iterator_find_end_addr_async); + if (!SUCCESS(state->__async_result)) { + async_return(state, state->__async_result); + } + } + while (state->itor->curr.addr == state->itor->end_addr && !state->itor->end_idx_valid) { @@ -4035,8 +4611,8 @@ btree_iterator_prepare_end_idx_async(btree_iterator_async_state *state, * them, keeping ~lookahead leaf extents of IO in flight. Within the * iterator's level (batch), the extents are in key order, so the cursor * advances in lockstep with consumption. Internal-node extents are skipped; - * blob extents are prefetched for height-0 scans. See btree_prefetch_cursor - * in btree.h. + * blob roots and tail extents are prefetched for height-0 scans. See + * btree_prefetch_cursor in btree.h. * ---------------------------------------------------------------------------- */ static inline uint64 @@ -4049,13 +4625,14 @@ btree_extent_base_addr(cache *cc, uint64 addr) static inline mini_meta_cursor_status btree_prefetch_cursor_step(btree_prefetch_cursor *pf, uint64 *extent_addr, - uint64 *batch) + uint64 *batch, + page_type *type) { mini_meta_cursor_status status = pf->going_forward ? mini_meta_cursor_next(&pf->meta_cursor) : mini_meta_cursor_prev(&pf->meta_cursor); if (status == MINI_META_CURSOR_ENTRY) { - mini_meta_cursor_curr(&pf->meta_cursor, extent_addr, batch); + mini_meta_cursor_curr(&pf->meta_cursor, extent_addr, batch, type, NULL); } return status; } @@ -4076,8 +4653,9 @@ btree_prefetch_cursor_fill(btree_iterator *itor) while (!pf->at_end && pf->prefetched_ahead < pf->depth) { uint64 extent_addr; uint64 batch; + page_type type; mini_meta_cursor_status status = - btree_prefetch_cursor_step(pf, &extent_addr, &batch); + btree_prefetch_cursor_step(pf, &extent_addr, &batch, &type); if (status == MINI_META_CURSOR_WOULD_BLOCK) { break; } @@ -4091,7 +4669,7 @@ btree_prefetch_cursor_fill(btree_iterator *itor) pf->at_end = TRUE; break; } - if (batch == pf->leaf_batch) { + if (type == itor->page_type && batch == pf->leaf_batch) { cache_prefetch(itor->cc, extent_addr, itor->page_type); pf->prefetched_ahead++; if (pf->going_forward && itor->end_addr != 0 @@ -4099,7 +4677,7 @@ btree_prefetch_cursor_fill(btree_iterator *itor) { pf->at_end = TRUE; } - } else if (pf->prefetch_blobs && batch < NUM_BLOB_BATCHES) { + } else if (pf->prefetch_blobs && type == PAGE_TYPE_BLOB) { cache_prefetch(itor->cc, extent_addr, PAGE_TYPE_BLOB); } } @@ -4975,7 +5553,7 @@ find_btree_node_and_get_idx_bounds_async(btree_iterator_async_state *state, state->itor->cc, state->itor->cfg, state->itor->root_addr, - state->format_version, + state->format, state->itor->page_type, state->target, NULL, @@ -5178,7 +5756,7 @@ btree_iterator_init_common(cache *cc, const btree_config *cfg, btree_iterator *itor, uint64 root_addr, - uint16 format_version, + btree_format format, page_type page_type, comparison min_key_comparison, key min_key, @@ -5192,7 +5770,7 @@ btree_iterator_init_common(cache *cc, { ZERO_CONTENTS(itor); platform_status version_rc = - btree_validate_root(root_addr, format_version, page_type); + btree_validate_root(root_addr, format, page_type); if (!SUCCESS(version_rc)) { return version_rc; } @@ -5218,8 +5796,13 @@ btree_iterator_init_common(cache *cc, start_key = max_key; } - itor->cc = cc; - itor->cfg = cfg; + itor->cc = cc; + itor->local_cfg = *cfg; + itor->local_cfg.cc = cc; + itor->local_cfg.blob_key_prefix_size = + page_type == PAGE_TYPE_BRANCH ? format.key_prefix_size + : btree_config_format(cfg).key_prefix_size; + itor->cfg = &itor->local_cfg; itor->root_addr = root_addr; itor->height = height; itor->copy_nodes = copy_nodes; @@ -5249,7 +5832,7 @@ btree_iterator_init(cache *cc, const btree_config *cfg, btree_iterator *itor, uint64 root_addr, - uint16 format_version, + btree_format format, page_type page_type, const btree_iterator_init_options *options) { @@ -5261,7 +5844,7 @@ btree_iterator_init(cache *cc, cfg, itor, root_addr, - format_version, + format, page_type, range.min_cmp, range.min_key, @@ -5300,7 +5883,7 @@ btree_iterator_init_async(btree_iterator_async_state *state) state->cfg, state->itor, state->root_addr, - state->format_version, + state->format, state->type, state->min_key_comparison, state->min_key, @@ -5315,6 +5898,7 @@ btree_iterator_init_async(btree_iterator_async_state *state) async_return(state, rc); } + state->cfg = state->itor->cfg; state->position_rule = state->start_type; async_await_subroutine(state, find_btree_node_and_get_idx_bounds_async); if (!SUCCESS(state->__async_result)) { @@ -5342,6 +5926,7 @@ btree_iterator_deinit(btree_iterator *itor) btree_iterator_release_curr(itor); if (itor->prefix_state != NULL) { key_buffer_deinit(&itor->prefix_state->decoded_key); + key_buffer_deinit(&itor->prefix_state->materialized_key); platform_free(PROCESS_PRIVATE_HEAP_ID, itor->prefix_state); itor->prefix_state = NULL; } @@ -5371,6 +5956,7 @@ btree_pack_setup_start(btree_pack_req *req) ZERO_ARRAY(req->edge_stats); ZERO_ARRAY(req->num_edges); ZERO_ARRAY(req->level_has_nodes); + req->leaf_has_blob_key = FALSE; ZERO_ARRAY(req->next_page_no); // we create a root here, but we won't build it with the rest @@ -5595,6 +6181,9 @@ btree_pack_compress_leaf(btree_pack_req *req, btree_hdr *hdr) key first = btree_get_tuple_key(req->cfg, hdr, 0); key last = btree_get_tuple_key(req->cfg, hdr, hdr->num_entries - 1); uint64 length = MIN(key_length(first), key_length(last)); + if (req->leaf_has_blob_key) { + length = MIN(length, req->cfg->blob_key_prefix_size); + } uint64 prefix = 0; while (prefix < length && ((const char *)key_data(first))[prefix] @@ -5618,7 +6207,8 @@ btree_pack_compress_leaf(btree_pack_req *req, btree_hdr *hdr) key suffix = key_create(FALSE, key_length(full) - prefix, (const char *)key_data(full) + prefix); - bool32 fits = btree_set_leaf_entry( + suffix.is_blob = full.is_blob; + bool32 fits = btree_set_leaf_entry( req->cfg, packed, i, suffix, leaf_entry_message(req->cc, entry)); platform_assert(fits); } @@ -5637,19 +6227,23 @@ btree_pack_compress_leaf(btree_pack_req *req, btree_hdr *hdr) static bool32 btree_pack_append_leaf(btree_pack_req *req, btree_hdr *hdr, - key full, - message msg) + const key *full, + const message *msg) { - key suffix = full; + key suffix = *full; + if (full->is_blob && hdr->prefix_length > req->cfg->blob_key_prefix_size) { + return FALSE; + } if (hdr->prefix_length != 0) { - if (btree_compare_prefix(req->cfg, hdr, full) != 0) { + if (btree_compare_prefix(req->cfg, hdr, *full) != 0) { return FALSE; } - suffix = key_create(FALSE, - key_length(full) - hdr->prefix_length, - (const char *)key_data(full) + hdr->prefix_length); + suffix = key_create(FALSE, + key_length(*full) - hdr->prefix_length, + (const char *)key_data(*full) + hdr->prefix_length); + suffix.is_blob = full->is_blob; } - return btree_set_leaf_entry(req->cfg, hdr, hdr->num_entries, suffix, msg); + return btree_set_leaf_entry(req->cfg, hdr, hdr->num_entries, suffix, *msg); } static inline platform_status @@ -5663,7 +6257,8 @@ btree_pack_loop(btree_pack_req *req, // IN/OUT return STATUS_INVALID_STATE; } - uint64 key_size = key_length(tuple_key); + key logical_key = tuple_key; + uint64 key_size = key_length(tuple_key); if (req->num_tuples >= BTREE_MAX_KNOWN_COUNTER || key_size > BTREE_MAX_KNOWN_COUNTER - req->key_bytes) { @@ -5711,12 +6306,20 @@ btree_pack_loop(btree_pack_req *req, // IN/OUT } } + if (key_size > MAX_INLINE_KEY_SIZE(req->cfg->page_size)) { + rc = btree_key_encode( + req->cfg, &req->mini, tuple_key, &req->key_blob_buffer); + if (!SUCCESS(rc)) { + return rc; + } + tuple_key = key_buffer_key(&req->key_blob_buffer); + } btree_node *leaf = btree_pack_get_current_node(req, 0); bool32 appended = - leaf != NULL && btree_pack_append_leaf(req, leaf->hdr, tuple_key, msg); + leaf != NULL && btree_pack_append_leaf(req, leaf->hdr, &tuple_key, &msg); if (!appended && leaf != NULL && btree_pack_compress_leaf(req, leaf->hdr)) { - appended = btree_pack_append_leaf(req, leaf->hdr, tuple_key, msg); + appended = btree_pack_append_leaf(req, leaf->hdr, &tuple_key, &msg); } if (!appended) { rc = btree_pack_create_next_node(req, 0, tuple_key, &leaf); @@ -5726,6 +6329,10 @@ btree_pack_loop(btree_pack_req *req, // IN/OUT bool32 result = btree_set_leaf_entry(req->cfg, leaf->hdr, 0, tuple_key, msg); platform_assert(result); + req->leaf_has_blob_key = FALSE; + } + if (tuple_key.is_blob) { + req->leaf_has_blob_key = TRUE; } btree_pivot_stats *leaf_stats = btree_pack_get_current_node_stats(req, 0); @@ -5738,7 +6345,7 @@ btree_pack_loop(btree_pack_req *req, // IN/OUT if (req->fingerprint_arr != NULL) { platform_assert(req->num_tuples < req->max_tuples); req->fingerprint_arr[req->num_tuples] = - data_key_hash(req->cfg->data_cfg, tuple_key, req->seed); + data_key_hash(req->cfg->data_cfg, logical_key, req->seed); } req->num_tuples++; @@ -5843,7 +6450,11 @@ btree_pack_abort(btree_pack_req *req) platform_status btree_pack(btree_pack_req *req) { - platform_status rc = btree_pack_setup_start(req); + platform_status rc = btree_config_validate(req->cfg); + if (!SUCCESS(rc)) { + return rc; + } + rc = btree_pack_setup_start(req); if (!SUCCESS(rc)) { return rc; } @@ -5929,7 +6540,7 @@ platform_status btree_count_in_range(cache *cc, const btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, key min_key, key max_key, btree_pivot_stats *stats) @@ -5940,11 +6551,16 @@ btree_count_in_range(cache *cc, memset(stats, 0, sizeof(*stats)); platform_status version_rc = - btree_validate_root(root_addr, format_version, PAGE_TYPE_BRANCH); + btree_validate_root(root_addr, format, PAGE_TYPE_BRANCH); if (!SUCCESS(version_rc)) { return version_rc; } + btree_config local_cfg = *cfg; + local_cfg.cc = cc; + local_cfg.blob_key_prefix_size = format.key_prefix_size; + cfg = &local_cfg; + platform_status rc = btree_get_rank(cc, cfg, root_addr, min_key, &min_stats); if (!SUCCESS(rc)) { return rc; @@ -5973,7 +6589,7 @@ platform_status btree_count_in_range_by_iterator(cache *cc, btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, key min_key, key max_key, btree_pivot_stats *stats) @@ -5985,7 +6601,7 @@ btree_count_in_range_by_iterator(cache *cc, cfg, &btree_itor, root_addr, - format_version, + format, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .lower_limit = key_greater_than_or_equal(min_key), @@ -6085,7 +6701,7 @@ btree_print_index_entry(platform_log_handle *log_handle, "[%2lu]: key=%s\n", entry_num, key_string(dcfg, index_entry_key(entry))); - btree_pivot_data decoded = {.child_addr = entry->pivot_data.child_addr, + btree_pivot_data decoded = {.child_addr = entry->child_addr, .stats = index_entry_stats(entry)}; btree_print_btree_pivot_data(log_handle, &decoded); } @@ -6433,7 +7049,9 @@ btree_print_tree(platform_log_handle *log_handle, uint64 root_addr, page_type type) { - btree_print_subtree(log_handle, cc, cfg, root_addr, type); + btree_config local_cfg = *cfg; + local_cfg.cc = cc; + btree_print_subtree(log_handle, cc, &local_cfg, root_addr, type); } void @@ -6512,13 +7130,26 @@ btree_verify_node(cache *cc, goto out; } } - key first = btree_get_tuple_key(cfg, node.hdr, idx); - key second = btree_get_tuple_key(cfg, node.hdr, idx + 1); - // A user comparator may require full keys even when its ordering - // is lexical. Compressed entries contain only suffixes. - int cmp = node.hdr->prefix_length != 0 - ? slice_lex_cmp(key_slice(first), key_slice(second)) - : btree_key_compare(cfg, first, second); + DECLARE_AUTO_KEY_BUFFER(first_buffer, PROCESS_PRIVATE_HEAP_ID); + key first; + rc = btree_leaf_full_key(cfg, + node.hdr, + btree_get_leaf_entry(cfg, node.hdr, idx), + &first_buffer, + &first); + int cmp = 0; + if (SUCCESS(rc)) { + rc = btree_compare_leaf_key_checked( + cfg, + node.hdr, + first, + btree_get_leaf_entry(cfg, node.hdr, idx + 1), + &cmp); + } + if (!SUCCESS(rc)) { + btree_node_unget(cc, cfg, &node); + goto out; + } if (cmp >= 0) { platform_error_log("out of order tuples\n"); platform_error_log("addr: %lu idx %2u\n", node.addr, idx); @@ -6552,11 +7183,13 @@ btree_verify_node(cache *cc, goto out; } } - if (btree_key_compare(cfg, - btree_get_pivot(cfg, node.hdr, idx), - btree_get_pivot(cfg, node.hdr, idx + 1)) - >= 0) - { + int cmp; + rc = btree_key_compare_checked( + cfg, + btree_get_pivot(cfg, node.hdr, idx), + btree_get_pivot(cfg, node.hdr, idx + 1), + &cmp); + if (!SUCCESS(rc) || cmp >= 0) { btree_node_unget(cc, cfg, &child); btree_node_unget(cc, cfg, &node); btree_print_tree(Platform_error_log_handle, cc, cfg, addr, type); @@ -6576,14 +7209,16 @@ btree_verify_node(cache *cc, goto out; } } - if (0 < idx - && btree_compare_leaf_key( - cfg, - child.hdr, - btree_get_pivot(cfg, node.hdr, idx), - btree_get_leaf_entry(cfg, child.hdr, 0)) - != 0) - { + int cmp = 0; + if (0 < idx) { + rc = btree_compare_leaf_key_checked( + cfg, + child.hdr, + btree_get_pivot(cfg, node.hdr, idx), + btree_get_leaf_entry(cfg, child.hdr, 0), + &cmp); + } + if (!SUCCESS(rc) || cmp != 0) { platform_error_log( "pivot key doesn't match in child and parent\n"); platform_error_log("addr: %lu idx %u\n", node.addr, idx); @@ -6614,15 +7249,17 @@ btree_verify_node(cache *cc, goto out; } } - if (idx != btree_num_entries(node.hdr) - 1 - && btree_compare_leaf_key( - cfg, - child.hdr, - btree_get_pivot(cfg, node.hdr, idx + 1), - btree_get_leaf_entry( - cfg, child.hdr, btree_num_entries(child.hdr) - 1)) - < 0) - { + int cmp = 0; + if (idx != btree_num_entries(node.hdr) - 1) { + rc = btree_compare_leaf_key_checked( + cfg, + child.hdr, + btree_get_pivot(cfg, node.hdr, idx + 1), + btree_get_leaf_entry( + cfg, child.hdr, btree_num_entries(child.hdr) - 1), + &cmp); + } + if (!SUCCESS(rc) || cmp < 0) { platform_error_log("child tuple larger than parent bound\n"); platform_error_log("addr: %lu idx %u\n", node.addr, idx); platform_error_log("child addr: %lu idx %u\n", child.addr, idx); @@ -6638,7 +7275,6 @@ btree_verify_node(cache *cc, child.addr, child.hdr, type); - platform_assert(0); btree_node_unget(cc, cfg, &child); btree_node_unget(cc, cfg, &node); goto out; @@ -6664,14 +7300,16 @@ btree_verify_node(cache *cc, goto out; } } - if (idx != btree_num_entries(node.hdr) - 1 - && btree_key_compare( - cfg, - btree_get_pivot(cfg, node.hdr, idx + 1), - btree_get_pivot( - cfg, child.hdr, btree_num_entries(child.hdr) - 1)) - < 0) - { + int cmp = 0; + if (idx != btree_num_entries(node.hdr) - 1) { + rc = btree_key_compare_checked( + cfg, + btree_get_pivot(cfg, node.hdr, idx + 1), + btree_get_pivot( + cfg, child.hdr, btree_num_entries(child.hdr) - 1), + &cmp); + } + if (!SUCCESS(rc) || cmp < 0) { platform_error_log("child pivot larger than parent bound\n"); platform_error_log("addr: %lu idx %u\n", node.addr, idx); platform_error_log("child addr: %lu idx %u\n", child.addr, idx); @@ -6687,7 +7325,6 @@ btree_verify_node(cache *cc, child.addr, child.hdr, type); - platform_assert(0); btree_node_unget(cc, cfg, &child); btree_node_unget(cc, cfg, &node); goto out; @@ -6712,7 +7349,11 @@ btree_verify_node(cache *cc, bool32 btree_verify_tree(cache *cc, btree_config *cfg, uint64 addr, page_type type) { - return btree_verify_node(cc, cfg, addr, type, TRUE); + btree_config local_cfg = *cfg; + local_cfg.cc = cc; + local_cfg.blob_key_prefix_size = + cfg->data_cfg->key_compare_is_lexical ? cfg->blob_key_prefix_size : 0; + return btree_verify_node(cc, &local_cfg, addr, type, TRUE); } void @@ -6739,7 +7380,7 @@ btree_print_lookup(cache *cc, // IN bool32 found; if (type == PAGE_TYPE_BRANCH) { rc = btree_find_pivot_checked( - cfg, node.hdr, target, &found, &child_idx, NULL); + cfg, node.hdr, &target, &found, &child_idx, NULL); if (!SUCCESS(rc)) { btree_print_invalid_node(Platform_default_log_handle, node.addr); btree_node_unget(cc, cfg, &node); @@ -6811,14 +7452,16 @@ btree_config_init(btree_config *btree_cfg, cache_config *cache_cfg, data_config *data_cfg) { + btree_cfg->cc = NULL; + btree_cfg->blob_key_prefix_size = + data_cfg->key_compare_is_lexical ? BTREE_DEFAULT_KEY_PREFIX_SIZE : 0; btree_cfg->cache_cfg = cache_cfg; btree_cfg->data_cfg = data_cfg; btree_cfg->page_size = cache_config_page_size(cache_cfg); btree_cfg->extent_size = cache_config_extent_size(cache_cfg); - btree_cfg->blob_cfg = (blob_build_config){ - .extent_batch = 0, - .page_batch = 1, - .subpage_batch = 2, + btree_cfg->blob_cfg = (mini_blob_config){ + .page_batch = 0, + .subpage_batch = 1, .blob_max_descriptor_size = BLOB_DEFAULT_MAX_DESCRIPTOR_SIZE, }; @@ -6829,3 +7472,20 @@ btree_config_init(btree_config *btree_cfg, + max_inline_msg_size + sizeof(table_entry); platform_assert(max_entry_space < (page_size - sizeof(btree_hdr)) / 2); } + +platform_status +btree_config_validate(const btree_config *cfg) +{ + uint64 limit = MAX_INLINE_KEY_SIZE(btree_page_size(cfg)); + uint64 descriptor = cfg->blob_cfg.blob_max_descriptor_size + ? cfg->blob_cfg.blob_max_descriptor_size + : BLOB_DEFAULT_MAX_DESCRIPTOR_SIZE; + uint64 prefix = + cfg->data_cfg->key_compare_is_lexical ? cfg->blob_key_prefix_size : 0; + if (descriptor < BLOB_MIN_MAX_DESCRIPTOR_SIZE || descriptor > limit + || prefix > limit - descriptor) + { + return STATUS_BAD_PARAM; + } + return STATUS_OK; +} diff --git a/src/btree.h b/src/btree.h index 3fe0f867b..10498eada 100644 --- a/src/btree.h +++ b/src/btree.h @@ -13,7 +13,7 @@ #include "platform_log.h" #include "platform_typed_alloc.h" #include "async.h" -#include "blob_build.h" +#include "mini_blob.h" #include "mini_allocator.h" #include "iterator.h" #include "lookup_result.h" @@ -34,15 +34,20 @@ #define BTREE_MAX_HEIGHT (8) /* - * Immutable branch versions travel with the root address. Query and recovery - * entry points validate the version before interpreting branch pages. The - * Version 2 adds the common-prefix length to the page header and encodes - * branch leaf keys as suffixes when that length is nonzero. The construction - * opt-in is not required to read existing prefix-compressed leaves. - * The version argument is ignored for PAGE_TYPE_MEMTABLE (an in-memory - * structure). + * A branch pointer carries the version of the entire B-tree format and its + * fixed blob-key prefix length. Version 4 stores contiguous counters after + * each index key, with an entry flag selecting 32-bit or 64-bit fields. */ -#define BTREE_BRANCH_FORMAT_VERSION (2) +#define BTREE_BRANCH_FORMAT_VERSION (6) +#define BTREE_DEFAULT_KEY_PREFIX_SIZE UINT64_C(256) + +typedef struct ONDISK btree_format { + uint16 version; + uint16 key_prefix_size; +} btree_format; + +#define BTREE_DEFAULT_FORMAT \ + ((btree_format){BTREE_BRANCH_FORMAT_VERSION, BTREE_DEFAULT_KEY_PREFIX_SIZE}) /* * Mini-allocator uses separate batches for each height of the BTree. @@ -68,7 +73,6 @@ _Static_assert(BTREE_MAX_HEIGHT + NUM_BLOB_BATCHES <= MINI_MAX_BATCHES, * * (This limit is indirectly 'disk-resident' as it affects the node's layout.) */ -#define MAX_INLINE_KEY_SIZE(page_size) ((page_size) / 8) // Bytes /* * Size of messages are limited so that a single split will always enable an @@ -93,13 +97,28 @@ _Static_assert(BTREE_MAX_HEIGHT + NUM_BLOB_BATCHES <= MINI_MAX_BATCHES, *---------------------------------------------------------------------- */ typedef struct btree_config { - cache_config *cache_cfg; - data_config *data_cfg; - uint64 page_size; - uint64 extent_size; - blob_build_config blob_cfg; + cache_config *cache_cfg; + data_config *data_cfg; + uint64 page_size; + uint64 extent_size; + mini_blob_config blob_cfg; + uint64 blob_key_prefix_size; + cache *cc; /* Per-operation copy: used only for stored blob keys. */ } btree_config; +static inline btree_format +btree_config_format(const btree_config *cfg) +{ + return (btree_format){ + .version = BTREE_BRANCH_FORMAT_VERSION, + .key_prefix_size = + cfg->data_cfg->key_compare_is_lexical ? cfg->blob_key_prefix_size : 0, + }; +} + +platform_status +btree_config_validate(const btree_config *cfg); + typedef struct ONDISK btree_hdr btree_hdr; typedef struct btree_node { @@ -133,8 +152,8 @@ btree_scratch_size(const btree_config *cfg) * Iterators at (height > 0) return this struct as a value for each pivot. * ************************************************************************* */ -/* In-memory counters. Index entries retain the legacy narrow representation - * unless a flagged high-word extension is needed; see btree_private.h. */ +/* In-memory counters are always 64-bit. The on-disk representation uses three + * contiguous 32-bit or 64-bit fields; see btree_private.h. */ typedef struct btree_pivot_stats { uint64 num_kvs; uint64 key_bytes; @@ -193,6 +212,7 @@ typedef struct btree_iterator { iterator super; cache *cc; const btree_config *cfg; + btree_config local_cfg; uint32 height; page_type page_type; // Active memtable iterators copy nodes here and release page locks. @@ -229,6 +249,7 @@ typedef struct btree_pack_req { // inputs to the pack cache *cc; const btree_config *cfg; + btree_config local_cfg; iterator *itor; // the itor which is being packed uint64 max_tuples; unsigned int seed; // seed used for calculating filter_hash @@ -242,14 +263,17 @@ typedef struct btree_pack_req { bool32 level_has_nodes[BTREE_MAX_HEIGHT]; uint64 next_page_no[BTREE_MAX_HEIGHT]; merge_accumulator blob_buffer; + key_buffer key_blob_buffer; + bool32 leaf_has_blob_key; // Current packing leaf only. mini_allocator mini; // output of the compaction - uint64 root_addr; // root address of the output tree - uint64 num_tuples; // no. of tuples in the output tree - uint64 key_bytes; // total size of keys in tuples of the output tree - uint64 message_bytes; // total size of msgs in tuples of the output tree + btree_format format; + uint64 root_addr; // root address of the output tree + uint64 num_tuples; // no. of tuples in the output tree + uint64 key_bytes; // total size of keys in tuples of the output tree + uint64 message_bytes; // total size of msgs in tuples of the output tree } btree_pack_req; typedef platform_status (*btree_insert_callback_fn)(void *arg); @@ -316,7 +340,7 @@ btree_create(cache *cc, * also validate before reading pages. Refcount operations below require a * validated root or one freshly created by this module. */ platform_status -btree_validate_root(uint64 root_addr, uint16 format_version, page_type type); +btree_validate_root(uint64 root_addr, btree_format format, page_type type); __attribute__((warn_unused_result)) platform_status btree_inc_ref(cache *cc, const btree_config *cfg, uint64 root_addr); @@ -331,7 +355,7 @@ platform_status btree_get_root_height(cache *cc, const btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, page_type type, uint32 *height); @@ -343,7 +367,7 @@ platform_status btree_recover_allocations(cache *cc, const btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, page_type type); void @@ -353,7 +377,7 @@ platform_status btree_lookup(cache *cc, btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, page_type type, key target, lookup_result *result); @@ -368,9 +392,9 @@ platform_status btree_lookup_and_merge(cache *cc, const btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, page_type type, - key target, + const key *target, lookup_result *result, bool32 *local_found); @@ -389,13 +413,14 @@ DEFINE_ASYNC_STATE(btree_lookup_async_state, 3, param, cache *, cc, param, const btree_config *, cfg, param, uint64, root_addr, - param, uint16, format_version, + param, btree_format, format, param, page_type, type, param, key, target, param, lookup_result *, result, param, async_callback_fn, callback, param, void *, callback_arg, local, platform_status, __async_result, + local, btree_config, local_cfg, local, uint16, stop_at_height, local, btree_pivot_stats *, stats, local, bool32, get_node, @@ -410,18 +435,18 @@ btree_lookup_and_merge_async_state_init(btree_lookup_async_state *state, cache *cc, const btree_config *cfg, uint64 root_addr, - uint16 format_version, - page_type type, - key target, - lookup_result *result, - async_callback_fn callback, - void *callback_arg) + btree_format format, + page_type type, + key target, + lookup_result *result, + async_callback_fn callback, + void *callback_arg) { btree_lookup_async_state_init(state, cc, cfg, root_addr, - format_version, + format, type, target, result, @@ -455,7 +480,7 @@ btree_iterator_init(cache *cc, const btree_config *cfg, btree_iterator *itor, uint64 root_addr, - uint16 format_version, + btree_format format, page_type page_type, const btree_iterator_init_options *options); @@ -465,7 +490,7 @@ DEFINE_ASYNC_STATE(btree_iterator_async_state, 6, param, const btree_config *, cfg, param, btree_iterator *, itor, param, uint64, root_addr, - param, uint16, format_version, + param, btree_format, format, param, page_type, type, param, comparison, min_key_comparison, param, key, min_key, @@ -530,15 +555,25 @@ btree_pack_req_init_full(btree_pack_req *req, platform_heap_id hid, const btree_pack_req_init_options *options) { - unsigned int seed = options->seed; - bool32 collect_fingerprints = options->collect_fingerprints; + unsigned int seed = options->seed; + platform_status rc = btree_config_validate(cfg); + if (!SUCCESS(rc)) { + return rc; + } + bool32 collect_fingerprints = options->collect_fingerprints; memset(req, 0, sizeof(*req)); - req->cc = cc; - req->cfg = cfg; + req->cc = cc; + req->local_cfg = *cfg; + req->local_cfg.cc = cc; + req->local_cfg.blob_key_prefix_size = + btree_config_format(cfg).key_prefix_size; + req->cfg = &req->local_cfg; req->itor = itor; req->max_tuples = max_tuples; req->seed = seed; + req->format = btree_config_format(cfg); merge_accumulator_init(&req->blob_buffer, hid); + key_buffer_init(&req->key_blob_buffer, hid); if (collect_fingerprints && cfg->data_cfg->key_hash != NULL && max_tuples > 0) { @@ -558,6 +593,7 @@ static inline void btree_pack_req_deinit(btree_pack_req *req, platform_heap_id hid) { merge_accumulator_deinit(&req->blob_buffer); + key_buffer_deinit(&req->key_blob_buffer); if (req->fingerprint_arr) { platform_free(hid, req->fingerprint_arr); } @@ -570,7 +606,7 @@ platform_status btree_count_in_range(cache *cc, const btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, key min_key, key max_key, btree_pivot_stats *stats); @@ -579,7 +615,7 @@ platform_status btree_count_in_range_by_iterator(cache *cc, btree_config *cfg, uint64 root_addr, - uint16 format_version, + btree_format format, key min_key, key max_key, btree_pivot_stats *stats); @@ -644,13 +680,42 @@ btree_config_init(btree_config *btree_cfg, cache_config *cache_cfg, data_config *data_cfg); +platform_status +btree_key_validate(const btree_config *cfg, key k); + +platform_status +btree_key_materialize(const btree_config *cfg, key k, key_buffer *result); + +platform_status +btree_key_encode(const btree_config *cfg, + mini_allocator *mini, + key logical, + key_buffer *result); + +platform_status +btree_blob_key_compare(const btree_config *cfg, key a, key b, int *cmp); + +/* Keep ordinary comparisons in the caller; only blob keys need fallible I/O. */ +static inline platform_status +btree_key_compare_checked(const btree_config *cfg, key a, key b, int *cmp) +{ + if (!a.is_blob && !b.is_blob) { + *cmp = data_key_compare(cfg->data_cfg, a, b); + return STATUS_OK; + } + return btree_blob_key_compare(cfg, a, b, cmp); +} + // robj: I propose making all the following functions private to // btree.c static inline int btree_key_compare(const btree_config *cfg, key key1, key key2) { - return data_key_compare(cfg->data_cfg, key1, key2); + int cmp; + platform_status rc = btree_key_compare_checked(cfg, key1, key2, &cmp); + platform_assert_status_ok(rc); + return cmp; } static inline void diff --git a/src/btree_key.c b/src/btree_key.c new file mode 100644 index 000000000..972e7846f --- /dev/null +++ b/src/btree_key.c @@ -0,0 +1,279 @@ +// Copyright 2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "btree.h" + +/* These views describe logical keys; descriptor bytes never participate in + * ordering. P comes from the owning tree, not from an individual entry. */ +typedef struct { + slice prefix; + slice descriptor; + uint64 length; + bool32 blob; +} btree_key_view; + +static platform_status +btree_key_view_init(const btree_config *cfg, key k, btree_key_view *view) +{ + *view = (btree_key_view){ + .prefix = key_slice(k), .length = key_length(k), .blob = k.is_blob}; + if (!k.is_blob) { + return STATUS_OK; + } + uint64 p = cfg->blob_key_prefix_size; + if (cfg->cc == NULL || p > key_length(k) || key_length(k) - p < sizeof(blob)) + { + return STATUS_INVALID_STATE; + } + view->prefix = slice_create(p, key_data(k)); + view->descriptor = + slice_create(key_length(k) - p, (const char *)key_data(k) + p); + parsed_blob parsed; + platform_status rc = + blob_parse_descriptor(cfg->cc, view->descriptor, &parsed); + if (!SUCCESS(rc)) { + return rc; + } + view->length = blob_length(view->descriptor); + if (view->length <= MAX_INLINE_KEY_SIZE(cfg->page_size) || view->length <= p) + { + return STATUS_INVALID_STATE; + } + return STATUS_OK; +} + +/* Keep descriptor parsing and its scratch frame out of inline-key searches, + * including when link-time optimization can see this definition. */ +__attribute__((noinline)) platform_status +btree_key_validate(const btree_config *cfg, key k) +{ + if (!key_is_user_key(k)) { + return k.is_blob ? STATUS_INVALID_STATE : STATUS_OK; + } + if (key_length(k) > MAX_INLINE_KEY_SIZE(cfg->page_size)) { + return STATUS_INVALID_STATE; + } + btree_key_view view; + return btree_key_view_init(cfg, k, &view); +} + +platform_status +btree_key_materialize(const btree_config *cfg, key stored, key_buffer *result) +{ + if (!stored.is_blob) { + return key_buffer_copy_key(result, stored); + } + btree_key_view view; + platform_status rc = btree_key_view_init(cfg, stored, &view); + if (!SUCCESS(rc)) { + return rc; + } + rc = blob_materialize_full(cfg->cc, view.descriptor, &result->wb); + if (SUCCESS(rc)) { + result->kind = USER_KEY; + result->is_query_key = stored.is_query_key; + result->is_blob = FALSE; + if (slice_length(view.prefix) + && memcmp(key_buffer_data(result), + slice_data(view.prefix), + slice_length(view.prefix)) + != 0) + { + return STATUS_INVALID_STATE; + } + } + return rc; +} + +platform_status +btree_key_encode(const btree_config *cfg, + mini_allocator *mini, + key logical, + key_buffer *result) +{ + platform_assert(!logical.is_blob); + if (key_length(logical) <= MAX_INLINE_KEY_SIZE(cfg->page_size)) { + return key_buffer_copy_key(result, logical); + } + auto_writable_buffer descriptor; + writable_buffer_init(&descriptor, result->wb.heap_id); + platform_status rc = mini_blob_build( + &cfg->blob_cfg, cfg->cc, mini, key_slice(logical), &descriptor); + if (!SUCCESS(rc)) { + return rc; + } + uint64 p = cfg->blob_key_prefix_size; + uint64 d = writable_buffer_length(&descriptor); + if (p > MAX_INLINE_KEY_SIZE(cfg->page_size) + || d > MAX_INLINE_KEY_SIZE(cfg->page_size) - p) + { + return STATUS_LIMIT_EXCEEDED; + } + rc = key_buffer_resize(result, p + d); + if (!SUCCESS(rc)) { + return rc; + } + if (p) { + memcpy(key_buffer_data(result), key_data(logical), p); + } + memcpy((char *)key_buffer_data(result) + p, + writable_buffer_data(&descriptor), + d); + result->is_blob = TRUE; + result->is_query_key = FALSE; + return STATUS_OK; +} + +typedef struct { + const btree_key_view *view; + blob_page_iterator iter; + uint64 offset; + bool32 live; +} btree_key_cursor; + +static platform_status +btree_key_cursor_init(const btree_config *cfg, + btree_key_cursor *c, + const btree_key_view *view, + uint64 offset) +{ + *c = (btree_key_cursor){.view = view, .offset = offset}; + if (!view->blob || offset == view->length) { + return STATUS_OK; + } + platform_status rc = + blob_page_iterator_init(cfg->cc, + &c->iter, + view->descriptor, + offset, + BLOB_PAGE_ITERATOR_MODE_NO_PREFETCH); + c->live = SUCCESS(rc); + return rc; +} + +static void +btree_key_cursor_deinit(btree_key_cursor *c) +{ + if (c->live) { + blob_page_iterator_deinit(&c->iter); + } +} + +static platform_status +btree_key_cursor_get(btree_key_cursor *c, slice *part) +{ + if (!c->view->blob) { + *part = + slice_create(c->view->length - c->offset, + (const char *)slice_data(c->view->prefix) + c->offset); + return STATUS_OK; + } + uint64 offset; + return blob_page_iterator_get_curr(&c->iter, &offset, part); +} + +static void +btree_key_cursor_advance(btree_key_cursor *c, uint64 bytes) +{ + c->offset += bytes; + if (c->live) { + blob_page_iterator_advance_bytes(&c->iter, bytes); + } +} + +static platform_status +btree_key_compare_streamed(const btree_config *cfg, + const btree_key_view *a, + const btree_key_view *b, + uint64 offset, + int *cmp) +{ + btree_key_cursor left = {0}, right = {0}; + platform_status rc = btree_key_cursor_init(cfg, &left, a, offset); + if (!SUCCESS(rc)) { + goto out; + } + rc = btree_key_cursor_init(cfg, &right, b, offset); + if (!SUCCESS(rc)) { + goto out; + } + uint64 end = MIN(a->length, b->length); + *cmp = 0; + while (offset < end) { + slice lpart, rpart; + rc = btree_key_cursor_get(&left, &lpart); + if (!SUCCESS(rc)) { + goto out; + } + rc = btree_key_cursor_get(&right, &rpart); + if (!SUCCESS(rc)) { + goto out; + } + uint64 bytes = + MIN(MIN(slice_length(lpart), slice_length(rpart)), end - offset); + if (bytes == 0) { + rc = STATUS_INVALID_STATE; + goto out; + } + *cmp = memcmp(slice_data(lpart), slice_data(rpart), bytes); + if (*cmp) { + goto out; + } + offset += bytes; + if (offset < end) { + btree_key_cursor_advance(&left, bytes); + btree_key_cursor_advance(&right, bytes); + } + } + *cmp = (a->length > b->length) - (a->length < b->length); +out: + btree_key_cursor_deinit(&right); + btree_key_cursor_deinit(&left); + return rc; +} + +/* The inline comparison is in btree_key_compare_checked(). Do not pull the + * blob cursor and materialization paths into every ordinary pivot search. */ +__attribute__((noinline)) platform_status +btree_blob_key_compare(const btree_config *cfg, key a, key b, int *cmp) +{ + if (!key_is_user_key(a) || !key_is_user_key(b) || (!a.is_blob && !b.is_blob)) + { + *cmp = data_key_compare(cfg->data_cfg, a, b); + return STATUS_OK; + } + btree_key_view av, bv; + platform_status rc = btree_key_view_init(cfg, a, &av); + if (!SUCCESS(rc)) { + return rc; + } + rc = btree_key_view_init(cfg, b, &bv); + if (!SUCCESS(rc)) { + return rc; + } + if (cfg->data_cfg->key_compare_is_lexical) { + uint64 common = MIN(slice_length(av.prefix), slice_length(bv.prefix)); + *cmp = common + ? memcmp(slice_data(av.prefix), slice_data(bv.prefix), common) + : 0; + if (*cmp || common == MIN(av.length, bv.length)) { + if (!*cmp) { + *cmp = (av.length > bv.length) - (av.length < bv.length); + } + return STATUS_OK; + } + return btree_key_compare_streamed(cfg, &av, &bv, common, cmp); + } + DECLARE_AUTO_KEY_BUFFER(left, PROCESS_PRIVATE_HEAP_ID); + DECLARE_AUTO_KEY_BUFFER(right, PROCESS_PRIVATE_HEAP_ID); + rc = btree_key_materialize(cfg, a, &left); + if (!SUCCESS(rc)) { + return rc; + } + rc = btree_key_materialize(cfg, b, &right); + if (SUCCESS(rc)) { + *cmp = data_key_compare( + cfg->data_cfg, key_buffer_key(&left), key_buffer_key(&right)); + } + return rc; +} diff --git a/src/btree_private.h b/src/btree_private.h index c39fb10df..c495cc939 100644 --- a/src/btree_private.h +++ b/src/btree_private.h @@ -74,83 +74,83 @@ struct ONDISK btree_hdr { * ************************************************************************* */ #define BTREE_INDEX_WIDE_STATS (1 << 0) -#define BTREE_LEGACY_UNKNOWN_COUNTER UINT32_C(0x7fffffff) -#define BTREE_WIDE_STATS_BYTES (3 * sizeof(uint32)) +#define BTREE_NARROW_UNKNOWN_COUNTER UINT32_MAX +typedef struct ONDISK btree_pivot_stats32 { + uint32 num_kvs; + uint32 key_bytes; + uint32 message_bytes; +} btree_pivot_stats32; + +_Static_assert(sizeof(btree_pivot_stats) == 3 * sizeof(uint64), + "wide on-disk counters must be contiguous"); + +/* The key has the same offset in both encodings. Immediately after its bytes + * are three contiguous uint32 or uint64 counters, selected by the flags byte. + * Access counters with memcpy because the key need not end at an aligned + * address. + */ typedef struct ONDISK index_entry { - struct ONDISK { - uint64 child_addr; - struct ONDISK { - uint32 num_kvs; - uint32 key_bytes; - uint32 message_bytes; - } stats; - } pivot_data; + uint64 child_addr; ondisk_key pivot; - /* With BTREE_INDEX_WIDE_STATS, three high uint32 words follow the key. - * All three counters then contain literal 64-bit values. Otherwise the - * layout and unknown sentinel are exactly the original on-disk format. */ } index_entry; static inline bool32 btree_stats_need_wide(btree_pivot_stats stats) { - return (stats.num_kvs >= BTREE_LEGACY_UNKNOWN_COUNTER + return (stats.num_kvs >= BTREE_NARROW_UNKNOWN_COUNTER && stats.num_kvs != UINT64_MAX) - || (stats.key_bytes >= BTREE_LEGACY_UNKNOWN_COUNTER + || (stats.key_bytes >= BTREE_NARROW_UNKNOWN_COUNTER && stats.key_bytes != UINT64_MAX) - || (stats.message_bytes >= BTREE_LEGACY_UNKNOWN_COUNTER + || (stats.message_bytes >= BTREE_NARROW_UNKNOWN_COUNTER && stats.message_bytes != UINT64_MAX); } +static inline uint64 +btree_stats_size(btree_pivot_stats stats) +{ + return btree_stats_need_wide(stats) ? sizeof(btree_pivot_stats) + : sizeof(btree_pivot_stats32); +} + static inline btree_pivot_stats index_entry_stats(const index_entry *entry) { - uint64 values[3] = {entry->pivot_data.stats.num_kvs, - entry->pivot_data.stats.key_bytes, - entry->pivot_data.stats.message_bytes}; + const void *data = + entry->pivot.bytes + sizeof_ondisk_key_data(&entry->pivot); if (entry->pivot.flags & BTREE_INDEX_WIDE_STATS) { - uint32 high[3]; - memcpy(high, - entry->pivot.bytes + sizeof_ondisk_key_data(&entry->pivot), - sizeof(high)); - for (uint64 i = 0; i < 3; i++) { - values[i] |= (uint64)high[i] << 32; - } - } else { - for (uint64 i = 0; i < 3; i++) { - if (values[i] == BTREE_LEGACY_UNKNOWN_COUNTER) { - values[i] = UINT64_MAX; - } - } + btree_pivot_stats stats; + memcpy(&stats, data, sizeof(stats)); + return stats; } - return (btree_pivot_stats){values[0], values[1], values[2]}; + btree_pivot_stats32 stats; + memcpy(&stats, data, sizeof(stats)); + return (btree_pivot_stats){ + .num_kvs = stats.num_kvs == BTREE_NARROW_UNKNOWN_COUNTER ? UINT64_MAX + : stats.num_kvs, + .key_bytes = stats.key_bytes == BTREE_NARROW_UNKNOWN_COUNTER + ? UINT64_MAX + : stats.key_bytes, + .message_bytes = stats.message_bytes == BTREE_NARROW_UNKNOWN_COUNTER + ? UINT64_MAX + : stats.message_bytes, + }; } static inline void index_entry_set_stats(index_entry *entry, btree_pivot_stats stats) { - uint64 values[3] = {stats.num_kvs, stats.key_bytes, stats.message_bytes}; + void *data = entry->pivot.bytes + sizeof_ondisk_key_data(&entry->pivot); if (btree_stats_need_wide(stats)) { - uint32 high[3]; - for (uint64 i = 0; i < 3; i++) { - high[i] = values[i] >> 32; - } - memcpy(entry->pivot.bytes + sizeof_ondisk_key_data(&entry->pivot), - high, - sizeof(high)); - entry->pivot.flags = BTREE_INDEX_WIDE_STATS; + entry->pivot.flags |= BTREE_INDEX_WIDE_STATS; + memcpy(data, &stats, sizeof(stats)); } else { - entry->pivot.flags = ONDISK_KEY_DEFAULT_FLAGS; - for (uint64 i = 0; i < 3; i++) { - if (values[i] == UINT64_MAX) { - values[i] = BTREE_LEGACY_UNKNOWN_COUNTER; - } - } + entry->pivot.flags &= ~BTREE_INDEX_WIDE_STATS; + /* Casting UINT64_MAX preserves the narrow unknown sentinel. */ + btree_pivot_stats32 narrow = { + stats.num_kvs, stats.key_bytes, stats.message_bytes}; + memcpy(data, &narrow, sizeof(narrow)); } - entry->pivot_data.stats.num_kvs = values[0]; - entry->pivot_data.stats.key_bytes = values[1]; - entry->pivot_data.stats.message_bytes = values[2]; } /* @@ -177,14 +177,15 @@ typedef struct leaf_incorporate_spec { } msg; } leaf_incorporate_spec; +/* Borrowed inputs must remain valid through this call and not alias spec. */ platform_status btree_create_leaf_incorporate_spec(const btree_config *cfg, cache *cc, mini_allocator *mini, platform_heap_id heap_id, btree_hdr *hdr, - key tuple_key, - message message, + const key *tuple_key, + const message *message, leaf_incorporate_spec *spec); bool32 @@ -444,8 +445,8 @@ sizeof_index_entry(const index_entry *entry) { return sizeof(*entry) + sizeof_ondisk_key_data(&entry->pivot) + ((entry->pivot.flags & BTREE_INDEX_WIDE_STATS) - ? BTREE_WIDE_STATS_BYTES - : 0); + ? sizeof(btree_pivot_stats) + : sizeof(btree_pivot_stats32)); } static inline uint64 @@ -463,7 +464,7 @@ index_entry_key(const index_entry *entry) static inline uint64 index_entry_child_addr(const index_entry *entry) { - return entry->pivot_data.child_addr; + return entry->child_addr; } static inline key @@ -560,7 +561,8 @@ btree_leaf_full_key(const btree_config *cfg, memcpy((char *)key_buffer_data(buffer) + hdr->prefix_length, entry->key_and_message, entry->key_length); - *result = key_buffer_key(buffer); + buffer->is_blob = (entry->key_flags & ONDISK_KEY_IS_BLOB) != 0; + *result = key_buffer_key(buffer); return STATUS_OK; } diff --git a/src/core.c b/src/core.c index 6374c0b69..8286ddaf2 100644 --- a/src/core.c +++ b/src/core.c @@ -1241,7 +1241,7 @@ core_memtable_iterator_init(core_handle *spl, spl->cfg.btree_cfg, itor, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &(const btree_iterator_init_options){0}); } @@ -1819,9 +1819,9 @@ core_memtable_lookup(core_handle *spl, return btree_lookup_and_merge(cc, cfg, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + btree_config_format(cfg), type, - target, + &target, result, NULL); } @@ -1910,7 +1910,7 @@ core_start_btree_iterator_init_async( core_btree_iterator_init_async_context *ctxt, btree_iterator *itor, uint64 root_addr, - uint16 format_version, + btree_format format, page_type page_type, comparison min_key_comparison, key min_key, @@ -1926,7 +1926,7 @@ core_start_btree_iterator_init_async( spl->cfg.btree_cfg, itor, root_addr, - format_version, + format, page_type, min_key_comparison, min_key, @@ -2277,8 +2277,8 @@ core_range_iterator_init_span(core_handle *spl, } range_itor->branch[range_itor->num_branches].addr = root_addr; - range_itor->branch[range_itor->num_branches].format_version = - compacted ? BTREE_BRANCH_FORMAT_VERSION : 0; + range_itor->branch[range_itor->num_branches].format = + btree_config_format(spl->cfg.btree_cfg); range_itor->branch[range_itor->num_branches].type = compacted ? PAGE_TYPE_BRANCH : PAGE_TYPE_MEMTABLE; range_itor->num_branches++; @@ -2387,7 +2387,7 @@ core_range_iterator_init_span(core_handle *spl, &init_ctxt[i], btree_itor, branch_addr, - range_itor->branch[branch_no].format_version, + range_itor->branch[branch_no].format, page_type, range_itor->local_min_key_comparison, key_buffer_key(&range_itor->local_min_key), @@ -2809,9 +2809,9 @@ core_fetch_and_update_catch_up(core_handle *spl, rc = btree_lookup_and_merge(spl->cc, spl->cfg.btree_cfg, refs[i].root_addr, - BTREE_BRANCH_FORMAT_VERSION, + btree_config_format(spl->cfg.btree_cfg), refs[i].type, - tuple_key, + &tuple_key, catch_up_result, NULL); if (!SUCCESS(rc)) { @@ -5066,7 +5066,7 @@ core_print_lookup(core_handle *spl, key target, platform_log_handle *log_handle) rc = btree_lookup(spl->cc, spl->cfg.btree_cfg, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, target, &lookup); diff --git a/src/data_blob_build.c b/src/data_blob_build.c index a59c1ab47..d802118c7 100644 --- a/src/data_blob_build.c +++ b/src/data_blob_build.c @@ -11,36 +11,36 @@ #include "poison.h" platform_status -message_to_blob(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - message msg, - merge_accumulator *ma) +message_to_blob(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + message msg, + merge_accumulator *ma) { platform_assert(!message_is_blob(msg)); ma->type = message_class(msg); ma->cc = cc; - return blob_build(cfg, cc, mini, message_slice(msg), &ma->data); + return mini_blob_build(cfg, cc, mini, message_slice(msg), &ma->data); } platform_status -message_clone(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - message msg, - merge_accumulator *result) +message_clone(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + message msg, + merge_accumulator *result) { platform_assert(message_is_blob(msg)); result->type = message_class(msg); result->cc = cc; - return blob_clone(cfg, cc, mini, message_slice(msg), &result->data); + return mini_blob_clone(cfg, cc, mini, message_slice(msg), &result->data); } platform_status -merge_accumulator_convert_to_blob(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - merge_accumulator *ma) +merge_accumulator_convert_to_blob(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + merge_accumulator *ma) { if (merge_accumulator_is_blob(ma)) { return STATUS_OK; @@ -48,8 +48,8 @@ merge_accumulator_convert_to_blob(const blob_build_config *cfg, writable_buffer blob; writable_buffer_init(&blob, ma->data.heap_id); - platform_status rc = - blob_build(cfg, cc, mini, writable_buffer_to_slice(&ma->data), &blob); + platform_status rc = mini_blob_build( + cfg, cc, mini, writable_buffer_to_slice(&ma->data), &blob); if (!SUCCESS(rc)) { writable_buffer_deinit(&blob); return rc; diff --git a/src/data_blob_build.h b/src/data_blob_build.h index 7bd9fddfa..725d9985d 100644 --- a/src/data_blob_build.h +++ b/src/data_blob_build.h @@ -9,25 +9,25 @@ #pragma once -#include "blob_build.h" +#include "mini_blob.h" #include "data_internal.h" platform_status -message_to_blob(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - message msg, - merge_accumulator *ma); +message_to_blob(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + message msg, + merge_accumulator *ma); platform_status -message_clone(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - message msg, - merge_accumulator *result); +message_clone(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + message msg, + merge_accumulator *result); platform_status -merge_accumulator_convert_to_blob(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - merge_accumulator *ma); +merge_accumulator_convert_to_blob(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + merge_accumulator *ma); diff --git a/src/data_internal.h b/src/data_internal.h index ac48e2982..a817369e4 100644 --- a/src/data_internal.h +++ b/src/data_internal.h @@ -41,11 +41,18 @@ typedef enum { POSITIVE_INFINITY = 1, } key_type; +// Shared inline-key policy for B-trees, trunk pivots, and the log. +#define MAX_INLINE_KEY_SIZE(page_size) ((page_size) / 8) // Bytes + typedef struct key { key_type kind; - bool32 is_query_key; - slice user_slice; + uint16 is_query_key; + // Internal stored-key encoding; never passed to callbacks. + uint16 is_blob; + slice user_slice; } key; +_Static_assert(sizeof(key) == sizeof(slice) + 8, + "Blob flags must not enlarge the frequently copied key"); #define NEGATIVE_INFINITY_KEY \ ((key){.kind = NEGATIVE_INFINITY, \ @@ -110,7 +117,7 @@ key_create(bool32 is_query_key, uint64 length, const void *data) static inline bool32 key_equals(key a, key b) { - return a.kind == b.kind + return a.kind == b.kind && a.is_blob == b.is_blob && IMPLIES(a.kind == USER_KEY, slice_equals(a.user_slice, b.user_slice)); } @@ -154,6 +161,7 @@ key_copy_contents(void *dst, key k) typedef struct { key_type kind; bool32 is_query_key; + bool32 is_blob; writable_buffer wb; } key_buffer; @@ -169,8 +177,10 @@ key_buffer_key(const key_buffer *kb) } else if (kb->kind == POSITIVE_INFINITY) { return POSITIVE_INFINITY_KEY; } else { - return key_create_from_slice(kb->is_query_key, - writable_buffer_to_slice(&kb->wb)); + key result = key_create_from_slice(kb->is_query_key, + writable_buffer_to_slice(&kb->wb)); + result.is_blob = kb->is_blob; + return result; } } @@ -185,6 +195,7 @@ key_buffer_init(key_buffer *kb, platform_heap_id hid) { kb->kind = USER_KEY; kb->is_query_key = FALSE; + kb->is_blob = FALSE; writable_buffer_init(&kb->wb, hid); writable_buffer_resize(&kb->wb, 0); } @@ -194,6 +205,7 @@ key_buffer_copy_slice(key_buffer *kb, bool32 is_query_key, slice src) { kb->kind = USER_KEY; kb->is_query_key = is_query_key; + kb->is_blob = FALSE; return writable_buffer_copy_slice(&kb->wb, src); } @@ -205,7 +217,10 @@ key_buffer_copy_key(key_buffer *kb, key src) } else if (key_is_positive_infinity(src)) { kb->kind = POSITIVE_INFINITY; } else { - return key_buffer_copy_slice(kb, src.is_query_key, key_slice(src)); + platform_status rc = + key_buffer_copy_slice(kb, src.is_query_key, key_slice(src)); + kb->is_blob = src.is_blob; + return rc; } return STATUS_OK; } @@ -227,7 +242,8 @@ key_buffer_length(key_buffer *kb) static inline platform_status key_buffer_resize(key_buffer *kb, uint64 length) { - kb->kind = USER_KEY; + kb->kind = USER_KEY; + kb->is_blob = FALSE; return writable_buffer_resize(&kb->wb, length); } @@ -274,6 +290,8 @@ typedef struct ONDISK ondisk_key { ((((ondisk_key_length)1) << ONDISK_KEY_LENGTH_BITS) - 2) #define ONDISK_KEY_DEFAULT_FLAGS (0) +/* Bit 0 is reserved for B-tree index statistics; it is invalid in leaf keys. */ +#define ONDISK_KEY_IS_BLOB (1 << 1) /* How big is the flexible array field of the given ondisk_key */ static inline uint64 @@ -306,14 +324,16 @@ ondisk_key_to_key(const ondisk_key *odk) } else if (odk->length == ONDISK_KEY_POSITIVE_INFINITY) { return POSITIVE_INFINITY_KEY; } else { - return key_create(FALSE, odk->length, odk->bytes); + return (key){.kind = USER_KEY, + .is_blob = (odk->flags & ONDISK_KEY_IS_BLOB) != 0, + .user_slice = slice_create(odk->length, odk->bytes)}; } } static inline void copy_key_to_ondisk_key(ondisk_key *odk, key k) { - odk->flags = ONDISK_KEY_DEFAULT_FLAGS; + odk->flags = k.is_blob ? ONDISK_KEY_IS_BLOB : ONDISK_KEY_DEFAULT_FLAGS; if (key_is_user_key(k)) { odk->length = key_length(k); memcpy(odk->bytes, key_data(k), key_length(k)); @@ -490,7 +510,10 @@ ondisk_tuple_required_data_capacity(key k, message msg) static inline key ondisk_tuple_key(const ondisk_tuple *odt) { - return key_create(FALSE, odt->key_length, odt->key_and_message); + return (key){.kind = USER_KEY, + .is_blob = (odt->key_flags & ONDISK_KEY_IS_BLOB) != 0, + .user_slice = + slice_create(odt->key_length, odt->key_and_message)}; } static inline message_type @@ -531,7 +554,7 @@ copy_tuple_to_ondisk_tuple(ondisk_tuple *odt, key k, message msg) { debug_assert(key_is_user_key(k)); odt->key_length = key_length(k); - odt->key_flags = ONDISK_KEY_DEFAULT_FLAGS; + odt->key_flags = k.is_blob ? ONDISK_KEY_IS_BLOB : ONDISK_KEY_DEFAULT_FLAGS; memcpy(odt->key_and_message, key_data(k), key_length(k)); copy_message_to_ondisk_tuple(odt, msg); } diff --git a/src/log_data.h b/src/log_data.h index 2e609b6fe..4f9f24b10 100644 --- a/src/log_data.h +++ b/src/log_data.h @@ -9,7 +9,7 @@ #include "splinterdb/public_platform.h" -#define SHARD_LOG_FORMAT_VERSION (1) +#define SHARD_LOG_FORMAT_VERSION (4) typedef struct log_nonce { uint64 high; diff --git a/src/memtable.h b/src/memtable.h index 0e7c9f527..a989960a6 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -410,6 +410,10 @@ memtable_ok_to_lookup_compacted(memtable *mt) bool32 memtable_is_empty(memtable_context *mt_ctxt); +/* Fullness counts allocated and attached extents, including shared storage. */ +bool32 +memtable_is_full(const memtable_config *cfg, memtable *mt); + static inline bool32 memtable_verify(cache *cc, memtable *mt) { diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 82f7e2680..e6a13cf56 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -38,6 +38,7 @@ typedef struct ONDISK mini_meta_hdr { uint64 meta_page_no; uint64 pos; uint32 num_entries; + uint64 num_charged_extents; uint16 format_version; char entry_buffer[]; } mini_meta_hdr; @@ -61,17 +62,20 @@ typedef struct ONDISK mini_meta_hdr { * originating mini_allocator batch: * * bits [ 0: 7] batch (one of MINI_MAX_BATCHES batches) - * bits [ 8:15] type (page_type) + * bits [ 8:11] type (page_type) + * bits [12:15] height (blob pointer depth; zero for other types) * bits [16:63] extent number (extent_addr / extent_size) */ typedef struct ONDISK meta_entry { uint64 packed; } meta_entry; -#define META_ENTRY_BATCH_BITS (8) -#define META_ENTRY_TYPE_BITS (8) -#define META_ENTRY_EXTENT_BITS \ - (64 - META_ENTRY_BATCH_BITS - META_ENTRY_TYPE_BITS) +#define META_ENTRY_BATCH_BITS (8) +#define META_ENTRY_TYPE_BITS (4) +#define META_ENTRY_HEIGHT_BITS (4) +#define META_ENTRY_EXTENT_SHIFT \ + (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS + META_ENTRY_HEIGHT_BITS) +#define META_ENTRY_EXTENT_BITS (64 - META_ENTRY_EXTENT_SHIFT) _Static_assert(MINI_MAX_BATCHES <= (1 << META_ENTRY_BATCH_BITS), "mini_allocator batch number does not fit in a meta_entry"); @@ -91,11 +95,20 @@ meta_entry_type(const meta_entry *entry) & ((1 << META_ENTRY_TYPE_BITS) - 1)); } +static inline uint64 +meta_entry_height(const meta_entry *entry) +{ + return (entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS)) + & ((1 << META_ENTRY_HEIGHT_BITS) - 1); +} + +_Static_assert(BLOB_MAX_HEIGHT <= (1 << META_ENTRY_HEIGHT_BITS), + "blob height does not fit in a meta_entry"); + static inline uint64 meta_entry_extent_addr(cache *cc, const meta_entry *entry) { - uint64 extent_number = - entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); + uint64 extent_number = entry->packed >> (META_ENTRY_EXTENT_SHIFT); return extent_number * cache_extent_size(cc); } @@ -104,7 +117,8 @@ meta_entry_pack(cache *cc, meta_entry *entry, uint64 extent_addr, page_type type, - uint64 batch) + uint64 batch, + uint64 height) { platform_assert(cc != NULL); platform_assert(entry != NULL); @@ -116,10 +130,12 @@ meta_entry_pack(cache *cc, platform_assert(extent_number < (1ULL << META_ENTRY_EXTENT_BITS)); platform_assert(PAGE_TYPE_FIRST <= type && type < NUM_PAGE_TYPES); platform_assert(batch < MINI_MAX_BATCHES); + platform_assert(height < BLOB_MAX_HEIGHT); + platform_assert(type == PAGE_TYPE_BLOB || height == 0); - entry->packed = - (extent_number << (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS)) - | ((uint64)type << META_ENTRY_BATCH_BITS) | batch; + entry->packed = (extent_number << (META_ENTRY_EXTENT_SHIFT)) + | (height << (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS)) + | ((uint64)type << META_ENTRY_BATCH_BITS) | batch; } static meta_entry * @@ -176,13 +192,14 @@ mini_init_meta_page(mini_allocator *mini, page_handle *meta_page, uint64 meta_page_no) { - mini_meta_hdr *hdr = (mini_meta_hdr *)page_handle_data(meta_page); - hdr->next_meta_addr = 0; - hdr->prev_meta_addr = 0; - hdr->meta_page_no = meta_page_no; - hdr->pos = offsetof(typeof(*hdr), entry_buffer); - hdr->num_entries = 0; - hdr->format_version = MINI_ALLOCATOR_FORMAT_VERSION; + mini_meta_hdr *hdr = (mini_meta_hdr *)page_handle_data(meta_page); + hdr->next_meta_addr = 0; + hdr->prev_meta_addr = 0; + hdr->meta_page_no = meta_page_no; + hdr->pos = offsetof(typeof(*hdr), entry_buffer); + hdr->num_entries = 0; + hdr->num_charged_extents = 0; + hdr->format_version = MINI_ALLOCATOR_FORMAT_VERSION; } /* @@ -421,7 +438,9 @@ mini_append_entry_to_page(mini_allocator *mini, page_handle *meta_page, uint64 extent_addr, page_type type, - uint64 batch) + uint64 batch, + uint64 height, + uint64 charge) { platform_assert(mini != NULL); platform_assert(mini->cc != NULL); @@ -435,15 +454,18 @@ mini_append_entry_to_page(mini_allocator *mini, mini_meta_hdr *hdr = (mini_meta_hdr *)page_handle_data(meta_page); - if (!entry_fits_in_page(page_size, hdr->pos, sizeof(meta_entry))) { + if (!entry_fits_in_page(page_size, hdr->pos, sizeof(meta_entry)) + || charge > UINT64_MAX - hdr->num_charged_extents) + { return FALSE; } meta_entry *new_entry = pointer_byte_offset(hdr, hdr->pos); - meta_entry_pack(mini->cc, new_entry, extent_addr, type, batch); + meta_entry_pack(mini->cc, new_entry, extent_addr, type, batch, height); hdr->pos += sizeof(meta_entry); hdr->num_entries++; + hdr->num_charged_extents += charge; return TRUE; } @@ -535,7 +557,13 @@ mini_set_prev_meta_addr(mini_allocator *mini, } static platform_status -mini_append_entry(mini_allocator *mini, uint64 batch, uint64 next_addr) +mini_append_entry(mini_allocator *mini, + uint64 batch, + uint64 next_addr, + page_type type, + uint64 height, + uint64 charge, + uint64 *entry_meta_addr) { page_handle meta_storage; page_handle *meta_page = &meta_storage; @@ -545,7 +573,7 @@ mini_append_entry(mini_allocator *mini, uint64 batch, uint64 next_addr) } bool32 success; success = mini_append_entry_to_page( - mini, meta_page, next_addr, mini->types[batch], batch); + mini, meta_page, next_addr, type, batch, height, charge); if (!success) { // need to allocate a new meta page uint64 old_meta_tail = @@ -580,23 +608,44 @@ mini_append_entry(mini_allocator *mini, uint64 batch, uint64 next_addr) mini_set_prev_meta_addr(mini, meta_page, last_meta_addr); success = mini_append_entry_to_page( - mini, meta_page, next_addr, mini->types[batch], batch); + mini, meta_page, next_addr, type, batch, height, charge); if (mini->pinned) { cache_pin(mini->cc, meta_page); } debug_assert(success); } - // Record the meta page that now holds this extent's entry, so btree nodes - // allocated from this extent can point straight at it (see - // mini_current_extent_meta_page). Safe: the caller holds the batch lock, so - // mini->meta_tail cannot advance under us until we unlock the page below. - mini->cur_extent_meta_page[batch] = - __atomic_load_n(&mini->meta_tail, __ATOMIC_RELAXED); + if (entry_meta_addr != NULL) { + *entry_meta_addr = page_handle_addr(meta_page); + } mini_full_unlock_meta_page(mini, meta_page); return STATUS_OK; } +platform_status +mini_adopt_blob_ref(mini_allocator *mini, + uint64 batch, + blob_ref ref, + uint64 num_extents) +{ + platform_status rc = blob_ref_validate(mini->cc, ref); + if (!SUCCESS(rc)) { + return rc; + } + if (batch >= mini->num_batches || mini->types[batch] != PAGE_TYPE_BLOB + || num_extents == 0 || (ref.height == 0 && num_extents != 1) + || (ref.height != 0 && num_extents <= ref.height)) + { + return STATUS_BAD_PARAM; + } + rc = mini_append_entry( + mini, batch, ref.addr, PAGE_TYPE_BLOB, ref.height, num_extents, NULL); + if (SUCCESS(rc)) { + __sync_fetch_and_add(&mini->num_extents, num_extents); + } + return rc; +} + platform_status mini_alloc_bytes(mini_allocator *mini, uint64 batch, @@ -644,7 +693,13 @@ mini_alloc_bytes(mini_allocator *mini, } next_addr = extent_addr; - rc = mini_append_entry(mini, batch, next_addr); + rc = mini_append_entry(mini, + batch, + next_addr, + mini->types[batch], + 0, + 1, + &mini->cur_extent_meta_page[batch]); if (!SUCCESS(rc)) { refcount ref = allocator_dec_ref(mini->al, new_next_extent, mini->types[batch]); @@ -736,12 +791,16 @@ mini_attach_extent(mini_allocator *mini, uint64 batch, uint64 addr) mini_unlock_batch_set_next_addr(mini, batch, old_next_addr); return rc; } - rc = mini_append_entry(mini, batch, addr); + rc = mini_append_entry(mini, batch, addr, mini->types[batch], 0, 1, NULL); if (!SUCCESS(rc)) { (void)allocator_dec_ref(mini->al, addr, mini->types[batch]); mini_unlock_batch_set_next_addr(mini, batch, old_next_addr); return rc; } + /* Memtable fullness bounds referenced data, even when extents are shared. + * Charge each successfully recorded reference, not just fresh allocations. + */ + __sync_fetch_and_add(&mini->num_extents, 1); mini_unlock_batch_set_next_addr(mini, batch, old_next_addr); return STATUS_OK; } @@ -974,13 +1033,14 @@ mini_validate_metadata_stream(cache *cc, uint64 meta_head, page_type type) return mini_for_each_meta_page(cc, meta_head, type, NULL, NULL, NULL); } -/* mini_for_each(): call a function on each allocated extent in the - * mini_allocator (not including the extents used by the mini_allocator itself). +/* mini_for_each(): visit owned extents and blob roots in one metadata pass + * (not including the extents used by the mini allocator itself). */ -typedef void (*mini_for_each_fn)(cache *cc, - page_type type, - uint64 extent_addr, - void *arg); +typedef platform_status (*mini_for_each_fn)(cache *cc, + page_type type, + uint64 extent_addr, + uint64 height, + void *arg); typedef struct for_each_func { mini_for_each_fn func; @@ -993,16 +1053,18 @@ mini_for_each_meta_page_func(cache *cc, page_handle *meta_page, void *arg) { - for_each_func *fef = (for_each_func *)arg; - - uint64 num_meta_entries = mini_num_entries(meta_page); - meta_entry *entry = first_entry(meta_page); - for (uint64 i = 0; i < num_meta_entries; i++) { - fef->func(cc, - meta_entry_type(entry), - meta_entry_extent_addr(cc, entry), - fef->arg); - entry = next_entry(entry); + for_each_func *fef = arg; + uint64 num_entries = mini_num_entries(meta_page); + meta_entry *entry = first_entry(meta_page); + for (uint64 i = 0; i < num_entries; i++, entry = next_entry(entry)) { + platform_status rc = fef->func(cc, + meta_entry_type(entry), + meta_entry_extent_addr(cc, entry), + meta_entry_height(entry), + fef->arg); + if (!SUCCESS(rc)) { + return rc; + } } return STATUS_OK; } @@ -1019,7 +1081,6 @@ mini_for_each(cache *cc, cc, meta_head, type, NULL, mini_for_each_meta_page_func, &fef); } - /* *----------------------------------------------------------------------------- * mini_[inc,dec]_ref -- @@ -1043,9 +1104,16 @@ mini_inc_ref(cache *cc, uint64 meta_head) cache_get_allocator(cc), base_addr(cc, meta_head), NULL); } -static void -mini_dealloc_extent(cache *cc, page_type type, uint64 base_addr, void *out) +static platform_status +mini_dealloc_extent(cache *cc, + page_type type, + uint64 base_addr, + uint64 height, + void *out) { + if (type == PAGE_TYPE_BLOB) { + return blob_ref_release(cc, blob_ref_create(base_addr, height)); + } allocator *al = cache_get_allocator(cc); refcount ref = allocator_dec_ref(al, base_addr, type); if (ref == AL_NO_REFS) { @@ -1055,6 +1123,7 @@ mini_dealloc_extent(cache *cc, page_type type, uint64 base_addr, void *out) } else { platform_assert(ref != AL_FREE); } + return STATUS_OK; } refcount @@ -1095,7 +1164,8 @@ mini_dec_ref(cache *cc, uint64 meta_head, page_type type) *----------------------------------------------------------------------------- * mini_prefetch -- * - * Prefetches all extents in the mini allocator. + * Prefetches ordinary extents and blob roots recorded by the mini + * allocator. Blob descendants are read ahead by the blob iterator. * * Results: * None. @@ -1104,10 +1174,15 @@ mini_dec_ref(cache *cc, uint64 meta_head, page_type type) * Standard cache side effects. *----------------------------------------------------------------------------- */ -static void -mini_prefetch_extent(cache *cc, page_type type, uint64 base_addr, void *out) +static platform_status +mini_prefetch_extent(cache *cc, + page_type type, + uint64 base_addr, + uint64 height, + void *out) { cache_prefetch(cc, base_addr, type); + return STATUS_OK; } void @@ -1203,7 +1278,9 @@ mini_validate_meta_header(const mini_meta_hdr *hdr, uint64 expected_pos = first_entry_offset + (uint64)hdr->num_entries * sizeof(meta_entry); - if (hdr->pos != expected_pos) { + if (hdr->pos != expected_pos || hdr->num_charged_extents < hdr->num_entries + || (hdr->num_entries == 0 && hdr->num_charged_extents != 0)) + { return mini_metadata_corruption("metadata entry position is invalid", meta_addr); } @@ -1278,13 +1355,14 @@ mini_validate_meta_entry(const allocator_config *cfg, const meta_entry *entry, uint64 meta_addr) { - uint64 batch = meta_entry_batch(entry); - page_type extent_type = meta_entry_type(entry); - uint64 extent_number = - entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); + uint64 batch = meta_entry_batch(entry); + page_type extent_type = meta_entry_type(entry); + uint64 extent_number = entry->packed >> (META_ENTRY_EXTENT_SHIFT); if (batch >= MINI_MAX_BATCHES || extent_type < PAGE_TYPE_FIRST - || extent_type >= NUM_PAGE_TYPES || extent_number == 0 + || extent_type >= NUM_PAGE_TYPES + || (extent_type != PAGE_TYPE_BLOB && meta_entry_height(entry) != 0) + || extent_number == 0 || extent_number > UINT64_MAX / cfg->io_cfg->extent_size) { return mini_metadata_corruption("metadata extent entry is invalid", @@ -1396,7 +1474,8 @@ mini_recover_visit_meta_extent(cache *cc, uint64 extent_addr, void *arg) { - mini_recover_state *state = (mini_recover_state *)arg; + for_each_func *visitor = arg; + mini_recover_state *state = visitor->arg; if (!mini_metadata_valid_extent_addr(state->cfg, extent_addr)) { return mini_metadata_corruption("metadata extent is out of range", @@ -1405,38 +1484,18 @@ mini_recover_visit_meta_extent(cache *cc, return allocator_recovery_record_reference(state->al, extent_addr, type); } -/* - * The "func" hook mini_for_each_meta_page() calls once per meta page, with the - * page held. Validates the page and every data-extent entry on it before - * trusting any of it, and records a reference for each data extent found (its - * type is self-describing on disk, unlike a meta page's). - */ static platform_status -mini_recover_visit_meta_page(cache *cc, - page_type type, - page_handle *meta_page, - void *arg) +mini_recover_extent(cache *cc, + page_type type, + uint64 extent_addr, + uint64 height, + void *arg) { - mini_recover_state *state = (mini_recover_state *)arg; - (void)type; - - mini_meta_hdr *hdr = (mini_meta_hdr *)page_handle_data(meta_page); - - meta_entry *entry = first_entry(meta_page); - for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { - page_type extent_type = meta_entry_type(entry); - uint64 extent_addr = meta_entry_extent_addr(cc, entry); - - platform_status rc = allocator_recovery_record_reference( - state->al, extent_addr, extent_type); - if (!SUCCESS(rc)) { - return rc; - } - - entry = next_entry(entry); - } - - return STATUS_OK; + mini_recover_state *state = arg; + return type == PAGE_TYPE_BLOB + ? blob_ref_recover(cc, blob_ref_create(extent_addr, height)) + : allocator_recovery_record_reference( + state->al, extent_addr, type); } /* @@ -1446,16 +1505,16 @@ mini_recover_visit_meta_page(cache *cc, * trusting allocator refcounts -- the discovery primitive crash recovery * uses to reconstruct them. Records one reference for every metadata * extent and every data-extent entry found in the on-disk mini metadata - * stream (data entries are not deduplicated: repeated entries mean - * repeated references, and are recorded exactly as often as they occur). + * stream. Repeated extent and blob-root records represent repeated incoming + * references. A shared blob pointer extent's children are enumerated once. * * meta_type is the page type of meta_head's own chain; each data extent's * type is read from its own metadata entry. * * Validates the metadata-page chain, page-header bounds, page types, * batches, and extent addresses before using them. A malformed on-disk - * stream returns STATUS_INVALID_STATE. This function performs no writes - * and does not use allocator refcounts to decide what to traverse. + * stream returns STATUS_INVALID_STATE. This function performs no writes; + * blob recovery uses rebuilt refcounts to discover each subtree once. * * This is a physical enumeration only. Recovering logical reference * multiplicity, and deduplicating references shared by distinct mini @@ -1482,14 +1541,15 @@ mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type) return mini_metadata_corruption("metadata head is not a page", meta_head); } - mini_recover_state state = {.al = al, .cfg = cfg}; + mini_recover_state state = {.al = al, .cfg = cfg}; + for_each_func visitor = {mini_recover_extent, &state}; return mini_for_each_meta_page(cc, meta_head, meta_type, mini_recover_visit_meta_extent, - mini_recover_visit_meta_page, - &state); + mini_for_each_meta_page_func, + &visitor); } platform_status @@ -1569,8 +1629,7 @@ mini_meta_cursor_init(mini_meta_cursor *cursor, uint64 target_extent_number = target_extent_addr / cfg->io_cfg->extent_size; while (cursor->entry_idx < cursor->num_entries) { meta_entry *entry = first_entry(&cursor->meta_page) + cursor->entry_idx; - uint64 extent_number = - entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); + uint64 extent_number = entry->packed >> (META_ENTRY_EXTENT_SHIFT); if (extent_number == target_extent_number) { /* The scan only compares fixed-width extent numbers. Validate the * matching entry before exposing its remaining fields through curr; @@ -1610,7 +1669,9 @@ mini_meta_cursor_deinit(mini_meta_cursor *cursor) void mini_meta_cursor_curr(mini_meta_cursor *cursor, uint64 *extent_addr, - uint64 *batch) + uint64 *batch, + page_type *type, + uint64 *height) { platform_assert(cursor != NULL); platform_assert(cursor->cc != NULL); @@ -1624,14 +1685,21 @@ mini_meta_cursor_curr(mini_meta_cursor *cursor, meta_entry *entry = first_entry(&cursor->meta_page) + cursor->entry_idx; *extent_addr = meta_entry_extent_addr(cursor->cc, entry); *batch = meta_entry_batch(entry); + if (type != NULL) { + *type = meta_entry_type(entry); + } + if (height != NULL) { + *height = meta_entry_height(entry); + } } -mini_meta_cursor_status -mini_meta_cursor_next(mini_meta_cursor *cursor) +static mini_meta_cursor_status +mini_meta_cursor_next_internal(mini_meta_cursor *cursor) { if (cursor == NULL || cursor->cc == NULL || page_handle_is_null(&cursor->meta_page) - || cursor->entry_idx >= cursor->num_entries) + || (cursor->num_entries != 0 + && cursor->entry_idx >= cursor->num_entries)) { return MINI_META_CURSOR_INVALID; } @@ -1646,6 +1714,7 @@ mini_meta_cursor_next(mini_meta_cursor *cursor) return MINI_META_CURSOR_ENTRY; } +next_page:; uint64 next_meta_addr = mini_get_next_meta_addr(&cursor->meta_page); if (next_meta_addr == 0) { return MINI_META_CURSOR_END; @@ -1681,7 +1750,7 @@ mini_meta_cursor_next(mini_meta_cursor *cursor) cursor->meta_page_no + 1, TRUE); uint64 next_num_entries = SUCCESS(rc) ? mini_num_entries(next_page) : 0; - if (!SUCCESS(rc) || next_num_entries == 0) { + if (!SUCCESS(rc)) { cache_unget(cursor->cc, next_page); return MINI_META_CURSOR_INVALID; } @@ -1692,18 +1761,22 @@ mini_meta_cursor_next(mini_meta_cursor *cursor) cursor->meta_page_no++; cursor->num_entries = next_num_entries; cursor->entry_idx = 0; + if (next_num_entries == 0) { + goto next_page; + } if (!SUCCESS(mini_meta_cursor_validate_entry(cursor, 0))) { return MINI_META_CURSOR_INVALID; } return MINI_META_CURSOR_ENTRY; } -mini_meta_cursor_status -mini_meta_cursor_prev(mini_meta_cursor *cursor) +static mini_meta_cursor_status +mini_meta_cursor_prev_internal(mini_meta_cursor *cursor) { if (cursor == NULL || cursor->cc == NULL || page_handle_is_null(&cursor->meta_page) - || cursor->entry_idx >= cursor->num_entries) + || (cursor->num_entries != 0 + && cursor->entry_idx >= cursor->num_entries)) { return MINI_META_CURSOR_INVALID; } @@ -1718,6 +1791,7 @@ mini_meta_cursor_prev(mini_meta_cursor *cursor) return MINI_META_CURSOR_ENTRY; } +prev_page:; mini_meta_hdr *hdr = (mini_meta_hdr *)page_handle_data(&cursor->meta_page); uint64 prev_addr = hdr->prev_meta_addr; if (prev_addr == 0) { @@ -1755,9 +1829,7 @@ mini_meta_cursor_prev(mini_meta_cursor *cursor) rc = mini_validate_next_meta_addr( cfg, page_handle_addr(prev_page), prev_hdr->next_meta_addr); } - if (!SUCCESS(rc) || prev_hdr->next_meta_addr != cursor->meta_addr - || prev_num_entries == 0) - { + if (!SUCCESS(rc) || prev_hdr->next_meta_addr != cursor->meta_addr) { cache_unget(cursor->cc, prev_page); return MINI_META_CURSOR_INVALID; } @@ -1767,18 +1839,66 @@ mini_meta_cursor_prev(mini_meta_cursor *cursor) cursor->meta_addr = prev_addr; cursor->meta_page_no--; cursor->num_entries = prev_num_entries; - cursor->entry_idx = cursor->num_entries - 1; + cursor->entry_idx = prev_num_entries ? prev_num_entries - 1 : 0; + if (prev_num_entries == 0) { + goto prev_page; + } if (!SUCCESS(mini_meta_cursor_validate_entry(cursor, cursor->entry_idx))) { return MINI_META_CURSOR_INVALID; } return MINI_META_CURSOR_ENTRY; } -static void -space_use_add_extent(cache *cc, page_type type, uint64 extent_addr, void *out) +/* Cross-page movement uses a temporary cursor so skipped blob-only pages do + * not change curr() on END, WOULD_BLOCK, or corruption. */ +static mini_meta_cursor_status +mini_meta_cursor_move(mini_meta_cursor *cursor, bool32 forward) +{ + if (cursor == NULL || cursor->cc == NULL + || page_handle_is_null(&cursor->meta_page) + || cursor->entry_idx >= cursor->num_entries) + { + return MINI_META_CURSOR_INVALID; + } + if ((forward && cursor->entry_idx + 1 < cursor->num_entries) + || (!forward && cursor->entry_idx > 0)) + { + return forward ? mini_meta_cursor_next_internal(cursor) + : mini_meta_cursor_prev_internal(cursor); + } + mini_meta_cursor candidate = *cursor; + candidate.meta_page = (page_handle){0}; + platform_status rc = cache_get(cursor->cc, + cursor->meta_addr, + FALSE, + cursor->meta_type, + &candidate.meta_page); + if (!SUCCESS(rc)) { + return STATUS_IS_EQ(rc, STATUS_BUSY) ? MINI_META_CURSOR_WOULD_BLOCK + : MINI_META_CURSOR_INVALID; + } + mini_meta_cursor_status status = + forward ? mini_meta_cursor_next_internal(&candidate) + : mini_meta_cursor_prev_internal(&candidate); + if (status == MINI_META_CURSOR_ENTRY) { + mini_meta_cursor_deinit(cursor); + *cursor = candidate; + } else { + mini_meta_cursor_deinit(&candidate); + } + return status; +} + +mini_meta_cursor_status +mini_meta_cursor_next(mini_meta_cursor *cursor) +{ + return mini_meta_cursor_move(cursor, TRUE); +} + +mini_meta_cursor_status +mini_meta_cursor_prev(mini_meta_cursor *cursor) { - uint64 *sum = (uint64 *)out; - *sum += cache_extent_size(cc); + return mini_meta_cursor_move(cursor, FALSE); } static platform_status @@ -1787,8 +1907,17 @@ space_use_add_meta_page(cache *cc, page_handle *meta_page, void *out) { - uint64 *sum = (uint64 *)out; - *sum += cache_page_size(cc); + uint64 *sum = out; + mini_meta_hdr *hdr = (mini_meta_hdr *)page_handle_data(meta_page); + uint64 page_size = cache_page_size(cc); + uint64 extent_size = cache_extent_size(cc); + if (*sum > UINT64_MAX - page_size + || hdr->num_charged_extents + > (UINT64_MAX - *sum - page_size) / extent_size) + { + return STATUS_LIMIT_EXCEEDED; + } + *sum += page_size + hdr->num_charged_extents * extent_size; return STATUS_OK; } @@ -1796,21 +1925,14 @@ platform_status mini_space_use_bytes(cache *cc, uint64 meta_head, page_type type, uint64 *bytes) { uint64 total = 0; - platform_status rc = - mini_for_each(cc, meta_head, type, space_use_add_extent, &total); - if (!SUCCESS(rc)) { - return rc; - } - rc = mini_for_each_meta_page( + platform_status rc = mini_for_each_meta_page( cc, meta_head, type, NULL, space_use_add_meta_page, &total); - if (!SUCCESS(rc)) { - return rc; + if (SUCCESS(rc)) { + *bytes = total; } - *bytes = total; - return STATUS_OK; + return rc; } - /* *----------------------------------------------------------------------------- * mini_print -- @@ -1859,10 +1981,11 @@ mini_print(cache *cc, uint64 meta_head, page_type type) uint64 num_entries = mini_num_entries(meta_page); meta_entry *entry = first_entry(meta_page); for (uint64 i = 0; i < num_entries; i++) { - platform_default_log("| %3lu | %35lu | %s\n", + platform_default_log("| %3lu | %35lu | %s height=%lu\n", i, meta_entry_extent_addr(cc, entry), - page_type_str[meta_entry_type(entry)]); + page_type_str[meta_entry_type(entry)], + meta_entry_height(entry)); entry = next_entry(entry); } platform_default_log("|-------------------------------------------|\n"); diff --git a/src/mini_allocator.h b/src/mini_allocator.h index 27cf6af56..9f2d39720 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -12,12 +12,15 @@ * such as reference counting and deallocation. A single mini allocator can * manage batches with different page types; the metadata records each * extent's page type so bulk operations can deallocate mixed trees/blobs. + * Blob roots use the same entry stream, with their pointer depth encoded + * alongside the page type. The blob module reclaims their descendants. */ #pragma once #include "allocator.h" #include "cache.h" +#include "blob.h" /* * Mini-allocator breaks extents into pages. The pages are fed out of separate @@ -27,7 +30,7 @@ * contiguity. */ #define MINI_MAX_BATCHES 16 -#define MINI_ALLOCATOR_FORMAT_VERSION (1) +#define MINI_ALLOCATOR_FORMAT_VERSION (3) /* * mini_allocator: Mini-allocator context. @@ -41,6 +44,8 @@ typedef struct mini_allocator { page_type meta_type; page_type types[MINI_MAX_BATCHES]; + // Allocation/attachment charges since initialization, including reserves. + // Repeated attachments count separately for memtable admission accounting. uint64 num_extents; uint64 num_batches; volatile uint64 next_addr[MINI_MAX_BATCHES]; @@ -107,6 +112,16 @@ mini_alloc_extent(mini_allocator *mini, platform_status mini_attach_extent(mini_allocator *mini, uint64 batch, uint64 addr); +/* Transfer one already-owned blob subtree reference to this container. + * Failure leaves ownership with the caller. Charges all referenced extents, + * including shared descendants, without walking the subtree. The caller + * supplies the charge from the descriptor via blob_root_num_extents(). */ +platform_status +mini_adopt_blob_ref(mini_allocator *mini, + uint64 batch, + blob_ref ref, + uint64 num_extents); + uint64 mini_next_addr(mini_allocator *mini, uint64 batch); @@ -151,10 +166,10 @@ mini_recover_references(cache *cc, uint64 meta_head, page_type type); /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a - * finalized mini_allocator. Entries from all batches are interleaved in - * allocation order; the caller filters by batch as needed (each entry reports - * its batch). The btree iterator uses this to read extent addresses ahead of or - * behind itself for prefetching. + * finalized mini_allocator. Entries from all batches, including blob roots, + * are interleaved in recording order; the caller can filter by batch/type. + * Blob descendants are not separate entries. The btree iterator uses this to + * read extent addresses ahead of or behind itself for prefetching. * * The cursor holds a read reference on the meta page it is currently reading; * call mini_meta_cursor_deinit() to release it. The cursor is non-blocking: it @@ -195,11 +210,15 @@ mini_meta_cursor_init(mini_meta_cursor *cursor, void mini_meta_cursor_deinit(mini_meta_cursor *cursor); -// Get the current extent entry. Requires a successful init, next, or prev. +// Get the current extent entry, including blob roots in recording order. +// type and height are optional. Height is zero for non-blob and data extents. +// Requires a successful init, next, or prev. void mini_meta_cursor_curr(mini_meta_cursor *cursor, uint64 *extent_addr, - uint64 *batch); + uint64 *batch, + page_type *type, + uint64 *height); // Move to the next extent entry in allocation order. Non-blocking: returns // MINI_META_CURSOR_WOULD_BLOCK (and issues a prefetch for it) if the next meta diff --git a/src/mini_blob.c b/src/mini_blob.c new file mode 100644 index 000000000..b52028fe2 --- /dev/null +++ b/src/mini_blob.c @@ -0,0 +1,139 @@ +// Copyright 2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 +#include "mini_blob.h" +#include "poison.h" + +typedef struct mini_blob_tails { + const mini_blob_config *cfg; + mini_allocator *mini; +} mini_blob_tails; + +static blob_build_config +mini_blob_build_config(const mini_blob_config *cfg) +{ + return (blob_build_config){.alignment = cfg->alignment, + .blob_max_descriptor_size = + cfg->blob_max_descriptor_size}; +} + +platform_status +mini_blob_descriptor_size_bound(const mini_blob_config *cfg, + uint64 extent_size, + uint64 length, + uint64 *size) +{ + if (cfg == NULL) { + return STATUS_BAD_PARAM; + } + blob_build_config build = mini_blob_build_config(cfg); + return blob_descriptor_size_bound(&build, extent_size, length, size); +} + +static platform_status +mini_blob_allocate_tail(void *arg, + uint64 bytes, + uint64 alignment, + uint64 boundary, + uint64 addrs[2]) +{ + mini_blob_tails *tails = arg; + mini_allocator *mini = tails->mini; + cache *cc = mini->cc; + uint64 batch = + boundary == 0 ? tails->cfg->page_batch : tails->cfg->subpage_batch; + platform_status rc = + mini_alloc_bytes(mini, batch, bytes, alignment, boundary, addrs, NULL); + if (!SUCCESS(rc)) { + return rc; + } + uint64 page_size = cache_page_size(cc); + uint64 extent_size = cache_extent_size(cc); + uint64 remaining = bytes; + /* Initialize every new page while the batch is locked. Subsequent tail + * allocations may immediately reuse the last partial page. */ + for (uint64 i = 0; i < 2 && remaining; i++) { + uint64 count = MIN(remaining, extent_size - addrs[i] % extent_size); + uint64 end = addrs[i] + count; + uint64 page_addr = (addrs[i] + page_size - 1) / page_size * page_size; + for (; page_addr < end; page_addr += page_size) { + page_handle storage; + page_handle *page = + cache_alloc(cc, page_addr, PAGE_TYPE_BLOB, &storage); + platform_assert(page != NULL); + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + } + remaining -= count; + } + mini_alloc_bytes_finish(mini, batch); + return STATUS_OK; +} + +static platform_status +mini_blob_adopt(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + writable_buffer *result) +{ + parsed_blob parsed; + platform_status rc = + blob_parse_descriptor(cc, writable_buffer_to_slice(result), &parsed); + platform_assert_status_ok(rc); + uint64 i = 0; + for (; i < parsed.num_extents; i++) { + rc = mini_adopt_blob_ref(mini, + cfg->page_batch, + blob_get_ref(cc, &parsed, i), + blob_root_num_extents(cc, &parsed, i)); + if (!SUCCESS(rc)) { + break; + } + } + for (; i < parsed.num_extents; i++) { + (void)blob_ref_release(cc, blob_get_ref(cc, &parsed, i)); + } + return rc; +} + +static platform_status +mini_blob_construct(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + slice data, + writable_buffer *result, + bool32 clone) +{ + blob_build_config build = mini_blob_build_config(cfg); + mini_blob_tails state = {cfg, mini}; + blob_tail_allocator tails = {mini_blob_allocate_tail, &state}; + platform_status rc = clone ? blob_clone(&build, cc, &tails, data, result) + : blob_build(&build, cc, &tails, data, result); + if (SUCCESS(rc)) { + rc = mini_blob_adopt(cfg, cc, mini, result); + } + if (!SUCCESS(rc)) { + (void)writable_buffer_resize(result, 0); + } + return rc; +} + +platform_status +mini_blob_build(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + slice data, + writable_buffer *result) +{ + return mini_blob_construct(cfg, cc, mini, data, result, FALSE); +} + +platform_status +mini_blob_clone(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + slice source, + writable_buffer *result) +{ + return mini_blob_construct(cfg, cc, mini, source, result, TRUE); +} diff --git a/src/mini_blob.h b/src/mini_blob.h new file mode 100644 index 000000000..d4c6e4e8d --- /dev/null +++ b/src/mini_blob.h @@ -0,0 +1,40 @@ +// Copyright 2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "blob_build.h" +#include "mini_allocator.h" + +#define NUM_BLOB_BATCHES (2) + +/* Mini-allocator adapter configuration. The blob builder itself has no batch + * numbers or mini-allocator dependency. */ +typedef struct mini_blob_config { + uint64 page_batch; + uint64 subpage_batch; + uint64 alignment; + uint64 blob_max_descriptor_size; +} mini_blob_config; + +platform_status +mini_blob_descriptor_size_bound(const mini_blob_config *cfg, + uint64 extent_size, + uint64 length, + uint64 *size); + +/* Successful results are borrowed from mini. On failure any allocations or + * already-adopted roots remain tracked by mini for bulk reclamation; all roots + * not adopted by mini are released. */ +platform_status +mini_blob_build(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + slice data, + writable_buffer *result); + +platform_status +mini_blob_clone(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + slice source, + writable_buffer *result); diff --git a/src/shard_log.c b/src/shard_log.c index 9ccef1189..ab78795ff 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -487,7 +487,7 @@ shard_log_validate_page_records(cache *cc, message_type message_class = ondisk_tuple_message_class(tuple); if (le->memtable_generation == INVALID_LOG_GENERATION || le->leaf_generation == INVALID_LOG_GENERATION - || tuple->key_flags != ONDISK_KEY_DEFAULT_FLAGS + || (tuple->key_flags & ~ONDISK_KEY_IS_BLOB) != 0 || tuple->key_length == ONDISK_KEY_NEGATIVE_INFINITY || tuple->key_length == ONDISK_KEY_POSITIVE_INFINITY || (tuple->flags @@ -504,6 +504,14 @@ shard_log_validate_page_records(cache *cc, return STATUS_INVALID_STATE; } + if (tuple->key_flags & ONDISK_KEY_IS_BLOB) { + parsed_blob descriptor; + rc = blob_parse_descriptor( + cc, key_slice(log_entry_key(le)), &descriptor); + if (!SUCCESS(rc)) { + return rc; + } + } if (ondisk_tuple_message_is_blob(tuple)) { parsed_blob blob_descriptor; rc = blob_parse_descriptor( @@ -550,7 +558,15 @@ shard_log_measure_page_records(cache *cc, if (le->memtable_generation < first_needed_generation) { le = (log_entry *)((char *)le + sizeof_log_entry(le)); continue; - } else if (log_entry_message_is_blob(le)) { + } + key stored_key = log_entry_key(le); + if (stored_key.is_blob) { + rc = blob_validate(cc, key_slice(stored_key)); + if (!SUCCESS(rc)) { + return rc; + } + } + if (log_entry_message_is_blob(le)) { message msg = log_entry_message(cc, le); rc = message_validate(msg); if (!SUCCESS(rc)) { @@ -1132,46 +1148,52 @@ shard_log_graduate_final_buffer(shard_log *log, shard_log_group *group) log, group, shard_log_get_thread_data(group, 0), TRUE); } +/* + * Bound the encoded record before allocating either blob. Log keys carry only + * a descriptor: generation order, not key order, determines replay order. + */ static platform_status shard_log_validate_reservation_entry(shard_log *log, key tuple_key, message msg, - uint64 *entry_size_out) + bool32 force_blob, + bool32 *blob_key, + bool32 *blob_message) { + if (tuple_key.is_blob || !key_is_user_key(tuple_key)) { + return STATUS_BAD_PARAM; + } uint64 key_size = key_length(tuple_key); uint64 message_size = message_length(msg); - uint64 max_entry_size = - shard_log_page_size(log->cfg) - sizeof(shard_log_hdr); - - if (key_size > UINT16_MAX || key_size > max_entry_size - sizeof(log_entry)) { - return STATUS_LIMIT_EXCEEDED; - } - - uint64 entry_size = log_entry_required_capacity(tuple_key, msg); - if (message_size <= UINT16_MAX && entry_size <= max_entry_size) { - *entry_size_out = entry_size; - return STATUS_OK; + uint64 capacity = + shard_log_page_size(log->cfg) - sizeof(shard_log_hdr) - sizeof(log_entry); + *blob_key = key_size > MAX_INLINE_KEY_SIZE(shard_log_page_size(log->cfg)); + if (*blob_key) { + platform_status rc = mini_blob_descriptor_size_bound( + &log->cfg->blob_cfg, cache_extent_size(log->cc), key_size, &key_size); + if (!SUCCESS(rc)) { + return rc; + } } - - if (message_is_blob(msg)) { + if (key_size >= ONDISK_KEY_POSITIVE_INFINITY || key_size > capacity) { return STATUS_LIMIT_EXCEEDED; } - - uint64 descriptor_size; - platform_status rc = blob_descriptor_size_bound(&log->cfg->blob_cfg, - cache_extent_size(log->cc), - message_size, - &descriptor_size); - if (!SUCCESS(rc)) { - return rc; + *blob_message = message_is_blob(msg) || force_blob + || message_size > UINT16_MAX + || message_size > capacity - key_size; + if (*blob_message && !message_is_blob(msg)) { + platform_status rc = + mini_blob_descriptor_size_bound(&log->cfg->blob_cfg, + cache_extent_size(log->cc), + message_size, + &message_size); + if (!SUCCESS(rc)) { + return rc; + } } - if (descriptor_size > UINT16_MAX - || descriptor_size > max_entry_size - sizeof(log_entry) - key_size) - { + if (message_size > UINT16_MAX || message_size > capacity - key_size) { return STATUS_LIMIT_EXCEEDED; } - - *entry_size_out = entry_size; return STATUS_OK; } @@ -1179,6 +1201,7 @@ typedef struct shard_log_prepared_write { key tuple_key; message msg; merge_accumulator log_blob; + writable_buffer *key_blob; // Allocated only for large keys. uint64 entry_size; uint64 blob_pages; threadid owner_tid; @@ -1212,6 +1235,11 @@ shard_log_prepared_state_deinit(log_write_prepared *prepared) if (state->log_blob_inited) { merge_accumulator_deinit(&state->log_blob); } + if (state->key_blob != NULL) { + platform_heap_id hid = state->key_blob->heap_id; + writable_buffer_deinit(state->key_blob); + platform_free(hid, state->key_blob); + } ZERO_CONTENTS(state); prepared->log = NULL; } @@ -1227,21 +1255,38 @@ shard_log_write_prepare(log_handle *logh, threadid tid = platform_get_tid(); shard_log_prepared_write *state = shard_log_prepared_state(prepared); - uint64 entry_size; - platform_status rc = - shard_log_validate_reservation_entry(log, tuple_key, msg, &entry_size); + bool32 blob_key, blob_message; + platform_status rc = shard_log_validate_reservation_entry( + log, tuple_key, msg, force_blob, &blob_key, &blob_message); if (!SUCCESS(rc)) { return rc; } - uint64 page_size = shard_log_page_size(log->cfg); - uint64 max_entry_size = page_size - sizeof(shard_log_hdr); - bool32 input_is_blob = message_is_blob(msg); + uint64 page_size = shard_log_page_size(log->cfg); + if (blob_key) { + state->key_blob = TYPED_MALLOC(platform_get_heap_id(), state->key_blob); + if (state->key_blob == NULL) { + rc = STATUS_NO_MEMORY; + goto fail; + } + writable_buffer_init(state->key_blob, platform_get_heap_id()); + rc = mini_blob_build(&log->cfg->blob_cfg, + log->cc, + &log->mini, + key_slice(tuple_key), + state->key_blob); + if (!SUCCESS(rc)) { + goto fail; + } + tuple_key = key_create_from_slice( + FALSE, writable_buffer_to_slice(state->key_blob)); + tuple_key.is_blob = TRUE; + } - if (input_is_blob || force_blob || max_entry_size < entry_size) { + if (blob_message) { merge_accumulator_init(&state->log_blob, platform_get_heap_id()); state->log_blob_inited = TRUE; - if (input_is_blob) { + if (message_is_blob(msg)) { rc = message_clone( &log->cfg->blob_cfg, log->cc, &log->mini, msg, &state->log_blob); } else { @@ -1252,12 +1297,9 @@ shard_log_write_prepare(log_handle *logh, goto fail; } msg = merge_accumulator_to_message(&state->log_blob); - rc = - shard_log_validate_reservation_entry(log, tuple_key, msg, &entry_size); - if (!SUCCESS(rc)) { - goto fail; - } } + uint64 entry_size = log_entry_required_capacity(tuple_key, msg); + platform_assert(entry_size <= page_size - sizeof(shard_log_hdr)); /* * Pin the current group while enrolling prepared blob writes and making @@ -1274,7 +1316,10 @@ shard_log_write_prepare(log_handle *logh, } shard_log_thread_data *thread_data = shard_log_get_thread_data(group, tid); - if (state->log_blob_inited) { + if (state->key_blob != NULL) { + rc = blob_writeback(log->cc, key_slice(tuple_key), &thread_data->wbset); + } + if (SUCCESS(rc) && state->log_blob_inited) { rc = blob_writeback(log->cc, message_slice(msg), &thread_data->wbset); } @@ -1302,6 +1347,13 @@ shard_log_write_prepare(log_handle *logh, uint64 bytes = blob_length(message_slice(msg)); state->blob_pages = bytes / page_size + (bytes % page_size != 0); } + if (tuple_key.is_blob) { + uint64 bytes = blob_length(key_slice(tuple_key)); + uint64 pages = bytes / page_size + (bytes % page_size != 0); + state->blob_pages = pages > UINT64_MAX - state->blob_pages + ? UINT64_MAX + : state->blob_pages + pages; + } state->tuple_key = tuple_key; state->msg = msg; state->entry_size = entry_size; @@ -2217,18 +2269,26 @@ shard_log_recover_page_blob_allocations(cache *cc, log_entry *le = first_log_entry(page_handle_data(page)); platform_status rc = STATUS_OK; for (uint64 i = 0; i < hdr->num_entries; i++) { - if (le->memtable_generation >= first_needed_generation - && log_entry_message_is_blob(le)) - { - message msg = log_entry_message(cc, le); - rc = blob_recover_allocations(cc, message_slice(msg)); - if (STATUS_IS_EQ(rc, STATUS_INVALID_STATE) - || STATUS_IS_EQ(rc, STATUS_IO_ERROR)) - { - *first_invalid_group = MIN(*first_invalid_group, hdr->group_id); - rc = STATUS_OK; - } else if (!SUCCESS(rc)) { - return rc; + if (le->memtable_generation >= first_needed_generation) { + key stored_key = log_entry_key(le); + slice descriptors[2]; + uint64 count = 0; + if (stored_key.is_blob) { + descriptors[count++] = key_slice(stored_key); + } + if (log_entry_message_is_blob(le)) { + descriptors[count++] = message_slice(log_entry_message(cc, le)); + } + for (uint64 j = 0; j < count; j++) { + rc = blob_recover_allocations(cc, descriptors[j]); + if (STATUS_IS_EQ(rc, STATUS_INVALID_STATE) + || STATUS_IS_EQ(rc, STATUS_IO_ERROR)) + { + *first_invalid_group = MIN(*first_invalid_group, hdr->group_id); + rc = STATUS_OK; + } else if (!SUCCESS(rc)) { + return rc; + } } } le = (log_entry *)((char *)le + sizeof_log_entry(le)); @@ -2579,6 +2639,10 @@ shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) { shard_log_iterator *itor = (shard_log_iterator *)itorh; *curr_key = log_entry_key(itor->entries[itor->pos]); + if (curr_key->is_blob) { + *curr_key = key_create_from_slice( + FALSE, writable_buffer_to_slice(&itor->key_buffer)); + } *msg = log_entry_message(itor->cc, itor->entries[itor->pos]); } @@ -2607,12 +2671,25 @@ shard_log_iterator_can_next(iterator *itorh) return itor->pos < itor->num_entries; } +static platform_status +shard_log_iterator_load_key(shard_log_iterator *itor) +{ + if (itor->pos < itor->num_entries) { + key stored_key = log_entry_key(itor->entries[itor->pos]); + if (stored_key.is_blob) { + return blob_materialize_full( + itor->cc, key_slice(stored_key), &itor->key_buffer); + } + } + return STATUS_OK; +} + static platform_status shard_log_iterator_next(iterator *itorh) { shard_log_iterator *itor = (shard_log_iterator *)itorh; itor->pos++; - return STATUS_OK; + return shard_log_iterator_load_key(itor); } /* @@ -2632,10 +2709,9 @@ shard_log_config_init(shard_log_config *log_cfg, log_cfg->data_cfg = data_cfg; log_cfg->seed = HASH_SEED; log_cfg->group_size_pages = SHARD_LOG_DEFAULT_GROUP_SIZE_PAGES; - log_cfg->blob_cfg = (blob_build_config){ - .extent_batch = 1, - .page_batch = 2, - .subpage_batch = 3, + log_cfg->blob_cfg = (mini_blob_config){ + .page_batch = 1, + .subpage_batch = 2, .alignment = cache_config_page_size(cache_cfg), .blob_max_descriptor_size = BLOB_DEFAULT_MAX_DESCRIPTOR_SIZE, }; @@ -2686,7 +2762,9 @@ shard_log_print(shard_log *log) for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { platform_default_log( "%s -- %s%s : memtable=%lu leaf=%lu\n", - key_string(dcfg, log_entry_key(le)), + log_entry_key(le).is_blob + ? "(blob key)" + : key_string(dcfg, log_entry_key(le)), log_entry_message_is_blob(le) ? "(blob) " : "", message_string(dcfg, log_entry_message(cc, le)), le->memtable_generation, @@ -2711,6 +2789,7 @@ shard_log_iterator_deinit(log_iterator *itorh) if (itor->entries != NULL) { platform_free(hid, itor->entries); } + writable_buffer_deinit(&itor->key_buffer); platform_free(hid, itor); // the handle, from shard_log_iterator_create() } @@ -2793,6 +2872,7 @@ shard_log_iterator_init(cache *cc, bool32 readable_pages[MAX_PAGES_PER_EXTENT]; memset(itor, 0, sizeof(shard_log_iterator)); + writable_buffer_init(&itor->key_buffer, hid); itor->super.super.ops = &shard_log_iterator_ops; // generic iterator itor->super.ops = &shard_log_log_iterator_ops; // log_iterator itor->heap_id = hid; @@ -3122,7 +3202,13 @@ shard_log_iterator_init(cache *cc, &tmp); } - return STATUS_OK; + rc = shard_log_iterator_load_key(itor); + if (!SUCCESS(rc)) { + writable_buffer_deinit(&itor->key_buffer); + platform_free(hid, itor->entries); + platform_free(hid, itor->contents); + } + return rc; } platform_status diff --git a/src/shard_log.h b/src/shard_log.h index b0c2e5dbd..ac047a638 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -15,7 +15,7 @@ #include "cache.h" #include "iterator.h" #include "splinterdb/data.h" -#include "blob_build.h" +#include "mini_blob.h" #include "mini_allocator.h" #include "platform_mutex.h" #include "writeback_set.h" @@ -30,11 +30,11 @@ shard_log_validate_format_version(uint16 format_version); * Configuration structure to set up the sharded log sub-system. */ typedef struct shard_log_config { - cache_config *cache_cfg; - data_config *data_cfg; - uint64 seed; - uint64 group_size_pages; // UINT64_MAX disables automatic cuts - blob_build_config blob_cfg; + cache_config *cache_cfg; + data_config *data_cfg; + uint64 seed; + uint64 group_size_pages; // UINT64_MAX disables automatic cuts + mini_blob_config blob_cfg; } shard_log_config; typedef enum shard_log_close_mode { @@ -254,6 +254,8 @@ typedef struct shard_log_iterator { log_entry **entries; uint64 num_entries; uint64 pos; + // Logical bytes for the current blob key; curr() itself cannot fail. + writable_buffer key_buffer; // Whether the replayable records run all the way to an end-of-stream marker. bool32 stream_complete; } shard_log_iterator; diff --git a/src/splinterdb.c b/src/splinterdb.c index 0dbd53b8a..5f47bbeff 100644 --- a/src/splinterdb.c +++ b/src/splinterdb.c @@ -15,7 +15,7 @@ #include "splinterdb/splinterdb.h" #include "clockcache.h" -#include "blob_build.h" +#include "mini_blob.h" #include "data_internal.h" #include "rc_allocator.h" #include "superblock.h" @@ -210,6 +210,9 @@ splinterdb_config_set_defaults(splinterdb_config *cfg) if (!cfg->blob_max_descriptor_size) { cfg->blob_max_descriptor_size = BLOB_DEFAULT_MAX_DESCRIPTOR_SIZE; } + if (!cfg->blob_key_prefix_size) { + cfg->blob_key_prefix_size = BTREE_DEFAULT_KEY_PREFIX_SIZE; + } if (!cfg->btree_rough_count_height) { cfg->btree_rough_count_height = 1; } @@ -467,6 +470,13 @@ splinterdb_init_config(const splinterdb_config *kvs_cfg, // IN kvs->btree_cfg.blob_cfg.blob_max_descriptor_size = cfg.blob_max_descriptor_size; + kvs->btree_cfg.blob_key_prefix_size = + cfg.blob_key_prefix_size == UINT64_MAX ? 0 : cfg.blob_key_prefix_size; + rc = btree_config_validate(&kvs->btree_cfg); + if (!SUCCESS(rc)) { + return rc; + } + trunk_config_init(&kvs->trunk_node_cfg, kvs->data_cfg, &kvs->btree_cfg, @@ -798,7 +808,7 @@ splinterdb_get_cost_stats(splinterdb *kvs, splinterdb_cost_stats *stats) status = btree_get_root_height(mt_ctxt->cc, mt_ctxt->cfg.btree_cfg, memtable_root_addr(&mt_ctxt->mt[mt_no]), - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &height); if (!SUCCESS(status)) { @@ -1118,10 +1128,10 @@ splinterdb_blob_create(splinterdb *kvs, slice value, message *out) if (slice_length(value) > allocator_get_config(al)->capacity) { return ENOSPC; } - blob_build_config cfg = kvs->btree_cfg.blob_cfg; - cfg.extent_batch = cfg.page_batch = cfg.subpage_batch = 0; + mini_blob_config cfg = kvs->btree_cfg.blob_cfg; + cfg.page_batch = cfg.subpage_batch = 0; uint64 descriptor_capacity; - platform_status status = blob_descriptor_size_bound( + platform_status status = mini_blob_descriptor_size_bound( &cfg, cache_extent_size(cc), slice_length(value), &descriptor_capacity); if (!SUCCESS(status)) { return platform_status_to_int(status); @@ -1155,7 +1165,7 @@ splinterdb_blob_create(splinterdb *kvs, slice value, message *out) // A private mini allocator isolates the caller's lifetime from memtable // rotations and log reclamation. One batch suffices for this one blob. - status = blob_build(&cfg, cc, &mini, value, &descriptor); + status = mini_blob_build(&cfg, cc, &mini, value, &descriptor); mini_release(&mini); if (!SUCCESS(status)) { mini_dec_ref(cc, handle->meta_head, PAGE_TYPE_BLOB); diff --git a/src/superblock.h b/src/superblock.h index fff86abab..35ef4d6b7 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -48,7 +48,8 @@ * descriptors carry their format version and fingerprint size; v8 adds * independent structure versions and complete routing-filter/log read config. */ -#define SUPERBLOCK_FORMAT_VERSION (8) +/* v9 requires hierarchical blob ownership and mini-allocator format 2. */ +#define SUPERBLOCK_FORMAT_VERSION (10) /* The two physical superblock copies live at pages 0 and 1. */ #define SUPERBLOCK_NUM_SLOTS (2) diff --git a/src/trunk.c b/src/trunk.c index b27d7a7a7..7840c4eaf 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -42,8 +42,14 @@ struct trunk_pivot { trunk_pivot_stats stats; uint64 child_addr; // Index of the oldest bundle that is live for this pivot - uint64 inflight_bundle_start; - ondisk_key key; + uint64 inflight_bundle_start; + key key; + /* The serialized blob key is borrowed under the same source-root lifetime + * as child/branch references. Serialization clones it before publication. + * Its bytes follow the materialized logical key in bytes[]. */ + uint16 blob_key_length; + uint16 blob_key_prefix_size; + uint8 bytes[]; }; typedef VECTOR(trunk_ondisk_node_ref *) trunk_ondisk_node_ref_vector; @@ -60,7 +66,9 @@ typedef struct ONDISK trunk_ondisk_node { uint16 num_pivots; // On disk, inflight bundles are ordered from newest to oldest. uint16 num_inflight_bundles; + uint16 blob_key_prefix_size; uint32 inflight_bundles_offset; + uint32 tail_offset; uint32 pivot_offsets[]; } trunk_ondisk_node; @@ -214,11 +222,10 @@ struct pending_gc { ***************************************************/ static branch_ref -create_branch_ref(uint64 addr) +create_branch_ref(uint64 addr, btree_format format) { - return (branch_ref){.addr = addr, - .format_version = - addr == 0 ? 0 : BTREE_BRANCH_FORMAT_VERSION}; + return (branch_ref){.addr = addr, + .format = addr == 0 ? (btree_format){0} : format}; } static uint64 @@ -442,21 +449,36 @@ trunk_pivot_stats_add(trunk_pivot_stats a, ((trunk_pivot_stats){.num_kv_bytes = 0, .num_tuples = 0}) static trunk_pivot * -trunk_pivot_create(platform_heap_id hid, - key k, - uint64 child_addr, - uint64 inflight_bundle_start, - trunk_pivot_stats prereceive_stats, - trunk_pivot_stats stats) -{ +trunk_pivot_create_with_blob(platform_heap_id hid, + key k, + uint64 child_addr, + uint64 inflight_bundle_start, + trunk_pivot_stats prereceive_stats, + trunk_pivot_stats stats, + key stored, + uint16 prefix_size) +{ + uint64 stored_size = stored.is_blob ? key_length(stored) : 0; + if (key_length(k) > UINT64_MAX - stored_size) { + return NULL; + } trunk_pivot *result = TYPED_FLEXIBLE_STRUCT_ZALLOC( - hid, result, key.bytes, ondisk_key_required_data_capacity(k)); + hid, result, bytes, key_length(k) + stored_size); if (result == NULL) { platform_error_log( "%s():%d: TYPED_FLEXIBLE_STRUCT_ZALLOC() failed", __func__, __LINE__); return NULL; } - copy_key_to_ondisk_key(&result->key, k); + result->key = k; + if (key_is_user_key(k)) { + memcpy(result->bytes, key_data(k), key_length(k)); + result->key.user_slice = slice_create(key_length(k), result->bytes); + } + result->blob_key_length = stored_size; + result->blob_key_prefix_size = prefix_size; + if (stored_size) { + memcpy(result->bytes + key_length(k), key_data(stored), stored_size); + } result->child_addr = child_addr; result->inflight_bundle_start = inflight_bundle_start; platform_assert(trunk_pivot_stats_are_nonnegative(prereceive_stats)); @@ -466,15 +488,47 @@ trunk_pivot_create(platform_heap_id hid, return result; } +static key +trunk_pivot_blob_key(const trunk_pivot *pivot) +{ + if (pivot->blob_key_length == 0) { + return NULL_KEY; + } + key stored = key_create( + FALSE, pivot->blob_key_length, pivot->bytes + key_length(pivot->key)); + stored.is_blob = TRUE; + return stored; +} + +static trunk_pivot * +trunk_pivot_create(platform_heap_id hid, + key k, + uint64 child_addr, + uint64 inflight_bundle_start, + trunk_pivot_stats prereceive_stats, + trunk_pivot_stats stats) +{ + return trunk_pivot_create_with_blob(hid, + k, + child_addr, + inflight_bundle_start, + prereceive_stats, + stats, + NULL_KEY, + 0); +} + static trunk_pivot * trunk_pivot_copy(const trunk_pivot *src, platform_heap_id hid) { - return trunk_pivot_create(hid, - ondisk_key_to_key(&src->key), - src->child_addr, - src->inflight_bundle_start, - src->prereceive_stats, - src->stats); + return trunk_pivot_create_with_blob(hid, + src->key, + src->child_addr, + src->inflight_bundle_start, + src->prereceive_stats, + src->stats, + trunk_pivot_blob_key(src), + src->blob_key_prefix_size); } static void @@ -486,7 +540,7 @@ trunk_pivot_destroy(trunk_pivot *pvt, platform_heap_id hid) static key trunk_pivot_key(const trunk_pivot *pvt) { - return ondisk_key_to_key(&pvt->key); + return pvt->key; } static uint64 @@ -1023,6 +1077,27 @@ trunk_ondisk_pivot_key(trunk_ondisk_pivot *odp) * them. ********************************************************/ +static btree_config +trunk_blob_key_config(const trunk_context *context, + const trunk_ondisk_node_handle *handle) +{ + btree_config cfg = *context->cfg->btree_cfg; + cfg.cc = handle->cc; + cfg.page_size = handle->page_size; + const trunk_ondisk_node *header = + (const trunk_ondisk_node *)page_handle_data(&handle->header_page); + cfg.blob_key_prefix_size = header->blob_key_prefix_size; + return cfg; +} + +static uint64 +trunk_ondisk_node_metadata_size(const trunk_ondisk_node_handle *handle) +{ + const trunk_ondisk_node *header = + (const trunk_ondisk_node *)page_handle_data(&handle->header_page); + return header->tail_offset; +} + static platform_status trunk_ondisk_node_read_header(trunk_ondisk_node_handle *handle) { @@ -1040,6 +1115,10 @@ trunk_ondisk_node_read_header(trunk_ondisk_node_handle *handle) (page_size - sizeof(*header)) / sizeof(header->pivot_offsets[0]); if (header->height >= TRUNK_MAX_HEIGHT || header->num_pivots < 2 || max_pivots < header->num_pivots + || (header->height == 0 && header->num_pivots != 2) + || header->tail_offset < page_size || header->tail_offset > extent_size + || header->tail_offset % page_size != 0 + || header->blob_key_prefix_size > MAX_INLINE_KEY_SIZE(page_size) || extent_size / sizeof(trunk_ondisk_bundle) < header->num_inflight_bundles) { @@ -1226,7 +1305,7 @@ trunk_ondisk_node_handle_setup_content_page(trunk_ondisk_node_handle *handle, page_handle *page) { uint64 page_size = handle->page_size; - if (handle->extent_size <= offset) { + if (trunk_ondisk_node_metadata_size(handle) <= offset) { return STATUS_INVALID_STATE; } @@ -1274,7 +1353,9 @@ trunk_ondisk_node_handle_setup_content_page_async( uint64 page_size = state->handlep->page_size; - if (state->handlep->extent_size <= state->work.node.offset) { + if (trunk_ondisk_node_metadata_size(state->handlep) + <= state->work.node.offset) + { state->rc = STATUS_INVALID_STATE; async_return(state); } @@ -1352,7 +1433,7 @@ trunk_ondisk_node_read_pivot_offset(trunk_ondisk_node_handle *handle, + (uint64)header->num_pivots * sizeof(header->pivot_offsets[0]); uint64 offset = header->pivot_offsets[pivot_num]; uint64 page_size = handle->page_size; - if (offset < table_end || handle->extent_size <= offset + if (offset < table_end || trunk_ondisk_node_metadata_size(handle) <= offset || page_size - (offset % page_size) < sizeof(trunk_ondisk_pivot)) { return STATUS_INVALID_STATE; @@ -1361,6 +1442,72 @@ trunk_ondisk_node_read_pivot_offset(trunk_ondisk_node_handle *handle, return STATUS_OK; } +static bool32 +trunk_ondisk_key_readable(const trunk_ondisk_node_handle *handle, + const ondisk_key *key) +{ + if ((key->flags & ~ONDISK_KEY_IS_BLOB) != 0 + || sizeof_ondisk_key_data(key) > MAX_INLINE_KEY_SIZE(handle->page_size)) + { + return FALSE; + } + if (key->flags & ONDISK_KEY_IS_BLOB) { + if (key->length == ONDISK_KEY_NEGATIVE_INFINITY + || key->length == ONDISK_KEY_POSITIVE_INFINITY) + { + return FALSE; + } + const trunk_ondisk_node *header = + (const trunk_ondisk_node *)page_handle_data(&handle->header_page); + return key->length >= header->blob_key_prefix_size + sizeof(blob); + } + return TRUE; +} + +/* A node extent mixes metadata and tail pages, but no physical page has + * both types. Reject references across that boundary before entering cache. */ +static platform_status +trunk_validate_blob_pivot(const trunk_ondisk_node_handle *handle, + const ondisk_key *stored) +{ + if (!(stored->flags & ONDISK_KEY_IS_BLOB)) { + return STATUS_OK; + } + const trunk_ondisk_node *header = + (const trunk_ondisk_node *)page_handle_data(&handle->header_page); + uint64 prefix = header->blob_key_prefix_size; + btree_config cfg = {.cc = handle->cc, + .page_size = handle->page_size, + .blob_key_prefix_size = prefix}; + platform_status rc = btree_key_validate(&cfg, ondisk_key_to_key(stored)); + if (!SUCCESS(rc)) { + return rc; + } + parsed_blob parsed; + rc = blob_parse_descriptor( + handle->cc, + slice_create(stored->length - prefix, stored->bytes + prefix), + &parsed); + if (!SUCCESS(rc)) { + return rc; + } + uint64 node_addr = page_handle_addr(&handle->header_page); + for (uint64 i = 0; i < parsed.num_extents; i++) { + if (blob_get_ref(handle->cc, &parsed, i).addr == node_addr) { + return STATUS_INVALID_STATE; + } + } + for (uint64 i = 0; i < ARRAY_SIZE(parsed.leftovers); i++) { + uint64 addr = parsed.leftovers[i].addr; + if (parsed.leftovers[i].length != 0 && addr >= node_addr + && addr - node_addr < header->tail_offset) + { + return STATUS_INVALID_STATE; + } + } + return STATUS_OK; +} + static platform_status trunk_ondisk_node_read_pivot_key(trunk_ondisk_node_handle *handle, uint64 pivot_num, @@ -1392,12 +1539,16 @@ trunk_ondisk_node_read_pivot_key(trunk_ondisk_node_handle *handle, trunk_ondisk_pivot *pivot = (trunk_ondisk_pivot *)(page_handle_data(&handle->pivot_page) + page_offset); - if (pivot->key.flags != ONDISK_KEY_DEFAULT_FLAGS + if (!trunk_ondisk_key_readable(handle, &pivot->key) || sizeof_ondisk_key_data(&pivot->key) > page_size - page_offset - sizeof(*pivot)) { return STATUS_INVALID_STATE; } + rc = trunk_validate_blob_pivot(handle, &pivot->key); + if (!SUCCESS(rc)) { + return rc; + } *out = pivot; return STATUS_OK; } @@ -1415,7 +1566,7 @@ trunk_ondisk_node_read_pivot_fields(trunk_ondisk_node_handle *handle, return STATUS_INVALID_STATE; } if (pivot_num + 1 < header->num_pivots) { - if (header->height == 0 && pivot->child_addr != 0) { + if ((header->height == 0) != (pivot->child_addr == 0)) { return STATUS_INVALID_STATE; } } @@ -1483,7 +1634,7 @@ trunk_ondisk_node_read_pivot_key_async(trunk_merge_lookup_async_state *state, - content_page_offset(state->handlep, &state->handlep->pivot_page); uint64 page_size = state->handlep->page_size; if (page_size - page_offset < sizeof(*state->pivot) - || state->pivot->key.flags != ONDISK_KEY_DEFAULT_FLAGS + || !trunk_ondisk_key_readable(state->handlep, &state->pivot->key) || sizeof_ondisk_key_data(&state->pivot->key) > page_size - page_offset - sizeof(*state->pivot)) { @@ -1491,14 +1642,33 @@ trunk_ondisk_node_read_pivot_key_async(trunk_merge_lookup_async_state *state, state->rc = STATUS_INVALID_STATE; async_return(state); } - state->rc = STATUS_OK; + state->rc = trunk_validate_blob_pivot(state->handlep, &state->pivot->key); + if (!SUCCESS(state->rc)) { + state->pivot = NULL; + } async_return(state); } static platform_status -trunk_ondisk_node_get_pivot_key(trunk_ondisk_node_handle *handle, +trunk_compare_stored_key(const trunk_context *context, + const trunk_ondisk_node_handle *handle, + key query, + key stored, + int *cmp) +{ + if (!stored.is_blob) { + *cmp = data_key_compare(context->cfg->data_cfg, query, stored); + return STATUS_OK; + } + btree_config cfg = trunk_blob_key_config(context, handle); + return btree_key_compare_checked(&cfg, query, stored, cmp); +} + +static platform_status +trunk_ondisk_node_get_pivot_key(const trunk_context *context, + trunk_ondisk_node_handle *handle, uint64 pivot_num, - key *k) + key_buffer *k) { trunk_ondisk_pivot *odp; platform_status rc = @@ -1508,8 +1678,12 @@ trunk_ondisk_node_get_pivot_key(trunk_ondisk_node_handle *handle, "%s():%d: ondisk_node_get_pivot() failed", __func__, __LINE__); return rc; } - *k = ondisk_key_to_key(&odp->key); - return STATUS_OK; + key stored = ondisk_key_to_key(&odp->key); + if (!stored.is_blob) { + return key_buffer_copy_key(k, stored); + } + btree_config cfg = trunk_blob_key_config(context, handle); + return btree_key_materialize(&cfg, stored, k); } static bool32 @@ -1577,7 +1751,7 @@ trunk_ondisk_node_read_pivot_bundle(trunk_ondisk_node_handle *handle, uintptr_t pivot_addr = (uintptr_t)pivot; if (pivot_addr < page_start || pivot_addr - page_start > page_size || page_size - (pivot_addr - page_start) < sizeof(*pivot) - || pivot->key.flags != ONDISK_KEY_DEFAULT_FLAGS) + || !trunk_ondisk_key_readable(handle, &pivot->key)) { return NULL; } @@ -1603,7 +1777,7 @@ trunk_ondisk_node_bundle_at_offset(trunk_ondisk_node_handle *handle, { *out = NULL; uint64 page_size = handle->page_size; - uint64 extent_size = handle->extent_size; + uint64 extent_size = trunk_ondisk_node_metadata_size(handle); if (extent_size <= offset) { return STATUS_INVALID_STATE; } @@ -1689,7 +1863,7 @@ trunk_ondisk_node_bundle_at_offset_async(trunk_merge_lookup_async_state *state, uint64 depth) { uint64 page_size = state->handlep->page_size; - uint64 extent_size = state->handlep->extent_size; + uint64 extent_size = trunk_ondisk_node_metadata_size(state->handlep); async_begin(state, depth); @@ -1801,7 +1975,7 @@ trunk_ondisk_node_get_first_inflight_bundle(trunk_ondisk_node_handle *handle, uint64 table_end = sizeof(*header) + (uint64)header->num_pivots * sizeof(header->pivot_offsets[0]); - if (offset < table_end || handle->extent_size <= offset + if (offset < table_end || trunk_ondisk_node_metadata_size(handle) <= offset || offset <= header->pivot_offsets[header->num_pivots - 1]) { *bndl = NULL; @@ -1843,7 +2017,8 @@ trunk_ondisk_node_get_first_inflight_bundle_async( sizeof(*header) + (uint64)header->num_pivots * sizeof(header->pivot_offsets[0]); if (state->work.node.offset < table_end - || state->handlep->extent_size <= state->work.node.offset + || trunk_ondisk_node_metadata_size(state->handlep) + <= state->work.node.offset || state->work.node.offset <= header->pivot_offsets[header->num_pivots - 1]) { @@ -1900,7 +2075,9 @@ trunk_ondisk_node_get_next_inflight_bundle_async( } static platform_status -trunk_pivot_deserialize(platform_heap_id hid, +trunk_pivot_deserialize(const trunk_context *context, + bool32 materialize, + platform_heap_id hid, trunk_ondisk_node_handle *handle, uint64 i, trunk_pivot **pivot) @@ -1922,12 +2099,28 @@ trunk_pivot_deserialize(platform_heap_id hid, } else { inflight_bundle_start = 0; } - *pivot = trunk_pivot_create(hid, - trunk_ondisk_pivot_key(odp), - odp->child_addr, - inflight_bundle_start, - odp->stats, - odp->stats); + key stored = trunk_ondisk_pivot_key(odp); + DECLARE_AUTO_KEY_BUFFER(logical, hid); + key k = stored; + if (stored.is_blob) { + btree_config cfg = trunk_blob_key_config(context, handle); + rc = btree_key_validate(&cfg, stored); + if (SUCCESS(rc) && materialize) { + rc = btree_key_materialize(&cfg, stored, &logical); + k = key_buffer_key(&logical); + } + if (!SUCCESS(rc)) { + return rc; + } + } + *pivot = trunk_pivot_create_with_blob(hid, + k, + odp->child_addr, + inflight_bundle_start, + odp->stats, + odp->stats, + stored, + header->blob_key_prefix_size); return *pivot == NULL ? STATUS_NO_MEMORY : STATUS_OK; } @@ -1950,7 +2143,7 @@ bundle_deserialize(bundle *bndl, platform_heap_id hid, trunk_ondisk_bundle *odb) for (uint64 i = 0; i < odb->num_branches; i++) { rc = btree_validate_root(odb->branches[i].addr, - odb->branches[i].format_version, + odb->branches[i].format, trunk_ondisk_bundle_branch_type(odb)); if (!SUCCESS(rc)) { bundle_deinit(bndl); @@ -1964,9 +2157,10 @@ bundle_deserialize(bundle *bndl, platform_heap_id hid, trunk_ondisk_bundle *odb) } static platform_status -trunk_node_deserialize(const trunk_context *context, - uint64 addr, - trunk_node *result) +trunk_node_deserialize_internal(const trunk_context *context, + uint64 addr, + trunk_node *result, + bool32 materialize) { platform_status rc; trunk_ondisk_node_handle handle; @@ -2016,7 +2210,8 @@ trunk_node_deserialize(const trunk_context *context, for (uint64 i = 0; i < header->num_pivots; i++) { trunk_pivot *imp; - rc = trunk_pivot_deserialize(PROCESS_PRIVATE_HEAP_ID, &handle, i, &imp); + rc = trunk_pivot_deserialize( + context, materialize, PROCESS_PRIVATE_HEAP_ID, &handle, i, &imp); if (!SUCCESS(rc)) { platform_error_log( "%s():%d: pivot_deserialize() failed", __func__, __LINE__); @@ -2097,10 +2292,17 @@ trunk_node_deserialize(const trunk_context *context, trunk_node_init( result, height, pivots, pivot_bundles, num_old_bundles, inflight_bundles); + bool32 encoded = FALSE; + if (!materialize) { + for (uint64 i = 0; i < vector_length(&result->pivots); i++) { + encoded |= trunk_pivot_key(vector_get(&result->pivots, i)).is_blob; + } + } bool32 well_formed = - trunk_node_is_leaf(result) - ? trunk_node_is_well_formed_leaf(context->cfg->data_cfg, result) - : trunk_node_is_well_formed_index(context->cfg->data_cfg, result); + encoded + || (trunk_node_is_leaf(result) + ? trunk_node_is_well_formed_leaf(context->cfg->data_cfg, result) + : trunk_node_is_well_formed_index(context->cfg->data_cfg, result)); if (!well_formed) { trunk_node_deinit(result, context); return STATUS_INVALID_STATE; @@ -2119,6 +2321,14 @@ trunk_node_deserialize(const trunk_context *context, return rc; } +static platform_status +trunk_node_deserialize(const trunk_context *context, + uint64 addr, + trunk_node *result) +{ + return trunk_node_deserialize_internal(context, addr, result, TRUE); +} + __attribute__((warn_unused_result)) static platform_status bundle_inc_all_branch_refs(const trunk_context *context, bundle *bndl) { @@ -2194,6 +2404,144 @@ bundle_dec_all_refs(trunk_context *context, bundle *bndl) // cache_unget(context->cc, page); // } +/* Each serialized node owns its blob roots and its tail extents. Tails in the + * node's own extent need no additional reference; overflow extents are owned + * once per node, even when several pivots use them. */ +typedef VECTOR(uint64) trunk_tail_extent_vector; + +static platform_status +trunk_node_blob_references(const trunk_context *context, + trunk_node *node, + uint64 node_addr, + bool32 recover) +{ + trunk_tail_extent_vector tails; + vector_init(&tails, PROCESS_PRIVATE_HEAP_ID); + platform_status rc = STATUS_OK; + uint64 extent_size = cache_extent_size(context->cc); + for (uint64 i = 0; i < vector_length(&node->pivots); i++) { + trunk_pivot *pvt = vector_get(&node->pivots, i); + key stored = trunk_pivot_blob_key(pvt); + if (!stored.is_blob) { + continue; + } + uint64 prefix = pvt->blob_key_prefix_size; + slice descriptor = slice_create(key_length(stored) - prefix, + (const char *)key_data(stored) + prefix); + parsed_blob parsed; + rc = blob_parse_descriptor(context->cc, descriptor, &parsed); + if (!SUCCESS(rc)) { + goto out; + } + for (uint64 j = 0; j < ARRAY_SIZE(parsed.leftovers); j++) { + if (parsed.leftovers[j].length == 0) { + continue; + } + uint64 addr = parsed.leftovers[j].addr / extent_size * extent_size; + if (addr == node_addr) { + continue; + } + bool32 seen = FALSE; + for (uint64 k = 0; k < vector_length(&tails); k++) { + seen |= addr == vector_get(&tails, k); + } + if (!seen) { + rc = vector_append(&tails, addr); + if (!SUCCESS(rc)) { + goto out; + } + } + } + } + for (uint64 i = 0; i < vector_length(&node->pivots); i++) { + trunk_pivot *pvt = vector_get(&node->pivots, i); + key stored = trunk_pivot_blob_key(pvt); + if (!stored.is_blob) { + continue; + } + uint64 prefix = pvt->blob_key_prefix_size; + slice descriptor = slice_create(key_length(stored) - prefix, + (const char *)key_data(stored) + prefix); + parsed_blob parsed; + platform_status parse_rc = + blob_parse_descriptor(context->cc, descriptor, &parsed); + platform_assert_status_ok(parse_rc); + for (uint64 j = 0; j < parsed.num_extents; j++) { + blob_ref ref = blob_get_ref(context->cc, &parsed, j); + platform_status ref_rc = recover ? blob_ref_recover(context->cc, ref) + : blob_ref_release(context->cc, ref); + if (SUCCESS(rc)) { + rc = ref_rc; + } + if (recover && !SUCCESS(rc)) { + goto out; + } + } + } + for (uint64 i = 0; i < vector_length(&tails); i++) { + uint64 addr = vector_get(&tails, i); + platform_status tail_rc = STATUS_OK; + if (recover) { + tail_rc = allocator_recovery_record_reference( + context->al, addr, PAGE_TYPE_BLOB); + } else { + refcount ref = allocator_dec_ref(context->al, addr, PAGE_TYPE_BLOB); + if (ref == AL_NO_REFS) { + cache_extent_discard(context->cc, addr, PAGE_TYPE_BLOB); + ref = allocator_dec_ref(context->al, addr, PAGE_TYPE_BLOB); + platform_assert(ref == AL_FREE); + } + } + if (SUCCESS(rc)) { + rc = tail_rc; + } + if (recover && !SUCCESS(rc)) { + goto out; + } + } +out: + vector_deinit(&tails); + return rc; +} + +static platform_status +trunk_node_materialize_keys(const trunk_context *context, trunk_node *node) +{ + for (uint64 i = 0; i < vector_length(&node->pivots); i++) { + trunk_pivot *pvt = vector_get(&node->pivots, i); + if (!pvt->key.is_blob) { + continue; + } + btree_config cfg = *context->cfg->btree_cfg; + cfg.cc = context->cc; + cfg.blob_key_prefix_size = pvt->blob_key_prefix_size; + DECLARE_AUTO_KEY_BUFFER(logical, PROCESS_PRIVATE_HEAP_ID); + platform_status rc = btree_key_materialize(&cfg, pvt->key, &logical); + if (!SUCCESS(rc)) { + return rc; + } + trunk_pivot *replacement = + trunk_pivot_create_with_blob(PROCESS_PRIVATE_HEAP_ID, + key_buffer_key(&logical), + pvt->child_addr, + pvt->inflight_bundle_start, + pvt->prereceive_stats, + pvt->stats, + trunk_pivot_blob_key(pvt), + pvt->blob_key_prefix_size); + if (replacement == NULL) { + return STATUS_NO_MEMORY; + } + *vector_get_ptr(&node->pivots, i) = replacement; + trunk_pivot_destroy(pvt, PROCESS_PRIVATE_HEAP_ID); + } + bool32 valid = + trunk_node_is_leaf(node) + ? trunk_node_is_well_formed_leaf(context->cfg->data_cfg, node) + : trunk_node_is_well_formed_index(context->cfg->data_cfg, node); + return valid ? STATUS_OK : STATUS_INVALID_STATE; +} + static platform_status trunk_ondisk_node_dec_ref(trunk_context *context, uint64 addr); @@ -2209,7 +2557,8 @@ trunk_ondisk_node_gc(trunk_context *context, uint64 expected_height) { trunk_node node; - platform_status rc = trunk_node_deserialize(context, addr, &node); + platform_status rc = + trunk_node_deserialize_internal(context, addr, &node, FALSE); if (SUCCESS(rc)) { rc = trunk_node_validate_expected_height(&node, expected_height); bool32 topology_valid = SUCCESS(rc); @@ -2257,6 +2606,13 @@ trunk_ondisk_node_gc(trunk_context *context, bundle_dec_all_refs(context, bndl); } } + if (topology_valid) { + platform_status blob_rc = + trunk_node_blob_references(context, &node, addr, FALSE); + if (SUCCESS(rc)) { + rc = blob_rc; + } + } trunk_node_deinit(&node, context); } else { platform_error_log("%s():%d: node_deserialize() failed: %s", @@ -2446,7 +2802,7 @@ trunk_recover_bundle_refs(const trunk_context *context, bundle *bndl) platform_status rc = btree_recover_allocations(context->cc, context->cfg->btree_cfg, branch_ref_addr(bref), - bref.format_version, + bref.format, type); if (!SUCCESS(rc)) { return rc; @@ -2471,7 +2827,8 @@ trunk_recover_node_refs(const trunk_context *context, uint64 expected_height) { trunk_node node; - platform_status rc = trunk_node_deserialize(context, addr, &node); + platform_status rc = + trunk_node_deserialize_internal(context, addr, &node, FALSE); if (!SUCCESS(rc)) { platform_error_log("trunk_recover_node_refs: cannot read node %lu: %s\n", addr, @@ -2484,6 +2841,15 @@ trunk_recover_node_refs(const trunk_context *context, return rc; } + rc = trunk_node_blob_references(context, &node, addr, TRUE); + if (SUCCESS(rc)) { + rc = trunk_node_materialize_keys(context, &node); + } + if (!SUCCESS(rc)) { + trunk_node_deinit(&node, context); + return rc; + } + uint64 num_children = trunk_node_is_leaf(&node) ? 0 : vector_length(&node.pivots) - 1; @@ -2606,10 +2972,10 @@ trunk_ondisk_node_ref_create(trunk_context *context, bool32 adopt, trunk_ondisk_node_ref **out) { - *out = NULL; - platform_heap_id hid = context->hid; - trunk_ondisk_node_ref *result = TYPED_FLEXIBLE_STRUCT_ZALLOC( - hid, result, key.bytes, ondisk_key_required_data_capacity(k)); + *out = NULL; + platform_heap_id hid = context->hid; + trunk_ondisk_node_ref *result = + TYPED_FLEXIBLE_STRUCT_ZALLOC(hid, result, bytes, key_length(k)); if (result == NULL) { platform_error_log( "%s():%d: TYPED_FLEXIBLE_STRUCT_ZALLOC() failed", __func__, __LINE__); @@ -2637,7 +3003,11 @@ trunk_ondisk_node_ref_create(trunk_context *context, return rc; } } - copy_key_to_ondisk_key(&result->key, k); + result->key = k; + if (key_is_user_key(k)) { + memcpy(result->bytes, key_data(k), key_length(k)); + result->key.user_slice = slice_create(key_length(k), result->bytes); + } *out = result; return STATUS_OK; } @@ -2657,19 +3027,13 @@ trunk_pivot_create_from_ondisk_node_ref(trunk_ondisk_node_ref *odnref, platform_heap_id hid) { return trunk_pivot_create(hid, - ondisk_key_to_key(&odnref->key), + odnref->key, odnref->ref.addr, 0, TRUNK_STATS_ZERO, TRUNK_STATS_ZERO); } -static uint64 -trunk_pivot_ondisk_size(trunk_pivot *pvt) -{ - return trunk_ondisk_pivot_size(trunk_pivot_key(pvt)); -} - static uint64 bundle_ondisk_size(bundle *bndl) { @@ -2680,6 +3044,7 @@ static void pivot_serialize(trunk_context *context, trunk_node *node, uint64 pivot_num, + key stored, trunk_ondisk_pivot *dest) { trunk_pivot *pvt = vector_get(&node->pivots, pivot_num); @@ -2692,7 +3057,7 @@ pivot_serialize(trunk_context *context, } else { dest->num_live_inflight_bundles = 0; } - copy_key_to_ondisk_key(&dest->key, trunk_pivot_key(pvt)); + copy_key_to_ondisk_key(&dest->key, stored); } static void @@ -2810,6 +3175,211 @@ trunk_node_record_and_report_maxes(trunk_context *context, trunk_node *node) } } +typedef struct trunk_blob_tails { + trunk_context *context; + uint64 next_addr; + uint64 limit; + trunk_tail_extent_vector overflow; +} trunk_blob_tails; + +static platform_status +trunk_blob_allocate_tail(void *arg, + uint64 bytes, + uint64 alignment, + uint64 boundary, + uint64 addrs[2]) +{ + trunk_blob_tails *tails = arg; + cache *cc = tails->context->cc; + uint64 extent_size = cache_extent_size(cc); + uint64 page_size = cache_page_size(cc); + platform_assert(bytes <= extent_size); + addrs[0] = addrs[1] = 0; + alignment = MAX(alignment, 1); + uint64 addr = ROUNDUP(tails->next_addr, alignment); + if (boundary && addr % boundary + bytes > boundary) { + addr = tails->limit; + } + uint64 remaining = bytes; + for (uint64 fragment = 0; remaining != 0; fragment++) { + platform_assert(fragment < 2); + if (addr >= tails->limit) { + platform_status rc = vector_ensure_capacity( + &tails->overflow, vector_length(&tails->overflow) + 1); + if (!SUCCESS(rc)) { + return rc; + } + rc = allocator_alloc(tails->context->al, &addr, PAGE_TYPE_BLOB); + if (!SUCCESS(rc)) { + return rc; + } + rc = vector_append(&tails->overflow, addr); + platform_assert_status_ok(rc); + tails->limit = addr + extent_size; + } + uint64 count = MIN(remaining, tails->limit - addr); + addrs[fragment] = addr; + for (uint64 page_addr = ROUNDUP(addr, page_size); + page_addr < addr + count; + page_addr += page_size) + { + page_handle storage; + page_handle *page = + cache_alloc(cc, page_addr, PAGE_TYPE_BLOB, &storage); + if (page == NULL) { + return STATUS_NO_MEMORY; + } + memset(page_handle_data(page), 0, page_size); + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + } + addr += count; + remaining -= count; + } + tails->next_addr = addr; + return STATUS_OK; +} + +static blob_build_config +trunk_blob_build_config(const trunk_context *context) +{ + return (blob_build_config){ + .alignment = context->cfg->btree_cfg->blob_cfg.alignment, + .blob_max_descriptor_size = + context->cfg->btree_cfg->blob_cfg.blob_max_descriptor_size, + }; +} + +static platform_status +trunk_pivot_storage_capacity(const trunk_context *context, + trunk_pivot *pivot, + uint64 *size) +{ + *size = key_length(pivot->key); + if (*size <= MAX_INLINE_KEY_SIZE(cache_page_size(context->cc))) { + return STATUS_OK; + } + blob_build_config cfg = trunk_blob_build_config(context); + platform_status rc = blob_descriptor_size_bound( + &cfg, cache_extent_size(context->cc), *size, size); + if (SUCCESS(rc)) { + *size += btree_config_format(context->cfg->btree_cfg).key_prefix_size; + if (*size > MAX_INLINE_KEY_SIZE(cache_page_size(context->cc))) { + rc = STATUS_LIMIT_EXCEEDED; + } + } + return rc; +} + +static platform_status +trunk_layout_add(uint64 page_size, + uint64 extent_size, + uint64 bytes, + uint64 *offset) +{ + if (bytes > page_size) { + return STATUS_LIMIT_EXCEEDED; + } + if (*offset % page_size + bytes > page_size) { + *offset = ROUNDUP(*offset, page_size); + } + if (*offset > extent_size || bytes > extent_size - *offset) { + return STATUS_LIMIT_EXCEEDED; + } + *offset += bytes; + return STATUS_OK; +} + +/* Reserve a bound on each descriptor's size: splitting a tail across an + * extent boundary can add an address. Any slack stays in the node body. */ +static platform_status +trunk_node_layout(const trunk_context *context, + trunk_node *node, + uint64 *size, + bool32 *has_blobs) +{ + uint64 page_size = cache_page_size(context->cc); + uint64 extent_size = cache_extent_size(context->cc); + *size = + sizeof(trunk_ondisk_node) + vector_length(&node->pivots) * sizeof(uint32); + *has_blobs = FALSE; + for (uint64 i = 0; i < vector_length(&node->pivots); i++) { + trunk_pivot *pivot = vector_get(&node->pivots, i); + *has_blobs |= key_length(pivot->key) > MAX_INLINE_KEY_SIZE(page_size); + uint64 capacity; + platform_status rc = + trunk_pivot_storage_capacity(context, pivot, &capacity); + if (!SUCCESS(rc)) { + return rc; + } + uint64 bytes = sizeof(trunk_ondisk_pivot) + capacity; + if (i + 1 < vector_length(&node->pivots)) { + bytes += bundle_ondisk_size(vector_get_ptr(&node->pivot_bundles, i)); + } + rc = trunk_layout_add(page_size, extent_size, bytes, size); + if (!SUCCESS(rc)) { + return rc; + } + } + int64 first = trunk_node_first_live_inflight_bundle(node); + for (int64 i = vector_length(&node->inflight_bundles) - 1; i >= first; i--) { + uint64 bytes = + bundle_ondisk_size(vector_get_ptr(&node->inflight_bundles, i)); + platform_status rc = + trunk_layout_add(page_size, extent_size, bytes, size); + if (!SUCCESS(rc)) { + return rc; + } + } + return STATUS_OK; +} + +static platform_status +trunk_encode_blob_pivot(trunk_context *context, + trunk_blob_tails *tails, + trunk_pivot *pivot, + key_buffer *result) +{ + blob_build_config cfg = trunk_blob_build_config(context); + blob_tail_allocator tail_allocator = {trunk_blob_allocate_tail, tails}; + auto_writable_buffer descriptor; + writable_buffer_init(&descriptor, context->hid); + key source = trunk_pivot_blob_key(pivot); + platform_status rc; + if (source.is_blob) { + uint64 p = pivot->blob_key_prefix_size; + slice source_descriptor = slice_create( + key_length(source) - p, (const char *)key_data(source) + p); + rc = blob_clone( + &cfg, context->cc, &tail_allocator, source_descriptor, &descriptor); + } else { + rc = blob_build(&cfg, + context->cc, + &tail_allocator, + key_slice(pivot->key), + &descriptor); + } + if (!SUCCESS(rc)) { + return rc; + } + uint64 prefix = btree_config_format(context->cfg->btree_cfg).key_prefix_size; + uint64 size = prefix + writable_buffer_length(&descriptor); + rc = key_buffer_resize(result, size); + if (!SUCCESS(rc)) { + platform_status release_rc = + blob_release_refs(context->cc, writable_buffer_to_slice(&descriptor)); + trunk_record_allocator_cleanup_error(context, release_rc); + return rc; + } + memcpy(key_buffer_data(result), key_data(pivot->key), prefix); + memcpy((char *)key_buffer_data(result) + prefix, + writable_buffer_data(&descriptor), + writable_buffer_length(&descriptor)); + result->is_blob = TRUE; + return STATUS_OK; +} + __attribute__((warn_unused_result)) static platform_status trunk_node_serialize(trunk_context *context, trunk_node *node, @@ -2823,6 +3393,11 @@ trunk_node_serialize(trunk_context *context, page_handle *current_page = NULL; trunk_ondisk_node_ref *result = NULL; threadid tid = platform_get_tid(); + key_buffer *blob_keys = NULL; + uint64 node_bytes; + bool32 has_blobs; + trunk_blob_tails tails = {.context = context}; + vector_init(&tails.overflow, context->hid); if (trunk_node_num_children(node) > trunk_max_fanout(cache_page_size(context->cc))) @@ -2864,6 +3439,21 @@ trunk_node_serialize(trunk_context *context, trunk_node_is_well_formed_index(context->cfg->data_cfg, node)); } + rc = trunk_node_layout(context, node, &node_bytes, &has_blobs); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (has_blobs) { + blob_keys = TYPED_ARRAY_ZALLOC( + context->hid, blob_keys, vector_length(&node->pivots)); + if (blob_keys == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + for (uint64 i = 0; i < vector_length(&node->pivots); i++) { + key_buffer_init(&blob_keys[i], context->hid); + } + } rc = allocator_alloc(context->al, &header_addr, PAGE_TYPE_TRUNK); if (!SUCCESS(rc)) { platform_error_log("%s():%d: allocator_alloc() failed: %s", @@ -2873,6 +3463,22 @@ trunk_node_serialize(trunk_context *context, goto cleanup; } + tails.next_addr = + header_addr + ROUNDUP(node_bytes, cache_page_size(context->cc)); + tails.limit = header_addr + cache_extent_size(context->cc); + if (has_blobs) { + for (uint64 i = 0; i < vector_length(&node->pivots); i++) { + trunk_pivot *pivot = vector_get(&node->pivots, i); + if (key_length(pivot->key) + > MAX_INLINE_KEY_SIZE(cache_page_size(context->cc))) + { + rc = trunk_encode_blob_pivot(context, &tails, pivot, &blob_keys[i]); + if (!SUCCESS(rc)) { + goto cleanup; + } + } + } + } header_page = cache_alloc(context->cc, header_addr, PAGE_TYPE_TRUNK, &header_storage); if (header_page == NULL) { @@ -2886,8 +3492,11 @@ trunk_node_serialize(trunk_context *context, trunk_ondisk_node *odnode = (trunk_ondisk_node *)page_handle_data(header_page); - odnode->height = node->height; - odnode->num_pivots = vector_length(&node->pivots); + odnode->blob_key_prefix_size = + btree_config_format(context->cfg->btree_cfg).key_prefix_size; + odnode->tail_offset = ROUNDUP(node_bytes, cache_page_size(context->cc)); + odnode->height = node->height; + odnode->num_pivots = vector_length(&node->pivots); odnode->num_inflight_bundles = vector_length(&node->inflight_bundles) - min_inflight_bundle_start; @@ -2896,8 +3505,16 @@ trunk_node_serialize(trunk_context *context, sizeof(*odnode) + sizeof(odnode->pivot_offsets[0]) * odnode->num_pivots; for (uint64 i = 0; i < vector_length(&node->pivots); i++) { - uint64 pivot_size = trunk_pivot_ondisk_size(vector_get(&node->pivots, i)); - uint64 required_space = pivot_size; + trunk_pivot *pivot = vector_get(&node->pivots, i); + key stored = blob_keys != NULL && blob_keys[i].is_blob + ? key_buffer_key(&blob_keys[i]) + : pivot->key; + uint64 capacity; + rc = trunk_pivot_storage_capacity(context, pivot, &capacity); + platform_assert_status_ok(rc); + uint64 pivot_size = trunk_ondisk_pivot_size(stored); + uint64 required_space = sizeof(trunk_ondisk_pivot) + capacity; + uint64 padding = required_space - pivot_size; bundle *pivot_bundle; uint64 bundle_size; @@ -2937,6 +3554,7 @@ trunk_node_serialize(trunk_context *context, context, node, i, + stored, (trunk_ondisk_pivot *)(page_handle_data(current_page) + page_offset)); page_offset += pivot_size; if (i < vector_length(&node->pivots) - 1) { @@ -2945,6 +3563,8 @@ trunk_node_serialize(trunk_context *context, + page_offset)); page_offset += bundle_size; } + memset(page_handle_data(current_page) + page_offset, 0, padding); + page_offset += padding; } odnode->inflight_bundles_offset = 0; @@ -2989,9 +3609,9 @@ trunk_node_serialize(trunk_context *context, goto cleanup; } - // Acquire every outgoing reference before publishing the new node. This - // helper unwinds its acquired prefix on failure; the serialized bytes still - // describe borrowed objects until the entire acquisition succeeds. + // Blob clones already own their references. Acquire the child and branch + // references before publication; this helper unwinds its acquired prefix + // on failure, and cleanup below releases the blobs and tail storage. rc = trunk_node_inc_all_refs(context, node); if (!SUCCESS(rc)) { goto cleanup; @@ -3020,6 +3640,13 @@ trunk_node_serialize(trunk_context *context, cache_unget(context->cc, header_page); *out = result; + if (blob_keys != NULL) { + for (uint64 i = 0; i < vector_length(&node->pivots); i++) { + key_buffer_deinit(&blob_keys[i]); + } + platform_free(context->hid, blob_keys); + } + vector_deinit(&tails.overflow); return STATUS_OK; cleanup: @@ -3038,6 +3665,31 @@ trunk_node_serialize(trunk_context *context, cache_unclaim(context->cc, header_page); cache_unget(context->cc, header_page); } + if (blob_keys != NULL) { + for (uint64 i = 0; i < vector_length(&node->pivots); i++) { + if (blob_keys[i].is_blob) { + uint64 prefix = + btree_config_format(context->cfg->btree_cfg).key_prefix_size; + slice descriptor = + slice_create(key_buffer_length(&blob_keys[i]) - prefix, + (char *)key_buffer_data(&blob_keys[i]) + prefix); + platform_status release_rc = + blob_release_refs(context->cc, descriptor); + trunk_record_allocator_cleanup_error(context, release_rc); + } + key_buffer_deinit(&blob_keys[i]); + } + platform_free(context->hid, blob_keys); + } + for (uint64 i = 0; i < vector_length(&tails.overflow); i++) { + uint64 addr = vector_get(&tails.overflow, i); + refcount ref = allocator_dec_ref(context->al, addr, PAGE_TYPE_BLOB); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(context->cc, addr, PAGE_TYPE_BLOB); + ref = allocator_dec_ref(context->al, addr, PAGE_TYPE_BLOB); + platform_assert(ref == AL_FREE); + } + vector_deinit(&tails.overflow); if (header_addr != 0) { // The unpublished extent may be only partially serialized. Discard it // directly, without traversing references we have not acquired. @@ -3119,14 +3771,14 @@ trunk_branch_merger_init(trunk_branch_merger *merger, static platform_status trunk_branch_merger_add_branch(trunk_branch_merger *merger, uint64 addr, - uint16 format_version, + btree_format format, page_type type) { platform_assert(merger != NULL); platform_assert(addr != 0); platform_assert(PAGE_TYPE_FIRST <= type && type < NUM_PAGE_TYPES); - trunk_branch_info branch = {addr, type, format_version}; + trunk_branch_info branch = {addr, type, format}; platform_status rc = vector_append(&merger->branches, branch); if (!SUCCESS(rc)) { platform_error_log("%s():%d: vector_append() failed: %s", @@ -3157,10 +3809,8 @@ trunk_branch_merger_add_branches(trunk_branch_merger *merger, } for (uint64 i = 0; i < num_branches; i++) { - rc = trunk_branch_merger_add_branch(merger, - branches[i].addr, - branches[i].format_version, - branches[i].type); + rc = trunk_branch_merger_add_branch( + merger, branches[i].addr, branches[i].format, branches[i].type); if (!SUCCESS(rc)) { platform_error_log("%s():%d: btree_merger_add_branch() failed: %s", __func__, @@ -3194,7 +3844,7 @@ trunk_branch_merger_add_bundle(trunk_branch_merger *merger, branch_ref bref = vector_get(&routed->branches, i); rc = trunk_branch_merger_add_branch(merger, branch_ref_addr(bref), - bref.format_version, + bref.format, bundle_branch_type(routed)); if (!SUCCESS(rc)) { platform_error_log("%s():%d: btree_merger_add_branch() failed: %s", @@ -3249,7 +3899,7 @@ trunk_branch_merger_build_merge_itor(trunk_branch_merger *merger, rc = btree_get_root_height(cc, merger->cfg->btree_cfg, branch.addr, - branch.format_version, + branch.format, branch.type, &root_height); if (!SUCCESS(rc)) { @@ -3278,7 +3928,7 @@ trunk_branch_merger_build_merge_itor(trunk_branch_merger *merger, merger->cfg->btree_cfg, itor, branch.addr, - branch.format_version, + branch.format, branch.type, &(const btree_iterator_init_options){ .lower_limit = key_greater_than_or_equal(merger->min_key), @@ -4160,7 +4810,7 @@ bundle_compaction_create(trunk_context *context, return rc; } page_type type = bundle_branch_type(bndl); - trunk_branch_info bi = {bref.addr, type, bref.format_version}; + trunk_branch_info bi = {bref.addr, type, bref.format}; rc = vector_append(&result->input.branches, bi); platform_assert_status_ok(rc); __sync_fetch_and_add(&bc_incs, 1); @@ -4943,7 +5593,7 @@ compute_input_stats(trunk_context *context, platform_status rc = btree_count_in_range(context->cc, context->cfg->btree_cfg, bi.addr, - bi.format_version, + bi.format, lb, ub, &btree_stats); @@ -5069,7 +5719,7 @@ bundle_compaction_task(task *arg) platform_timestamp_elapsed(pack_start); } - bc->output.branch = create_branch_ref(pack_req.root_addr); + bc->output.branch = create_branch_ref(pack_req.root_addr, pack_req.format); bc->output.stats = (trunk_pivot_stats){ .num_tuples = pack_req.num_tuples, .num_kv_bytes = pack_req.key_bytes + pack_req.message_bytes}; @@ -5278,7 +5928,7 @@ accumulate_branch_tuple_counts_in_range(branch_ref bref, platform_status rc = btree_count_in_range(context->cc, context->cfg->btree_cfg, branch_ref_addr(bref), - bref.format_version, + bref.format, minkey, maxkey, &stats); @@ -5483,7 +6133,7 @@ leaf_estimate_unique_keys(trunk_context *context, rc = btree_count_in_range(context->cc, context->cfg->btree_cfg, bundle_branch(bndl, 0).addr, - bundle_branch(bndl, 0).format_version, + bundle_branch(bndl, 0).format, trunk_node_pivot_min_key(leaf), trunk_node_pivot_max_key(leaf), &stats); @@ -5505,14 +6155,13 @@ leaf_estimate_unique_keys(trunk_context *context, platform_assert(bundle_num_branches(&pivot_bundle) <= 1); if (bundle_num_branches(&pivot_bundle) == 1) { btree_pivot_stats stats; - rc = - btree_count_in_range(context->cc, - context->cfg->btree_cfg, - bundle_branch(&pivot_bundle, 0).addr, - bundle_branch(&pivot_bundle, 0).format_version, - trunk_node_pivot_min_key(leaf), - trunk_node_pivot_max_key(leaf), - &stats); + rc = btree_count_in_range(context->cc, + context->cfg->btree_cfg, + bundle_branch(&pivot_bundle, 0).addr, + bundle_branch(&pivot_bundle, 0).format, + trunk_node_pivot_min_key(leaf), + trunk_node_pivot_max_key(leaf), + &stats); if (!SUCCESS(rc)) { goto cleanup; } @@ -5678,8 +6327,8 @@ leaf_split_select_pivots(trunk_context *context, platform_status rc; trunk_pivot *first = vector_get(&leaf->pivots, 0); trunk_pivot *last = vector_get(&leaf->pivots, 1); - key min_key = ondisk_key_to_key(&first->key); - key max_key = ondisk_key_to_key(&last->key); + key min_key = first->key; + key max_key = last->key; trunk_branch_merger merger; trunk_branch_merger_init(&merger, @@ -6793,7 +7442,8 @@ trunk_flush_prepare(trunk_context *context, vector_init(&new_node_refs, PROCESS_PRIVATE_HEAP_ID); if (branch_addr != 0) { - branch_ref branch = create_branch_ref(branch_addr); + branch_ref branch = create_branch_ref( + branch_addr, btree_config_format(context->cfg->btree_cfg)); // Construct a vector of inflight bundles with one singleton bundle for // the new branch. @@ -7051,7 +7701,11 @@ trunk_ondisk_node_find_pivot(const trunk_context *context, return rc; } key mid_key = trunk_ondisk_pivot_key(mid_pivot); - int cmp = data_key_compare(context->cfg->data_cfg, tgt, mid_key); + int cmp; + rc = trunk_compare_stored_key(context, handle, tgt, mid_key, &cmp); + if (!SUCCESS(rc)) { + return rc; + } if (cmp < 0) { max = mid; } else { @@ -7139,8 +7793,14 @@ trunk_ondisk_node_find_pivot_async(trunk_merge_lookup_async_state *state, async_return(state); } key mid_key = trunk_ondisk_pivot_key(state->pivot); - int cmp = - data_key_compare(state->context->cfg->data_cfg, state->tgt, mid_key); + int cmp; + /* As in the B-tree, the uncommon blob comparison currently performs + * synchronous reads. All failures propagate through the async result. */ + state->rc = trunk_compare_stored_key( + state->context, state->handlep, state->tgt, mid_key, &cmp); + if (!SUCCESS(state->rc)) { + async_return(state); + } if (cmp < 0) { state->work.node.max = state->work.node.pivot_num; } else { @@ -7247,9 +7907,9 @@ trunk_ondisk_bundle_merge_lookup(trunk_context *context, rc = btree_lookup_and_merge(context->cc, context->cfg->btree_cfg, branch_ref_addr(bndl->branches[idx]), - bndl->branches[idx].format_version, + bndl->branches[idx].format, trunk_ondisk_bundle_branch_type(bndl), - tgt, + &tgt, result, &branch_found); if (!SUCCESS(rc)) { @@ -7280,9 +7940,9 @@ trunk_ondisk_bundle_merge_lookup(trunk_context *context, rc = btree_lookup_and_merge(context->cc, context->cfg->btree_cfg, branch_ref_addr(bndl->branches[idx]), - bndl->branches[idx].format_version, + bndl->branches[idx].format, trunk_ondisk_bundle_branch_type(bndl), - tgt, + &tgt, &temp_result, &branch_found); if (!SUCCESS(rc)) { @@ -7388,7 +8048,7 @@ trunk_ondisk_bundle_merge_lookup_async(trunk_merge_lookup_async_state *state, state->context->cc, state->context->cfg->btree_cfg, branch_ref_addr(state->bndl->branches[state->work.bundle.idx]), - state->bndl->branches[state->work.bundle.idx].format_version, + state->bndl->branches[state->work.bundle.idx].format, trunk_ondisk_bundle_branch_type(state->bndl), state->tgt, state->result, @@ -7425,9 +8085,9 @@ trunk_ondisk_bundle_merge_lookup_async(trunk_merge_lookup_async_state *state, state->context->cc, state->context->cfg->btree_cfg, branch_ref_addr(state->bndl->branches[state->work.bundle.idx]), - state->bndl->branches[state->work.bundle.idx].format_version, + state->bndl->branches[state->work.bundle.idx].format, trunk_ondisk_bundle_branch_type(state->bndl), - state->tgt, + &state->tgt, &temp_result, NULL); if (!SUCCESS(rc)) { @@ -7503,10 +8163,12 @@ trunk_merge_lookup(trunk_context *context, } if (log) { - platform_log( - log, - "pivot: %s\n", - key_string(context->cfg->data_cfg, trunk_ondisk_pivot_key(pivot))); + platform_log(log, + "pivot: %s\n", + (pivot->key.flags & ONDISK_KEY_IS_BLOB) + ? "" + : key_string(context->cfg->data_cfg, + trunk_ondisk_pivot_key(pivot))); } // Search the inflight bundles @@ -7639,8 +8301,10 @@ trunk_merge_lookup_async(trunk_merge_lookup_async_state *state) platform_log(state->log, "pivot_num: %lu pivot: %s\n", state->work.node.min, - key_string(state->context->cfg->data_cfg, - trunk_ondisk_pivot_key(state->pivot))); + (state->pivot->key.flags & ONDISK_KEY_IS_BLOB) + ? "" + : key_string(state->context->cfg->data_cfg, + trunk_ondisk_pivot_key(state->pivot))); } // Search the inflight bundles @@ -7775,7 +8439,7 @@ trunk_collect_bundle_branches(const trunk_context *context, } platform_status rc = btree_validate_root(bndl->branches[i].addr, - bndl->branches[i].format_version, + bndl->branches[i].format, trunk_ondisk_bundle_branch_type(bndl)); if (!SUCCESS(rc)) { /* References are acquired only after all selected roots validate. */ @@ -7785,9 +8449,9 @@ trunk_collect_bundle_branches(const trunk_context *context, *num_branches = bundle_start; return rc; } - branches[*num_branches].addr = branch_ref_addr(bndl->branches[i]); - branches[*num_branches].format_version = bndl->branches[i].format_version; - branches[*num_branches].type = trunk_ondisk_bundle_branch_type(bndl); + branches[*num_branches].addr = branch_ref_addr(bndl->branches[i]); + branches[*num_branches].format = bndl->branches[i].format; + branches[*num_branches].type = trunk_ondisk_bundle_branch_type(bndl); (*num_branches)++; } for (uint64 i = bundle_start; i < *num_branches; i++) { @@ -7926,37 +8590,21 @@ trunk_collect_branches(const trunk_context *context, trunk_ondisk_node_handle_deinit(&handle); handle = child_handle; } else { - key leaf_min_key; - key leaf_max_key; debug_assert(trunk_ondisk_node_num_pivots(&handle) == 2); - rc = trunk_ondisk_node_get_pivot_key(&handle, 0, &leaf_min_key); + rc = trunk_ondisk_node_get_pivot_key(context, &handle, 0, min_key); if (!SUCCESS(rc)) { platform_error_log("trunk_collect_branches: " "ondisk_node_get_pivot_key failed: %d\n", rc.r); goto cleanup; } - rc = trunk_ondisk_node_get_pivot_key(&handle, 1, &leaf_max_key); + rc = trunk_ondisk_node_get_pivot_key(context, &handle, 1, max_key); if (!SUCCESS(rc)) { platform_error_log("trunk_collect_branches: " "ondisk_node_get_pivot_key failed: %d\n", rc.r); goto cleanup; } - rc = key_buffer_copy_key(min_key, leaf_min_key); - if (!SUCCESS(rc)) { - platform_error_log("trunk_collect_branches: " - "key_buffer_copy_key failed: %d\n", - rc.r); - goto cleanup; - } - rc = key_buffer_copy_key(max_key, leaf_max_key); - if (!SUCCESS(rc)) { - platform_error_log("trunk_collect_branches: " - "key_buffer_copy_key failed: %d\n", - rc.r); - goto cleanup; - } trunk_ondisk_node_handle_deinit(&handle); } } diff --git a/src/trunk.h b/src/trunk.h index f6669b019..27a8e1349 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -19,7 +19,7 @@ #include "data_internal.h" #include "batch_rwlock.h" -#define TRUNK_FORMAT_VERSION (1) +#define TRUNK_FORMAT_VERSION (5) /* The superblock persists this version; the trunk owns its supported readers. */ @@ -136,12 +136,13 @@ typedef struct trunk_pivot_state_map { */ typedef struct trunk_ondisk_node_ref { ondisk_ref ref; - ondisk_key key; + key key; + uint8 bytes[]; } trunk_ondisk_node_ref; typedef struct ONDISK branch_ref { - uint64 addr; - uint16 format_version; + uint64 addr; + btree_format format; } branch_ref; typedef VECTOR(branch_ref) branch_ref_vector; @@ -352,9 +353,9 @@ trunk_merge_lookup(trunk_context *context, platform_log_handle *log); typedef struct trunk_branch_info { - uint64 addr; - page_type type; - uint16 format_version; + uint64 addr; + page_type type; + btree_format format; } trunk_branch_info; // On failure, releases every reference acquired by this call and restores diff --git a/tests/config.c b/tests/config.c index 0437283ae..2c7ea1e31 100644 --- a/tests/config.c +++ b/tests/config.c @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 #include "config.h" -#include "blob_build.h" +#include "mini_blob.h" +#include "btree.h" #include "platform_units.h" #include "platform_buffer.h" #include "platform_heap.h" @@ -82,6 +83,7 @@ config_set_defaults(master_config *cfg) .cache_capacity = GiB_TO_B(TEST_CONFIG_DEFAULT_CACHE_SIZE_GB), .btree_rough_count_height = 1, .blob_max_descriptor_size = BLOB_DEFAULT_MAX_DESCRIPTOR_SIZE, + .blob_key_prefix_size = BTREE_DEFAULT_KEY_PREFIX_SIZE, .filter_hash_size = TEST_CONFIG_DEFAULT_FILTER_HASH_SIZE, .filter_log_index_size = TEST_CONFIG_DEFAULT_FILTER_LOG_INDEX_SIZE, .use_log = FALSE, @@ -118,7 +120,8 @@ void config_usage() { platform_error_log("\nConfiguration: (default)\n"); - platform_error_log("\t--blob-max-descriptor-size (512 bytes)\n"); + platform_error_log("\t--blob-max-descriptor-size (256 bytes)\n"); + platform_error_log("\t--blob-key-prefix-size (256 bytes; 0 disables)\n"); platform_error_log("\t--page-size (%d)\n", TEST_CONFIG_DEFAULT_PAGE_SIZE); platform_error_log("\t--extent-size (%d)\n", TEST_CONFIG_DEFAULT_EXTENT_SIZE); @@ -404,6 +407,7 @@ config_parse(master_config *cfg, const uint8 num_config, int argc, char *argv[]) "blob-max-descriptor-size", cfg, blob_max_descriptor_size) { } + config_set_uint64("blob-key-prefix-size", cfg, blob_key_prefix_size) {} config_set_mib("checkpoint-log-size", cfg, checkpoint_log_size) {} config_set_gib("checkpoint-log-size", cfg, checkpoint_log_size) {} config_set_uint64( diff --git a/tests/config.h b/tests/config.h index 91452a6aa..f0caba2dd 100644 --- a/tests/config.h +++ b/tests/config.h @@ -67,6 +67,7 @@ typedef struct master_config { char cache_logfile[MAX_STRING_LENGTH]; uint64 blob_max_descriptor_size; + uint64 blob_key_prefix_size; // btree uint64 btree_rough_count_height; diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index bd8db29ff..4f11cf4ad 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -60,55 +60,52 @@ typedef struct test_memtable_context { uint64 max_generation; } test_memtable_context; -#define BTREE_HEIGHT_LIMIT_TEST_NUM_KEYS 3 -#define BTREE_HEIGHT_LIMIT_TEST_KEY_SIZE 2010 - -typedef struct btree_height_limit_iterator { - iterator super; - uint64 pos; - uint8 keys[BTREE_HEIGHT_LIMIT_TEST_NUM_KEYS] - [BTREE_HEIGHT_LIMIT_TEST_KEY_SIZE]; +#define BTREE_BLOB_KEY_TEST_NUM_KEYS 3 +#define BTREE_BLOB_KEY_TEST_KEY_SIZE 2010 + +typedef struct btree_blob_key_iterator { + iterator super; + uint64 pos; + uint8 keys[BTREE_BLOB_KEY_TEST_NUM_KEYS][BTREE_BLOB_KEY_TEST_KEY_SIZE]; data_handle data; -} btree_height_limit_iterator; +} btree_blob_key_iterator; static void -btree_height_limit_iterator_curr(iterator *base, key *curr_key, message *msg) +btree_blob_key_iterator_curr(iterator *base, key *curr_key, message *msg) { - btree_height_limit_iterator *itor = (btree_height_limit_iterator *)base; - platform_assert(itor->pos < BTREE_HEIGHT_LIMIT_TEST_NUM_KEYS); - *curr_key = key_create( - FALSE, BTREE_HEIGHT_LIMIT_TEST_KEY_SIZE, itor->keys[itor->pos]); + btree_blob_key_iterator *itor = (btree_blob_key_iterator *)base; + platform_assert(itor->pos < BTREE_BLOB_KEY_TEST_NUM_KEYS); + *curr_key = + key_create(FALSE, BTREE_BLOB_KEY_TEST_KEY_SIZE, itor->keys[itor->pos]); *msg = message_create_internal( MESSAGE_TYPE_INSERT, NULL, slice_create(sizeof(itor->data), &itor->data)); } static bool32 -btree_height_limit_iterator_can_curr(iterator *base) +btree_blob_key_iterator_can_curr(iterator *base) { - btree_height_limit_iterator *itor = (btree_height_limit_iterator *)base; - return itor->pos < BTREE_HEIGHT_LIMIT_TEST_NUM_KEYS; + btree_blob_key_iterator *itor = (btree_blob_key_iterator *)base; + return itor->pos < BTREE_BLOB_KEY_TEST_NUM_KEYS; } static platform_status -btree_height_limit_iterator_next(iterator *base) +btree_blob_key_iterator_next(iterator *base) { - btree_height_limit_iterator *itor = (btree_height_limit_iterator *)base; - platform_assert(itor->pos < BTREE_HEIGHT_LIMIT_TEST_NUM_KEYS); + btree_blob_key_iterator *itor = (btree_blob_key_iterator *)base; + platform_assert(itor->pos < BTREE_BLOB_KEY_TEST_NUM_KEYS); itor->pos++; return STATUS_OK; } static platform_status -btree_height_limit_iterator_prev(iterator *base) +btree_blob_key_iterator_prev(iterator *base) { (void)base; return STATUS_NOTSUP; } static platform_status -btree_height_limit_iterator_seek(iterator *base, - comparison seek_type, - key seek_key) +btree_blob_key_iterator_seek(iterator *base, comparison seek_type, key seek_key) { (void)base; (void)seek_type; @@ -117,52 +114,109 @@ btree_height_limit_iterator_seek(iterator *base, } static void -btree_height_limit_iterator_print(iterator *base) +btree_blob_key_iterator_print(iterator *base) { (void)base; } -static const iterator_ops btree_height_limit_iterator_ops = { - .curr = btree_height_limit_iterator_curr, - .can_prev = btree_height_limit_iterator_can_curr, - .can_next = btree_height_limit_iterator_can_curr, - .next = btree_height_limit_iterator_next, - .prev = btree_height_limit_iterator_prev, - .seek = btree_height_limit_iterator_seek, - .print = btree_height_limit_iterator_print, +static const iterator_ops btree_blob_key_iterator_ops = { + .curr = btree_blob_key_iterator_curr, + .can_prev = btree_blob_key_iterator_can_curr, + .can_next = btree_blob_key_iterator_can_curr, + .next = btree_blob_key_iterator_next, + .prev = btree_blob_key_iterator_prev, + .seek = btree_blob_key_iterator_seek, + .print = btree_blob_key_iterator_print, }; -/* These keys fit two at a time in a leaf but only one at a time in an index - * page. Three tuples therefore keep producing two nodes at every level and - * exercise the fixed-height packer limit with a very small input. */ +/* + * Three keys above the inline limit should fit in one leaf using blob + * descriptors. Check that they round-trip and that aborting a partial pack + * releases its blob allocations. + */ static platform_status -test_btree_pack_height_limit(cache *cc, - const btree_config *cfg, - platform_heap_id hid) +test_btree_pack_blob_keys(cache *cc, btree_config *cfg, platform_heap_id hid) { platform_assert(btree_page_size(cfg) == 4096); - btree_height_limit_iterator itor = { - .super = {.ops = &btree_height_limit_iterator_ops}, + btree_blob_key_iterator itor = { + .super = {.ops = &btree_blob_key_iterator_ops}, .data = {.ref_count = 1}, }; for (uint64 i = 0; i < ARRAY_SIZE(itor.keys); i++) { memset(itor.keys[i], 'a' + i, sizeof(itor.keys[i])); } - btree_pack_req req; - platform_status rc = btree_pack_req_init(&req, cc, cfg, &itor.super, 0, hid); - if (!SUCCESS(rc)) { - return rc; - } - rc = btree_pack(&req); - bool32 passed = - STATUS_IS_EQ(rc, STATUS_LIMIT_EXCEEDED) && req.root_addr == 0; - if (SUCCESS(rc) && req.root_addr != 0) { - btree_dec_ref(cc, cfg, req.root_addr, PAGE_TYPE_BRANCH); + const uint64 tuple_limits[] = {0, BTREE_BLOB_KEY_TEST_NUM_KEYS - 1}; + for (uint64 limit_idx = 0; limit_idx < ARRAY_SIZE(tuple_limits); limit_idx++) + { + uint64 max_tuples = tuple_limits[limit_idx]; + uint64 allocated_before = allocator_in_use(cache_get_allocator(cc)); + itor.pos = 0; + + btree_pack_req req; + platform_status rc = + btree_pack_req_init(&req, cc, cfg, &itor.super, max_tuples, hid); + if (!SUCCESS(rc)) { + return rc; + } + rc = btree_pack(&req); + bool32 passed; + if (max_tuples != 0) { + passed = STATUS_IS_EQ(rc, STATUS_LIMIT_EXCEEDED) && req.root_addr == 0 + && req.num_tuples == max_tuples; + } else { + passed = SUCCESS(rc) && req.root_addr != 0 + && req.num_tuples == BTREE_BLOB_KEY_TEST_NUM_KEYS; + if (passed) { + uint32 height; + rc = btree_get_root_height( + cc, cfg, req.root_addr, req.format, PAGE_TYPE_BRANCH, &height); + passed = SUCCESS(rc) && height == 0; + } + + lookup_result result; + lookup_result_init( + &result, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + message expected = message_create_internal( + MESSAGE_TYPE_INSERT, + NULL, + slice_create(sizeof(itor.data), &itor.data)); + for (uint64 i = 0; passed && i < ARRAY_SIZE(itor.keys); i++) { + key target = key_create(FALSE, sizeof(itor.keys[i]), itor.keys[i]); + rc = btree_lookup(cc, + cfg, + req.root_addr, + req.format, + PAGE_TYPE_BRANCH, + target, + &result); + passed = SUCCESS(rc) && lookup_result_found(&result) + && message_lex_cmp(merge_accumulator_to_message( + lookup_result_accumulator(&result)), + expected) + == 0; + } + lookup_result_deinit(&result); + } + + if (req.root_addr != 0) { + btree_dec_ref(cc, cfg, req.root_addr, PAGE_TYPE_BRANCH); + } + btree_pack_req_deinit(&req, hid); + if (!passed + || allocator_in_use(cache_get_allocator(cc)) != allocated_before) + { + platform_error_log( + "btree blob-key pack test failed (max_tuples=%lu, status=%s)\n", + max_tuples, + platform_status_to_string(rc)); + return STATUS_TEST_FAILED; + } } - btree_pack_req_deinit(&req, hid); - return passed ? STATUS_OK : STATUS_TEST_FAILED; + + platform_default_log("btree blob-key pack and abort tests passed\n"); + return STATUS_OK; } btree_config * @@ -411,7 +465,7 @@ test_btree_lookup(cache *cc, lookup_result_init(&result, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); rc = btree_lookup( - cc, cfg, root_addr, BTREE_BRANCH_FORMAT_VERSION, type, target, &result); + cc, cfg, root_addr, BTREE_DEFAULT_FORMAT, type, target, &result); platform_assert_status_ok(rc); message data = @@ -649,7 +703,7 @@ test_btree_scan_once(cache *cc, btree_cfg, &itor, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, &(const btree_iterator_init_options){ .lower_limit = key_greater_than_or_equal(min_key), @@ -946,7 +1000,7 @@ test_btree_scan_perf(cache *cc, btree_cfg, &itor, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &(const btree_iterator_init_options){0}); if (!SUCCESS(rc)) { @@ -1174,7 +1228,7 @@ test_btree_async_lookup(cache *cc, cc, cfg, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, target, &async_ctxt->result, @@ -1245,7 +1299,7 @@ test_btree_lookup_status(cache *cc, lookup_result result; lookup_result_init(&result, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); platform_status rc = btree_lookup( - cc, cfg, root_addr, BTREE_BRANCH_FORMAT_VERSION, type, target, &result); + cc, cfg, root_addr, BTREE_DEFAULT_FORMAT, type, target, &result); lookup_result_deinit(&result); return rc; } @@ -1264,7 +1318,7 @@ test_btree_async_lookup_status(cache *cc, cc, cfg, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, target, &ctxt.result, @@ -1330,7 +1384,7 @@ test_btree_iterator_init_status(cache *cc, cfg, &itor, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .start_position = make_key_start_position(start_type, start_key)}); @@ -1347,7 +1401,7 @@ test_btree_iterator_init_status(cache *cc, cfg, &itor, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, greater_than_or_equal, NEGATIVE_INFINITY_KEY, @@ -1423,7 +1477,7 @@ test_btree_query_validation(cache *cc, cfg, &itor, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){0}); if (!SUCCESS(rc) || !iterator_can_curr(&itor.super)) { @@ -1560,7 +1614,7 @@ test_btree_query_validation(cache *cc, cfg, &sequence_itor, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .start_position = key_start_at_or_after(key_buffer_key(&first_key))}); @@ -1696,8 +1750,8 @@ test_btree_query_validation(cache *cc, child_idx = 0; } index_entry *index = btree_get_index_entry(cfg, hdr, child_idx); - uint64 saved_child_addr = index->pivot_data.child_addr; - index->pivot_data.child_addr = root_addr; + uint64 saved_child_addr = index->child_addr; + index->child_addr = root_addr; test_btree_unclaim_page(cc, page); bool32 height_rejected = @@ -1715,7 +1769,7 @@ test_btree_query_validation(cache *cc, cfg, &height_itor, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .lower_limit = key_greater_than_or_equal(key_buffer_key(&touched_key)), @@ -1727,10 +1781,10 @@ test_btree_query_validation(cache *cc, btree_iterator_deinit(&height_itor); } - *page = test_btree_claim_page(cc, root_addr, PAGE_TYPE_BRANCH); - hdr = (btree_hdr *)page_handle_data(page); - index = btree_get_index_entry(cfg, hdr, child_idx); - index->pivot_data.child_addr = saved_child_addr; + *page = test_btree_claim_page(cc, root_addr, PAGE_TYPE_BRANCH); + hdr = (btree_hdr *)page_handle_data(page); + index = btree_get_index_entry(cfg, hdr, child_idx); + index->child_addr = saved_child_addr; test_btree_unclaim_page(cc, page); if (!height_rejected || !iterator_rejected || !diagnostic_cycle_rejected || !test_btree_query_status_matches(cc, @@ -1885,7 +1939,7 @@ test_btree_basic(cache *cc, btree_cfg, &itor, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &(const btree_iterator_init_options){0}); platform_assert_status_ok(rc); @@ -2069,7 +2123,7 @@ test_btree_create_packed_trees(cache *cc, btree_cfg, &itor, memtable_root_addr(mt), - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &(const btree_iterator_init_options){0}); platform_assert_status_ok(rc); @@ -2120,7 +2174,7 @@ test_count_tuples_in_range(cache *cc, cfg, &itor, root_addr[i], - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, &(const btree_iterator_init_options){ .lower_limit = key_greater_than_or_equal(low_key), @@ -2217,7 +2271,7 @@ test_btree_print_all_keys(cache *cc, cfg, &itor, root_addr[i], - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, &(const btree_iterator_init_options){ .lower_limit = key_greater_than_or_equal(low_key), @@ -2294,7 +2348,7 @@ test_btree_merge_basic(cache *cc, btree_cfg, &btree_itor_arr[tree_no], root_addr[tree_no], - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .lower_limit = key_greater_than_or_equal(lo), @@ -2434,7 +2488,7 @@ test_btree_count_in_range(cache *cc, rc = btree_count_in_range(cc, btree_cfg, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, min_key, max_key, &stats); @@ -2523,7 +2577,7 @@ test_btree_rough_iterator(cache *cc, btree_cfg, &rough_btree_itor[tree_no], root_addr[tree_no], - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .copy_nodes = TRUE, .height = 1, .prefetch_lookahead = 1}); @@ -2684,7 +2738,7 @@ test_btree_merge_perf(cache *cc, btree_cfg, &btree_itor_arr[tree_no], root_addr[global_tree_no], - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .lower_limit = key_greater_than_or_equal(min_key), @@ -3057,7 +3111,7 @@ btree_test(int argc, char *argv[]) rc = test_memtable_generation_init(ccp, &test_cfg, hid); platform_assert_status_ok(rc); - rc = test_btree_pack_height_limit(ccp, mt_cfg->btree_cfg, hid); + rc = test_btree_pack_blob_keys(ccp, mt_cfg->btree_cfg, hid); platform_assert_status_ok(rc); uint64 max_tuples_per_memtable = diff --git a/tests/functional/filter_test.c b/tests/functional/filter_test.c index 390a09e2b..c032e7aec 100644 --- a/tests/functional/filter_test.c +++ b/tests/functional/filter_test.c @@ -30,6 +30,7 @@ typedef struct ONDISK test_mini_meta_hdr { uint64 meta_page_no; uint64 pos; uint32 num_entries; + uint64 num_charged_extents; uint16 format_version; char entry_buffer[]; } test_mini_meta_hdr; @@ -424,10 +425,11 @@ test_filter_malicious_metadata(cache *cc, memset(page_handle_data(second_meta_page), 0, page_size); test_mini_meta_hdr *second_meta_hdr = (test_mini_meta_hdr *)page_handle_data(second_meta_page); - second_meta_hdr->format_version = MINI_ALLOCATOR_FORMAT_VERSION; - second_meta_hdr->prev_meta_addr = filter.meta_head; - second_meta_hdr->meta_page_no = 1; - second_meta_hdr->num_entries = 1; + second_meta_hdr->format_version = MINI_ALLOCATOR_FORMAT_VERSION; + second_meta_hdr->prev_meta_addr = filter.meta_head; + second_meta_hdr->meta_page_no = 1; + second_meta_hdr->num_entries = 1; + second_meta_hdr->num_charged_extents = 1; second_meta_hdr->pos = offsetof(test_mini_meta_hdr, entry_buffer) + sizeof(test_mini_meta_entry); *(test_mini_meta_entry *)second_meta_hdr->entry_buffer = head_entry; @@ -568,6 +570,25 @@ test_filter_malicious_metadata(cache *cc, goto out; } + /* Non-blob entries may not smuggle a pointer height into the type byte. */ + *meta_page = test_filter_claim_page(cc, filter.meta_head, PAGE_TYPE_FILTER); + meta_hdr = (test_mini_meta_hdr *)page_handle_data(meta_page); + entry = (test_mini_meta_entry *)meta_hdr->entry_buffer; + entry->packed = saved_entry | (UINT64_C(1) << 12); + test_filter_unclaim_page(cc, meta_page); + cursor_rc = mini_meta_cursor_init( + &cursor, cc, PAGE_TYPE_FILTER, filter.meta_head, target_extent); + mini_meta_cursor_deinit(&cursor); + *meta_page = test_filter_claim_page(cc, filter.meta_head, PAGE_TYPE_FILTER); + meta_hdr = (test_mini_meta_hdr *)page_handle_data(meta_page); + entry = (test_mini_meta_entry *)meta_hdr->entry_buffer; + entry->packed = saved_entry; + test_filter_unclaim_page(cc, meta_page); + if (cursor_rc != MINI_META_CURSOR_INVALID) { + rc = STATUS_TEST_FAILED; + goto out; + } + rc = STATUS_OK; out: diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index af857ba71..879888c09 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -1374,6 +1374,19 @@ test_log_prepare_failure_does_not_poison(clockcache *cc, int append_rc = log_write(log, invalid_key, invalid_msg, 257, 0); platform_assert(append_rc == STATUS_INVALID_STATE.r); + /* Fail after a key blob has been adopted by the mini allocator, then abort + * a successful preparation. Neither operation may publish a record or + * strand a reservation. Bulk log reclamation owns both unused key blobs. */ + invalid_key = test_key(&keybuffer, TEST_RANDOM, 257, 0, 0, 65553, 0); + append_rc = log_write(log, invalid_key, invalid_msg, 257, 0); + platform_assert(append_rc == STATUS_INVALID_STATE.r); + log_write_prepared prepared = {0}; + message small = + message_create_internal(MESSAGE_TYPE_INSERT, NULL, slice_create(2, "ok")); + platform_assert_status_ok( + log_write_prepare(log, invalid_key, small, FALSE, &prepared)); + log_write_abort(&prepared); + /* Reuse the failed record's sequence number so verification stays dense. */ test_log_write_range(log, gen, hid, key_size, 257, 1); @@ -2355,7 +2368,8 @@ test_log_indirect_blob_recovery(clockcache *cc, io_handle *io, rc_allocator *al, shard_log_config *cfg, - platform_heap_id hid) + platform_heap_id hid, + bool32 as_key) { cache *cacheh = (cache *)cc; log_handle *log; @@ -2375,8 +2389,15 @@ test_log_indirect_blob_recovery(clockcache *cc, platform_assert(merge_accumulator_resize(&value, length)); merge_accumulator_set_class(&value, MESSAGE_TYPE_INSERT); memset(merge_accumulator_data(&value), 'B', length); + key large_key = key_create_from_slice( + FALSE, message_slice(merge_accumulator_to_message(&value))); platform_assert( - log_write(log, skey, merge_accumulator_to_message(&value), 1, 0) == 0); + log_write(log, + as_key ? large_key : skey, + as_key ? small : merge_accumulator_to_message(&value), + 1, + 0) + == 0); platform_assert_status_ok(log_make_durable(log)); platform_assert(log_write(log, skey, small, 2, 0) == 0); platform_assert_status_ok(log_seal(log)); @@ -2391,8 +2412,23 @@ test_log_indirect_blob_recovery(clockcache *cc, message msg; log_iterator_curr(iter, &returned_key, &msg); parsed_blob parsed; + slice descriptor = message_slice(msg); + if (as_key) { + platform_assert(!returned_key.is_blob); + platform_assert(key_length(returned_key) == length); + platform_assert(memcmp(slice_data(key_slice(returned_key)), + merge_accumulator_data(&value), + length) + == 0); + shard_log_iterator *concrete = (shard_log_iterator *)iter; + ondisk_tuple *tuple = + (ondisk_tuple *)((char *)concrete->entries[concrete->pos] + + 2 * sizeof(uint64)); + platform_assert(ondisk_tuple_key(tuple).is_blob); + descriptor = key_slice(ondisk_tuple_key(tuple)); + } platform_assert_status_ok( - blob_parse_descriptor(cacheh, message_slice(msg), &parsed)); + blob_parse_descriptor(cacheh, descriptor, &parsed)); platform_assert(parsed.height == 2); uint64 root = parsed.base->addrs[0]; log_iterator_deinit(iter); @@ -2942,7 +2978,9 @@ log_test(int argc, char *argv[]) cc, &system_cfg.cache_cfg, io, &al, &system_cfg.log_cfg, hid); test_log_indirect_blob_recovery( - cc, &system_cfg.cache_cfg, io, &al, &system_cfg.log_cfg, hid); + cc, &system_cfg.cache_cfg, io, &al, &system_cfg.log_cfg, hid, FALSE); + test_log_indirect_blob_recovery( + cc, &system_cfg.cache_cfg, io, &al, &system_cfg.log_cfg, hid, TRUE); rc = test_log_large_message((cache *)cc, &system_cfg.log_cfg, hid); platform_assert(rc == 0); diff --git a/tests/functional/test.h b/tests/functional/test.h index 201904cfa..d2208c7ff 100644 --- a/tests/functional/test.h +++ b/tests/functional/test.h @@ -326,6 +326,12 @@ test_config_init(system_config *system_cfg, // OUT &system_cfg->cache_cfg.super, system_cfg->data_cfg); system_cfg->btree_cfg.blob_cfg.blob_max_descriptor_size = blob_limit; + system_cfg->btree_cfg.blob_key_prefix_size = + master_cfg->blob_key_prefix_size; + rc = btree_config_validate(&system_cfg->btree_cfg); + if (!SUCCESS(rc)) { + return rc; + } trunk_config_init(&system_cfg->trunk_node_cfg, system_cfg->data_cfg, diff --git a/tests/unit/btree_stress_test.c b/tests/unit/btree_stress_test.c index b7ae6459d..1b5b50b55 100644 --- a/tests/unit/btree_stress_test.c +++ b/tests/unit/btree_stress_test.c @@ -310,7 +310,7 @@ CTEST2(btree_stress, insert_callback_failure_abandons_insert) rc = btree_lookup((cache *)&data->cc, &data->dbtree_cfg, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, tuple_key, &lookup); @@ -353,7 +353,7 @@ CTEST2(btree_stress, insert_callback_failure_after_root_growth) ASSERT_TRUE(SUCCESS(btree_get_root_height((cache *)&data->cc, &data->dbtree_cfg, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &height))); ASSERT_EQUAL_U(0, height); @@ -386,7 +386,7 @@ CTEST2(btree_stress, insert_callback_failure_after_root_growth) ASSERT_TRUE(SUCCESS(btree_get_root_height((cache *)&data->cc, &data->dbtree_cfg, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &height))); ASSERT_TRUE(height > 0); @@ -399,7 +399,7 @@ CTEST2(btree_stress, insert_callback_failure_after_root_growth) rc = btree_lookup((cache *)&data->cc, &data->dbtree_cfg, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, rejected_key, &lookup); @@ -548,8 +548,13 @@ CTEST2(btree_stress, insert_rebuild_failure_releases_parent_claim) lookup_result lookup; lookup_result_init(&lookup, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); - ASSERT_TRUE(SUCCESS(btree_lookup( - cc, cfg, root_addr, 0, PAGE_TYPE_MEMTABLE, target, &lookup))); + ASSERT_TRUE(SUCCESS(btree_lookup(cc, + cfg, + root_addr, + (btree_format){0}, + PAGE_TYPE_MEMTABLE, + target, + &lookup))); ASSERT_EQUAL(0, slice_lex_cmp(merge_accumulator_to_value( lookup_result_accumulator(&lookup)), @@ -800,7 +805,7 @@ CTEST2(btree_stress, overwrite_returns_old_value_after_tree_growth) rc = btree_lookup((cache *)&data->cc, &data->dbtree_cfg, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, gen_key(&data->dbtree_cfg, i, keybuf, bt_page_size), &verify_result); @@ -939,7 +944,7 @@ query_tests(cache *cc, btree_lookup(cc, cfg, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, gen_key(cfg, i, keybuf, bt_page_size), &result); @@ -1039,7 +1044,7 @@ iterator_tests(cache *cc, cfg, &dbiter, root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, &(const btree_iterator_init_options){ .start_position = key_start_at_or_after(start_key)}); @@ -1087,7 +1092,7 @@ iterator_seek_tests(cache *cc, cfg, &dbiter, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &(const btree_iterator_init_options){ .start_position = key_start_at_or_after(start_key)}); @@ -1130,7 +1135,7 @@ pack_tests(cache *cc, cfg, &dbiter, root_addr, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &(const btree_iterator_init_options){0}); ASSERT_TRUE(SUCCESS(rc)); @@ -1225,7 +1230,7 @@ bounds_init_with_copy(cache *cc, cfg, itor, root, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, &(const btree_iterator_init_options){ .lower_limit = make_key_lower_limit(lo_cmp, lo), @@ -1241,7 +1246,7 @@ bounds_init_with_copy(cache *cc, cfg, itor, root, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, type, lo_cmp, lo, @@ -1368,7 +1373,7 @@ iterator_endpoint_fault_init(cache *cc, cfg, itor, root, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .upper_limit = key_less_than(upper)}); @@ -1381,7 +1386,7 @@ iterator_endpoint_fault_init(cache *cc, cfg, itor, root, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, greater_than_or_equal, NEGATIVE_INFINITY_KEY, @@ -1402,6 +1407,170 @@ iterator_endpoint_fault_init(cache *cc, return btree_iterator_init_async_result(&state); } +/* Unbounded immutable scans defer endpoint traversal until the last page. + * Observe root acquisitions to catch an eager walk, then exercise the endpoint + * while crossing leaves in both directions and seeking away from the end. */ +CTEST2(btree_stress, unbounded_branch_iterator) +{ + cache *cc = (cache *)&data->cc; + btree_config *cfg = &data->dbtree_cfg; + mini_allocator mini; + const uint64 count = 2048; + uint64 source_root = test_btree_create(cc, cfg, &mini, PAGE_TYPE_MEMTABLE); + for (uint64 n = 0; n < count; n++) { + bounds_insert( + cc, cfg, data->hid, data->test_scratch, &mini, source_root, n); + } + btree_iterator source; + ASSERT_TRUE( + SUCCESS(btree_iterator_init(cc, + cfg, + &source, + source_root, + (btree_format){0}, + PAGE_TYPE_MEMTABLE, + &(const btree_iterator_init_options){0}))); + btree_pack_req req; + ASSERT_TRUE(SUCCESS( + btree_pack_req_init(&req, cc, cfg, &source.super, count, data->hid))); + ASSERT_TRUE(SUCCESS(btree_pack(&req))); + uint64 packed_root = req.root_addr; + btree_pack_req_deinit(&req, data->hid); + btree_iterator_deinit(&source); + + for (unsigned async = 0; async < 2; async++) { + for (unsigned copy = 0; copy < 2; copy++) { + for (unsigned reverse = 0; reverse < 2; reverse++) { + iterator_endpoint_fault observer = { + .cc = cc, + .original = cc->ops, + .ops = *cc->ops, + .root = packed_root, + }; + observer.ops.page_get = iterator_endpoint_fault_get; + observer.ops.page_get_async_result = + iterator_endpoint_fault_async_result; + active_iterator_endpoint_fault = &observer; + cc->ops = &observer.ops; + + uint64 lower_buf; + const int64 first = 13; + btree_iterator itor; + platform_status rc = + bounds_init_with_copy(cc, + cfg, + &itor, + packed_root, + bounds_key(first, &lower_buf), + greater_than_or_equal, + POSITIVE_INFINITY_KEY, + less_than, + reverse, + async, + PAGE_TYPE_BRANCH, + 0, + copy); + + cc->ops = observer.original; + active_iterator_endpoint_fault = NULL; + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_NOT_EQUAL(packed_root, itor.curr.addr); + ASSERT_EQUAL(reverse && !copy ? 2 : 1, observer.root_gets); + bool32 backwards = reverse; + for (unsigned direction = 0; direction < 2; direction++) { + int64 expected = backwards ? count - 1 : first; + for (uint64 n = first; n < count; n++) { + ASSERT_TRUE(iterator_can_curr(&itor.super)); + key k; + message msg; + iterator_curr(&itor.super, &k, &msg); + ASSERT_EQUAL(expected, bounds_key_number(k)); + expected += backwards ? -1 : 1; + rc = backwards ? iterator_prev(&itor.super) + : iterator_next(&itor.super); + ASSERT_TRUE(SUCCESS(rc)); + } + ASSERT_FALSE(iterator_can_curr(&itor.super)); + rc = backwards ? iterator_next(&itor.super) + : iterator_prev(&itor.super); + ASSERT_TRUE(SUCCESS(rc)); + backwards = !backwards; + } + + /* Seeking back from the last leaf must discard its cached bound. */ + uint64 seek_buf; + ASSERT_TRUE( + SUCCESS(iterator_seek(&itor.super, + greater_than_or_equal, + bounds_key(count - 1, &seek_buf)))); + ASSERT_TRUE(SUCCESS(iterator_next(&itor.super))); + ASSERT_FALSE(iterator_can_curr(&itor.super)); + ASSERT_TRUE(SUCCESS(iterator_seek(&itor.super, + greater_than_or_equal, + bounds_key(first, &seek_buf)))); + for (uint64 n = first; n < count; n++) { + ASSERT_TRUE(iterator_can_curr(&itor.super)); + key k; + message msg; + iterator_curr(&itor.super, &k, &msg); + ASSERT_EQUAL(n, bounds_key_number(k)); + ASSERT_TRUE(SUCCESS(iterator_next(&itor.super))); + } + ASSERT_FALSE(iterator_can_curr(&itor.super)); + btree_iterator_deinit(&itor); + } + } + } + /* A forged early last-page marker must still fail instead of producing a + * successful truncated scan. Endpoint lookup is deferred, not removed. */ + btree_iterator probe; + ASSERT_TRUE(SUCCESS(iterator_endpoint_fault_init( + cc, cfg, &probe, packed_root, POSITIVE_INFINITY_KEY, FALSE))); + uint64 first_leaf = probe.curr.addr; + btree_iterator_deinit(&probe); + page_handle page; + ASSERT_TRUE( + SUCCESS(cache_get(cc, first_leaf, TRUE, PAGE_TYPE_BRANCH, &page))); + ASSERT_TRUE(cache_try_claim(cc, &page)); + cache_lock(cc, &page); + btree_hdr *hdr = (btree_hdr *)page_handle_data(&page); + uint8 flags = hdr->flags; + hdr->flags |= BTREE_HDR_LAST_IN_LEVEL; + cache_unlock(cc, &page); + cache_unclaim(cc, &page); + cache_unget(cc, &page); + + for (unsigned async = 0; async < 2; async++) { + platform_status rc = iterator_endpoint_fault_init( + cc, cfg, &probe, packed_root, POSITIVE_INFINITY_KEY, async); + uint64 seen = 0; + while (iterator_can_curr(&probe.super)) { + seen++; + rc = iterator_next(&probe.super); + if (!SUCCESS(rc)) { + break; + } + } + btree_iterator_deinit(&probe); + ASSERT_FALSE(SUCCESS(rc)); + ASSERT_TRUE(seen < count); + } + + ASSERT_TRUE( + SUCCESS(cache_get(cc, first_leaf, TRUE, PAGE_TYPE_BRANCH, &page))); + ASSERT_TRUE(cache_try_claim(cc, &page)); + cache_lock(cc, &page); + hdr = (btree_hdr *)page_handle_data(&page); + hdr->flags = flags; + cache_unlock(cc, &page); + cache_unclaim(cc, &page); + cache_unget(cc, &page); + + btree_dec_ref(cc, cfg, packed_root, PAGE_TYPE_BRANCH); + mini_release(&mini); + btree_dec_ref(cc, cfg, source_root, PAGE_TYPE_MEMTABLE); +} + CTEST2(btree_stress, iterator_endpoint_error_is_returned_by_init) { cache *cc = (cache *)&data->cc; @@ -1420,7 +1589,7 @@ CTEST2(btree_stress, iterator_endpoint_error_is_returned_by_init) cfg, &source, source_root, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &(const btree_iterator_init_options){0}); ASSERT_TRUE(SUCCESS(rc)); @@ -1438,7 +1607,7 @@ CTEST2(btree_stress, iterator_endpoint_error_is_returned_by_init) cfg, &probe, packed_root, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){0}); ASSERT_TRUE(SUCCESS(rc)); @@ -1512,7 +1681,7 @@ CTEST2(btree_stress, count_in_range_returns_lookup_error) cfg, &source, source_root, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &(const btree_iterator_init_options){0}); ASSERT_TRUE(SUCCESS(rc)); @@ -1542,13 +1711,8 @@ CTEST2(btree_stress, count_in_range_returns_lookup_error) btree_pivot_stats stats = { .num_kvs = 1, .key_bytes = 1, .message_bytes = 1}; - rc = btree_count_in_range(cc, - cfg, - packed_root, - BTREE_BRANCH_FORMAT_VERSION, - min_key, - max_key, - &stats); + rc = btree_count_in_range( + cc, cfg, packed_root, BTREE_DEFAULT_FORMAT, min_key, max_key, &stats); cc->ops = fault.original; active_iterator_endpoint_fault = NULL; @@ -1822,7 +1986,7 @@ CTEST2(btree_stress, copied_iterator_split_on_release) cfg, &probe, root, - 0, + (btree_format){0}, PAGE_TYPE_MEMTABLE, &(const btree_iterator_init_options){0}); ASSERT_TRUE(SUCCESS(rc)); @@ -2012,3 +2176,640 @@ CTEST2(btree_stress, copied_iterator_prefetch) } } } + +/* Native keys deliberately differ after P, forcing blob reads for comparisons. + * Inline keys surround blobs so packing must both start a new leaf when C > P + * and restore unrestricted prefix compression after leaving the blob leaves. */ +#define NATIVE_KEY_COUNT 180 +#define NATIVE_INLINE_COUNT 40 + +static key +native_test_key(uint64 n, char *buf) +{ + bool32 inline_key = + n < NATIVE_INLINE_COUNT || n >= NATIVE_KEY_COUNT - NATIVE_INLINE_COUNT; + uint64 length = inline_key ? (n % 5 ? 400 : 512) + : n == NATIVE_INLINE_COUNT ? 513 + : (n % 11 ? 1024 : 12000); + memset(buf, 'a', length); + uint64 order = htobe64(n); + memcpy(buf + 300, &order, sizeof(order)); + return key_create(FALSE, length, buf); +} + + +static int +native_merge(const data_config *cfg, + slice logical_key, + message older, + merge_accumulator *newer) +{ + platform_assert(slice_length(logical_key) >= 400); + uint64 a, b; + memcpy(&a, message_data(older), sizeof(a)); + memcpy(&b, merge_accumulator_data(newer), sizeof(b)); + b += a; + memcpy(merge_accumulator_data(newer), &b, sizeof(b)); + merge_accumulator_set_class(newer, message_class(older)); + return 0; +} +static int +native_reverse_compare(const data_config *cfg, user_key a, user_key b) +{ + /* A descriptor or truncated key must never reach the user comparator. */ + platform_assert(slice_length(a.key) >= 400 && slice_length(b.key) >= 400); + return -slice_lex_cmp(a.key, b.key); +} + +static void +native_check_tree(cache *cc, + btree_config *cfg, + uint64 root, + btree_format format, + page_type type, + bool32 reverse) +{ + char keybuf[12000]; + lookup_result result; + lookup_result_init(&result, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + for (uint64 i = 0; i < NATIVE_KEY_COUNT; i++) { + key target = native_test_key(i, keybuf); + platform_status rc = + btree_lookup(cc, cfg, root, format, type, target, &result); + ASSERT_TRUE(SUCCESS(rc), "lookup %lu: %d", i, rc.r); + ASSERT_TRUE(lookup_result_found(&result)); + uint64 value; + memcpy(&value, merge_accumulator_data(&result.value), sizeof(value)); + ASSERT_EQUAL(i + (i >= 5 && (i - 5) % 31 == 0), value); + if (i % 17 == 0) { + btree_lookup_async_state state; + bool32 ready = FALSE; + btree_lookup_async_state_init(&state, + cc, + cfg, + root, + format, + type, + target, + &result, + bounds_ready, + &ready); + while (btree_lookup_async(&state) != ASYNC_STATUS_DONE) { + while (!__atomic_exchange_n(&ready, FALSE, __ATOMIC_ACQUIRE)) { + cache_cleanup(cc); + } + } + ASSERT_TRUE(SUCCESS(async_result(&state))); + ASSERT_TRUE(lookup_result_found(&result)); + } + } + lookup_result_deinit(&result); + for (bool32 copy = FALSE; copy <= TRUE; copy++) { + btree_iterator itor; + platform_status rc; + if (!copy) { + rc = btree_iterator_init( + cc, + cfg, + &itor, + root, + format, + type, + &(btree_iterator_init_options){.copy_nodes = copy}); + } else { + btree_iterator_async_state state; + bool32 ready = FALSE; + btree_iterator_async_state_init(&state, + cc, + cfg, + &itor, + root, + format, + type, + greater_than_or_equal, + NEGATIVE_INFINITY_KEY, + less_than, + POSITIVE_INFINITY_KEY, + greater_than_or_equal, + NEGATIVE_INFINITY_KEY, + copy, + 0, + 0, + bounds_ready, + &ready); + while (btree_iterator_init_async(&state) != ASYNC_STATUS_DONE) { + while (!__atomic_exchange_n(&ready, FALSE, __ATOMIC_ACQUIRE)) { + cache_cleanup(cc); + } + } + rc = btree_iterator_init_async_result(&state); + } + ASSERT_TRUE(SUCCESS(rc)); + for (uint64 i = 0; i < NATIVE_KEY_COUNT; i++) { + ASSERT_TRUE(iterator_can_curr(&itor.super)); + key actual, + expected = + native_test_key(reverse ? NATIVE_KEY_COUNT - 1 - i : i, keybuf); + message msg; + iterator_curr(&itor.super, &actual, &msg); + ASSERT_FALSE(actual.is_blob); + ASSERT_TRUE( + (slice_lex_cmp(key_slice(expected), key_slice(actual)) == 0), + "scan i=%lu actual_length=%lu expected_length=%lu type=%u copy=%u", + i, + key_length(actual), + key_length(expected), + type, + copy); + rc = iterator_next(&itor.super); + ASSERT_TRUE(SUCCESS(rc)); + } + ASSERT_FALSE(iterator_can_curr(&itor.super)); + for (uint64 i = NATIVE_KEY_COUNT; i > 0; i--) { + rc = iterator_prev(&itor.super); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_TRUE(iterator_can_curr(&itor.super)); + key actual, + expected = + native_test_key(reverse ? NATIVE_KEY_COUNT - i : i - 1, keybuf); + message msg; + iterator_curr(&itor.super, &actual, &msg); + ASSERT_TRUE( + (slice_lex_cmp(key_slice(expected), key_slice(actual)) == 0), + "scan i=%lu actual_length=%lu expected_length=%lu type=%u copy=%u", + i, + key_length(actual), + key_length(expected), + type, + copy); + } + key seek = native_test_key(71, keybuf); + rc = iterator_seek(&itor.super, greater_than_or_equal, seek); + ASSERT_TRUE(SUCCESS(rc)); + key actual; + message msg; + iterator_curr(&itor.super, &actual, &msg); + ASSERT_TRUE((slice_lex_cmp(key_slice(seek), key_slice(actual)) == 0)); + btree_iterator_deinit(&itor); + } +} + + +static const cache_ops *native_saved_ops; +static uint64 native_failed_reads; + +static platform_status +native_fail_blob_read(cache *cc, + uint64 addr, + bool32 blocking, + page_type type, + page_handle *page) +{ + if (type == PAGE_TYPE_BLOB) { + native_failed_reads++; + *page = (page_handle){0}; + return STATUS_TIMEDOUT; + } + return native_saved_ops->page_get(cc, addr, blocking, type, page); +} + +static void +native_check_read_failure(cache *cc, + btree_config *cfg, + uint64 root, + btree_format format, + page_type type) +{ + cache_ops fault_ops = *cc->ops; + native_saved_ops = cc->ops; + fault_ops.page_get = native_fail_blob_read; + native_failed_reads = 0; + lookup_result result; + lookup_result_init(&result, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + char buf[12000]; + key target = native_test_key(71, buf); + cc->ops = &fault_ops; + platform_status rc = + btree_lookup(cc, cfg, root, format, type, target, &result); + cc->ops = native_saved_ops; + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_TIMEDOUT)); + ASSERT_TRUE(native_failed_reads > 0); + lookup_result_deinit(&result); + + cc->ops = &fault_ops; + btree_iterator itor; + rc = + btree_iterator_init(cc, + cfg, + &itor, + root, + format, + type, + &(btree_iterator_init_options){ + .start_position = key_start_at_or_after(target)}); + cc->ops = native_saved_ops; + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_TIMEDOUT)); + btree_iterator_deinit(&itor); + + /* Prefix-only rejection must not fetch blob pages for lexical order. */ + if (cfg->data_cfg->key_compare_is_lexical + && (type == PAGE_TYPE_BRANCH ? format.key_prefix_size + : cfg->blob_key_prefix_size) + > 0) + { + buf[0] = 'z'; + lookup_result_init( + &result, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + native_failed_reads = 0; + cc->ops = &fault_ops; + rc = btree_lookup(cc, cfg, root, format, type, target, &result); + cc->ops = native_saved_ops; + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_FALSE(lookup_result_found(&result)); + ASSERT_EQUAL(0, native_failed_reads); + lookup_result_deinit(&result); + } +} + +static void +native_check_corrupt_leaf(cache *cc, + btree_config *cfg, + uint64 root, + btree_format format) +{ + char buf[12000]; + key target = native_test_key(71, buf); + btree_iterator holder; + ASSERT_TRUE(SUCCESS(btree_iterator_init( + cc, + cfg, + &holder, + root, + format, + PAGE_TYPE_BRANCH, + &(btree_iterator_init_options){.start_position = + key_start_at_or_after(target)}))); + uint64 leaf_addr = holder.curr.addr; + uint64 idx = holder.idx; + btree_iterator_deinit(&holder); + lookup_result result; + lookup_result_init(&result, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + for (uint64 fault = 0; fault < 2; fault++) { + page_handle page; + ASSERT_TRUE( + SUCCESS(cache_get(cc, leaf_addr, TRUE, PAGE_TYPE_BRANCH, &page))); + ASSERT_TRUE(cache_try_claim(cc, &page)); + cache_lock(cc, &page); + btree_hdr *hdr = (btree_hdr *)page_handle_data(&page); + leaf_entry *entry = btree_get_leaf_entry(cfg, hdr, idx); + uint8 flags = entry->key_flags; + uint16 common = hdr->prefix_length; + uint64 p = format.key_prefix_size - common; + blob *descriptor = (blob *)(entry->key_and_message + p); + uint64 logical_length = descriptor->length; + if (fault == 0) { + entry->key_flags |= BTREE_INDEX_WIDE_STATS; /* Invalid on leaves. */ + } else { + descriptor->length = 1; /* Inline-sized keys may not be blobbed. */ + } + cache_unlock(cc, &page); + cache_unclaim(cc, &page); + cache_unget(cc, &page); + + platform_status rc = + btree_lookup(cc, cfg, root, format, PAGE_TYPE_BRANCH, target, &result); + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE), + "corruption %lu: %d", + fault, + rc.r); + + ASSERT_TRUE( + SUCCESS(cache_get(cc, leaf_addr, TRUE, PAGE_TYPE_BRANCH, &page))); + ASSERT_TRUE(cache_try_claim(cc, &page)); + cache_lock(cc, &page); + hdr = (btree_hdr *)page_handle_data(&page); + hdr->prefix_length = common; + entry = btree_get_leaf_entry(cfg, hdr, idx); + entry->key_flags = flags; + descriptor = (blob *)(entry->key_and_message + p); + descriptor->length = logical_length; + cache_unlock(cc, &page); + cache_unclaim(cc, &page); + cache_unget(cc, &page); + } + lookup_result_deinit(&result); + /* A copied leaf containing blobs must reject C > P on the next entry, + * without relying on a search that might reject at the common prefix. */ + ASSERT_TRUE(SUCCESS(btree_iterator_init( + cc, + cfg, + &holder, + root, + format, + PAGE_TYPE_BRANCH, + &(btree_iterator_init_options){.copy_nodes = TRUE, + .start_position = + key_start_at_or_after(target)}))); + holder.curr.hdr->prefix_length = format.key_prefix_size + 1; + platform_status rc = iterator_next(&holder.super); + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); + ASSERT_FALSE(iterator_can_curr(&holder.super)); + btree_iterator_deinit(&holder); +} +static void +native_check_corrupt_index(cache *cc, + btree_config *cfg, + uint64 root, + btree_format format, + key target) +{ + lookup_result result; + lookup_result_init(&result, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + for (uint64 fault = 0; fault < 3; fault++) { + page_handle page; + ASSERT_TRUE(SUCCESS(cache_get(cc, root, TRUE, PAGE_TYPE_BRANCH, &page))); + ASSERT_TRUE(cache_try_claim(cc, &page)); + cache_lock(cc, &page); + btree_hdr *hdr = (btree_hdr *)page_handle_data(&page); + ASSERT_TRUE(hdr->height > 0); + index_entry *entry = btree_get_index_entry(cfg, hdr, 0); + uint8 flags = entry->pivot.flags; + uint16 key_length = entry->pivot.length; + ASSERT_EQUAL(0, flags); + ASSERT_FALSE(flags & BTREE_INDEX_WIDE_STATS); + ASSERT_EQUAL(cfg->page_size, hdr->offsets[0] + sizeof_index_entry(entry)); + if (fault < 2) { + // Widening this entry makes its counters extend beyond the page. + entry->pivot.flags |= fault == 0 ? BTREE_INDEX_WIDE_STATS : 0x80; + } else { + /* A longer key pushes its narrow counters past the page. The custom + * ordering's 400-byte first key still takes the ordinary-entry fast + * path; the lexical 512-byte key exercises the maximum-key check. */ + entry->pivot.length++; + } + cache_unlock(cc, &page); + cache_unclaim(cc, &page); + cache_unget(cc, &page); + + for (uint64 async = 0; async < 2; async++) { + platform_status rc = + async + ? async_call_sync_callback(cache_cleanup(cc), + btree_lookup_async, + cc, + cfg, + root, + format, + PAGE_TYPE_BRANCH, + target, + &result) + : btree_lookup( + cc, cfg, root, format, PAGE_TYPE_BRANCH, target, &result); + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); + } + + ASSERT_TRUE(SUCCESS(cache_get(cc, root, TRUE, PAGE_TYPE_BRANCH, &page))); + ASSERT_TRUE(cache_try_claim(cc, &page)); + cache_lock(cc, &page); + hdr = (btree_hdr *)page_handle_data(&page); + entry = pointer_byte_offset(hdr, hdr->offsets[0]); + entry->pivot.flags = flags; + entry->pivot.length = key_length; + cache_unlock(cc, &page); + cache_unclaim(cc, &page); + cache_unget(cc, &page); + } + lookup_result_deinit(&result); +} + +/* Mutation searches may trust memtable entries, but key-blob reads can still + * fail. Exercise both a root leaf and a tree with internal pivots; subsequent + * inserts and the full-tree verification also check that locks were released. + */ +static void +native_check_insert_read_failure(struct ctest_btree_stress_data *data, + btree_config *cfg, + mini_allocator *mini, + uint64 root) +{ + cache *cc = (cache *)&data->cc; + cache_ops fault_ops = *cc->ops; + native_saved_ops = cc->ops; + fault_ops.page_get = native_fail_blob_read; + native_failed_reads = 0; + char buf[12000]; + key target = native_test_key(71, buf); + /* Keep the incoming key inline so any blob read comes from comparison, + * rather than constructing a new key blob. Its ordinal sorts among blobs. */ + target = key_create(FALSE, 512, key_data(target)); + uint64 value = 0; + btree_insert_results results; + btree_insert_results_init(&results, NULL); + cc->ops = &fault_ops; + platform_status rc = btree_insert( + cc, + cfg, + data->hid, + data->test_scratch, + root, + mini, + target, + message_create_internal( + MESSAGE_TYPE_INSERT, NULL, slice_create(sizeof(value), &value)), + &results); + cc->ops = native_saved_ops; + btree_insert_results_deinit(&results); + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_TIMEDOUT)); + ASSERT_TRUE(native_failed_reads > 0); +} + +static void +native_blob_keys_test(struct ctest_btree_stress_data *data, + bool32 reverse, + uint64 prefix) +{ + cache *cc = (cache *)&data->cc; + btree_config cfg = data->dbtree_cfg; + data_config dc = *cfg.data_cfg; + dc.key_compare_is_lexical = !reverse; + if (reverse) { + dc.key_compare = native_reverse_compare; + } + cfg.data_cfg = &dc; + cfg.blob_key_prefix_size = prefix; + dc.merge_tuples = native_merge; + mini_allocator mini; + uint64 root = test_btree_create(cc, &cfg, &mini, PAGE_TYPE_MEMTABLE); + char keybuf[12000]; + uint64 total_bytes = 0; + for (uint64 i = 0; i < NATIVE_KEY_COUNT; i++) { + /* Permute insertion order to exercise both sides of splits. */ + uint64 n = (i * 73) % NATIVE_KEY_COUNT; + key k = native_test_key(n, keybuf); + total_bytes += key_length(k); + uint64 value = n; + btree_insert_results results; + btree_insert_results_init(&results, NULL); + platform_status rc = btree_insert( + cc, + &cfg, + data->hid, + data->test_scratch, + root, + &mini, + k, + message_create_internal( + MESSAGE_TYPE_INSERT, NULL, slice_create(sizeof(value), &value)), + &results); + btree_insert_results_deinit(&results); + ASSERT_TRUE(SUCCESS(rc), "insert %lu: %d", n, rc.r); + if (i == 1) { + native_check_insert_read_failure(data, &cfg, &mini, root); + } + } + native_check_insert_read_failure(data, &cfg, &mini, root); + /* Update callbacks receive materialized keys; old-result ownership survives + * the operation-local B-tree configuration. */ + for (uint64 n = 5; n < NATIVE_KEY_COUNT; n += 31) { + lookup_result old; + lookup_result_init(&old, &dc, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + btree_insert_results results; + btree_insert_results_init(&results, &old); + uint64 increment = 1; + ASSERT_TRUE(SUCCESS(btree_insert( + cc, + &cfg, + data->hid, + data->test_scratch, + root, + &mini, + native_test_key(n, keybuf), + message_create_internal( + MESSAGE_TYPE_UPDATE, NULL, slice_create(8, &increment)), + &results))); + uint64 old_value; + memcpy(&old_value, merge_accumulator_data(&old.value), sizeof(old_value)); + ASSERT_EQUAL(n, old_value); + btree_insert_results_deinit(&results); + lookup_result_deinit(&old); + } + native_check_read_failure( + cc, &cfg, root, (btree_format){0}, PAGE_TYPE_MEMTABLE); + native_check_tree( + cc, &cfg, root, (btree_format){0}, PAGE_TYPE_MEMTABLE, reverse); + btree_iterator source; + ASSERT_TRUE(SUCCESS(btree_iterator_init(cc, + &cfg, + &source, + root, + (btree_format){0}, + PAGE_TYPE_MEMTABLE, + &(btree_iterator_init_options){0}))); + btree_pack_req req; + ASSERT_TRUE(SUCCESS(btree_pack_req_init( + &req, cc, &cfg, &source.super, NATIVE_KEY_COUNT, data->hid))); + ASSERT_TRUE(SUCCESS(btree_pack(&req))); + uint64 packed_root = req.root_addr; + btree_format format = req.format; + ASSERT_EQUAL(reverse ? 0 : prefix, format.key_prefix_size); + ASSERT_EQUAL(total_bytes, req.key_bytes); + btree_pack_req_deinit(&req, data->hid); + btree_iterator_deinit(&source); + mini_release(&mini); + btree_dec_ref(cc, &cfg, root, PAGE_TYPE_MEMTABLE); + ASSERT_TRUE(btree_verify_tree(cc, &cfg, packed_root, PAGE_TYPE_BRANCH)); + + /* Read with a different construction policy; the branch's persisted P wins. + */ + cfg.blob_key_prefix_size = 64; + native_check_corrupt_index( + cc, + &cfg, + packed_root, + format, + native_test_key(reverse ? NATIVE_KEY_COUNT - 1 : 0, keybuf)); + native_check_corrupt_leaf(cc, &cfg, packed_root, format); + native_check_read_failure(cc, &cfg, packed_root, format, PAGE_TYPE_BRANCH); + native_check_tree(cc, &cfg, packed_root, format, PAGE_TYPE_BRANCH, reverse); + btree_pivot_stats stats; + ASSERT_TRUE(SUCCESS(btree_count_in_range(cc, + &cfg, + packed_root, + format, + NEGATIVE_INFINITY_KEY, + POSITIVE_INFINITY_KEY, + &stats))); + ASSERT_EQUAL(NATIVE_KEY_COUNT, stats.num_kvs); + ASSERT_EQUAL(total_bytes, stats.key_bytes); + ASSERT_EQUAL(NATIVE_KEY_COUNT * sizeof(uint64), stats.message_bytes); + ASSERT_TRUE(SUCCESS(btree_iterator_init(cc, + &cfg, + &source, + packed_root, + format, + PAGE_TYPE_BRANCH, + &(btree_iterator_init_options){0}))); + bool32 saw_wide_prefix = FALSE, saw_blob_prefix = FALSE; + bool32 saw_blob = FALSE, saw_wide_prefix_after_blob = FALSE; + for (uint64 i = 0; i < NATIVE_KEY_COUNT; i++) { + const leaf_entry *entry = + btree_get_leaf_entry(source.cfg, source.curr.hdr, source.idx); + key logical; + message msg; + iterator_curr(&source.super, &logical, &msg); + bool32 blobbed = key_length(logical) > MAX_INLINE_KEY_SIZE(cfg.page_size); + ASSERT_EQUAL(blobbed ? ONDISK_KEY_IS_BLOB : 0, entry->key_flags); + if (blobbed) { + saw_blob = TRUE; + ASSERT_TRUE(source.curr.hdr->prefix_length <= format.key_prefix_size); + uint64 suffix = + format.key_prefix_size - source.curr.hdr->prefix_length; + slice descriptor = slice_create(entry->key_length - suffix, + entry->key_and_message + suffix); + ASSERT_EQUAL(key_length(logical), blob_length(descriptor)); + ASSERT_TRUE(slice_length(descriptor) <= 256); + saw_blob_prefix |= + source.curr.hdr->prefix_length == format.key_prefix_size; + } else { + saw_wide_prefix |= + source.curr.hdr->prefix_length > format.key_prefix_size; + saw_wide_prefix_after_blob |= + saw_blob && source.curr.hdr->prefix_length > format.key_prefix_size; + } + ASSERT_TRUE(SUCCESS(iterator_next(&source.super))); + } + if (!reverse) { + ASSERT_TRUE(saw_wide_prefix); + ASSERT_TRUE(saw_blob_prefix); + ASSERT_TRUE(saw_wide_prefix_after_blob); + } + ASSERT_TRUE(SUCCESS(iterator_seek( + &source.super, greater_than_or_equal, NEGATIVE_INFINITY_KEY))); + ASSERT_TRUE(SUCCESS(btree_pack_req_init( + &req, cc, &cfg, &source.super, NATIVE_KEY_COUNT, data->hid))); + ASSERT_TRUE(SUCCESS(btree_pack(&req))); + uint64 repacked_root = req.root_addr; + btree_format repacked_format = req.format; + ASSERT_EQUAL(reverse ? 0 : 64, repacked_format.key_prefix_size); + btree_pack_req_deinit(&req, data->hid); + btree_iterator_deinit(&source); + btree_dec_ref(cc, &cfg, packed_root, PAGE_TYPE_BRANCH); + native_check_tree( + cc, &cfg, repacked_root, repacked_format, PAGE_TYPE_BRANCH, reverse); + btree_dec_ref(cc, &cfg, repacked_root, PAGE_TYPE_BRANCH); +} + +CTEST2(btree_stress, native_blob_keys_lexical) +{ + native_blob_keys_test(data, FALSE, 256); +} + +CTEST2(btree_stress, native_blob_keys_custom_order) +{ + native_blob_keys_test(data, TRUE, 256); +} + +CTEST2(btree_stress, native_blob_keys_without_prefix) +{ + native_blob_keys_test(data, FALSE, 0); +} diff --git a/tests/unit/btree_test.c b/tests/unit/btree_test.c index 505eec33d..690efa3f3 100644 --- a/tests/unit/btree_test.c +++ b/tests/unit/btree_test.c @@ -62,7 +62,7 @@ btree_leaf_incorporate_tuple(const btree_config *cfg, uint64 *generation) { platform_status rc = btree_create_leaf_incorporate_spec( - cfg, NULL, NULL, hid, hdr, tuple_key, msg, spec); + cfg, NULL, NULL, hid, hdr, &tuple_key, &msg, spec); ASSERT_TRUE(SUCCESS(rc)); return btree_try_perform_leaf_incorporate_spec(cfg, hdr, spec, generation); } @@ -218,12 +218,15 @@ CTEST2(btree, test_defragmented_index_bytes) * A NULL cache makes an accidental read fail, including in the async paths. */ static const struct { - uint64 addr; - uint16 format_version; + uint64 addr; + btree_format format; } invalid_branch_refs[] = { - {4096, BTREE_BRANCH_FORMAT_VERSION - 1}, - {4096, BTREE_BRANCH_FORMAT_VERSION + 1}, - {0, BTREE_BRANCH_FORMAT_VERSION}, + {4096, {1, 256}}, + {4096, {2, 256}}, + {4096, {3, 256}}, + {4096, {BTREE_BRANCH_FORMAT_VERSION + 1, 256}}, + {0, BTREE_DEFAULT_FORMAT}, + {4096, {BTREE_BRANCH_FORMAT_VERSION, UINT16_MAX}}, }; CTEST2(btree, test_invalid_branch_point_lookup) @@ -238,7 +241,7 @@ CTEST2(btree, test_invalid_branch_point_lookup) platform_status rc = btree_lookup(NULL, &data->dbtree_cfg, invalid_branch_refs[i].addr, - invalid_branch_refs[i].format_version, + invalid_branch_refs[i].format, PAGE_TYPE_BRANCH, target, &result); @@ -249,9 +252,9 @@ CTEST2(btree, test_invalid_branch_point_lookup) rc = btree_lookup_and_merge(NULL, &data->dbtree_cfg, invalid_branch_refs[i].addr, - invalid_branch_refs[i].format_version, + invalid_branch_refs[i].format, PAGE_TYPE_BRANCH, - target, + &target, &result, &local_found); ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); @@ -275,7 +278,7 @@ CTEST2(btree, test_invalid_branch_async_point_lookup) NULL, &data->dbtree_cfg, invalid_branch_refs[i].addr, - invalid_branch_refs[i].format_version, + invalid_branch_refs[i].format, PAGE_TYPE_BRANCH, target, &result, @@ -285,17 +288,16 @@ CTEST2(btree, test_invalid_branch_async_point_lookup) ASSERT_TRUE(STATUS_IS_EQ(async_result(&state), STATUS_INVALID_STATE)); ASSERT_FALSE(lookup_result_found(&result)); - btree_lookup_and_merge_async_state_init( - &state, - NULL, - &data->dbtree_cfg, - invalid_branch_refs[i].addr, - invalid_branch_refs[i].format_version, - PAGE_TYPE_BRANCH, - target, - &result, - NULL, - NULL); + btree_lookup_and_merge_async_state_init(&state, + NULL, + &data->dbtree_cfg, + invalid_branch_refs[i].addr, + invalid_branch_refs[i].format, + PAGE_TYPE_BRANCH, + target, + &result, + NULL, + NULL); ASSERT_EQUAL(ASYNC_STATUS_DONE, btree_lookup_and_merge_async(&state)); ASSERT_TRUE(STATUS_IS_EQ(async_result(&state), STATUS_INVALID_STATE)); ASSERT_FALSE(lookup_result_found(&result)); @@ -314,7 +316,7 @@ CTEST2(btree, test_invalid_branch_iterator) &data->dbtree_cfg, &itor, invalid_branch_refs[i].addr, - invalid_branch_refs[i].format_version, + invalid_branch_refs[i].format, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){.copy_nodes = copy_nodes, .prefetch_lookahead = 1}); @@ -328,7 +330,7 @@ CTEST2(btree, test_invalid_branch_iterator) &data->dbtree_cfg, &itor, invalid_branch_refs[i].addr, - invalid_branch_refs[i].format_version, + invalid_branch_refs[i].format, PAGE_TYPE_BRANCH, greater_than_or_equal, NEGATIVE_INFINITY_KEY, @@ -353,20 +355,19 @@ CTEST2(btree, test_invalid_branch_metadata_queries) { for (uint64 i = 0; i < ARRAY_SIZE(invalid_branch_refs); i++) { uint32 height; - platform_status rc = - btree_get_root_height(NULL, - &data->dbtree_cfg, - invalid_branch_refs[i].addr, - invalid_branch_refs[i].format_version, - PAGE_TYPE_BRANCH, - &height); + platform_status rc = btree_get_root_height(NULL, + &data->dbtree_cfg, + invalid_branch_refs[i].addr, + invalid_branch_refs[i].format, + PAGE_TYPE_BRANCH, + &height); ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); btree_pivot_stats stats; rc = btree_count_in_range(NULL, &data->dbtree_cfg, invalid_branch_refs[i].addr, - invalid_branch_refs[i].format_version, + invalid_branch_refs[i].format, NEGATIVE_INFINITY_KEY, POSITIVE_INFINITY_KEY, &stats); @@ -375,7 +376,7 @@ CTEST2(btree, test_invalid_branch_metadata_queries) rc = btree_recover_allocations(NULL, &data->dbtree_cfg, invalid_branch_refs[i].addr, - invalid_branch_refs[i].format_version, + invalid_branch_refs[i].format, PAGE_TYPE_BRANCH); ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); } @@ -530,7 +531,10 @@ index_hdr_tests(btree_config *cfg, btree_scratch *scratch, platform_heap_id hid) char *index_buffer = TYPED_MANUAL_MALLOC(hid, index_buffer, btree_page_size(cfg)); btree_hdr *hdr = (btree_hdr *)index_buffer; - int nkvs = 100; + int nkvs = (btree_page_size(cfg) - sizeof(*hdr)) + / (2 + * (sizeof(index_entry) + sizeof(btree_pivot_stats32) + + sizeof(uint64) + sizeof(table_entry))); bool32 rv = FALSE; int cmp_rv = 0; @@ -592,8 +596,12 @@ index_hdr_search_tests(btree_config *cfg, platform_heap_id hid) { char *leaf_buffer = TYPED_MANUAL_MALLOC(hid, leaf_buffer, btree_page_size(cfg)); - btree_hdr *hdr = (btree_hdr *)leaf_buffer; - int nkvs = 256; + btree_hdr *hdr = (btree_hdr *)leaf_buffer; + int nkvs = 2 + * MIN(128, + (btree_page_size(cfg) - sizeof(*hdr)) + / (sizeof(index_entry) + sizeof(btree_pivot_stats32) + 1 + + sizeof(table_entry))); btree_pivot_stats stats; memset(&stats, 0, sizeof(stats)); @@ -751,15 +759,9 @@ defragmented_leaf_bytes_tests(btree_config *cfg, leaf_incorporate_spec spec; message msg = message_create_internal( MESSAGE_TYPE_INSERT, NULL, slice_create(9, msgbuf)); - platform_status rc = - btree_create_leaf_incorporate_spec(cfg, - NULL, - NULL, - hid, - copy, - key_create(FALSE, 1, &new_keys[k]), - msg, - &spec); + key tuple_key = key_create(FALSE, 1, &new_keys[k]); + platform_status rc = btree_create_leaf_incorporate_spec( + cfg, NULL, NULL, hid, copy, &tuple_key, &msg, &spec); ASSERT_TRUE(SUCCESS(rc)); ASSERT_EQUAL(k == 0 ? ENTRY_DID_NOT_EXIST : ENTRY_STILL_EXISTS, spec.old_entry_state); @@ -798,7 +800,8 @@ defragmented_index_bytes_tests(btree_config *cfg, ASSERT_TRUE(btree_set_index_entry( cfg, hdr, i, key_create(FALSE, i % 5 + 1, keybuf), i, stats)); } - // Lengthening every third pivot leaves its old copy behind as garbage. + // Lengthening and widening every third pivot leaves garbage behind. + stats.message_bytes = UINT64_C(1) << 40; for (uint32 i = 0; i < nkvs; i += 3) { ASSERT_TRUE(btree_set_index_entry( cfg, hdr, i, key_create(FALSE, i % 5 + 8, keybuf), i, stats)); @@ -814,6 +817,109 @@ defragmented_index_bytes_tests(btree_config *cfg, btree_defragment_index(cfg, scratch, hdr); ASSERT_EQUAL(node_used_bytes(cfg, hdr), predicted); + for (uint32 i = 0; i < nkvs; i++) { + index_entry *entry = btree_get_index_entry(cfg, hdr, i); + btree_pivot_stats actual = index_entry_stats(entry); + ASSERT_EQUAL(i % 3 == 0 ? stats.message_bytes : 0, actual.message_bytes); + ASSERT_EQUAL(i, index_entry_child_addr(entry)); + ASSERT_EQUAL(i % 5 + (i % 3 == 0 ? 8 : 1), + key_length(index_entry_key(entry))); + } + + // Narrowing an existing wide entry must clear its flag and retain its key. + stats.message_bytes = 1; + for (uint32 i = 0; i < nkvs; i += 3) { + ASSERT_TRUE(btree_set_index_entry( + cfg, hdr, i, key_create(FALSE, i % 5 + 8, keybuf), i, stats)); + index_entry *entry = btree_get_index_entry(cfg, hdr, i); + ASSERT_FALSE(entry->pivot.flags & BTREE_INDEX_WIDE_STATS); + ASSERT_EQUAL(1, index_entry_stats(entry).message_bytes); + } + predicted = btree_defragmented_index_bytes(cfg, hdr); + btree_defragment_index(cfg, scratch, hdr); + ASSERT_EQUAL(node_used_bytes(cfg, hdr), predicted); + platform_free(hid, index_buffer); return 0; } + +CTEST2(btree, test_native_statistics_layout) +{ + ASSERT_EQUAL(3 * sizeof(uint32), sizeof(btree_pivot_stats32)); + ASSERT_EQUAL(3 * sizeof(uint64), sizeof(btree_pivot_stats)); + ASSERT_EQUAL(sizeof(uint64), offsetof(index_entry, pivot)); + const struct { + btree_pivot_stats stats; + bool32 wide; + } cases[] = { + {{0, 1, 2}, FALSE}, + {{UINT64_C(0x7fffffff), UINT32_MAX - 1, UINT64_MAX}, FALSE}, + {{UINT64_MAX, UINT64_MAX, UINT64_MAX}, FALSE}, + {{UINT32_MAX, 0, 0}, TRUE}, + {{0, UINT32_MAX, 0}, TRUE}, + {{0, 0, UINT32_MAX}, TRUE}, + {{(UINT64_C(1) << 40) + 7, (UINT64_C(1) << 53) + 9, UINT64_MAX}, TRUE}, + {{UINT64_MAX - 1, UINT64_MAX, 0}, TRUE}, + }; + // Three key bytes deliberately leave both counter representations unaligned. + char storage[sizeof(index_entry) + 3 + sizeof(btree_pivot_stats) + 1]; + for (uint64 i = 0; i < ARRAY_SIZE(cases); i++) { + for (uint8 blob_flag = 0; blob_flag <= ONDISK_KEY_IS_BLOB; + blob_flag += ONDISK_KEY_IS_BLOB) + { + memset(storage, 0xa5, sizeof(storage)); + index_entry *entry = (index_entry *)storage; + entry->pivot.length = 3; + // Start with the opposite width to exercise both flag transitions. + entry->pivot.flags = + blob_flag | (cases[i].wide ? 0 : BTREE_INDEX_WIDE_STATS); + memcpy(entry->pivot.bytes, "key", 3); + index_entry_set_stats(entry, cases[i].stats); + btree_pivot_stats actual = index_entry_stats(entry); + ASSERT_EQUAL(cases[i].stats.num_kvs, actual.num_kvs); + ASSERT_EQUAL(cases[i].stats.key_bytes, actual.key_bytes); + ASSERT_EQUAL(cases[i].stats.message_bytes, actual.message_bytes); + ASSERT_EQUAL(blob_flag | (cases[i].wide ? BTREE_INDEX_WIDE_STATS : 0), + entry->pivot.flags); + ASSERT_EQUAL(0, memcmp(entry->pivot.bytes, "key", 3)); + uint64 size = cases[i].wide ? 24 : 12; + ASSERT_EQUAL(sizeof(index_entry) + 3 + size, + sizeof_index_entry(entry)); + ASSERT_EQUAL(0xa5, (uint8)storage[sizeof_index_entry(entry)]); + if (cases[i].wide) { + uint64 raw[3]; + memcpy(raw, entry->pivot.bytes + 3, sizeof(raw)); + ASSERT_EQUAL(cases[i].stats.num_kvs, raw[0]); + ASSERT_EQUAL(cases[i].stats.key_bytes, raw[1]); + ASSERT_EQUAL(cases[i].stats.message_bytes, raw[2]); + } else { + uint32 raw[3]; + memcpy(raw, entry->pivot.bytes + 3, sizeof(raw)); + ASSERT_EQUAL((uint32)cases[i].stats.num_kvs, raw[0]); + ASSERT_EQUAL((uint32)cases[i].stats.key_bytes, raw[1]); + ASSERT_EQUAL((uint32)cases[i].stats.message_bytes, raw[2]); + } + } + } +} + +CTEST2(btree, test_blob_key_configuration) +{ + btree_config cfg = data->dbtree_cfg; + data_config dc = *cfg.data_cfg; + dc.key_compare_is_lexical = TRUE; + cfg.data_cfg = &dc; + cfg.blob_key_prefix_size = 256; + cfg.blob_cfg.blob_max_descriptor_size = 256; + ASSERT_TRUE(SUCCESS(btree_config_validate(&cfg))); + cfg.blob_key_prefix_size = 257; + ASSERT_TRUE(STATUS_IS_EQ(btree_config_validate(&cfg), STATUS_BAD_PARAM)); + cfg.blob_key_prefix_size = 0; + cfg.blob_cfg.blob_max_descriptor_size = 512; + ASSERT_TRUE(SUCCESS(btree_config_validate(&cfg))); + cfg.blob_key_prefix_size = UINT64_C(65536); + ASSERT_TRUE(STATUS_IS_EQ(btree_config_validate(&cfg), STATUS_BAD_PARAM)); + dc.key_compare_is_lexical = FALSE; + ASSERT_TRUE(SUCCESS(btree_config_validate(&cfg))); + ASSERT_EQUAL(0, btree_config_format(&cfg).key_prefix_size); +} diff --git a/tests/unit/btree_test_common.c b/tests/unit/btree_test_common.c index a115cf328..c18deced9 100644 --- a/tests/unit/btree_test_common.c +++ b/tests/unit/btree_test_common.c @@ -69,5 +69,8 @@ init_btree_config_from_master_config(btree_config *dbtree_cfg, data_config *data_cfg) { btree_config_init(dbtree_cfg, cache_cfg, data_cfg); - return 1; + dbtree_cfg->blob_cfg.blob_max_descriptor_size = + master_cfg->blob_max_descriptor_size; + dbtree_cfg->blob_key_prefix_size = master_cfg->blob_key_prefix_size; + return SUCCESS(btree_config_validate(dbtree_cfg)); } diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 0bbaa7ea4..0deda5baa 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -23,7 +23,7 @@ */ #include "core.h" #include "shard_log.h" -#include "blob_build.h" +#include "mini_blob.h" #include "clockcache.h" #include "allocator.h" #include "mini_allocator.h" @@ -42,7 +42,9 @@ typedef struct ONDISK test_trunk_node_hdr { uint16 height; uint16 num_pivots; uint16 num_inflight_bundles; + uint16 blob_key_prefix_size; uint32 inflight_bundles_offset; + uint32 tail_offset; uint32 pivot_offsets[]; } test_trunk_node_hdr; @@ -949,13 +951,13 @@ static const iterator_ops pack_abort_test_ops = { }; static platform_status -blob_checksum_test_build_and_check(const blob_build_config *cfg, - cache *cc, - mini_allocator *mini, - uint64 length, - writable_buffer *data, - writable_buffer *descriptor, - writable_buffer *materialized) +blob_checksum_test_build_and_check(const mini_blob_config *cfg, + cache *cc, + mini_allocator *mini, + uint64 length, + writable_buffer *data, + writable_buffer *descriptor, + writable_buffer *materialized) { platform_status rc = writable_buffer_resize(data, length); if (!SUCCESS(rc)) { @@ -963,7 +965,8 @@ blob_checksum_test_build_and_check(const blob_build_config *cfg, } blob_checksum_test_fill(data, length); - rc = blob_build(cfg, cc, mini, writable_buffer_to_slice(data), descriptor); + rc = mini_blob_build( + cfg, cc, mini, writable_buffer_to_slice(data), descriptor); if (!SUCCESS(rc)) { return rc; } @@ -1010,7 +1013,6 @@ blob_checksum_test_mini_init(cache *cc, mini_allocator *mini, uint64 *meta_head) page_type types[NUM_BLOB_BATCHES] = { PAGE_TYPE_BLOB, PAGE_TYPE_BLOB, - PAGE_TYPE_BLOB, }; platform_status rc = allocator_alloc(al, meta_head, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { @@ -1275,6 +1277,51 @@ CTEST2(splinter, test_retain_failure_log_handle_unwinds) ASSERT_EQUAL_U(initial, allocator_in_use(al)); } +CTEST2(splinter, test_mini_attachment_counts_toward_memtable_fullness) +{ + cache *cc = (cache *)data->clock_cache; + allocator *al = (allocator *)&data->al; + btree_config *cfg = &data->system_cfg->btree_cfg; + uint64 initial = allocator_in_use(al); + mini_allocator source; + uint64 source_head, addr; + ASSERT_TRUE( + SUCCESS(blob_checksum_test_mini_init(cc, &source, &source_head))); + ASSERT_TRUE(SUCCESS(mini_alloc_extent(&source, 0, &addr, NULL))); + memtable target = {0}; + ASSERT_TRUE(SUCCESS(btree_create( + cc, cfg, &target.mini, PAGE_TYPE_MEMTABLE, &target.root_addr))); + uint64 before = mini_num_extents(&target.mini); + memtable_config mt_cfg = {.max_extents_per_memtable = before + 2}; + ASSERT_FALSE(memtable_is_full(&mt_cfg, &target)); + + // A failed ownership acquisition must not charge capacity. + retain_test_saved_ops = al->ops; + allocator_ops fault_ops = *al->ops; + fault_ops.inc_ref = retain_test_fail; + retain_test_successes_before_failure = 0; + al->ops = &fault_ops; + platform_status rc = + mini_attach_extent(&target.mini, cfg->blob_cfg.page_batch, addr); + al->ops = retain_test_saved_ops; + ASSERT_TRUE(STATUS_IS_EQ(STATUS_NO_MEMORY, rc)); + ASSERT_EQUAL_U(before, mini_num_extents(&target.mini)); + + // Charge each reference, even when two logical blobs share one extent. + for (uint64 i = 1; i <= 2; i++) { + ASSERT_TRUE(SUCCESS( + mini_attach_extent(&target.mini, cfg->blob_cfg.page_batch, addr))); + ASSERT_EQUAL_U(before + i, mini_num_extents(&target.mini)); + ASSERT_EQUAL(i == 2, memtable_is_full(&mt_cfg, &target)); + ASSERT_EQUAL_U(AL_ONE_REF + i, allocator_get_refcount(al, addr)); + } + ASSERT_TRUE( + SUCCESS(blob_checksum_test_mini_deinit(cc, &source, source_head))); + mini_release(&target.mini); + ASSERT_TRUE(btree_dec_ref(cc, cfg, target.root_addr, PAGE_TYPE_MEMTABLE)); + ASSERT_EQUAL_U(initial, allocator_in_use(al)); +} + CTEST2(splinter, test_retain_failure_blob_lookup_and_old_value_are_atomic) { cache *cc = (cache *)data->clock_cache; @@ -1322,24 +1369,24 @@ CTEST2(splinter, test_retain_failure_blob_lookup_and_old_value_are_atomic) for (uint64 failure = 0; failure < 2; failure++) { retain_test_successes_before_failure = failure; al->ops = &fault_ops; - platform_status rc = - async ? async_call_sync_callback(cache_cleanup(cc), - btree_lookup_async, - cc, - cfg, - root, - BTREE_BRANCH_FORMAT_VERSION, - PAGE_TYPE_MEMTABLE, - tuple_key, - &result) - : btree_lookup(cc, - cfg, - root, - BTREE_BRANCH_FORMAT_VERSION, - PAGE_TYPE_MEMTABLE, - tuple_key, - &result); - al->ops = retain_test_saved_ops; + platform_status rc = async + ? async_call_sync_callback(cache_cleanup(cc), + btree_lookup_async, + cc, + cfg, + root, + (btree_format){0}, + PAGE_TYPE_MEMTABLE, + tuple_key, + &result) + : btree_lookup(cc, + cfg, + root, + (btree_format){0}, + PAGE_TYPE_MEMTABLE, + tuple_key, + &result); + al->ops = retain_test_saved_ops; ASSERT_TRUE(STATUS_IS_EQ(STATUS_NO_MEMORY, rc)); ASSERT_FALSE(lookup_result_found(&result)); ASSERT_TRUE(ondisk_ref_is_null(&result.blob_ref)); @@ -1378,7 +1425,7 @@ CTEST2(splinter, test_retain_failure_blob_lookup_and_old_value_are_atomic) ASSERT_TRUE(SUCCESS(btree_lookup(cc, cfg, root, - BTREE_BRANCH_FORMAT_VERSION, + (btree_format){0}, PAGE_TYPE_MEMTABLE, tuple_key, &result))); @@ -1425,17 +1472,15 @@ CTEST2(splinter, test_retain_failure_blob_clone_prefix_cleanup) SUCCESS(blob_checksum_test_mini_init(cc, &source, &source_head))); ASSERT_TRUE( SUCCESS(btree_create(cc, cfg, &target, PAGE_TYPE_MEMTABLE, &root))); - blob_build_config blob_cfg = {.extent_batch = 0, - .page_batch = 1, - .subpage_batch = 2, - .blob_max_descriptor_size = 512}; - writable_buffer bytes, descriptor; + mini_blob_config blob_cfg = { + .page_batch = 0, .subpage_batch = 1, .blob_max_descriptor_size = 512}; + writable_buffer bytes, descriptor; writable_buffer_init(&bytes, data->hid); writable_buffer_init(&descriptor, data->hid); ASSERT_TRUE( SUCCESS(writable_buffer_resize(&bytes, 2 * cache_extent_size(cc)))); blob_checksum_test_fill(&bytes, writable_buffer_length(&bytes)); - ASSERT_TRUE(SUCCESS(blob_build( + ASSERT_TRUE(SUCCESS(mini_blob_build( &blob_cfg, cc, &source, writable_buffer_to_slice(&bytes), &descriptor))); parsed_blob parsed; ASSERT_TRUE(SUCCESS(blob_parse_descriptor( @@ -1473,14 +1518,14 @@ CTEST2(splinter, test_retain_failure_blob_clone_prefix_cleanup) al->ops = retain_test_saved_ops; ASSERT_TRUE(STATUS_IS_EQ(STATUS_NO_MEMORY, rc)); ASSERT_EQUAL_U(0, callbacks); - ASSERT_EQUAL_U(AL_ONE_REF + 1, allocator_get_refcount(al, first)); + ASSERT_EQUAL_U(AL_ONE_REF, allocator_get_refcount(al, first)); ASSERT_EQUAL_U(AL_ONE_REF, allocator_get_refcount(al, second)); lookup_result result; lookup_result_init(&result, cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); ASSERT_TRUE(SUCCESS(btree_lookup(cc, cfg, root, - BTREE_BRANCH_FORMAT_VERSION, + (btree_format){0}, PAGE_TYPE_MEMTABLE, tuple_key, &result))); @@ -1498,7 +1543,7 @@ CTEST2(splinter, test_retain_failure_blob_clone_prefix_cleanup) ASSERT_TRUE(SUCCESS(btree_lookup(cc, cfg, root, - BTREE_BRANCH_FORMAT_VERSION, + (btree_format){0}, PAGE_TYPE_MEMTABLE, tuple_key, &result))); @@ -1525,11 +1570,10 @@ CTEST2(splinter, test_retain_failure_blob_clone_prefix_cleanup) CTEST2(splinter, test_blob_checksums) { - cache *cc = (cache *)data->clock_cache; - blob_build_config cfg = { - .extent_batch = 0, - .page_batch = 1, - .subpage_batch = 2, + cache *cc = (cache *)data->clock_cache; + mini_blob_config cfg = { + .page_batch = 0, + .subpage_batch = 1, .alignment = 0, }; @@ -1603,11 +1647,11 @@ CTEST2(splinter, test_blob_checksums) goto cleanup; } checked_clone_mini_live = TRUE; - rc = blob_clone(&cfg, - cc, - &checked_clone_mini, - writable_buffer_to_slice(&descriptor), - &checked_clone); + rc = mini_blob_clone(&cfg, + cc, + &checked_clone_mini, + writable_buffer_to_slice(&descriptor), + &checked_clone); if (!SUCCESS(rc)) { goto cleanup; } @@ -1662,7 +1706,7 @@ CTEST2(splinter, test_blob_checksums) platform_status materialize_rc = blob_materialize_full(cc, invalid, &materialized); platform_status clone_rc = - blob_clone(&cfg, cc, &checked_clone_mini, invalid, &invalid_clone); + mini_blob_clone(&cfg, cc, &checked_clone_mini, invalid, &invalid_clone); mutable_blob->format = expected_format; if (!STATUS_IS_EQ(validate_rc, STATUS_INVALID_STATE) || !STATUS_IS_EQ(materialize_rc, STATUS_INVALID_STATE) @@ -1678,7 +1722,7 @@ CTEST2(splinter, test_blob_checksums) validate_rc = blob_validate(cc, truncated); materialize_rc = blob_materialize_full(cc, truncated, &materialized); clone_rc = - blob_clone(&cfg, cc, &checked_clone_mini, truncated, &invalid_clone); + mini_blob_clone(&cfg, cc, &checked_clone_mini, truncated, &invalid_clone); if (!STATUS_IS_EQ(validate_rc, STATUS_INVALID_STATE) || !STATUS_IS_EQ(materialize_rc, STATUS_INVALID_STATE) || !STATUS_IS_EQ(clone_rc, STATUS_INVALID_STATE)) @@ -2133,7 +2177,7 @@ CTEST2(splinter, test_invalid_branch_reference_rejected) (test_trunk_bundle *)(page_handle_data(page) + offset % page_size); bundle->maplet = filter; if (invalid_field == 0) { - bundle->branches[0].format_version++; + bundle->branches[0].format.version++; } else { bundle->branches[0].addr = 0; } @@ -2252,9 +2296,9 @@ CTEST2(splinter, test_persisted_old_branch_version_rejected) uint64 bundle_offset = (char *)bundle - page_handle_data(&page); ASSERT_TRUE(bundle->num_branches != 0); branch_ref saved_branch = bundle->branches[0]; - ASSERT_EQUAL(BTREE_BRANCH_FORMAT_VERSION, saved_branch.format_version); + ASSERT_EQUAL(BTREE_BRANCH_FORMAT_VERSION, saved_branch.format.version); /* Version 1 predates the prefix_length field in the branch header. */ - bundle->branches[0].format_version = 1; + bundle->branches[0].format.version = 1; test_splinter_unclaim_page(cc, &page); ASSERT_TRUE(SUCCESS(cache_writeback_dirty(cc))); ASSERT_TRUE(SUCCESS(cache_durable_barrier(cc))); @@ -6846,10 +6890,10 @@ CTEST2(splinter, test_cache_read_errors_reach_core_operations) * allocating or multiplying a rounded capacity past UINT64_MAX. */ CTEST2(splinter, test_blob_descriptor_sizes) { - uint64 extent = cache_extent_size((cache *)data->clock_cache); - uint64 fanout = extent / sizeof(uint64); - uint64 roots = (512 - sizeof(blob)) / sizeof(uint64); - blob_build_config cfg = {.blob_max_descriptor_size = 512}; + uint64 extent = cache_extent_size((cache *)data->clock_cache); + uint64 fanout = extent / sizeof(uint64); + uint64 roots = (512 - sizeof(blob)) / sizeof(uint64); + mini_blob_config cfg = {.blob_max_descriptor_size = 512}; const struct { uint64 length; uint64 addresses; @@ -6862,21 +6906,22 @@ CTEST2(splinter, test_blob_descriptor_sizes) }; for (uint64 i = 0; i < ARRAY_SIZE(cases); i++) { uint64 size; - ASSERT_TRUE(SUCCESS( - blob_descriptor_size_bound(&cfg, extent, cases[i].length, &size))); + ASSERT_TRUE(SUCCESS(mini_blob_descriptor_size_bound( + &cfg, extent, cases[i].length, &size))); ASSERT_EQUAL(sizeof(blob) + cases[i].addresses * sizeof(uint64), size); } uint64 size; - ASSERT_TRUE( - SUCCESS(blob_descriptor_size_bound(&cfg, extent, UINT64_MAX, &size))); + ASSERT_TRUE(SUCCESS( + mini_blob_descriptor_size_bound(&cfg, extent, UINT64_MAX, &size))); ASSERT_TRUE(size <= 512); cfg.blob_max_descriptor_size = 0; - ASSERT_TRUE( - SUCCESS(blob_descriptor_size_bound(&cfg, extent, roots * extent, &size))); + roots = (BLOB_DEFAULT_MAX_DESCRIPTOR_SIZE - sizeof(blob)) / sizeof(uint64); + ASSERT_TRUE(SUCCESS( + mini_blob_descriptor_size_bound(&cfg, extent, roots * extent, &size))); ASSERT_EQUAL(sizeof(blob) + roots * sizeof(uint64), size); cfg.blob_max_descriptor_size = BLOB_MIN_MAX_DESCRIPTOR_SIZE - 1; ASSERT_TRUE( - STATUS_IS_EQ(blob_descriptor_size_bound(&cfg, extent, extent, &size), + STATUS_IS_EQ(mini_blob_descriptor_size_bound(&cfg, extent, extent, &size), STATUS_BAD_PARAM)); } @@ -6885,18 +6930,16 @@ CTEST2(splinter, test_blob_descriptor_sizes) * the clone owns every level, not just its descriptor's children. */ CTEST2(splinter, test_blob_trees) { - cache *cc = (cache *)data->clock_cache; - uint64 extent = cache_extent_size(cc); - uint64 page = cache_page_size(cc); - blob_build_config cfg = {.extent_batch = 0, - .page_batch = 1, - .subpage_batch = 2, - .blob_max_descriptor_size = 512}; - uint64 roots = (512 - sizeof(blob)) / sizeof(uint64); - uint64 lengths[] = {roots * extent - 1, - roots * extent, - roots * extent + 1, - (page / sizeof(uint64) + 1) * extent + 17}; + cache *cc = (cache *)data->clock_cache; + uint64 extent = cache_extent_size(cc); + uint64 page = cache_page_size(cc); + mini_blob_config cfg = { + .page_batch = 0, .subpage_batch = 1, .blob_max_descriptor_size = 512}; + uint64 roots = (512 - sizeof(blob)) / sizeof(uint64); + uint64 lengths[] = {roots * extent - 1, + roots * extent, + roots * extent + 1, + (page / sizeof(uint64) + 1) * extent + 17}; for (uint64 trial = 0; trial < ARRAY_SIZE(lengths); trial++) { uint64 length = lengths[trial]; mini_allocator source, target; @@ -6912,7 +6955,7 @@ CTEST2(splinter, test_blob_trees) writable_buffer_init(&result, data->hid); ASSERT_TRUE(SUCCESS(writable_buffer_resize(&bytes, length))); blob_checksum_test_fill(&bytes, length); - ASSERT_TRUE(SUCCESS(blob_build( + ASSERT_TRUE(SUCCESS(mini_blob_build( &cfg, cc, &source, writable_buffer_to_slice(&bytes), &desc))); slice descriptor = writable_buffer_to_slice(&desc); parsed_blob parsed; @@ -6920,8 +6963,8 @@ CTEST2(splinter, test_blob_trees) ASSERT_EQUAL(trial < 2 ? 1 : 2, parsed.height); ASSERT_TRUE(slice_length(descriptor) <= 512); uint64 bound; - ASSERT_TRUE( - SUCCESS(blob_descriptor_size_bound(&cfg, extent, length, &bound))); + ASSERT_TRUE(SUCCESS( + mini_blob_descriptor_size_bound(&cfg, extent, length, &bound))); ASSERT_EQUAL(slice_length(descriptor), bound); writeback_set set; @@ -6933,7 +6976,8 @@ CTEST2(splinter, test_blob_trees) ASSERT_TRUE(SUCCESS(writeback_set_wait(&set))); writeback_set_deinit(&set); ASSERT_TRUE(SUCCESS(blob_validate(cc, descriptor))); - ASSERT_TRUE(SUCCESS(blob_clone(&cfg, cc, &target, descriptor, &cloned))); + ASSERT_TRUE( + SUCCESS(mini_blob_clone(&cfg, cc, &target, descriptor, &cloned))); ASSERT_TRUE( SUCCESS(blob_checksum_test_mini_deinit(cc, &source, source_head))); ASSERT_TRUE( @@ -6985,8 +7029,8 @@ CTEST2(splinter, test_blob_trees) ASSERT_EQUAL(1, parsed.height); ASSERT_TRUE(writable_buffer_length(&desc) > 512); cfg.blob_max_descriptor_size = 512; - ASSERT_TRUE(SUCCESS( - blob_clone(&cfg, cc, &target, writable_buffer_to_slice(&desc), &cloned))); + ASSERT_TRUE(SUCCESS(mini_blob_clone( + &cfg, cc, &target, writable_buffer_to_slice(&desc), &cloned))); ASSERT_TRUE( SUCCESS(blob_checksum_test_mini_deinit(cc, &source, source_head))); ASSERT_TRUE(SUCCESS( @@ -7129,12 +7173,11 @@ CTEST2(splinter, test_blob_height_three_and_large_stats) } /* Force the encoder through two levels of pointer extents, sharing the * sparse data instead of materializing tens of GiB in the test process. */ - blob_build_config cfg = {.extent_batch = 0, - .page_batch = 1, - .subpage_batch = 2, - .blob_max_descriptor_size = - BLOB_MIN_MAX_DESCRIPTOR_SIZE}; - ASSERT_TRUE(SUCCESS(blob_clone( + mini_blob_config cfg = {.page_batch = 0, + .subpage_batch = 1, + .blob_max_descriptor_size = + BLOB_MIN_MAX_DESCRIPTOR_SIZE}; + ASSERT_TRUE(SUCCESS(mini_blob_clone( &cfg, cc, &target, writable_buffer_to_slice(&flat), &descriptor))); ASSERT_TRUE( SUCCESS(blob_checksum_test_mini_deinit(cc, &source, source_head))); @@ -7221,7 +7264,7 @@ CTEST2(splinter, test_blob_height_three_and_large_stats) ASSERT_TRUE(SUCCESS(btree_count_in_range(cc, req->cfg, req->root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, NEGATIVE_INFINITY_KEY, POSITIVE_INFINITY_KEY, &stats))); @@ -7236,3 +7279,379 @@ CTEST2(splinter, test_blob_height_three_and_large_stats) writable_buffer_deinit(&result); writable_buffer_deinit(&descriptor); } + +/* Sparse leaves exercise the builder at pointer-page and pointer-extent + * boundaries without materializing the logical multi-GiB value. */ +CTEST2(splinter, test_blob_pointer_termination) +{ + cache *cc = (cache *)data->clock_cache; + allocator *al = cache_get_allocator(cc); + uint64 initial = allocator_in_use(al); + uint64 extent = cache_extent_size(cc); + uint64 page_size = cache_page_size(cc); + uint64 per_page = page_size / sizeof(uint64); + uint64 fanout = extent / sizeof(uint64); + const uint64 lengths[] = { + per_page - 1, per_page, per_page + 1, fanout, fanout + 1}; + mini_blob_config cfg = {.page_batch = 0, + .subpage_batch = 1, + .blob_max_descriptor_size = + BLOB_MIN_MAX_DESCRIPTOR_SIZE}; + for (uint64 trial = 0; trial < ARRAY_SIZE(lengths); trial++) { + uint64 leaves = lengths[trial]; + mini_allocator source, target; + uint64 source_head, target_head; + ASSERT_TRUE( + SUCCESS(blob_checksum_test_mini_init(cc, &source, &source_head))); + ASSERT_TRUE( + SUCCESS(blob_checksum_test_mini_init(cc, &target, &target_head))); + auto_writable_buffer flat, descriptor; + writable_buffer_init(&flat, data->hid); + writable_buffer_init(&descriptor, data->hid); + ASSERT_TRUE(SUCCESS(writable_buffer_resize( + &flat, sizeof(blob) + leaves * sizeof(uint64)))); + blob *b = writable_buffer_data(&flat); + *b = (blob){.length = leaves * extent, .format = BLOB_FORMAT}; + for (uint64 i = 0; i < leaves; i++) { + uint64 addr; + ASSERT_TRUE(SUCCESS(mini_alloc_extent(&source, 0, &addr, NULL))); + b->addrs[i] = addr; + } + ASSERT_TRUE(SUCCESS(mini_blob_clone( + &cfg, cc, &target, writable_buffer_to_slice(&flat), &descriptor))); + parsed_blob parsed; + ASSERT_TRUE(SUCCESS(blob_parse_descriptor( + cc, writable_buffer_to_slice(&descriptor), &parsed))); + ASSERT_EQUAL(2, parsed.height); + for (uint64 i = 0; i < parsed.num_extents; i++) { + uint64 children = MIN(leaves - i * fanout, fanout); + uint64 slot = children < fanout ? children : children - 1; + uint64 addr = parsed.base->addrs[i] + slot * sizeof(uint64); + uint64 page_addr = addr - addr % page_size; + page_handle page; + ASSERT_TRUE( + SUCCESS(cache_get(cc, page_addr, TRUE, PAGE_TYPE_BLOB, &page))); + uint64 value; + memcpy( + &value, page_handle_data(&page) + addr % page_size, sizeof(value)); + cache_unget(cc, &page); + if (children < fanout) { + ASSERT_EQUAL(0, value); + if (children % per_page == 0) { + blob_tree_read_saved_ops = cc->ops; + cache_ops fault_ops = *cc->ops; + fault_ops.page_get = blob_tree_fail_get; + blob_tree_read_failed_addr = page_addr; + blob_tree_read_failures = 0; + cc->ops = &fault_ops; + platform_status rc = blob_walk_extents( + cc, writable_buffer_to_slice(&descriptor), FALSE, NULL, NULL); + cc->ops = blob_tree_read_saved_ops; + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_TIMEDOUT)); + ASSERT_EQUAL(1, blob_tree_read_failures); + } + page = test_splinter_claim_page(cc, page_addr, PAGE_TYPE_BLOB); + memcpy(page_handle_data(&page) + addr % page_size, + &b->addrs[0], + sizeof(uint64)); + test_splinter_unclaim_page(cc, &page); + ASSERT_TRUE(STATUS_IS_EQ( + blob_walk_extents( + cc, writable_buffer_to_slice(&descriptor), FALSE, NULL, NULL), + STATUS_INVALID_STATE)); + page = test_splinter_claim_page(cc, page_addr, PAGE_TYPE_BLOB); + memset( + page_handle_data(&page) + addr % page_size, 0, sizeof(uint64)); + test_splinter_unclaim_page(cc, &page); + } else { + ASSERT_NOT_EQUAL(0, value); + } + } + blob *stored = writable_buffer_data(&descriptor); + stored->format = BLOB_FORMAT - 1; + ASSERT_TRUE( + STATUS_IS_EQ(blob_parse_descriptor( + cc, writable_buffer_to_slice(&descriptor), &parsed), + STATUS_INVALID_STATE)); + stored->format = BLOB_FORMAT; + mini_release(&source); + mini_release(&target); + cache_flush(cc); + ASSERT_TRUE(SUCCESS(cache_evict(cc, FALSE))); + ASSERT_TRUE(SUCCESS(blob_walk_extents( + cc, writable_buffer_to_slice(&descriptor), TRUE, NULL, NULL))); + ASSERT_TRUE(SUCCESS(allocator_recovery_begin(al))); + ASSERT_TRUE( + SUCCESS(mini_recover_references(cc, source_head, PAGE_TYPE_MISC))); + ASSERT_TRUE( + SUCCESS(mini_recover_references(cc, target_head, PAGE_TYPE_MISC))); + allocator_recovery_finish(al); + for (uint64 i = 0; i < leaves; i++) { + ASSERT_EQUAL(AL_ONE_REF + 1, allocator_get_refcount(al, b->addrs[i])); + } + ASSERT_EQUAL(0, mini_dec_ref(cc, source_head, PAGE_TYPE_MISC)); + ASSERT_EQUAL(0, mini_dec_ref(cc, target_head, PAGE_TYPE_MISC)); + ASSERT_EQUAL(initial, allocator_in_use(al)); + ASSERT_TRUE(SUCCESS(allocator_reclamation_status(al))); + } +} + +/* A pointer extent owns its children once even when hundreds of container + * records reference it. Cursors must return those roots in order across + * metadata pages, preserving curr() on misses and at either end. */ +CTEST2(splinter, test_blob_ownership_graph_recovery_and_cursor) +{ + cache *cc = (cache *)data->clock_cache; + allocator *al = (allocator *)&data->al; + allocator_config *acfg = allocator_get_config(al); + uint64 initial = allocator_in_use(al); + uint64 extent = cache_extent_size(cc); + mini_allocator source, target; + uint64 source_head, target_head; + ASSERT_TRUE( + SUCCESS(blob_checksum_test_mini_init(cc, &source, &source_head))); + ASSERT_TRUE( + SUCCESS(blob_checksum_test_mini_init(cc, &target, &target_head))); + mini_blob_config cfg = {.page_batch = 0, + .subpage_batch = 1, + .blob_max_descriptor_size = + BLOB_MIN_MAX_DESCRIPTOR_SIZE}; + uint64 leaves = + (BLOB_MIN_MAX_DESCRIPTOR_SIZE - sizeof(blob)) / sizeof(uint64) + 1; + auto_writable_buffer bytes, descriptor, cloned; + writable_buffer_init(&bytes, data->hid); + writable_buffer_init(&descriptor, data->hid); + writable_buffer_init(&cloned, data->hid); + ASSERT_TRUE(SUCCESS(writable_buffer_resize(&bytes, leaves * extent))); + blob_checksum_test_fill(&bytes, leaves * extent); + ASSERT_TRUE(SUCCESS(mini_blob_build( + &cfg, cc, &source, writable_buffer_to_slice(&bytes), &descriptor))); + parsed_blob parsed; + ASSERT_TRUE(SUCCESS(blob_parse_descriptor( + cc, writable_buffer_to_slice(&descriptor), &parsed))); + ASSERT_EQUAL(2, parsed.height); + ASSERT_EQUAL(1, parsed.num_extents); + uint64 root = parsed.base->addrs[0]; + page_handle page; + ASSERT_TRUE(SUCCESS(cache_get(cc, root, TRUE, PAGE_TYPE_BLOB, &page))); + uint64 child; + memcpy(&child, page_handle_data(&page), sizeof(child)); + cache_unget(cc, &page); + ASSERT_EQUAL(AL_ONE_REF, allocator_get_refcount(al, child)); + + uint64 first, last; + ASSERT_TRUE(SUCCESS(mini_alloc_extent(&target, 0, &first, NULL))); + uint64 first_meta = mini_current_extent_meta_page(&target, 0); + uint64 before_charge = mini_num_extents(&target); + const uint64 copies = 600; + blob_tree_read_saved_ops = cc->ops; + cache_ops clone_ops = *cc->ops; + clone_ops.page_get = blob_tree_fail_get; + blob_tree_read_failed_addr = root; + blob_tree_read_failures = 0; + cc->ops = &clone_ops; + for (uint64 i = 0; i < copies; i++) { + ASSERT_TRUE(SUCCESS(mini_blob_clone( + &cfg, cc, &target, writable_buffer_to_slice(&descriptor), &cloned))); + } + cc->ops = blob_tree_read_saved_ops; + ASSERT_EQUAL(0, blob_tree_read_failures); + ASSERT_EQUAL(before_charge + copies * (leaves + 1), + mini_num_extents(&target)); + ASSERT_EQUAL(AL_ONE_REF + copies, allocator_get_refcount(al, root)); + ASSERT_EQUAL(AL_ONE_REF, allocator_get_refcount(al, child)); + ASSERT_TRUE(SUCCESS(mini_alloc_extent(&target, 0, &last, NULL))); + ASSERT_TRUE(mini_current_extent_meta_page(&target, 0) != first_meta); + mini_release(&source); + mini_release(&target); + uint64 space_bytes; + cc->ops = &clone_ops; + ASSERT_TRUE(SUCCESS( + mini_space_use_bytes(cc, target_head, PAGE_TYPE_MISC, &space_bytes))); + cc->ops = blob_tree_read_saved_ops; + ASSERT_EQUAL(0, blob_tree_read_failures); + ASSERT_EQUAL((copies * (leaves + 1) + 2) * extent + + (target.meta_tail - target_head + cache_page_size(cc)), + space_bytes); + ASSERT_TRUE(SUCCESS(mini_inc_ref(cc, source_head))); + + refcount *expected = + TYPED_ARRAY_MALLOC(data->hid, expected, acfg->extent_capacity); + ASSERT_NOT_NULL(expected); + for (uint64 i = 0; i < acfg->extent_capacity; i++) { + expected[i] = allocator_get_refcount(al, i * extent); + } + cache_flush(cc); + ASSERT_TRUE(SUCCESS(cache_evict(cc, FALSE))); + ASSERT_TRUE(SUCCESS(allocator_recovery_begin(al))); + ASSERT_TRUE( + SUCCESS(mini_recover_references(cc, target_head, PAGE_TYPE_MISC))); + ASSERT_TRUE( + SUCCESS(mini_recover_references(cc, source_head, PAGE_TYPE_MISC))); + ASSERT_TRUE( + SUCCESS(mini_recover_references(cc, source_head, PAGE_TYPE_MISC))); + allocator_recovery_finish(al); + for (uint64 i = 0; i < acfg->extent_capacity; i++) { + ASSERT_EQUAL(expected[i], + allocator_get_refcount(al, i * extent), + "blob graph extent %lu", + i); + } + platform_free(data->hid, expected); + ASSERT_TRUE(SUCCESS(cache_evict(cc, FALSE))); + mini_meta_cursor cursor; + mini_meta_cursor_status status; + do { + status = + mini_meta_cursor_init(&cursor, cc, PAGE_TYPE_MISC, first_meta, first); + } while (status == MINI_META_CURSOR_WOULD_BLOCK); + ASSERT_EQUAL(MINI_META_CURSOR_ENTRY, status); + uint64 addr, batch, height; + page_type type; + for (uint64 direction = 0; direction < 2; direction++) { + uint64 previous = direction == 0 ? first : last; + for (uint64 i = 0; i <= copies; i++) { + do { + status = direction == 0 ? mini_meta_cursor_next(&cursor) + : mini_meta_cursor_prev(&cursor); + if (status == MINI_META_CURSOR_WOULD_BLOCK) { + mini_meta_cursor_curr(&cursor, &addr, &batch, &type, &height); + ASSERT_EQUAL(previous, addr); + } + } while (status == MINI_META_CURSOR_WOULD_BLOCK); + ASSERT_EQUAL(MINI_META_CURSOR_ENTRY, status); + mini_meta_cursor_curr(&cursor, &addr, &batch, &type, &height); + uint64 expected_addr = + i < copies ? root : (direction == 0 ? last : first); + ASSERT_EQUAL(expected_addr, addr); + ASSERT_EQUAL(PAGE_TYPE_BLOB, type); + ASSERT_EQUAL(i < copies ? 1 : 0, height); + ASSERT_EQUAL(0, batch); + previous = addr; + } + status = direction == 0 ? mini_meta_cursor_next(&cursor) + : mini_meta_cursor_prev(&cursor); + ASSERT_EQUAL(MINI_META_CURSOR_END, status); + mini_meta_cursor_curr(&cursor, &addr, &batch, &type, &height); + ASSERT_EQUAL(previous, addr); + } + mini_meta_cursor_deinit(&cursor); + + ASSERT_EQUAL(1, mini_dec_ref(cc, source_head, PAGE_TYPE_MISC)); + ASSERT_EQUAL(0, mini_dec_ref(cc, source_head, PAGE_TYPE_MISC)); + ASSERT_EQUAL(AL_ONE_REF + copies - 1, allocator_get_refcount(al, root)); + ASSERT_TRUE(SUCCESS(blob_validate(cc, writable_buffer_to_slice(&cloned)))); + ASSERT_EQUAL(0, mini_dec_ref(cc, target_head, PAGE_TYPE_MISC)); + ASSERT_EQUAL(AL_FREE, allocator_get_refcount(al, root)); + ASSERT_EQUAL(AL_FREE, allocator_get_refcount(al, child)); + ASSERT_EQUAL(initial, allocator_in_use(al)); +} + +typedef struct blob_test_tail_container { + uint64 addr; + uint64 used; + uint64 capacity; + bool32 fail; +} blob_test_tail_container; + +static platform_status +blob_test_container_allocate(void *arg, + uint64 bytes, + uint64 alignment, + uint64 boundary, + uint64 addrs[2]) +{ + blob_test_tail_container *container = arg; + uint64 align = MAX(alignment, 1); + uint64 start = (container->used + align - 1) / align * align; + if (container->fail || start > container->capacity + || bytes > container->capacity - start) + { + return STATUS_NO_SPACE; + } + addrs[0] = container->addr + start; + addrs[1] = 0; + container->used = start + bytes; + return STATUS_OK; +} + +/* No mini allocator: two descriptors borrow disjoint tails within an existing + * container page. Clone must stage source bytes before locking that same page. + */ +CTEST2(splinter, test_blob_ownership_custom_tails_and_failures) +{ + cache *cc = (cache *)data->clock_cache; + allocator *al = (allocator *)&data->al; + uint64 initial = allocator_in_use(al); + uint64 extent = cache_extent_size(cc); + uint64 addr; + ASSERT_TRUE(SUCCESS(allocator_alloc(al, &addr, PAGE_TYPE_BLOB))); + page_handle page; + ASSERT_NOT_NULL(cache_alloc(cc, addr, PAGE_TYPE_BLOB, &page)); + memset(page_handle_data(&page), 0xa5, cache_page_size(cc)); + test_splinter_unclaim_page(cc, &page); + blob_test_tail_container container = {addr, 64, cache_page_size(cc), FALSE}; + blob_tail_allocator tails = {blob_test_container_allocate, &container}; + blob_build_config cfg = {.blob_max_descriptor_size = + BLOB_MIN_MAX_DESCRIPTOR_SIZE}; + auto_writable_buffer bytes, descriptor, clone, result; + writable_buffer_init(&bytes, data->hid); + writable_buffer_init(&descriptor, data->hid); + writable_buffer_init(&clone, data->hid); + writable_buffer_init(&result, data->hid); + ASSERT_TRUE(SUCCESS(writable_buffer_resize(&bytes, 900))); + blob_checksum_test_fill(&bytes, 900); + ASSERT_TRUE(SUCCESS(blob_build( + &cfg, cc, &tails, writable_buffer_to_slice(&bytes), &descriptor))); + ASSERT_TRUE(SUCCESS(blob_clone( + &cfg, cc, &tails, writable_buffer_to_slice(&descriptor), &clone))); + ASSERT_TRUE( + SUCCESS(blob_checksum_test_roundtrip(cc, + writable_buffer_to_slice(&clone), + writable_buffer_to_slice(&bytes), + &result))); + ASSERT_TRUE( + SUCCESS(blob_release_refs(cc, writable_buffer_to_slice(&descriptor)))); + ASSERT_TRUE( + SUCCESS(blob_release_refs(cc, writable_buffer_to_slice(&clone)))); + ASSERT_EQUAL(AL_ONE_REF, allocator_get_refcount(al, addr)); + ASSERT_TRUE(SUCCESS(cache_get(cc, addr, TRUE, PAGE_TYPE_BLOB, &page))); + for (uint64 i = 0; i < 64; i++) { + ASSERT_EQUAL(0xa5, (unsigned char)page_handle_data(&page)[i]); + } + cache_unget(cc, &page); + + container.fail = TRUE; + ASSERT_TRUE(SUCCESS(writable_buffer_resize(&bytes, extent + 900))); + blob_checksum_test_fill(&bytes, writable_buffer_length(&bytes)); + ASSERT_TRUE(STATUS_IS_EQ( + STATUS_NO_SPACE, + blob_build( + &cfg, cc, &tails, writable_buffer_to_slice(&bytes), &descriptor))); + ASSERT_EQUAL(initial + 1, allocator_in_use(al)); + + uint64 leaves = + (BLOB_MIN_MAX_DESCRIPTOR_SIZE - sizeof(blob)) / sizeof(uint64) + 1; + ASSERT_TRUE(SUCCESS(writable_buffer_resize(&bytes, leaves * extent))); + blob_checksum_test_fill(&bytes, writable_buffer_length(&bytes)); + for (uint64 allocations = 0; allocations <= leaves; allocations++) { + mini_init_saved_ops = al->ops; + allocator_ops ops = *al->ops; + ops.alloc = mini_init_fail_allocation; + mini_init_allocations_before_failure = allocations; + al->ops = &ops; + platform_status rc = blob_build( + &cfg, cc, NULL, writable_buffer_to_slice(&bytes), &descriptor); + al->ops = mini_init_saved_ops; + ASSERT_TRUE(STATUS_IS_EQ(STATUS_NO_SPACE, rc)); + ASSERT_EQUAL(initial + 1, + allocator_in_use(al), + "partial blob after %lu allocations", + allocations); + } + ASSERT_TRUE(SUCCESS(allocator_reclamation_status(al))); + ASSERT_EQUAL(AL_NO_REFS, allocator_dec_ref(al, addr, PAGE_TYPE_BLOB)); + cache_extent_discard(cc, addr, PAGE_TYPE_BLOB); + ASSERT_EQUAL(AL_FREE, allocator_dec_ref(al, addr, PAGE_TYPE_BLOB)); + ASSERT_EQUAL(initial, allocator_in_use(al)); +} diff --git a/tests/unit/splinterdb_quick_test.c b/tests/unit/splinterdb_quick_test.c index 9c8bd8e30..4258586bb 100644 --- a/tests/unit/splinterdb_quick_test.c +++ b/tests/unit/splinterdb_quick_test.c @@ -337,7 +337,8 @@ CTEST2(splinterdb_quick, test_get_config_returns_effective_config) ASSERT_EQUAL(IO_DEFAULT_PERMS, cfg->io_perms); ASSERT_EQUAL(IO_DEFAULT_ASYNC_QUEUE_DEPTH, cfg->io_async_queue_depth); - ASSERT_EQUAL(512, cfg->blob_max_descriptor_size); + ASSERT_EQUAL(256, cfg->blob_max_descriptor_size); + ASSERT_EQUAL(256, cfg->blob_key_prefix_size); ASSERT_EQUAL(1, cfg->btree_rough_count_height); ASSERT_EQUAL(27, cfg->filter_hash_size); ASSERT_EQUAL(9, cfg->filter_log_index_size); @@ -1290,7 +1291,7 @@ CTEST2(splinterdb_quick, test_apis_for_large_key) } /* - * Test case to verify core interfaces when key-size is too large. + * Keys just above the inline limit must use the same public operations. */ CTEST2(splinterdb_quick, test_key_size_gt_max_inline_key_size) { @@ -1301,14 +1302,21 @@ CTEST2(splinterdb_quick, test_key_size_gt_max_inline_key_size) int rc = splinterdb_insert( data->kvsb, too_large_key, message_create(sizeof("foo"), "foo"), NULL); - ASSERT_EQUAL(EINVAL, rc); + ASSERT_EQUAL(0, rc); splinterdb_lookup_result result; splinterdb_lookup_result_init(data->kvsb, &result); + rc = splinterdb_lookup(data->kvsb, too_large_key, &result); + ASSERT_EQUAL(0, rc); + assert_lookup_result_matches_slice(&result, + slice_create(sizeof("foo"), "foo")); + rc = splinterdb_delete(data->kvsb, too_large_key, NULL); - ASSERT_EQUAL(EINVAL, rc); + ASSERT_EQUAL(0, rc); + ASSERT_EQUAL(0, splinterdb_lookup(data->kvsb, too_large_key, &result)); + ASSERT_FALSE(splinterdb_lookup_found(&result)); splinterdb_lookup_result_deinit(&result); } @@ -1683,7 +1691,7 @@ CTEST2(splinterdb_quick, test_branch_prefix_pack_initializes_free_gap) core->cfg.btree_cfg, &scan, req.root_addr, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){0}))); btree_hdr *hdr = scan.curr.hdr; @@ -1767,7 +1775,7 @@ prefix_test_init_failure(core_handle *core, core->cfg.btree_cfg, &itor, branch->addr, - branch->format_version, + branch->format, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .start_position = key_start_at_or_after(start_key)}); @@ -1917,7 +1925,7 @@ prefix_variable_test_growth_failure(core_handle *core, core->cfg.btree_cfg, &itor, branch->addr, - branch->format_version, + branch->format, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){ .start_position = key_start_at_or_after(key_create_from_slice( @@ -2095,7 +2103,7 @@ prefix_test_malformed_leaf_entries(core_handle *core, trunk_branch_info *branch) core->cfg.btree_cfg, &itor, branch->addr, - branch->format_version, + branch->format, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){0}))); btree_hdr *hdr = itor.curr.hdr; @@ -2226,6 +2234,7 @@ CTEST2(splinterdb_quick, test_branch_prefix_roundtrip) ASSERT_EQUAL(0, splinterdb_close(&data->kvsb, FALSE)); data->default_data_cfg.super.key_compare_is_lexical = enabled; data->default_data_cfg.super.key_compare = prefix_test_compare; + data->cfg.blob_key_prefix_size = 128; ASSERT_EQUAL(0, splinterdb_create(&data->cfg, &data->kvsb)); char keybuf[64]; for (uint64 i = 0; i < count; i++) { @@ -2342,8 +2351,18 @@ CTEST2(splinterdb_quick, test_branch_prefix_roundtrip) ASSERT_EQUAL(0, splinterdb_close(&data->kvsb, FALSE)); /* Creation policy must not change interpretation of existing leaves. */ data->default_data_cfg.super.key_compare_is_lexical = !enabled; + data->cfg.blob_key_prefix_size = 64; ASSERT_EQUAL(0, splinterdb_open(&data->cfg, &data->kvsb)); core = (core_handle *)splinterdb_get_trunk_handle(data->kvsb); + ASSERT_TRUE(SUCCESS(core_range_iterator_init( + core, &probe, &(const core_range_iterator_init_options){0}))); + for (uint64 i = 0; i < probe.num_branches; i++) { + if (probe.branch[i].type == PAGE_TYPE_BRANCH) { + ASSERT_EQUAL(enabled ? 128 : 0, + probe.branch[i].format.key_prefix_size); + } + } + core_range_iterator_deinit(&probe); splinterdb_lookup_result result; splinterdb_lookup_result_init(data->kvsb, &result); for (uint64 i = 0; i < count; i++) { @@ -3797,7 +3816,7 @@ CTEST2(splinterdb_quick, test_immutable_branch_custom_comparator) ASSERT_TRUE(SUCCESS(btree_lookup(core->cc, core->cfg.btree_cfg, bt->root_addr, - probe.branch[branch].format_version, + probe.branch[branch].format, PAGE_TYPE_BRANCH, key_create_from_slice(TRUE, target), &result))); @@ -3808,7 +3827,7 @@ CTEST2(splinterdb_quick, test_immutable_branch_custom_comparator) core->cc, core->cfg.btree_cfg, bt->root_addr, - probe.branch[branch].format_version, + probe.branch[branch].format, PAGE_TYPE_BRANCH, key_create_from_slice(TRUE, target), &result))); @@ -3818,7 +3837,7 @@ CTEST2(splinterdb_quick, test_immutable_branch_custom_comparator) ASSERT_TRUE(SUCCESS(btree_lookup(core->cc, core->cfg.btree_cfg, bt->root_addr, - probe.branch[branch].format_version, + probe.branch[branch].format, PAGE_TYPE_BRANCH, key_create_from_slice(TRUE, target), &result))); @@ -3829,7 +3848,7 @@ CTEST2(splinterdb_quick, test_immutable_branch_custom_comparator) core->cc, core->cfg.btree_cfg, bt->root_addr, - probe.branch[branch].format_version, + probe.branch[branch].format, PAGE_TYPE_BRANCH, key_create_from_slice(TRUE, target), &result))); @@ -3849,16 +3868,15 @@ CTEST2(splinterdb_quick, test_immutable_branch_custom_comparator) memcpy(keybuf, entry->key_and_message, sizeof(keybuf)); key query = key_create_from_slice(TRUE, slice_create(sizeof(keybuf), keybuf)); - uint16 saved_length = entry->key_length; - entry->key_length = UINT16_MAX; - platform_status sync_rc = - btree_lookup(core->cc, - core->cfg.btree_cfg, - bt->root_addr, - probe.branch[branch].format_version, - PAGE_TYPE_BRANCH, - query, - &result); + uint16 saved_length = entry->key_length; + entry->key_length = UINT16_MAX; + platform_status sync_rc = btree_lookup(core->cc, + core->cfg.btree_cfg, + bt->root_addr, + probe.branch[branch].format, + PAGE_TYPE_BRANCH, + query, + &result); bool32 sync_found = lookup_result_found(&result); platform_status async_rc = async_call_sync_callback(cache_cleanup(core->cc), @@ -3866,7 +3884,7 @@ CTEST2(splinterdb_quick, test_immutable_branch_custom_comparator) core->cc, core->cfg.btree_cfg, bt->root_addr, - probe.branch[branch].format_version, + probe.branch[branch].format, PAGE_TYPE_BRANCH, query, &result); @@ -5591,3 +5609,321 @@ CTEST2(splinterdb_quick, test_blob_tree_crash_recovery) /* Keep the fixture's usual close path valid. */ ASSERT_EQUAL(0, splinterdb_open(&data->cfg, &data->kvsb)); } + +/* Exercise logical keys across all layers, including keys whose lengths do not + * fit the on-disk tuple's uint16 fields. Differences follow a long common + * prefix. */ +static slice +large_api_key(char *bytes, uint64 length, uint64 ordinal) +{ + memset(bytes, 'K', length); + for (uint64 i = 0; i < 8; i++) { + bytes[length - 8 + i] = ordinal >> (56 - 8 * i); + } + return slice_create(length, bytes); +} + +static int +large_api_reverse_compare(const data_config *cfg, user_key a, user_key b) +{ + platform_assert(slice_length(a.key) > 512); + platform_assert(slice_length(b.key) > 512); + return -slice_lex_cmp(a.key, b.key); +} + +static int +large_api_merge(const data_config *cfg, + slice k, + message old, + merge_accumulator *new) +{ + platform_assert(slice_length(k) > 512); + merge_accumulator_set_class(new, MESSAGE_TYPE_INSERT); + return 0; +} + +static int +large_api_merge_final(const data_config *cfg, slice k, merge_accumulator *new) +{ + platform_assert(slice_length(k) > 512); + merge_accumulator_set_class(new, MESSAGE_TYPE_INSERT); + return 0; +} + +CTEST2(splinterdb_quick, test_large_keys_public_operations) +{ + const uint64 lengths[] = {513, 4097, 65553, 2 * IO_DEFAULT_EXTENT_SIZE + 17}; + const uint64 count = 32; + char *bytes = malloc(lengths[ARRAY_SIZE(lengths) - 1]); + char *bound_bytes = malloc(lengths[ARRAY_SIZE(lengths) - 1]); + ASSERT_TRUE(bytes != NULL && bound_bytes != NULL); + for (uint64 custom = 0; custom < 2; custom++) { + for (uint64 logging = 0; logging < 2; logging++) { + for (uint64 n = 0; n < ARRAY_SIZE(lengths); n++) { + ASSERT_EQUAL(0, splinterdb_close(&data->kvsb, FALSE)); + default_data_config_init(&data->default_data_cfg.super); + data_config *dcfg = &data->default_data_cfg.super; + if (custom) { + dcfg->key_compare = large_api_reverse_compare; + dcfg->key_compare_is_lexical = FALSE; + } + dcfg->merge_tuples = large_api_merge; + dcfg->merge_tuples_final = large_api_merge_final; + data->cfg.use_log = logging; + data->cfg.blob_key_prefix_size = 256; + data->cfg.disk_size = 512 * Mega; + data->cfg.memtable_capacity = 8 * Mega; + ASSERT_EQUAL(0, splinterdb_create(&data->cfg, &data->kvsb)); + uint64 length = lengths[n]; + splinterdb_lookup_result result; + splinterdb_lookup_result_init(data->kvsb, &result); + for (uint64 i = 0; i < count; i++) { + slice k = large_api_key(bytes, length, i); + ASSERT_EQUAL( + 0, + splinterdb_insert( + data->kvsb, k, message_create(sizeof(i), &i), &result)); + ASSERT_FALSE(splinterdb_lookup_found(&result)); + } + /* Check memtable, branch, and reopened database views. */ + for (uint64 stage = 0; stage < 3; stage++) { + if (stage == 1) { + force_flush_current_memtable(data->kvsb); + } else if (stage == 2) { + splinterdb_lookup_result_deinit(&result); + ASSERT_EQUAL(0, splinterdb_close(&data->kvsb, FALSE)); + /* Old branches retain P=256 while the new instance uses + * P=128. Lookups must take the configuration fallback. */ + data->cfg.blob_key_prefix_size = 128; + ASSERT_EQUAL(0, splinterdb_open(&data->cfg, &data->kvsb)); + splinterdb_lookup_result_init(data->kvsb, &result); + } + for (uint64 i = 0; i < count; i++) { + ASSERT_EQUAL( + 0, + splinterdb_lookup( + data->kvsb, large_api_key(bytes, length, i), &result)); + assert_lookup_result_matches_slice( + &result, slice_create(sizeof(i), &i)); + } + ASSERT_EQUAL( + 0, + splinterdb_lookup( + data->kvsb, large_api_key(bytes, length, count), &result)); + ASSERT_FALSE(splinterdb_lookup_found(&result)); + for (uint64 reverse = 0; reverse < 2; reverse++) { + splinterdb_iterator *it; + ASSERT_EQUAL( + 0, + splinterdb_iterator_init( + data->kvsb, + &it, + .start_position = + reverse ? splinterdb_start_at_or_before(NULL_SLICE) + : splinterdb_start_at_or_after(NULL_SLICE))); + for (uint64 j = 0; j < count; j++) { + uint64 i = (custom ^ reverse) ? count - 1 - j : j; + ASSERT_TRUE(splinterdb_iterator_valid(it)); + slice k, value; + splinterdb_iterator_get_current(it, &k, &value); + ASSERT_EQUAL( + 0, slice_lex_cmp(k, large_api_key(bytes, length, i))); + ASSERT_EQUAL( + 0, slice_lex_cmp(value, slice_create(sizeof(i), &i))); + if (reverse) + splinterdb_iterator_prev(it); + else + splinterdb_iterator_next(it); + } + ASSERT_FALSE(splinterdb_iterator_valid(it)); + ASSERT_EQUAL(0, splinterdb_iterator_status(it)); + splinterdb_iterator_deinit(it); + } + slice lo = large_api_key(bytes, length, custom ? 23 : 8); + slice hi = large_api_key(bound_bytes, length, custom ? 7 : 24); + splinterdb_iterator *it; + ASSERT_EQUAL( + 0, + splinterdb_iterator_init( + data->kvsb, + &it, + .lower_limit = splinterdb_greater_than_or_equal(lo), + .upper_limit = splinterdb_less_than(hi))); + uint64 seen = 0; + while (splinterdb_iterator_valid(it)) { + seen++; + splinterdb_iterator_next(it); + } + ASSERT_EQUAL(16, seen); + ASSERT_EQUAL(0, splinterdb_iterator_status(it)); + splinterdb_iterator_deinit(it); + } + for (uint64 i = 0; i < count; i++) { + slice k = large_api_key(bytes, length, i); + uint64 replacement = count + i; + ASSERT_EQUAL( + 0, + splinterdb_update( + data->kvsb, + k, + message_create(sizeof(replacement), &replacement), + &result)); + assert_lookup_result_matches_slice(&result, + slice_create(sizeof(i), &i)); + ASSERT_EQUAL(0, splinterdb_delete(data->kvsb, k, &result)); + assert_lookup_result_matches_slice( + &result, slice_create(sizeof(replacement), &replacement)); + ASSERT_EQUAL(0, splinterdb_lookup(data->kvsb, k, &result)); + ASSERT_FALSE(splinterdb_lookup_found(&result)); + } + splinterdb_lookup_result_deinit(&result); + ASSERT_EQUAL(0, splinterdb_durable_barrier(data->kvsb, NULL)); + } + } + } + free(bytes); + free(bound_bytes); +} + +/* Acknowledged large keys survive an abrupt exit. A corrupt key tail discards + * its whole WAL group and every later group, including otherwise intact values. + */ +CTEST2(splinterdb_quick, test_large_key_crash_recovery) +{ + ASSERT_EQUAL(0, splinterdb_close(&data->kvsb, FALSE)); + data->cfg.disk_size = 384 * Mega; + data->cfg.memtable_capacity = 64 * Mega; + data->cfg.checkpoint_log_size_bytes = UINT64_MAX; + data->cfg.num_normal_bg_threads = 0; + data->cfg.num_memtable_bg_threads = 0; + const uint64 length = 2 * IO_DEFAULT_EXTENT_SIZE + 17; + char *bytes = malloc(length); + ASSERT_TRUE(bytes != NULL); + slice large = large_api_key(bytes, length, 42); + slice small = slice_create(2, "ok"); + slice prefix = slice_create(6, "prefix"); + slice suffix = slice_create(6, "suffix"); + const char *child_mode = getenv("SPLINTER_TEST_KEY_CRASH_CHILD"); + if (child_mode != NULL) { + data->cfg.use_log = child_mode[0] != '0'; + bool32 corrupt = child_mode[0] == '2'; + splinterdb *db; + if (splinterdb_create(&data->cfg, &db) != 0) + _exit(10); + if (splinterdb_insert(db, prefix, message_from_slice(small), NULL) != 0) + _exit(11); + if (splinterdb_durable_barrier(db, NULL) != 0) + _exit(12); + if (splinterdb_insert(db, large, message_from_slice(large), NULL) != 0) + _exit(13); + if (splinterdb_durable_barrier(db, NULL) != 0) + _exit(14); + if (splinterdb_insert(db, suffix, message_from_slice(small), NULL) != 0) + _exit(15); + if (splinterdb_durable_barrier(db, NULL) != 0) + _exit(16); + if (corrupt) { + const core_handle *core = splinterdb_get_trunk_handle(db); + cache *cc = (cache *)splinterdb_get_cache_handle(db); + log_iterator *it; + if (!SUCCESS( + shard_log_iterator_create(cc, + (shard_log_config *)core->cfg.log_cfg, + data->cfg.heap_id, + log_get_head(core->log), + 0, + &it))) + _exit(17); + uint64 addr = 0, offset = 0; + while (log_iterator_can_next(it)) { + shard_log_iterator *concrete = (shard_log_iterator *)it; + ondisk_tuple *tuple = + (ondisk_tuple *)((char *)concrete->entries[concrete->pos] + + 2 * sizeof(uint64)); + key stored = ondisk_tuple_key(tuple); + if (stored.is_blob) { + blob_page_iterator blob_it; + if (!SUCCESS(blob_page_iterator_init( + cc, + &blob_it, + key_slice(stored), + length - 1, + BLOB_PAGE_ITERATOR_MODE_NO_PREFETCH))) + _exit(18); + slice fragment; + uint64 ignored; + if (!SUCCESS(blob_page_iterator_get_curr( + &blob_it, &ignored, &fragment))) + _exit(19); + addr = blob_it.fragment.addr; + offset = blob_it.fragment.offset; + blob_page_iterator_deinit(&blob_it); + break; + } + if (!SUCCESS(log_iterator_next(it))) + _exit(20); + } + log_iterator_deinit(it); + if (!addr) + _exit(21); + page_handle page; + if (!SUCCESS(cache_get(cc, addr, TRUE, PAGE_TYPE_BLOB, &page))) + _exit(22); + if (!cache_try_claim(cc, &page)) + _exit(23); + cache_lock(cc, &page); + page_handle_data(&page)[offset] ^= 1; + cache_unlock(cc, &page); + cache_unclaim(cc, &page); + writeback_set set; + writeback_set_init(&set, cc, data->cfg.heap_id); + if (!SUCCESS(writeback_set_add_page(&set, &page, PAGE_TYPE_BLOB))) + _exit(24); + cache_unget(cc, &page); + if (!SUCCESS(writeback_set_wait(&set))) + _exit(25); + if (!SUCCESS(writeback_set_make_durable(&set))) + _exit(26); + writeback_set_deinit(&set); + } + _exit(0); + } + for (uint64 mode = 0; mode < 3; mode++) { + data->cfg.use_log = mode != 0; + pid_t pid = fork(); + ASSERT_TRUE(pid >= 0); + if (pid == 0) { + setenv("SPLINTER_TEST_KEY_CRASH_CHILD", + mode == 0 ? "0" + : mode == 1 ? "1" + : "2", + 1); + execl("/proc/self/exe", + "splinterdb_quick_test", + "test_large_key_crash_recovery", + (char *)NULL); + _exit(27); + } + int status; + ASSERT_EQUAL(pid, waitpid(pid, &status, 0)); + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQUAL(0, WEXITSTATUS(status)); + for (uint64 reopen = 0; reopen < 2; reopen++) { + ASSERT_EQUAL(0, splinterdb_open(&data->cfg, &data->kvsb)); + splinterdb_lookup_result result; + splinterdb_lookup_result_init(data->kvsb, &result); + ASSERT_EQUAL(0, splinterdb_lookup(data->kvsb, prefix, &result)); + assert_lookup_result_matches_slice(&result, small); + ASSERT_EQUAL(0, splinterdb_lookup(data->kvsb, large, &result)); + ASSERT_EQUAL(mode != 2, splinterdb_lookup_found(&result)); + if (mode != 2) + assert_lookup_result_matches_slice(&result, large); + ASSERT_EQUAL(0, splinterdb_lookup(data->kvsb, suffix, &result)); + ASSERT_EQUAL(mode != 2, splinterdb_lookup_found(&result)); + splinterdb_lookup_result_deinit(&result); + ASSERT_EQUAL(0, splinterdb_close(&data->kvsb, FALSE)); + } + } + free(bytes); + ASSERT_EQUAL(0, splinterdb_open(&data->cfg, &data->kvsb)); +} diff --git a/tests/unit/trunk_async_test.c b/tests/unit/trunk_async_test.c index f834cf617..2d3bc0f29 100644 --- a/tests/unit/trunk_async_test.c +++ b/tests/unit/trunk_async_test.c @@ -20,7 +20,9 @@ typedef struct ONDISK trunk_async_test_node { uint16 height; uint16 num_pivots; uint16 num_inflight_bundles; + uint16 blob_key_prefix_size; uint32 inflight_bundles_offset; + uint32 tail_offset; uint32 pivot_offsets[]; } trunk_async_test_node; @@ -171,6 +173,7 @@ trunk_async_test_pivot_lookup(bool32 fail) (trunk_async_test_node *)page_handle_data(&header_page); header->height = 0; header->num_pivots = 2; + header->tail_offset = TRUNK_ASYNC_TEST_EXTENT_SIZE; header->pivot_offsets[0] = TRUNK_ASYNC_TEST_PAGE_SIZE; header->pivot_offsets[1] = 2 * TRUNK_ASYNC_TEST_PAGE_SIZE; diff --git a/tests/unit/trunk_gc_test.c b/tests/unit/trunk_gc_test.c index 6f820bc44..866d848ca 100644 --- a/tests/unit/trunk_gc_test.c +++ b/tests/unit/trunk_gc_test.c @@ -22,7 +22,9 @@ typedef struct ONDISK gc_node_header { uint16 height; uint16 num_pivots; uint16 num_inflight_bundles; + uint16 blob_key_prefix_size; uint32 inflight_bundles_offset; + uint32 tail_offset; uint32 pivot_offsets[2]; } gc_node_header; @@ -196,6 +198,7 @@ gc_init_node(gc_cache *cc, uint64 index, uint64 child_addr) gc_node_header *header = (gc_node_header *)page_handle_data(page); header->height = child_addr != 0; header->num_pivots = 2; + header->tail_offset = GC_EXTENT_SIZE; header->pivot_offsets[0] = sizeof(*header); header->pivot_offsets[1] = sizeof(*header) + sizeof(gc_pivot) + sizeof(gc_bundle); diff --git a/tests/unit/trunk_split_test.c b/tests/unit/trunk_split_test.c index d517d6864..0e53f049b 100644 --- a/tests/unit/trunk_split_test.c +++ b/tests/unit/trunk_split_test.c @@ -4,6 +4,7 @@ #include "splinterdb/default_data_config.h" #include "splinterdb_tests_private.h" #include "platform_threads.h" +#include "rc_allocator.h" #include "ctest.h" typedef struct split_test_iterator { @@ -245,7 +246,7 @@ CTEST2(trunk_split, split_scan_propagates_io_error) data->cfg.btree_cfg, &itor, branch, - BTREE_BRANCH_FORMAT_VERSION, + BTREE_DEFAULT_FORMAT, PAGE_TYPE_BRANCH, &(const btree_iterator_init_options){0}))); uint64 first_page = itor.curr.addr; @@ -451,3 +452,597 @@ CTEST2(trunk_split, maplet_cow_retain_failure_restores_full_reference_map) platform_free(context->hid, counts); btree_dec_ref(context->cc, data->cfg.btree_cfg, branch, PAGE_TYPE_BRANCH); } + +#define TRUNK_BLOB_TEST_KEYS 16 + +typedef struct split_blob_iterator { + iterator super; + uint64 pos; + uint64 length; + char *bytes; + uint64 value; +} split_blob_iterator; + +static void +split_blob_curr(iterator *base, key *k, message *msg) +{ + split_blob_iterator *iter = (split_blob_iterator *)base; + iter->bytes[iter->length - 1] = iter->pos + 1; + iter->value = iter->pos; + *k = key_create(FALSE, iter->length, iter->bytes); + *msg = + message_create_internal(MESSAGE_TYPE_INSERT, + NULL, + slice_create(sizeof(iter->value), &iter->value)); +} + +static bool32 +split_blob_can_next(iterator *base) +{ + return ((split_blob_iterator *)base)->pos < TRUNK_BLOB_TEST_KEYS; +} + +static platform_status +split_blob_next(iterator *base) +{ + ((split_blob_iterator *)base)->pos++; + return STATUS_OK; +} + +static const iterator_ops split_blob_ops = { + .curr = split_blob_curr, + .can_next = split_blob_can_next, + .can_prev = split_test_can_prev, + .next = split_blob_next, +}; + +static uint64 +split_blob_pack(trunk_context *context, uint64 length) +{ + split_blob_iterator iter = { + .super = {.ops = &split_blob_ops}, + .length = length, + }; + iter.bytes = TYPED_ARRAY_MALLOC(context->hid, iter.bytes, length); + ASSERT_NOT_NULL(iter.bytes); + memset(iter.bytes, 'k', length); + btree_pack_req req; + ASSERT_TRUE(SUCCESS(btree_pack_req_init(&req, + context->cc, + context->cfg->btree_cfg, + &iter.super, + TRUNK_BLOB_TEST_KEYS, + context->hid, + .seed = 42))); + ASSERT_TRUE(SUCCESS(btree_pack(&req))); + uint64 addr = req.root_addr; + btree_pack_req_deinit(&req, context->hid); + platform_free(context->hid, iter.bytes); + return addr; +} + +static void +split_blob_ready(void *arg) +{ + __atomic_store_n((bool32 *)arg, TRUE, __ATOMIC_RELEASE); +} + +static void +split_blob_check(trunk_context *context, uint64 length) +{ + char *bytes = TYPED_ARRAY_MALLOC(context->hid, bytes, length); + ASSERT_NOT_NULL(bytes); + memset(bytes, 'k', length); + for (uint64 i = 0; i < TRUNK_BLOB_TEST_KEYS; i++) { + bytes[length - 1] = i + 1; + key k = key_create(FALSE, length, bytes); + trunk_ondisk_node_handle handle; + ASSERT_TRUE(SUCCESS(trunk_init_root_handle(context, &handle))); + for (uint64 async = 0; async < 2; async++) { + lookup_result result; + lookup_result_init( + &result, context->cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + platform_status rc; + if (async) { + bool32 ready = FALSE; + trunk_merge_lookup_async_state state; + trunk_merge_lookup_async_state_init(&state, + context, + &handle, + k, + &result, + NULL, + split_blob_ready, + &ready); + while (trunk_merge_lookup_async(&state) != ASYNC_STATUS_DONE) { + while (!__atomic_exchange_n(&ready, FALSE, __ATOMIC_ACQUIRE)) { + cache_cleanup(context->cc); + } + } + rc = state.__async_result; + } else { + rc = trunk_merge_lookup(context, &handle, k, &result, NULL); + } + ASSERT_TRUE( + SUCCESS(rc), "lookup key %lu async %lu: %d", i, async, rc.r); + ASSERT_TRUE(lookup_result_found(&result)); + uint64 value; + memcpy(&value, merge_accumulator_data(&result.value), sizeof(value)); + ASSERT_EQUAL(i, value); + lookup_result_deinit(&result); + } + key_buffer min, max; + key_buffer_init(&min, context->hid); + key_buffer_init(&max, context->hid); + trunk_branch_info branches[64]; + uint64 count = 0; + ASSERT_TRUE(SUCCESS(trunk_collect_branches(context, + &handle, + k, + greater_than_or_equal, + NULL, + ARRAY_SIZE(branches), + &count, + branches, + &min, + &max))); + ASSERT_TRUE( + data_key_compare(context->cfg->data_cfg, key_buffer_key(&min), k) + <= 0); + ASSERT_TRUE( + data_key_compare(context->cfg->data_cfg, k, key_buffer_key(&max)) < 0); + for (uint64 j = 0; j < count; j++) { + btree_dec_ref(context->cc, + context->cfg->btree_cfg, + branches[j].addr, + branches[j].type); + } + key_buffer_deinit(&min); + key_buffer_deinit(&max); + trunk_ondisk_node_handle_deinit(&handle); + } + platform_free(context->hid, bytes); +} + +typedef struct ONDISK split_blob_node_header { + uint16 height; + uint16 num_pivots; + uint16 num_inflight_bundles; + uint16 blob_key_prefix_size; + uint32 inflight_bundles_offset; + uint32 tail_offset; + uint32 pivot_offsets[]; +} split_blob_node_header; + +typedef struct ONDISK split_blob_pivot { + int64 num_kv_bytes; + int64 num_tuples; + uint64 child_addr; + uint64 num_live_inflight_bundles; + ondisk_key key; +} split_blob_pivot; + +static void +split_blob_root_descriptor(trunk_context *context, writable_buffer *descriptor) +{ + uint64 root = context->root->ref.addr; + page_handle header_page, pivot_page; + ASSERT_TRUE(SUCCESS( + cache_get(context->cc, root, TRUE, PAGE_TYPE_TRUNK, &header_page))); + split_blob_node_header *header = (void *)page_handle_data(&header_page); + ASSERT_TRUE(header->height > 0); + if (!context->cfg->data_cfg->key_compare_is_lexical) { + ASSERT_EQUAL(0, header->blob_key_prefix_size); + } + ASSERT_TRUE(header->num_pivots > 2); + uint64 page_size = cache_page_size(context->cc); + uint64 offset = header->pivot_offsets[1]; + ASSERT_TRUE(SUCCESS(cache_get(context->cc, + root + offset / page_size * page_size, + TRUE, + PAGE_TYPE_TRUNK, + &pivot_page))); + split_blob_pivot *pivot = + (void *)((char *)page_handle_data(&pivot_page) + offset % page_size); + ASSERT_EQUAL(ONDISK_KEY_IS_BLOB, pivot->key.flags); + ASSERT_TRUE(SUCCESS(writable_buffer_copy_slice( + descriptor, + slice_create(pivot->key.length - header->blob_key_prefix_size, + pivot->key.bytes + header->blob_key_prefix_size)))); + cache_unget(context->cc, &pivot_page); + cache_unget(context->cc, &header_page); +} + +static int +split_blob_compare(const data_config *cfg, user_key a, user_key b) +{ + ASSERT_TRUE(slice_length(a.key) > 512); + ASSERT_TRUE(slice_length(b.key) > 512); + return slice_lex_cmp(a.key, b.key); +} + +static page_handle * +split_blob_fail_tail_page(cache *cc, + uint64 addr, + page_type type, + page_handle *page) +{ + if (type == PAGE_TYPE_BLOB + && ++split_test_alloc_calls == split_test_fail_alloc_at) + { + return NULL; + } + return split_test_real_ops->page_alloc(cc, addr, type, page); +} + +static void +split_blob_incorporate_failures(trunk_context *context, uint64 branch) +{ + uint64 old_root = context->root->ref.addr; + uint64 extent_size = cache_extent_size(context->cc); + uint64 count = allocator_get_config(context->al)->capacity / extent_size; + refcount *expected = TYPED_ARRAY_MALLOC(context->hid, expected, count); + ASSERT_NOT_NULL(expected); + for (uint64 i = 0; i < count; i++) { + expected[i] = allocator_get_refcount(context->al, i * extent_size); + } + uint64 allocated = allocator_in_use(context->al); + split_test_real_ops = context->cc->ops; + cache_ops cache_faults = *split_test_real_ops; + cache_faults.page_alloc = split_blob_fail_tail_page; + split_test_real_allocator_ops = context->al->ops; + allocator_ops allocator_faults = *split_test_real_allocator_ops; + allocator_faults.inc_ref = split_test_inc_ref; + for (uint64 mode = 0; mode < 2; mode++) { + bool32 succeeded = FALSE; + for (uint64 at = 1; at < 200; at++) { + split_test_alloc_calls = split_test_retain_calls = 0; + split_test_fail_alloc_at = split_test_fail_retain_at = at; + split_test_fail_limit = at % 2 == 0; + if (mode == 0) { + context->cc->ops = &cache_faults; + } else { + context->al->ops = &allocator_faults; + } + platform_status rc = trunk_incorporate_prepare(context, branch); + context->cc->ops = split_test_real_ops; + context->al->ops = split_test_real_allocator_ops; + if (SUCCESS(rc)) { + ASSERT_TRUE(at > 2); + if (mode == 0) { + // A successful prepare owns a provisional tree; release it by + // committing it before beginning the independent retain sweep. + trunk_incorporate_commit(context); + trunk_incorporate_cleanup(context); + old_root = context->root->ref.addr; + allocated = allocator_in_use(context->al); + for (uint64 i = 0; i < count; i++) { + expected[i] = + allocator_get_refcount(context->al, i * extent_size); + } + } else { + trunk_incorporate_commit(context); + trunk_incorporate_cleanup(context); + } + succeeded = TRUE; + break; + } + ASSERT_EQUAL(mode && split_test_fail_limit ? STATUS_LIMIT_EXCEEDED.r + : STATUS_NO_MEMORY.r, + rc.r); + ASSERT_EQUAL(old_root, context->root->ref.addr); + ASSERT_NULL(context->post_incorporation_root); + ASSERT_EQUAL(allocated, allocator_in_use(context->al)); + for (uint64 i = 0; i < count; i++) { + ASSERT_EQUAL(expected[i], + allocator_get_refcount(context->al, i * extent_size), + "mode %lu failure %lu extent %lu", + mode, + at, + i); + } + ASSERT_TRUE(SUCCESS(context->allocator_cleanup_status)); + } + ASSERT_TRUE(succeeded); + } + split_test_fail_limit = FALSE; + platform_free(context->hid, expected); +} + +static void +split_blob_exercise(trunk_context *context, uint64 length, bool32 custom) +{ + const trunk_config *saved = context->cfg; + trunk_config cfg = *saved; + btree_config bcfg = *cfg.btree_cfg; + data_config dc = *cfg.data_cfg; + if (custom) { + dc.key_compare = split_blob_compare; + dc.key_compare_is_lexical = FALSE; + /* The writer must suppress prefixes for custom comparators even when + * the configured lexical prefix budget is nonzero. */ + bcfg.blob_key_prefix_size = 256; + } + bcfg.data_cfg = &dc; + cfg.data_cfg = &dc; + cfg.btree_cfg = &bcfg; + cfg.incorporation_size_kv_bytes = length + sizeof(uint64); + cfg.target_fanout = 4; + cfg.branch_rough_count_height = 0; + context->cfg = &cfg; + uint64 branch = split_blob_pack(context, length); + split_test_incorporate(context, branch); + btree_dec_ref(context->cc, &bcfg, branch, PAGE_TYPE_BRANCH); + ASSERT_TRUE(SUCCESS(task_perform_until_quiescent(context->ts))); + split_blob_check(context, length); + + auto_writable_buffer old_descriptor, new_descriptor; + writable_buffer_init(&old_descriptor, context->hid); + writable_buffer_init(&new_descriptor, context->hid); + split_blob_root_descriptor(context, &old_descriptor); + parsed_blob old_blob; + ASSERT_TRUE(SUCCESS(blob_parse_descriptor( + context->cc, writable_buffer_to_slice(&old_descriptor), &old_blob))); + uint64 extent_size = cache_extent_size(context->cc); + uint64 old_root = context->root->ref.addr; + ASSERT_EQUAL(old_root, + old_blob.leftovers[0].addr / extent_size * extent_size); + trunk_snapshot snapshot; + ASSERT_TRUE(SUCCESS(trunk_snapshot_create(context, &snapshot))); + + // Copy the existing pivots while changing only the root's bundle list. + cfg.incorporation_size_kv_bytes = UINT64_MAX; + branch = split_blob_pack(context, length); + if (old_blob.num_extents != 0) { + split_blob_incorporate_failures(context, branch); + } else { + split_test_incorporate(context, branch); + } + btree_dec_ref(context->cc, &bcfg, branch, PAGE_TYPE_BRANCH); + split_blob_root_descriptor(context, &new_descriptor); + parsed_blob new_blob; + ASSERT_TRUE(SUCCESS(blob_parse_descriptor( + context->cc, writable_buffer_to_slice(&new_descriptor), &new_blob))); + ASSERT_EQUAL(old_blob.num_extents, new_blob.num_extents); + for (uint64 i = 0; i < old_blob.num_extents; i++) { + ASSERT_EQUAL(old_blob.base->addrs[i], new_blob.base->addrs[i]); + } + ASSERT_NOT_EQUAL(old_blob.leftovers[0].addr, new_blob.leftovers[0].addr); + ASSERT_EQUAL(context->root->ref.addr, + new_blob.leftovers[0].addr / extent_size * extent_size); + ASSERT_TRUE(SUCCESS(trunk_snapshot_release(context, &snapshot))); + ASSERT_EQUAL(AL_FREE, allocator_get_refcount(context->al, old_root)); + cache_flush(context->cc); + cache_evict(context->cc, FALSE); + split_blob_check(context, length); + ASSERT_TRUE(SUCCESS(task_perform_until_quiescent(context->ts))); + ASSERT_TRUE(SUCCESS(context->allocator_cleanup_status)); + context->cfg = saved; +} + +CTEST2(trunk_split, blob_pivots_copy_colocated_tails) +{ + split_blob_exercise(data->context, 4096 + 37, FALSE); +} + +CTEST2(trunk_split, blob_pivots_copy_overflow_tails) +{ + split_blob_exercise( + data->context, cache_extent_size(data->context->cc) * 3 / 2 + 37, FALSE); +} + +CTEST2(trunk_split, blob_pivots_custom_comparator) +{ + split_blob_exercise(data->context, 65536 + 37, TRUE); +} + +CTEST2(trunk_split, blob_pivots_recovery_rebuild) +{ + uint64 length = cache_extent_size(data->context->cc) * 3 / 2 + 37; + split_blob_exercise(data->context, length, FALSE); + core_handle *core = (core_handle *)splinterdb_get_trunk_handle(data->db); + core_config cfg = core->cfg; + allocator *al = core->al; + cache *cc = core->cc; + task_system *ts = core->ts; + io_handle *io = core->superblock.io; + platform_heap_id hid = core->heap_id; + allocator_root_id id = core->id; + data->context->cfg = data->original_cfg; + ASSERT_TRUE(SUCCESS(core_unmount(core, FALSE))); + + allocator_config *acfg = allocator_get_config(al); + superblock_context sb; + ASSERT_TRUE(SUCCESS(superblock_context_init(&sb, io, acfg, hid))); + ASSERT_TRUE(SUCCESS(superblock_mount(&sb, acfg))); + superblock_tree_record record; + superblock_get_tree_record(&sb, &record); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(record.live_log)); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(record.sealed_log)); + superblock_snapshot_tree( + &sb, record.root_addr, record.first_unincorporated_generation); + ASSERT_TRUE(SUCCESS(superblock_make_durable(&sb))); + superblock_context_deinit(&sb); + ASSERT_TRUE(SUCCESS(rc_allocator_release_snapshot((rc_allocator *)al))); + + uint64 extent_size = cache_extent_size(cc); + uint64 count = acfg->capacity / extent_size; + refcount *expected = TYPED_ARRAY_MALLOC(hid, expected, count); + ASSERT_NOT_NULL(expected); + for (uint64 i = 0; i < count; i++) { + expected[i] = allocator_get_refcount(al, i * extent_size); + } + uint64 allocated = allocator_in_use(al); + for (uint64 round = 0; round < 2; round++) { + cache_evict(cc, FALSE); + ASSERT_TRUE(SUCCESS(allocator_recovery_begin(al))); + ASSERT_TRUE(SUCCESS(trunk_recover_allocations( + cfg.trunk_node_cfg, cc, hid, record.root_addr))); + allocator_recovery_finish(al); + ASSERT_EQUAL(allocated, allocator_in_use(al)); + for (uint64 i = 0; i < count; i++) { + ASSERT_EQUAL(expected[i], + allocator_get_refcount(al, i * extent_size), + "round %lu extent %lu", + round, + i); + } + } + platform_free(hid, expected); + ASSERT_TRUE(SUCCESS(core_mount(core, &cfg, al, cc, io, ts, id, hid))); + data->context = &core->trunk_context; + data->original_cfg = data->context->cfg; + data->cfg = *data->original_cfg; + data->context->cfg = &data->cfg; + split_blob_check(data->context, length); +} + +static platform_status +split_blob_fail_read(cache *cc, + uint64 addr, + bool32 blocking, + page_type type, + page_handle *page) +{ + if (type == PAGE_TYPE_BLOB) { + *page = (page_handle){0}; + return STATUS_IO_ERROR; + } + return split_test_real_ops->page_get(cc, addr, blocking, type, page); +} + +CTEST2(trunk_split, blob_pivots_propagate_read_failures) +{ + trunk_context *context = data->context; + uint64 length = 65536 + 37; + data->cfg.incorporation_size_kv_bytes = length; + data->cfg.target_fanout = 4; + data->cfg.branch_rough_count_height = 0; + uint64 branch = split_blob_pack(context, length); + split_test_incorporate(context, branch); + ASSERT_TRUE(SUCCESS(task_perform_until_quiescent(context->ts))); + uint64 allocated = allocator_in_use(context->al); + uint64 old_root = context->root->ref.addr; + char *bytes = TYPED_ARRAY_MALLOC(context->hid, bytes, length); + ASSERT_NOT_NULL(bytes); + memset(bytes, 'k', length); + bytes[length - 1] = TRUNK_BLOB_TEST_KEYS; + key query = key_create(FALSE, length, bytes); + split_test_real_ops = context->cc->ops; + cache_ops ops = *split_test_real_ops; + ops.page_get = split_blob_fail_read; + for (uint64 async = 0; async < 2; async++) { + trunk_ondisk_node_handle handle; + ASSERT_TRUE(SUCCESS(trunk_init_root_handle(context, &handle))); + lookup_result result; + lookup_result_init( + &result, context->cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + context->cc->ops = &ops; + platform_status rc; + if (async) { + bool32 ready = FALSE; + trunk_merge_lookup_async_state state; + trunk_merge_lookup_async_state_init(&state, + context, + &handle, + query, + &result, + NULL, + split_blob_ready, + &ready); + while (trunk_merge_lookup_async(&state) != ASYNC_STATUS_DONE) { + while (!__atomic_exchange_n(&ready, FALSE, __ATOMIC_ACQUIRE)) { + cache_cleanup(context->cc); + } + } + rc = state.__async_result; + } else { + rc = trunk_merge_lookup(context, &handle, query, &result, NULL); + } + context->cc->ops = split_test_real_ops; + ASSERT_EQUAL(STATUS_IO_ERROR.r, rc.r); + lookup_result_deinit(&result); + trunk_ondisk_node_handle_deinit(&handle); + } + context->cc->ops = &ops; + platform_status rc = trunk_incorporate_prepare(context, branch); + context->cc->ops = split_test_real_ops; + ASSERT_EQUAL(STATUS_IO_ERROR.r, rc.r); + ASSERT_EQUAL(old_root, context->root->ref.addr); + ASSERT_NULL(context->post_incorporation_root); + ASSERT_EQUAL(allocated, allocator_in_use(context->al)); + platform_free(context->hid, bytes); + btree_dec_ref(context->cc, data->cfg.btree_cfg, branch, PAGE_TYPE_BRANCH); + split_blob_check(context, length); +} + +CTEST2(trunk_split, blob_pivots_reject_metadata_as_tail) +{ + trunk_context *context = data->context; + uint64 length = 65536 + 37; + data->cfg.incorporation_size_kv_bytes = length; + data->cfg.target_fanout = 4; + data->cfg.branch_rough_count_height = 0; + uint64 branch = split_blob_pack(context, length); + split_test_incorporate(context, branch); + btree_dec_ref(context->cc, data->cfg.btree_cfg, branch, PAGE_TYPE_BRANCH); + ASSERT_TRUE(SUCCESS(task_perform_until_quiescent(context->ts))); + + uint64 root = context->root->ref.addr; + page_handle page; + ASSERT_TRUE( + SUCCESS(cache_get(context->cc, root, TRUE, PAGE_TYPE_TRUNK, &page))); + split_blob_node_header *header = (void *)page_handle_data(&page); + ASSERT_TRUE(header->num_pivots > 2); + uint64 offset = header->pivot_offsets[1]; + uint64 prefix = header->blob_key_prefix_size; + cache_unget(context->cc, &page); + uint64 page_size = cache_page_size(context->cc); + uint64 addr = root + offset / page_size * page_size; + ASSERT_TRUE( + SUCCESS(cache_get(context->cc, addr, TRUE, PAGE_TYPE_TRUNK, &page))); + split_blob_pivot *pivot = + (void *)(page_handle_data(&page) + offset % page_size); + ASSERT_EQUAL(ONDISK_KEY_IS_BLOB, pivot->key.flags); + blob *descriptor = (void *)(pivot->key.bytes + prefix); + parsed_blob parsed; + ASSERT_TRUE(SUCCESS(blob_parse_descriptor( + context->cc, + slice_create(pivot->key.length - prefix, descriptor), + &parsed))); + ASSERT_EQUAL(0, parsed.num_extents); + uint64 saved_addr = descriptor->addrs[0]; + ASSERT_TRUE(cache_try_claim(context->cc, &page)); + cache_lock(context->cc, &page); + descriptor->addrs[0] = root; + cache_unlock(context->cc, &page); + cache_unclaim(context->cc, &page); + cache_unget(context->cc, &page); + + char *bytes = TYPED_ARRAY_MALLOC(context->hid, bytes, length); + ASSERT_NOT_NULL(bytes); + memset(bytes, 'k', length); + bytes[length - 1] = 1; + trunk_ondisk_node_handle handle; + ASSERT_TRUE(SUCCESS(trunk_init_root_handle(context, &handle))); + lookup_result result; + lookup_result_init( + &result, context->cfg->data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + platform_status rc = trunk_merge_lookup( + context, &handle, key_create(FALSE, length, bytes), &result, NULL); + trunk_ondisk_node_handle_deinit(&handle); + lookup_result_deinit(&result); + platform_free(context->hid, bytes); + ASSERT_TRUE( + SUCCESS(cache_get(context->cc, addr, TRUE, PAGE_TYPE_TRUNK, &page))); + pivot = (void *)(page_handle_data(&page) + offset % page_size); + descriptor = (void *)(pivot->key.bytes + prefix); + ASSERT_TRUE(cache_try_claim(context->cc, &page)); + cache_lock(context->cc, &page); + descriptor->addrs[0] = saved_addr; + cache_unlock(context->cc, &page); + cache_unclaim(context->cc, &page); + cache_unget(context->cc, &page); + ASSERT_EQUAL(STATUS_INVALID_STATE.r, rc.r); + split_blob_check(context, length); +}