Robj/public blobs

Makefile

@@ -530,6 +530,11 @@ $(BINDIR)/$(UNITDIR)/splinterdb_quick_test: $(COMMON_TESTOBJ)
                                             $(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \
                                             $(LIBDIR)/libsplinterdb.so
 
+$(BINDIR)/$(UNITDIR)/splinterdb_blob_test: $(COMMON_TESTOBJ) \
+                                          $(COMMON_UNIT_TESTOBJ) \
+                                          $(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \
+                                          $(LIBDIR)/libsplinterdb.so
+
 $(BINDIR)/$(UNITDIR)/splinterdb_stress_test: $(COMMON_TESTOBJ)                             \
                                              $(COMMON_UNIT_TESTOBJ)                        \
                                              $(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \
@@ -616,6 +621,7 @@ unit/merge_test:                   $(BINDIR)/$(UNITDIR)/merge_test
 unit/trunk_split_test:             $(BINDIR)/$(UNITDIR)/trunk_split_test
 unit/splinter_test:                $(BINDIR)/$(UNITDIR)/splinter_test
 unit/splinterdb_quick_test:        $(BINDIR)/$(UNITDIR)/splinterdb_quick_test
+unit/splinterdb_blob_test:         $(BINDIR)/$(UNITDIR)/splinterdb_blob_test
 unit/splinterdb_stress_test:       $(BINDIR)/$(UNITDIR)/splinterdb_stress_test
 unit/notification_test:           $(BINDIR)/$(UNITDIR)/notification_test
 unit/splinterdb_optimize_test:     $(BINDIR)/$(UNITDIR)/splinterdb_optimize_test

benchmarks/allocation_table/kvbench_controlled.c

@@ -850,8 +850,10 @@ load_worker(void *arg)
       make_key(2 * id, key);
       make_value(id, 0, val);
       uint64_t t0 = now_ns();
-      int      rc = splinterdb_insert(
-         kvs, slice_create(KEY_SIZE, key), slice_create(VALUE_SIZE, val), NULL);
+      int      rc = splinterdb_insert(kvs,
+                                 slice_create(KEY_SIZE, key),
+                                 message_create(VALUE_SIZE, val),
+                                 NULL);
       uint64_t t1 = now_ns();
       if (rc) {
          w->errors++;
@@ -1133,8 +1135,10 @@ ow_worker(void *arg)
       make_key(2 * id, key);
       make_value(id, generation, val);
       uint64_t t0 = now_ns();
-      int      rc = splinterdb_insert(
-         kvs, slice_create(KEY_SIZE, key), slice_create(VALUE_SIZE, val), NULL);
+      int      rc = splinterdb_insert(kvs,
+                                 slice_create(KEY_SIZE, key),
+                                 message_create(VALUE_SIZE, val),
+                                 NULL);
       uint64_t t1 = now_ns();
       n++;
       atomic_store_explicit(&w->progress, n, memory_order_relaxed);

benchmarks/allocation_table/prepare_controlled.py

@@ -44,9 +44,12 @@ def main():
         close_flags = (["-DKVBENCH_CLOSE_HAS_FORCE"]
                        if "splinterdb_close(splinterdb **kvs, bool32 force)" in
                        (source / "include/splinterdb/splinterdb.h").read_text() else [])
-        command = flags + close_flags + ["-I" + str(source / "include"),
-                                         str(harness), str(library), "-laio", "-lxxhash", "-lm",
-                                         "-o", str(binary)]
+        value_flags = ([] if "message_create(" in
+                       (source / "include/splinterdb/data.h").read_text()
+                       else ["-Dmessage_create=slice_create"])
+        command = flags + close_flags + value_flags + [
+            "-I" + str(source / "include"), str(harness), str(library),
+            "-laio", "-lxxhash", "-lm", "-o", str(binary)]
         subprocess.run(command, check=True)
         manifest["variants"][name] = dict(item, binary=str(binary), sha256=sha(binary),
                                           link_command=command)

benchmarks/allocation_table/prepare_final.py

@@ -128,9 +128,13 @@ def main():
         close_flags = (["-DKVBENCH_CLOSE_HAS_FORCE"]
                        if "splinterdb_close(splinterdb **kvs, bool32 force)" in
                        (source / "include/splinterdb/splinterdb.h").read_text() else [])
-        link = flags + close_flags + ["-DKVBENCH_SPACE_MODE", "-I" + str(source / "include"),
-                                      str(harness), str(library), "-laio", "-lxxhash", "-lm",
-                                      "-o", str(binary)]
+        value_flags = ([] if "message_create(" in
+                       (source / "include/splinterdb/data.h").read_text()
+                       else ["-Dmessage_create=slice_create"])
+        link = flags + close_flags + value_flags + [
+            "-DKVBENCH_SPACE_MODE", "-I" + str(source / "include"),
+            str(harness), str(library), "-laio", "-lxxhash", "-lm",
+            "-o", str(binary)]
         run(link)
         manifest["variants"][name] = dict(
             source=str(source), revision=baseline_commit if name == "original" else current_commit,

benchmarks/barrier_bench.c

@@ -106,7 +106,7 @@ run_workload(splinterdb_config *cfg,
          memcpy(value, &id, sizeof(id));
          checked(splinterdb_insert(db,
                                    slice_create(sizeof(key), key),
-                                   slice_create(sizeof(value), value),
+                                   message_create(sizeof(value), value),
                                    NULL),
                  "insert");
       }

benchmarks/btree_prefix_bench.c

@@ -472,7 +472,7 @@ make_inputs(benchmark *b)
       }
       b->tuples[i] = (tuple){
          .tuple_key = generated[2 * i].tuple_key,
-         .msg       = message_create(
+         .msg       = message_create_internal(
             MESSAGE_TYPE_INSERT, NULL, slice_create(o->value_bytes, v))};
       b->misses[i] = generated[2 * i + 1].tuple_key;
    }
@@ -658,7 +658,7 @@ run_inserts(benchmark *b)
          }
          verify_tree(b, root, PAGE_TYPE_MEMTABLE, false);
          for (uint64 i = 0; i < b->opt.keys; i++)
-            b->tuples[i].msg = message_create(
+            b->tuples[i].msg = message_create_internal(
                MESSAGE_TYPE_INSERT,
                NULL,
                slice_create(b->opt.value_bytes,
@@ -700,7 +700,7 @@ run_inserts(benchmark *b)
       }
       if (overwrite) {
          for (uint64 i = 0; i < b->opt.keys; i++)
-            b->tuples[i].msg = message_create(
+            b->tuples[i].msg = message_create_internal(
                MESSAGE_TYPE_INSERT,
                NULL,
                slice_create(b->opt.value_bytes,

benchmarks/build_barrier_bench.py

@@ -23,6 +23,8 @@
     command += ['-DHAVE_ASYNC_BARRIER', '-DHAVE_SENTINEL_NOTIFICATION']
 elif 'splinterdb_durable_barrier_async' in header.read_text():
     command.append('-DHAVE_ASYNC_BARRIER')
+if 'message_create(' not in (tree / 'include/splinterdb/data.h').read_text():
+    command.append('-Dmessage_create=slice_create')
 command += ['-I', str(tree / 'include'), str(source),
             str(library), '-laio', '-lxxhash', '-lm',
             '-lpthread', '-o', str(binary)]

benchmarks/build_btree_prefix_bench.py

@@ -131,6 +131,8 @@ def main():
                        '-I', str(tree / 'src/platform_linux'), str(source_copy),
                        identity['library'], '-laio', '-lxxhash', '-lm', '-lpthread',
                        '-o', str(binary)]
+            if 'message_create_internal(' not in (tree / 'src/data_internal.h').read_text():
+                command.append('-Dmessage_create_internal=message_create')
             log = artifact_dir / f'{name}-compile.log'
             invoke(command, log, environment)
             verify_profile(profiles[name])

benchmarks/build_prefix_bench.py

@@ -117,6 +117,8 @@
            '-DKVBENCH_SPACE_MODE']
 if 'key_compare_is_lexical' in (tree / 'include/splinterdb/data.h').read_text():
     command.append('-DKVBENCH_PREFIX_SUPPORT')
+if 'message_create(' not in (tree / 'include/splinterdb/data.h').read_text():
+    command.append('-Dmessage_create=slice_create')
 command += ['-I', str(tree / 'include'), str(generated),
             str(tree / a.build_root / 'release/lib/libsplinterdb.a'), '-laio', '-lxxhash', '-lm',
             '-lpthread', '-o', str(binary)]

benchmarks/comparison/build.sh

@@ -29,20 +29,32 @@ CF='-D_GNU_SOURCE -ggdb -Wall -pthread -Wfatal-errors -Werror -Wvla -DXXH_STATIC
 LF='-ggdb -pthread -O3 -ffast-math -flto'
 LIBS='-laio -lxxhash -lm -lpthread'
 
-cc $CF $LF -DKVBENCH_CLOSE_HAS_FORCE \
+head_value_flags=()
+main_value_flags=()
+for name in head main; do
+   if ! python3 -c 'import pathlib, sys; sys.exit("message_create(" not in pathlib.Path(sys.argv[1]).read_text())' "$OUT/src/$name/include/splinterdb/data.h"; then
+      if [ "$name" = head ]; then
+         head_value_flags=(-Dmessage_create=slice_create)
+      else
+         main_value_flags=(-Dmessage_create=slice_create)
+      fi
+   fi
+done
+
+cc $CF $LF "${head_value_flags[@]}" -DKVBENCH_CLOSE_HAS_FORCE \
    -I "$OUT/src/head/include" "$ROOT/benchmarks/kvbench.c" \
    "$OUT/src/head/build/release/lib/libsplinterdb.a" $LIBS \
    -o "$OUT/bin/kvbench-head"
-cc $CF $LF \
+cc $CF $LF "${main_value_flags[@]}" \
    -I "$OUT/src/main/include" "$ROOT/benchmarks/kvbench.c" \
    "$OUT/src/main/build/release/lib/libsplinterdb.a" $LIBS \
    -o "$OUT/bin/kvbench-main"
 
-cc $CF $LF -I "$OUT/src/head/include" -I "$OUT/src/head/src" \
+cc $CF $LF "${head_value_flags[@]}" -I "$OUT/src/head/include" -I "$OUT/src/head/src" \
    -I "$OUT/src/head/src/platform_linux" "$ROOT/benchmarks/insert_benchmark.c" \
    "$OUT/src/head/build/release/lib/libsplinterdb.a" $LIBS \
    -o "$OUT/bin/insert_benchmark-head"
-cc $CF $LF -DKVBENCH_LEGACY_CLOSE -DKVBENCH_NO_TRACE \
+cc $CF $LF "${main_value_flags[@]}" -DKVBENCH_LEGACY_CLOSE -DKVBENCH_NO_TRACE \
    -DKVBENCH_NO_DURABLE_BARRIER -I "$OUT/src/main/include" \
    -I "$OUT/src/main/src" -I "$OUT/src/main/src/platform_linux" \
    "$ROOT/benchmarks/insert_benchmark.c" \

benchmarks/hash_range_benchmark.c

@@ -205,7 +205,7 @@ main(int argc, char **argv)
          memcpy(value, key, sizeof(key));
          check(splinterdb_insert(db,
                                  slice_create(sizeof(key), key),
-                                 slice_create(sizeof(value), value),
+                                 message_create(sizeof(value), value),
                                  NULL));
       }
    }

benchmarks/insert_benchmark.c

@@ -304,7 +304,7 @@ run_worker(void *arg)
       } else {
          rc = splinterdb_insert(b->db,
                                 slice_create(o->key_size, key),
-                                slice_create(o->value_size, value),
+                                message_create(o->value_size, value),
                                 NULL);
       }
       uint64_t end = now_ns();

benchmarks/kvbench.c

@@ -728,8 +728,10 @@ load_worker(void *arg)
       make_key(2 * id, key);
       make_value(id, 0, val);
       uint64_t t0 = now_ns();
-      int      rc = splinterdb_insert(
-         kvs, slice_create(KEY_SIZE, key), slice_create(VALUE_SIZE, val), NULL);
+      int      rc = splinterdb_insert(kvs,
+                                 slice_create(KEY_SIZE, key),
+                                 message_create(VALUE_SIZE, val),
+                                 NULL);
       uint64_t t1 = now_ns();
       if (rc) {
          w->errors++;
@@ -1011,8 +1013,10 @@ ow_worker(void *arg)
       make_key(2 * id, key);
       make_value(id, generation, val);
       uint64_t t0 = now_ns();
-      int      rc = splinterdb_insert(
-         kvs, slice_create(KEY_SIZE, key), slice_create(VALUE_SIZE, val), NULL);
+      int      rc = splinterdb_insert(kvs,
+                                 slice_create(KEY_SIZE, key),
+                                 message_create(VALUE_SIZE, val),
+                                 NULL);
       uint64_t t1 = now_ns();
       n++;
       atomic_store_explicit(&w->progress, n, memory_order_relaxed);

benchmarks/optimize_notification_bench.c

@@ -105,7 +105,7 @@ main(int argc, char **argv)
       memcpy(value, &id, sizeof(id));
       checked(splinterdb_insert(db,
                                 slice_create(sizeof(key), key),
-                                slice_create(sizeof(value), value),
+                                message_create(sizeof(value), value),
                                 NULL),
               "insert");
    }

docs/usage.md

@@ -69,6 +69,76 @@ looks out-of-date, please open an issue or pull request.
   functional tests.
 
 
+## Passing values and reusable blobs
+
+Both `splinterdb_insert()` and `splinterdb_update()` accept a public `message`.
+Use `message_create(length, pointer)` for direct bytes, or
+`message_from_slice(slice)` for an existing slice. These constructors borrow
+the input without allocation or copying. Keys and lookup results still use
+slices. Existing callers can replace value-side `slice_create()` calls with
+`message_create()`.
+
+The called operation determines the message type: insert always overwrites,
+and update always applies a delta through the configured merge callbacks.
+Neither modifies the caller's message. Its `type` field is ignored on input,
+so the same message can be passed to either operation.
+
+Use `splinterdb_blob_create()` to copy bytes into blob storage once, then pass
+the returned message to either operation. Full extents are shared; tails may
+be copied on insertion or compaction. Small values may be copied entirely,
+and each created blob has allocation overhead, so this API is most useful
+for large repeated payloads.
+
+```c
+int
+insert_shared_values(splinterdb *db, slice bytes, slice first_key, slice second_key)
+{
+   message value;
+   int rc = splinterdb_blob_create(db, bytes, &value);
+   if (rc != 0) {
+      return rc;
+   }
+
+   rc = splinterdb_insert(db, first_key, value, NULL);
+   if (rc == 0) {
+      rc = splinterdb_insert(db, second_key, value, NULL);
+   }
+   splinterdb_blob_release(&value);
+   if (rc == 0) {
+      rc = splinterdb_durable_barrier(db, NULL);
+   }
+   return rc;
+}
+```
+
+A reusable delta is passed as `splinterdb_update(db, key, value, NULL)`.
+It is processed by the configured merge callbacks just like a direct delta;
+the resulting value need not retain the delta's blob storage.
+
+The input bytes may be released after blob creation succeeds. Releasing the
+message drops the caller's ownership and resets that message to an empty
+direct message; successfully inserted values or deltas retain their own
+ownership. Operations are independent: if a second operation fails, a first
+successful operation remains applied.
+
+Copying a message only borrows the same payload. Release each created blob
+exactly once, after all uses of all copies finish; release invalidates the
+other copies. Callback messages are borrowed and must not be released.
+Do not change a blob message's `cc`, `data`, or descriptor bytes.
+For a blob, `message_data()`, `message_length()`, and `message_slice()`
+describe its encoded descriptor, not its logical bytes.
+
+A blob belongs to the open database instance that created it. Threads may
+reuse it concurrently, but release must not race any use. Blob messages cannot
+be serialized or reused after close/reopen. Release all created blobs before
+`splinterdb_close(&db, FALSE)`, which returns `EBUSY` while they remain live.
+Forced close releases them and invalidates every outstanding copy.
+
+Creating a standalone blob does not establish crash durability. Inserted
+references follow ordinary update and durability-barrier semantics. Recovery
+reclaims storage that no recovered data references. Keys retain the existing
+size limit; large-key support is separate.
+
 ## Example programs
 - [`tests/unit/splinterdb_quick_test.c`](../tests/unit/splinterdb_quick_test.c) covers
    various basic operations on a single thread. The

examples/splinterdb_custom_ipv4_addr_sortcmp_example.c

@@ -318,7 +318,8 @@ do_inserts(splinterdb *spl_handle, kv_pair *kv_pairs, int num_kv_pairs)
          slice_create(strlen(kv_pairs[ictr].kv_key), kv_pairs[ictr].kv_key);
       slice value = slice_create(WWW_PING_SIZE(&kv_pairs[ictr].kv_val),
                                  (const char *)&kv_pairs[ictr].kv_val);
-      int   rc    = splinterdb_insert(spl_handle, key, value, NULL);
+      int   rc =
+         splinterdb_insert(spl_handle, key, message_from_slice(value), NULL);
       if (rc) {
          printf(
             "Insert of key '%s' failed; rc=%d\n", kv_pairs[ictr].kv_key, rc);

examples/splinterdb_intro_example.c

@@ -48,45 +48,46 @@ main()
    const char *fruit = "apple";
    const char *descr = "An apple a day keeps the doctor away!";
    slice       key   = slice_create((size_t)strlen(fruit), fruit);
-   slice       value = slice_create((size_t)strlen(descr), descr);
+   message     value = message_create((size_t)strlen(descr), descr);
 
    rc = splinterdb_insert(spl_handle, key, value, NULL);
    printf("Inserted key '%s'\n", fruit);
 
    fruit = "Orange";
    descr = "Is a good source of vitamin-C.";
    key   = slice_create((size_t)strlen(fruit), fruit);
-   value = slice_create((size_t)strlen(descr), descr);
+   value = message_create((size_t)strlen(descr), descr);
    rc    = splinterdb_insert(spl_handle, key, value, NULL);
    printf("Inserted key '%s'\n", fruit);
 
    fruit = "Mango";
    descr = "Mango is the king of fruits.";
    key   = slice_create((size_t)strlen(fruit), fruit);
-   value = slice_create((size_t)strlen(descr), descr);
+   value = message_create((size_t)strlen(descr), descr);
    rc    = splinterdb_insert(spl_handle, key, value, NULL);
    printf("Inserted key '%s'\n", fruit);
 
    // Retrieve a key-value pair.
+   slice                    lookup_value;
    splinterdb_lookup_result result;
    splinterdb_lookup_result_init(spl_handle, &result);
 
    fruit = "Orange";
    key   = slice_create((size_t)strlen(fruit), fruit);
    rc    = splinterdb_lookup(spl_handle, key, &result);
-   rc    = splinterdb_lookup_result_value(&result, &value);
+   rc    = splinterdb_lookup_result_value(&result, &lookup_value);
    if (!rc) {
       printf("Found key: '%s', value: '%.*s'\n",
              fruit,
-             (int)slice_length(value),
-             (char *)slice_data(value));
+             (int)slice_length(lookup_value),
+             (char *)slice_data(lookup_value));
    }
 
    // Handling non-existent keys
    fruit = "Banana";
    key   = slice_create((size_t)strlen(fruit), fruit);
    rc    = splinterdb_lookup(spl_handle, key, &result);
-   rc    = splinterdb_lookup_result_value(&result, &value);
+   rc    = splinterdb_lookup_result_value(&result, &lookup_value);
    if (rc) {
       printf("Key: '%s' not found. (rc=%d)\n", fruit, rc);
    }

examples/splinterdb_iterators_example.c

@@ -112,7 +112,8 @@ do_inserts(splinterdb *spl_handle, kv_pair *kv_pairs, int num_kv_pairs)
          slice_create(strlen(kv_pairs[ictr].kv_key), kv_pairs[ictr].kv_key);
       slice value =
          slice_create(strlen(kv_pairs[ictr].kv_val), kv_pairs[ictr].kv_val);
-      int rc = splinterdb_insert(spl_handle, key, value, NULL);
+      int rc =
+         splinterdb_insert(spl_handle, key, message_from_slice(value), NULL);
       if (rc) {
          printf(
             "Insert for key='%s' failed, rc=%d\n", kv_pairs[ictr].kv_key, rc);

examples/splinterdb_wide_values_example.c

@@ -62,7 +62,7 @@ main()
 
       slice key   = slice_create(strlen(key_buf), key_buf);
       slice value = slice_create(val_len, val_buf);
-      rc          = splinterdb_insert(spl_handle, key, value, NULL);
+      rc = splinterdb_insert(spl_handle, key, message_from_slice(value), NULL);
       if (rc) {
          break;
       }