diff --git a/src/btree.c b/src/btree.c index 8a0bed7e0..28b23d5bd 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2966,15 +2966,15 @@ btree_lookup_node_async(btree_lookup_async_state *state, uint64 depth) state->__async_result = STATUS_OK; async_await_call(state, cache_get_async, - &state->cache_get_state, + &state->work.cache_get_state, state->cc, state->node.addr, state->type, state->callback, state->callback_arg); { - platform_status rc = cache_get_async_state_result(&state->cache_get_state, - &state->node.page); + platform_status rc = cache_get_async_state_result( + &state->work.cache_get_state, &state->node.page); if (!SUCCESS(rc)) { async_return(state, rc); } @@ -3058,14 +3058,14 @@ btree_lookup_node_async(btree_lookup_async_state *state, uint64 depth) async_await_call(state, cache_get_async, - &state->cache_get_state, + &state->work.cache_get_state, state->cc, state->child_node.addr, state->type, state->callback, state->callback_arg); platform_status get_rc = cache_get_async_state_result( - &state->cache_get_state, &state->child_node.page); + &state->work.cache_get_state, &state->child_node.page); if (!SUCCESS(get_rc)) { btree_node_unget(state->cc, state->cfg, &state->node); async_return(state, get_rc); @@ -3114,8 +3114,8 @@ btree_lookup_node_async(btree_lookup_async_state *state, uint64 depth) * * OUT Parameters: * - state->node: the node found - * - state->found_key: the key found in the leaf, or NULL_KEY otherwise. - * - state->msg: the message of the target + * - state->work.found_key: the key found in the leaf, or NULL_KEY otherwise. + * - state->work.msg: the message of the target * * LOCAL Variables: * - state->stats: the stats of the node found @@ -3136,7 +3136,7 @@ btree_lookup_with_ref_async(btree_lookup_async_state *state, uint64 depth) async_return(state); } - state->found_key = NULL_KEY; + state->work.found_key = NULL_KEY; bool32 found; int64 idx; const leaf_entry *entry = NULL; @@ -3163,12 +3163,12 @@ btree_lookup_with_ref_async(btree_lookup_async_state *state, uint64 depth) } else { entry = btree_get_leaf_entry(state->cfg, state->node.hdr, idx); } - state->found_key = state->node.hdr->prefix_length == 0 - ? leaf_entry_key((leaf_entry *)entry) - : key_create(FALSE, - key_length(state->target), - key_data(state->target)); - state->msg = leaf_entry_message(state->cc, (leaf_entry *)entry); + state->work.found_key = state->node.hdr->prefix_length == 0 + ? leaf_entry_key((leaf_entry *)entry) + : key_create(FALSE, + key_length(state->target), + key_data(state->target)); + state->work.msg = leaf_entry_message(state->cc, (leaf_entry *)entry); } else { btree_node_unget(state->cc, state->cfg, &state->node); } @@ -3300,7 +3300,7 @@ btree_lookup_and_merge(cache *cc, // IN * - state->target: the key to look up * * OUT Parameters: - * - state->found_key: the key found in the leaf, or NULL_KEY otherwise. + * - state->work.found_key: the key found in the leaf, or NULL_KEY otherwise. * * LOCAL Variables: * - state->node: the node found @@ -3308,7 +3308,7 @@ btree_lookup_and_merge(cache *cc, // IN * - state->stop_at_height: the height to stop at * - state->h: the height of the current node * - state->child_node: the child node - * - state->msg: the message of the target + * - state->work.msg: the message of the target */ async_status btree_lookup_and_merge_async(btree_lookup_async_state *state) @@ -3328,16 +3328,16 @@ btree_lookup_and_merge_async(btree_lookup_async_state *state) } platform_status rc = STATUS_OK; - if (!key_is_null(state->found_key)) { + if (!key_is_null(state->work.found_key)) { ondisk_ref blob_ref = ONDISK_REF_NULL; - if (message_is_blob(state->msg)) { + if (message_is_blob(state->work.msg)) { rc = btree_blob_ref_init( &blob_ref, state->cc, state->cfg, state->root_addr, state->type); } if (SUCCESS(rc)) { rc = lookup_result_update(state->result, - state->found_key, - state->msg, + state->work.found_key, + state->work.msg, ondisk_ref_is_null(&blob_ref) ? NULL : &blob_ref); } diff --git a/src/btree.h b/src/btree.h index 9c4a7bc12..3fe0f867b 100644 --- a/src/btree.h +++ b/src/btree.h @@ -374,6 +374,16 @@ btree_lookup_and_merge(cache *cc, lookup_result *result, bool32 *local_found); +/* Traversal has consumed the cache result before the leaf tuple is decoded. + * Keep node ownership outside this union until the tuple has been merged. */ +typedef union btree_lookup_scratch { + page_get_async_state_buffer cache_get_state; + struct { + key found_key; + message msg; + }; +} btree_lookup_scratch; + // clang-format off DEFINE_ASYNC_STATE(btree_lookup_async_state, 3, param, cache *, cc, @@ -392,9 +402,7 @@ DEFINE_ASYNC_STATE(btree_lookup_async_state, 3, local, btree_node, node, local, btree_node, child_node, local, uint32, h, - local, key, found_key, - local, message, msg, - local, page_get_async_state_buffer, cache_get_state) + local, btree_lookup_scratch, work) // clang-format on static inline void diff --git a/src/cache.h b/src/cache.h index e5e51fa9f..83e1d2547 100644 --- a/src/cache.h +++ b/src/cache.h @@ -131,7 +131,7 @@ cache_config_extent_page(const cache_config *cfg, uint64 extent_addr, uint64 i) return extent_addr + i * cache_config_page_size(cfg); } -#define PAGE_GET_ASYNC_STATE_BUFFER_SIZE (2048) +#define PAGE_GET_ASYNC_STATE_BUFFER_SIZE (320) typedef union page_get_async_state_payload { uint8 bytes[PAGE_GET_ASYNC_STATE_BUFFER_SIZE]; uint64 align_uint64; diff --git a/src/clockcache.c b/src/clockcache.c index 5276b203c..474b09230 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -1257,8 +1257,8 @@ clockcache_try_claim_writeback(clockcache *cc, } struct async_io_state { - clockcache *cc; - io_async_state_buffer iostate; + clockcache *cc; + io_async_state_extent_buffer iostate; }; /* Bounded by the width of the bitvector that tracks the reserve. */ @@ -1467,13 +1467,14 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) async_io_state *state = clockcache_io_state_acquire(cc); - state->cc = cc; - platform_status rc = io_async_state_init(state->iostate, - cc->io, - io_async_pwritev, - first_addr, - clockcache_write_callback, - state); + state->cc = cc; + platform_status rc = + io_async_state_init_extent(state->iostate, + cc->io, + io_async_pwritev, + first_addr, + clockcache_write_callback, + state); if (!SUCCESS(rc)) { platform_error_log("clockcache_batch_start_writeback: " "io_async_state_init failed: %s\n", @@ -2512,6 +2513,13 @@ clockcache_get(clockcache *cc, * Get addr if addr is at entry_number. Returns TRUE if successful. */ +/* Waiting for another loader and issuing our own read are separate branches. + * An issued read completes this get; it never returns to the waiter arm. */ +typedef union clockcache_get_scratch { + io_async_state_page_buffer iostate; + async_waiter wait_node; +} clockcache_get_scratch; + // clang-format off DEFINE_ASYNC_STATE(clockcache_get_async_state, 3, param, clockcache *, cc, @@ -2525,10 +2533,8 @@ DEFINE_ASYNC_STATE(clockcache_get_async_state, 3, local, threadid, tid, local, uint64, entry_number, local, clockcache_entry *, entry, - local, platform_status, rc, - local, io_async_state_buffer, iostate, local, uint64, io_start_time, - local, async_waiter, wait_node) + local, clockcache_get_scratch, work) // clang-format on _Static_assert(sizeof(clockcache_get_async_state) @@ -2581,7 +2587,7 @@ clockcache_get_in_cache_async(clockcache_get_async_state *state, uint64 depth) !clockcache_test_flag(state->cc, state->entry_number, CC_LOADING), state, &state->entry->waiters, - &state->wait_node, + &state->work.wait_node, state->callback, state->callback_arg); @@ -2623,10 +2629,10 @@ clockcache_get_from_disk_async_callback(void *arg) // Check whether we are done. If not, this will enqueue us for a future // callback so we can check again. - if (io_async_run(state->iostate) != ASYNC_STATUS_DONE) { + if (io_async_run(state->work.iostate) != ASYNC_STATUS_DONE) { return; } - platform_status rc = io_async_state_get_result(state->iostate); + platform_status rc = io_async_state_get_result(state->work.iostate); if (!SUCCESS(rc)) { platform_error_log("clockcache_get_from_disk_async_callback: async read " "failed for addr %lu, entry %lu: %s\n", @@ -2643,6 +2649,8 @@ clockcache_get_from_disk_async(clockcache_get_async_state *state, uint64 depth) { async_begin(state, depth); + platform_status rc; + /* A miss must be allocated before it can reserve a cache entry or issue I/O. * Resident hits inherit allocation validity from their existing mapping. */ @@ -2663,28 +2671,26 @@ clockcache_get_from_disk_async(clockcache_get_async_state *state, uint64 depth) // The normal idiom for async functions is to just pass the callback to the // async child, but we pass a wrapper function so that we can always clear // the CC_LOADING flag, even if our caller abandoned us. - state->rc = io_async_state_init(state->iostate, - state->cc->io, - io_async_preadv, - state->addr, - clockcache_get_from_disk_async_callback, - state); - if (!SUCCESS(state->rc)) { - clockcache_finish_load( - state->cc, state->addr, state->entry_number, state->rc); - state->result_status = state->rc; + rc = io_async_state_init_page(state->work.iostate, + state->cc->io, + io_async_preadv, + state->addr, + clockcache_get_from_disk_async_callback, + state); + if (!SUCCESS(rc)) { + clockcache_finish_load(state->cc, state->addr, state->entry_number, rc); + state->result_status = rc; clockcache_dec_ref(state->cc, state->entry_number, state->tid); state->succeeded = TRUE; async_return(state); } - state->rc = - io_async_state_append_page(state->iostate, state->entry->page.data); - if (!SUCCESS(state->rc)) { - io_async_state_deinit(state->iostate); - clockcache_finish_load( - state->cc, state->addr, state->entry_number, state->rc); - state->result_status = state->rc; + rc = + io_async_state_append_page(state->work.iostate, state->entry->page.data); + if (!SUCCESS(rc)) { + io_async_state_deinit(state->work.iostate); + clockcache_finish_load(state->cc, state->addr, state->entry_number, rc); + state->result_status = rc; clockcache_dec_ref(state->cc, state->entry_number, state->tid); state->succeeded = TRUE; async_return(state); @@ -2694,25 +2700,24 @@ clockcache_get_from_disk_async(clockcache_get_async_state *state, uint64 depth) state->io_start_time = platform_get_timestamp(); } - async_await(state, io_async_run(state->iostate) == ASYNC_STATUS_DONE); - state->rc = io_async_state_get_result(state->iostate); + async_await(state, io_async_run(state->work.iostate) == ASYNC_STATUS_DONE); + rc = io_async_state_get_result(state->work.iostate); /* Hard submission failures complete synchronously without invoking the * callback. On a callback-driven completion, the wrapper already published * readiness before notifying our caller. */ if (clockcache_test_flag(state->cc, state->entry_number, CC_LOADING)) { - clockcache_finish_load( - state->cc, state->addr, state->entry_number, state->rc); + clockcache_finish_load(state->cc, state->addr, state->entry_number, rc); } - if (!SUCCESS(state->rc)) { + if (!SUCCESS(rc)) { platform_error_log("clockcache_get_from_disk_async: async read failed " "for addr %lu, entry %lu: %s\n", state->addr, state->entry_number, - platform_status_to_string(state->rc)); + platform_status_to_string(rc)); } - io_async_state_deinit(state->iostate); - if (!SUCCESS(state->rc)) { - state->result_status = state->rc; + io_async_state_deinit(state->work.iostate); + if (!SUCCESS(rc)) { + state->result_status = rc; clockcache_dec_ref(state->cc, state->entry_number, state->tid); state->succeeded = TRUE; async_return(state); @@ -3078,12 +3083,12 @@ clockcache_writeback_page(clockcache *cc, state = clockcache_io_state_acquire(cc); state->cc = cc; - status = io_async_state_init(state->iostate, - cc->io, - io_async_pwritev, - addr, - clockcache_write_callback, - state); + status = io_async_state_init_extent(state->iostate, + cc->io, + io_async_pwritev, + addr, + clockcache_write_callback, + state); if (!SUCCESS(status)) { platform_error_log("clockcache_writeback_page: io_async_state_init " "failed for addr %lu, entry %u, type %u: %s\n", @@ -3173,15 +3178,16 @@ clockcache_writeback_extent(clockcache *cc, } if (state == NULL) { - req_addr = page_addr; - state = clockcache_io_state_acquire(cc); - state->cc = cc; - platform_status rc = io_async_state_init(state->iostate, - cc->io, - io_async_pwritev, - req_addr, - clockcache_write_callback, - state); + req_addr = page_addr; + state = clockcache_io_state_acquire(cc); + state->cc = cc; + platform_status rc = + io_async_state_init_extent(state->iostate, + cc->io, + io_async_pwritev, + req_addr, + clockcache_write_callback, + state); if (!SUCCESS(rc)) { platform_error_log("clockcache_writeback_extent: " "io_async_state_init failed for extent addr " @@ -3582,12 +3588,12 @@ clockcache_prefetch_pages(clockcache *cc, } state->cc = cc; platform_status rc = - io_async_state_init(state->iostate, - cc->io, - io_async_preadv, - addr, - clockcache_prefetch_callback, - state); + io_async_state_init_extent(state->iostate, + cc->io, + io_async_preadv, + addr, + clockcache_prefetch_callback, + state); if (!SUCCESS(rc)) { platform_error_log("clockcache_prefetch_pages: " "io_async_state_init failed for first " @@ -4112,7 +4118,7 @@ clockcache_get_async_state_init_virtual(void *payload, // the waiter on the wait queue, because the async wait system has internal // instrumentation that can catch double enqueues, which could break if we // re-init before submission. - async_waiter_init(&state->wait_node); + async_waiter_init(&state->work.wait_node); } static async_status diff --git a/src/platform_linux/async.h b/src/platform_linux/async.h index bbd6dd31a..e0204d599 100644 --- a/src/platform_linux/async.h +++ b/src/platform_linux/async.h @@ -823,7 +823,6 @@ async_call_sync_callback_function(void *arg) #define DEFINE_ASYNC_STATE(name, height, ...) \ _Static_assert(0 < height, "height must be greater than 0"); \ typedef struct name { \ - int __async_state_lock; \ async_state __async_state_stack[height]; \ DEFINE_STATE_STRUCT_FIELDS(__VA_ARGS__) \ } name; \ diff --git a/src/platform_linux/laio.c b/src/platform_linux/laio.c index 054c75776..4f331862b 100644 --- a/src/platform_linux/laio.c +++ b/src/platform_linux/laio.c @@ -502,19 +502,26 @@ typedef struct laio_async_state { void *callback_arg; async_waiter waiter_node; io_process_context *pctx; - platform_status rc; struct iocb req; - struct iocb *reqs[1]; long status; uint64 iovlen; + uint64 iov_capacity; struct iovec iovs[]; } laio_async_state; -_Static_assert( - sizeof(laio_async_state) <= IO_ASYNC_STATE_BUFFER_SIZE, - "laio_async_read_state is to large for IO_ASYNC_STATE_BUFFER_SIZE"); -_Static_assert(_Alignof(laio_async_state) <= _Alignof(io_async_state_buffer), - "io_async_state_buffer has insufficient alignment"); +_Static_assert(sizeof(laio_async_state) + sizeof(struct iovec) + <= IO_ASYNC_STATE_PAGE_BUFFER_SIZE, + "page I/O state buffer is too small"); +_Static_assert(sizeof(laio_async_state) + + IO_DEFAULT_PAGES_PER_EXTENT * sizeof(struct iovec) + <= IO_ASYNC_STATE_EXTENT_BUFFER_SIZE, + "extent I/O state buffer is too small"); +_Static_assert(_Alignof(laio_async_state) + <= _Alignof(io_async_state_page_buffer), + "page I/O state buffer has insufficient alignment"); +_Static_assert(_Alignof(laio_async_state) + <= _Alignof(io_async_state_extent_buffer), + "extent I/O state buffer has insufficient alignment"); static void laio_async_state_deinit(io_async_state *ios) @@ -525,15 +532,12 @@ static platform_status laio_async_state_append_page(io_async_state *ios, void *buf) { laio_async_state *lios = (laio_async_state *)ios; - uint64 pages_per_extent = - lios->io->cfg->extent_size / lios->io->cfg->page_size; - - if (lios->iovlen == pages_per_extent) { + if (lios->iovlen >= lios->iov_capacity) { platform_error_log("laio_async_state_append_page: request for addr %lu " "already has %lu pages, limit %lu\n", lios->addr, lios->iovlen, - pages_per_extent); + lios->iov_capacity); return STATUS_LIMIT_EXCEEDED; } @@ -645,7 +649,7 @@ laio_async_run(io_async_state *gios) // ({ // async_yield_if( // ios, - // (submit_status = io_submit(ios->pctx->ctx, 1, ios->reqs)) == 1); + // (submit_status = io_submit(ios->pctx->ctx, 1, &req)) == 1); // submit_status != EAGAIN; // }), // ios, @@ -663,8 +667,8 @@ laio_async_run(io_async_state *gios) } io_process_context *pctx = ios->pctx; - - submit_status = io_submit(pctx->ctx, 1, ios->reqs); + struct iocb *req = &ios->req; + submit_status = io_submit(pctx->ctx, 1, &req); if (submit_status == 1) { // Successfully submitted, which means that our state was stored on the @@ -763,6 +767,7 @@ static io_async_state_ops laio_async_state_ops = { static platform_status laio_async_state_init(io_async_state *state, + uint64 buffer_size, io_handle *gio, io_async_cmd cmd, uint64 addr, @@ -772,6 +777,12 @@ laio_async_state_init(io_async_state *state, laio_async_state *ios = (laio_async_state *)state; laio_handle *io = (laio_handle *)gio; + if (buffer_size < sizeof(*ios) + sizeof(struct iovec)) { + return STATUS_BAD_PARAM; + } + uint64 pages_per_extent = io->cfg->extent_size / io->cfg->page_size; + uint64 capacity = (buffer_size - sizeof(*ios)) / sizeof(struct iovec); + ios->super.ops = &laio_async_state_ops; ios->__async_state_stack[0] = ASYNC_STATE_INIT; ios->io = io; @@ -780,9 +791,9 @@ laio_async_state_init(io_async_state *state, ios->callback = callback; ios->callback_arg = callback_arg; async_waiter_init(&ios->waiter_node); - ios->reqs[0] = &ios->req; - ios->iovlen = 0; - ios->status = 0; + ios->iovlen = 0; + ios->iov_capacity = MIN(capacity, pages_per_extent); + ios->status = 0; return STATUS_OK; } @@ -1108,7 +1119,7 @@ laio_config_valid(io_config *cfg) } uint64 pages_per_extent = cfg->extent_size / cfg->page_size; - if (IO_ASYNC_STATE_BUFFER_SIZE + if (IO_ASYNC_STATE_EXTENT_BUFFER_SIZE < sizeof(laio_async_state) + pages_per_extent * sizeof(struct iovec)) { return STATUS_BAD_PARAM; diff --git a/src/platform_linux/platform_io.h b/src/platform_linux/platform_io.h index f279e34a0..aef94e5b5 100644 --- a/src/platform_linux/platform_io.h +++ b/src/platform_linux/platform_io.h @@ -70,13 +70,19 @@ typedef platform_status (*io_range_is_readable_fn)(io_handle *io, uint64 bytes, bool32 *readable); -#define IO_ASYNC_STATE_BUFFER_SIZE (1024) -/* Backends store their native async state in this opaque byte buffer. Its - * alignment must hold when embedded after a smaller field in another state. */ -typedef uint8 io_async_state_buffer[IO_ASYNC_STATE_BUFFER_SIZE] +#define IO_ASYNC_STATE_PAGE_BUFFER_SIZE (208) +#define IO_ASYNC_STATE_EXTENT_BUFFER_SIZE (704) +/* Both buffers hold the same backend state, with room for one page or a full + * extent of iovecs. Preserve alignment when embedded in another structure. */ +typedef uint8 io_async_state_page_buffer[IO_ASYNC_STATE_PAGE_BUFFER_SIZE] + __attribute__((aligned(_Alignof(max_align_t)))); +typedef uint8 io_async_state_extent_buffer[IO_ASYNC_STATE_EXTENT_BUFFER_SIZE] __attribute__((aligned(_Alignof(max_align_t)))); typedef enum { io_async_preadv, io_async_pwritev } io_async_cmd; +/* The backend must reject storage too small for one iovec and bound subsequent + * appends by both buffer_size and the configured pages per extent. */ typedef platform_status (*io_async_state_init_fn)(io_async_state *state, + uint64 buffer_size, io_handle *io, io_async_cmd cmd, uint64 addr, @@ -177,48 +183,72 @@ io_write(io_handle *io, void *buf, uint64 bytes, uint64 addr) } static inline platform_status -io_async_state_init(io_async_state_buffer buffer, - io_handle *io, - io_async_cmd cmd, - uint64 addr, - async_callback_fn callback, - void *callback_arg) +io_async_state_init_page(io_async_state_page_buffer buffer, + io_handle *io, + io_async_cmd cmd, + uint64 addr, + async_callback_fn callback, + void *callback_arg) +{ + io_async_state *state = (io_async_state *)buffer; + return io->ops->async_state_init(state, + IO_ASYNC_STATE_PAGE_BUFFER_SIZE, + io, + cmd, + addr, + callback, + callback_arg); +} + +static inline platform_status +io_async_state_init_extent(io_async_state_extent_buffer buffer, + io_handle *io, + io_async_cmd cmd, + uint64 addr, + async_callback_fn callback, + void *callback_arg) { io_async_state *state = (io_async_state *)buffer; - return io->ops->async_state_init( - state, io, cmd, addr, callback, callback_arg); + return io->ops->async_state_init(state, + IO_ASYNC_STATE_EXTENT_BUFFER_SIZE, + io, + cmd, + addr, + callback, + callback_arg); } +/* The remaining operations accept either initialized buffer type. */ static inline void -io_async_state_deinit(io_async_state_buffer buffer) +io_async_state_deinit(void *buffer) { io_async_state *state = (io_async_state *)buffer; return state->ops->deinit(state); } static inline platform_status -io_async_state_append_page(io_async_state_buffer buffer, void *buf) +io_async_state_append_page(void *buffer, void *buf) { io_async_state *state = (io_async_state *)buffer; return state->ops->append_page(state, buf); } static inline const struct iovec * -io_async_state_get_iovec(io_async_state_buffer buffer, uint64 *iovlen) +io_async_state_get_iovec(void *buffer, uint64 *iovlen) { io_async_state *state = (io_async_state *)buffer; return state->ops->get_iovec(state, iovlen); } static inline async_status -io_async_run(io_async_state_buffer buffer) +io_async_run(void *buffer) { io_async_state *state = (io_async_state *)buffer; return state->ops->run(state); } static inline platform_status -io_async_state_get_result(io_async_state_buffer buffer) +io_async_state_get_result(void *buffer) { io_async_state *state = (io_async_state *)buffer; return state->ops->get_result(state); diff --git a/src/routing_filter.c b/src/routing_filter.c index 8a4149636..26296e0cd 100644 --- a/src/routing_filter.c +++ b/src/routing_filter.c @@ -1385,56 +1385,56 @@ routing_filter_estimate_unique_fp(cache *cc, static inline async_status routing_get_header_async(routing_filter_lookup_async_state *state, uint64 depth) { - async_begin(state, depth); + // Recompute these locals on every invocation, before the resume jump. + const uint64 page_size = state->cfg->page_size; + const uint64 addrs_per_page = page_size / sizeof(uint64); - state->page_size = state->cfg->page_size; - state->addrs_per_page = state->page_size / sizeof(uint64); - if (state->addrs_per_page == 0) { + async_begin(state, depth); + if (addrs_per_page == 0) { async_return(state, STATUS_INVALID_STATE); } - uint64 index_page_no = state->index / state->addrs_per_page; - if (index_page_no > (UINT64_MAX - state->filter.addr) / state->page_size) { + uint64 index_page_no = state->index / addrs_per_page; + if (index_page_no > (UINT64_MAX - state->filter.addr) / page_size) { async_return(state, STATUS_INVALID_STATE); } - state->index_addr = state->filter.addr + state->page_size * index_page_no; + uint64 index_addr = state->filter.addr + page_size * index_page_no; async_await_call(state, cache_get_async, &state->cache_get_state, state->cc, - state->index_addr, + index_addr, PAGE_TYPE_FILTER, state->callback, state->callback_arg); { - platform_status rc = cache_get_async_state_result(&state->cache_get_state, - &state->index_page); + platform_status rc = + cache_get_async_state_result(&state->cache_get_state, &state->page); if (!SUCCESS(rc)) { async_return(state, rc); } } - state->hdr_raw_addr = ((uint64 *)page_handle_data( - &state->index_page))[state->index % state->addrs_per_page]; - cache_unget(state->cc, &state->index_page); - state->index_page = (page_handle){0}; + state->hdr_raw_addr = + ((uint64 *)page_handle_data(&state->page))[state->index % addrs_per_page]; + cache_unget(state->cc, &state->page); + state->page = (page_handle){0}; if (state->hdr_raw_addr == 0 || state->hdr_raw_addr % sizeof(uint32) != 0) { async_return(state, STATUS_INVALID_STATE); } - state->header_addr = - state->hdr_raw_addr - (state->hdr_raw_addr % state->page_size); + uint64 header_addr = state->hdr_raw_addr - (state->hdr_raw_addr % page_size); async_await_call(state, cache_get_async, &state->cache_get_state, state->cc, - state->header_addr, + header_addr, PAGE_TYPE_FILTER, state->callback, state->callback_arg); { - platform_status rc = cache_get_async_state_result(&state->cache_get_state, - &state->filter_page); + platform_status rc = + cache_get_async_state_result(&state->cache_get_state, &state->page); if (!SUCCESS(rc)) { async_return(state, rc); } @@ -1467,7 +1467,6 @@ routing_filter_lookup_async(routing_filter_lookup_async_state *state) state->fp = data_key_hash( state->cfg->data_cfg, state->target, state->filter.hash_seed); state->fp >>= 32 - state->filter.fingerprint_size; - state->remainder_size = params.remainder_size; state->index = routing_get_index(state->fp << state->filter.value_size, params.index_remainder_and_value_size); if (state->index >= params.num_indices) { @@ -1483,13 +1482,13 @@ routing_filter_lookup_async(routing_filter_lookup_async_state *state) * before using the header page returned by the subroutine. */ rc = routing_filter_get_params(state->cfg, &state->filter, ¶ms); if (!SUCCESS(rc)) { - routing_unget_header(state->cc, &state->filter_page); + routing_unget_header(state->cc, &state->page); async_return(state, rc); } routing_header_view view; rc = routing_header_view_init(state->cfg, 1U << state->filter.log_index_size, - &state->filter_page, + &state->page, state->hdr_raw_addr, params.remainder_and_value_size, &view); @@ -1500,7 +1499,7 @@ routing_filter_lookup_async(routing_filter_lookup_async_state *state) &view, state->found_values); } - routing_unget_header(state->cc, &state->filter_page); + routing_unget_header(state->cc, &state->page); async_return(state, rc); } diff --git a/src/routing_filter.h b/src/routing_filter.h index dd3f93c9a..7cca96ef9 100644 --- a/src/routing_filter.h +++ b/src/routing_filter.h @@ -239,17 +239,10 @@ DEFINE_ASYNC_STATE(routing_filter_lookup_async_state, 2, param, void *, callback_arg, local, platform_status, __async_result, local, uint32, fp, - local, uint32, remainder_size, - local, uint32, bucket, local, uint32, index, - local, routing_hdr *, hdr, - local, page_handle, filter_page, - local, uint64, page_size, - local, uint64, addrs_per_page, - local, uint64, index_addr, + // Reused for the index page, then the header page after releasing the index. + local, page_handle, page, local, uint64, hdr_raw_addr, - local, uint64, header_addr, - local, page_handle, index_page, local, page_get_async_state_buffer, cache_get_state) // clang-format on diff --git a/src/trunk.c b/src/trunk.c index 932eab691..b27d7a7a7 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -1099,7 +1099,7 @@ trunk_ondisk_node_handle_init(trunk_ondisk_node_handle *handle, * - state->pivot->child_addr: the address of the node * * OUT Parameters: - * - state->child_handle: the ondisk_node_handle + * - state->work.node.child_handle: the ondisk_node_handle * - state->rc: the return code */ static async_status @@ -1108,37 +1108,43 @@ trunk_ondisk_node_handle_init_async(trunk_merge_lookup_async_state *state, { async_begin(state, depth); - state->child_handle.cc = state->context->cc; - state->child_handle.page_size = cache_page_size(state->context->cc); - state->child_handle.extent_size = cache_extent_size(state->context->cc); - state->child_handle.header_page = (page_handle){0}; - state->child_handle.pivot_page = (page_handle){0}; - state->child_handle.inflight_bundle_page = (page_handle){0}; + state->work.node.child_handle.cc = state->context->cc; + state->work.node.child_handle.page_size = + cache_page_size(state->context->cc); + state->work.node.child_handle.extent_size = + cache_extent_size(state->context->cc); + state->work.node.child_handle.header_page = (page_handle){0}; + state->work.node.child_handle.pivot_page = (page_handle){0}; + state->work.node.child_handle.inflight_bundle_page = (page_handle){0}; async_await_call(state, cache_get_async, - &state->cache_get_state, + &state->work.node.cache_get_state, state->context->cc, state->pivot->child_addr, PAGE_TYPE_TRUNK, state->callback, state->callback_arg); - state->rc = cache_get_async_state_result(&state->cache_get_state, - &state->child_handle.header_page); + state->rc = + cache_get_async_state_result(&state->work.node.cache_get_state, + &state->work.node.child_handle.header_page); if (!SUCCESS(state->rc)) { async_return(state); } - state->rc = trunk_ondisk_node_read_header(&state->child_handle); + state->rc = trunk_ondisk_node_read_header(&state->work.node.child_handle); if (!SUCCESS(state->rc)) { - cache_unget(state->context->cc, &state->child_handle.header_page); - state->child_handle.header_page = (page_handle){0}; + cache_unget(state->context->cc, + &state->work.node.child_handle.header_page); + state->work.node.child_handle.header_page = (page_handle){0}; async_return(state); } - state->rc = state->height == 0 ? STATUS_INVALID_STATE - : trunk_ondisk_node_check_expected_height( - &state->child_handle, state->height - 1); + state->rc = state->height == 0 + ? STATUS_INVALID_STATE + : trunk_ondisk_node_check_expected_height( + &state->work.node.child_handle, state->height - 1); if (!SUCCESS(state->rc)) { - cache_unget(state->context->cc, &state->child_handle.header_page); - state->child_handle.header_page = (page_handle){0}; + cache_unget(state->context->cc, + &state->work.node.child_handle.header_page); + state->work.node.child_handle.header_page = (page_handle){0}; async_return(state); } async_return(state); @@ -1248,16 +1254,16 @@ trunk_ondisk_node_handle_setup_content_page(trunk_ondisk_node_handle *handle, /* * IN Parameters: * - state->handlep: the ondisk_node_handle - * - state->offset: the offset of the page to get + * - state->work.node.offset: the offset of the page to get * * IN/OUT Parameters: - * - state->page: Pointer to the page pointer in the handle to set up. + * - state->work.node.page: Pointer to the page pointer in the handle to set up. * * OUT Parameters: * - state->rc: the return code * * LOCAL Variables: - * - state->cache_get_state: the state of the cache_get() operation + * - state->work.node.cache_get_state: the state of the cache_get() operation */ static async_status trunk_ondisk_node_handle_setup_content_page_async( @@ -1268,41 +1274,44 @@ trunk_ondisk_node_handle_setup_content_page_async( uint64 page_size = state->handlep->page_size; - if (state->handlep->extent_size <= state->offset) { + if (state->handlep->extent_size <= state->work.node.offset) { state->rc = STATUS_INVALID_STATE; async_return(state); } - if (offset_is_in_content_page(state->handlep, state->page, state->offset)) { + if (offset_is_in_content_page( + state->handlep, state->work.node.page, state->work.node.offset)) + { state->rc = STATUS_OK; async_return(state); } - if (!page_handle_is_null(state->page) - && !page_handle_same_page(state->page, &state->handlep->header_page)) + if (!page_handle_is_null(state->work.node.page) + && !page_handle_same_page(state->work.node.page, + &state->handlep->header_page)) { - cache_unget(state->handlep->cc, state->page); - *state->page = (page_handle){0}; + cache_unget(state->handlep->cc, state->work.node.page); + *state->work.node.page = (page_handle){0}; } - if (state->offset < page_size) { - *state->page = state->handlep->header_page; - state->rc = STATUS_OK; + if (state->work.node.offset < page_size) { + *state->work.node.page = state->handlep->header_page; + state->rc = STATUS_OK; async_return(state); } else { - uint64 addr = - page_handle_addr(&state->handlep->header_page) + state->offset; + uint64 addr = page_handle_addr(&state->handlep->header_page) + + state->work.node.offset; addr -= (addr % page_size); async_await_call(state, cache_get_async, - &state->cache_get_state, + &state->work.node.cache_get_state, state->handlep->cc, addr, PAGE_TYPE_TRUNK, state->callback, state->callback_arg); - state->rc = - cache_get_async_state_result(&state->cache_get_state, state->page); + state->rc = cache_get_async_state_result( + &state->work.node.cache_get_state, state->work.node.page); if (!SUCCESS(state->rc)) { async_return(state); } @@ -1429,16 +1438,16 @@ trunk_ondisk_node_read_pivot(trunk_ondisk_node_handle *handle, /* * IN Parameters: * - state->handlep: the ondisk_node_handle - * - state->pivot_num: the pivot number to get + * - state->work.node.pivot_num: the pivot number to get * * OUT Parameters: * - state->pivot: the pivot * - state->rc: the return code * * LOCAL Variables: - * - state->offset: the offset of the pivot - * - state->page: Pointer to the page pointer in the handle to set up. - * - state->cache_get_state: the state of the cache_get() operation + * - state->work.node.offset: the offset of the pivot + * - state->work.node.page: Pointer to the page pointer in the handle to set up. + * - state->work.node.cache_get_state: the state of the cache_get() operation */ static async_status trunk_ondisk_node_read_pivot_key_async(trunk_merge_lookup_async_state *state, @@ -1447,12 +1456,12 @@ trunk_ondisk_node_read_pivot_key_async(trunk_merge_lookup_async_state *state, async_begin(state, depth); state->rc = trunk_ondisk_node_read_pivot_offset( - state->handlep, state->pivot_num, &state->offset); + state->handlep, state->work.node.pivot_num, &state->work.node.offset); if (!SUCCESS(state->rc)) { state->pivot = NULL; async_return(state); } - state->page = &state->handlep->pivot_page; + state->work.node.page = &state->handlep->pivot_page; async_await_subroutine(state, trunk_ondisk_node_handle_setup_content_page_async); if (!SUCCESS(state->rc)) { @@ -1466,11 +1475,11 @@ trunk_ondisk_node_read_pivot_key_async(trunk_merge_lookup_async_state *state, } state->pivot = (trunk_ondisk_pivot *)(page_handle_data(&state->handlep->pivot_page) - + state->offset + + state->work.node.offset - content_page_offset( state->handlep, &state->handlep->pivot_page)); uint64 page_offset = - state->offset + state->work.node.offset - content_page_offset(state->handlep, &state->handlep->pivot_page); uint64 page_size = state->handlep->page_size; if (page_size - page_offset < sizeof(*state->pivot) @@ -1665,15 +1674,15 @@ trunk_ondisk_node_bundle_at_offset(trunk_ondisk_node_handle *handle, /* * IN Parameters: * - state->handlep: the ondisk_node_handle - * - state->offset: the offset of the bundle + * - state->work.node.offset: the offset of the bundle * * OUT Parameters: * - state->bndl: the bundle * - state->rc: the return code * * LOCAL Variables: - * - state->page: Pointer to the page pointer in the handle to set up. - * - state->cache_get_state: the state of the cache_get() operation + * - state->work.node.page: Pointer to the page pointer in the handle to set up. + * - state->work.node.cache_get_state: the state of the cache_get() operation */ static async_status trunk_ondisk_node_bundle_at_offset_async(trunk_merge_lookup_async_state *state, @@ -1684,7 +1693,7 @@ trunk_ondisk_node_bundle_at_offset_async(trunk_merge_lookup_async_state *state, async_begin(state, depth); - if (extent_size <= state->offset) { + if (extent_size <= state->work.node.offset) { state->bndl = NULL; state->rc = STATUS_INVALID_STATE; async_return(state); @@ -1692,16 +1701,19 @@ trunk_ondisk_node_bundle_at_offset_async(trunk_merge_lookup_async_state *state, /* If there's not enough room for a bundle header, skip to the next * page. */ - if (page_size - (state->offset % page_size) < sizeof(trunk_ondisk_bundle)) { - state->offset += page_size - (state->offset % page_size); - if (extent_size <= state->offset) { + if (page_size - (state->work.node.offset % page_size) + < sizeof(trunk_ondisk_bundle)) + { + state->work.node.offset += + page_size - (state->work.node.offset % page_size); + if (extent_size <= state->work.node.offset) { state->bndl = NULL; state->rc = STATUS_INVALID_STATE; async_return(state); } } - state->page = &state->handlep->inflight_bundle_page; + state->work.node.page = &state->handlep->inflight_bundle_page; async_await_subroutine(state, trunk_ondisk_node_handle_setup_content_page_async); if (!SUCCESS(state->rc)) { @@ -1716,12 +1728,12 @@ trunk_ondisk_node_bundle_at_offset_async(trunk_merge_lookup_async_state *state, state->bndl = (trunk_ondisk_bundle *)(page_handle_data( &state->handlep->inflight_bundle_page) - + state->offset + + state->work.node.offset - content_page_offset( state->handlep, &state->handlep->inflight_bundle_page)); uint64 page_offset = - state->offset + state->work.node.offset - content_page_offset(state->handlep, &state->handlep->inflight_bundle_page); if (page_size - page_offset < sizeof(*state->bndl)) { @@ -1733,13 +1745,14 @@ trunk_ondisk_node_bundle_at_offset_async(trunk_merge_lookup_async_state *state, /* If there wasn't enough room for this bundle on this page, then we would * have zeroed the remaining bytes and put the bundle on the next page. */ if (state->bndl->num_branches == 0) { - state->offset += page_size - (state->offset % page_size); - if (extent_size <= state->offset) { + state->work.node.offset += + page_size - (state->work.node.offset % page_size); + if (extent_size <= state->work.node.offset) { state->bndl = NULL; state->rc = STATUS_INVALID_STATE; async_return(state); } - state->page = &state->handlep->inflight_bundle_page; + state->work.node.page = &state->handlep->inflight_bundle_page; async_await_subroutine(state, trunk_ondisk_node_handle_setup_content_page_async); if (!SUCCESS(state->rc)) { @@ -1754,11 +1767,11 @@ trunk_ondisk_node_bundle_at_offset_async(trunk_merge_lookup_async_state *state, state->bndl = (trunk_ondisk_bundle *)(page_handle_data( &state->handlep->inflight_bundle_page) - + state->offset + + state->work.node.offset - content_page_offset( state->handlep, &state->handlep->inflight_bundle_page)); - page_offset = state->offset + page_offset = state->work.node.offset - content_page_offset( state->handlep, &state->handlep->inflight_bundle_page); } @@ -1806,9 +1819,9 @@ trunk_ondisk_node_get_first_inflight_bundle(trunk_ondisk_node_handle *handle, * - state->rc: the return code * * LOCAL Variables: - * - state->offset: the offset of the bundle - * - state->page: Pointer to the page pointer in the handle to set up. - * - state->cache_get_state: the state of the cache_get() operation + * - state->work.node.offset: the offset of the bundle + * - state->work.node.page: Pointer to the page pointer in the handle to set up. + * - state->work.node.cache_get_state: the state of the cache_get() operation */ static async_status trunk_ondisk_node_get_first_inflight_bundle_async( @@ -1825,12 +1838,14 @@ trunk_ondisk_node_get_first_inflight_bundle_async( : STATUS_INVALID_STATE; async_return(state); } - state->offset = header->inflight_bundles_offset; + state->work.node.offset = header->inflight_bundles_offset; uint64 table_end = sizeof(*header) + (uint64)header->num_pivots * sizeof(header->pivot_offsets[0]); - if (state->offset < table_end || state->handlep->extent_size <= state->offset - || state->offset <= header->pivot_offsets[header->num_pivots - 1]) + if (state->work.node.offset < table_end + || state->handlep->extent_size <= state->work.node.offset + || state->work.node.offset + <= header->pivot_offsets[header->num_pivots - 1]) { state->bndl = NULL; state->rc = STATUS_INVALID_STATE; @@ -1864,9 +1879,9 @@ trunk_ondisk_node_get_next_inflight_bundle(trunk_ondisk_node_handle *handle, * - state->rc: the return code * * LOCAL Variables: - * - state->offset: the offset of the bundle - * - state->page: Pointer to the page pointer in the handle to set up. - * - state->cache_get_state: the state of the cache_get() operation + * - state->work.node.offset: the offset of the bundle + * - state->work.node.page: Pointer to the page pointer in the handle to set up. + * - state->work.node.cache_get_state: the state of the cache_get() operation */ static async_status trunk_ondisk_node_get_next_inflight_bundle_async( @@ -1874,11 +1889,12 @@ trunk_ondisk_node_get_next_inflight_bundle_async( uint64 depth) { async_begin(state, depth); - state->offset = ((char *)state->bndl) - - page_handle_data(&state->handlep->inflight_bundle_page) - + content_page_offset(state->handlep, - &state->handlep->inflight_bundle_page) - + sizeof_trunk_ondisk_bundle(state->bndl); + state->work.node.offset = + ((char *)state->bndl) + - page_handle_data(&state->handlep->inflight_bundle_page) + + content_page_offset(state->handlep, + &state->handlep->inflight_bundle_page) + + sizeof_trunk_ondisk_bundle(state->bndl); async_await_subroutine(state, trunk_ondisk_node_bundle_at_offset_async); async_return(state); } @@ -7089,22 +7105,18 @@ trunk_ondisk_node_find_pivot(const trunk_context *context, * state->context: the trunk node context * state->handlep: the ondisk node handle * state->tgt: the target key - * //state->cmp: the comparison to use * * OUT Parameters: * state->pivot: the pivot found * state->rc: the return code * * LOCAL Variables: - * state->min: the minimum pivot index - * state->max: the maximum pivot index - * state->min_pivot: the minimum pivot found - * state->last_cmp: the last comparison result - * state->mid: the mid pivot index - * state->pivot_num: the pivot number - * state->offset: the offset - * state->page: the page - * state->cache_get_state: the cache get state + * state->work.node.min: the minimum pivot index + * state->work.node.max: the maximum pivot index + * state->work.node.pivot_num: the pivot number + * state->work.node.offset: the offset + * state->work.node.page: the page + * state->work.node.cache_get_state: the cache get state */ static async_status trunk_ondisk_node_find_pivot_async(trunk_merge_lookup_async_state *state, @@ -7112,14 +7124,13 @@ trunk_ondisk_node_find_pivot_async(trunk_merge_lookup_async_state *state, { async_begin(state, depth); - state->min = 0; - state->max = trunk_ondisk_node_num_pivots(state->handlep) - 1; + state->work.node.min = 0; + state->work.node.max = trunk_ondisk_node_num_pivots(state->handlep) - 1; // invariant: pivot[min] <= tgt < pivot[max] - state->min_pivot = NULL; - while (state->min + 1 < state->max) { - state->mid = (state->min + state->max) / 2; - state->pivot_num = state->mid; + while (state->work.node.min + 1 < state->work.node.max) { + state->work.node.pivot_num = + (state->work.node.min + state->work.node.max) / 2; async_await_subroutine(state, trunk_ondisk_node_read_pivot_key_async); if (!SUCCESS(state->rc)) { platform_error_log("ondisk_node_find_pivot_async: " @@ -7131,39 +7142,25 @@ trunk_ondisk_node_find_pivot_async(trunk_merge_lookup_async_state *state, int cmp = data_key_compare(state->context->cfg->data_cfg, state->tgt, mid_key); if (cmp < 0) { - state->max = state->mid; + state->work.node.max = state->work.node.pivot_num; } else { - state->min = state->mid; - state->min_pivot = state->pivot; - state->last_cmp = cmp; + state->work.node.min = state->work.node.pivot_num; } } - /* 0 < min means we executed the loop at least once. - last_cmp == 0 means we found an exact match at pivot[mid], and we then - assigned mid to min, which means that pivot[min] == tgt. - */ - // if (0 < state->min && state->last_cmp == 0 && state->cmp == less_than) { - // state->min--; - // state->min_pivot = ondisk_node_get_pivot(state->handlep, state->min); - // } - /* As in the synchronous path, reload the winner so state->pivot cannot * outlive the content page that supplied it. */ - state->pivot_num = state->min; + state->work.node.pivot_num = state->work.node.min; async_await_subroutine(state, trunk_ondisk_node_read_pivot_key_async); if (!SUCCESS(state->rc)) { - state->min_pivot = NULL; async_return(state); } state->rc = trunk_ondisk_node_read_pivot_fields( - state->handlep, state->pivot_num, state->pivot); + state->handlep, state->work.node.pivot_num, state->pivot); if (!SUCCESS(state->rc)) { - state->pivot = NULL; - state->min_pivot = NULL; + state->pivot = NULL; async_return(state); } - state->min_pivot = state->pivot; - state->rc = STATUS_OK; + state->rc = STATUS_OK; async_return(state); } @@ -7333,19 +7330,19 @@ trunk_ondisk_bundle_merge_lookup_async(trunk_merge_lookup_async_state *state, state->rc = STATUS_INVALID_STATE; async_return(state); } - state->found_values = state->bndl->num_branches == 1 ? 1 : 0; + state->work.bundle.found_values = state->bndl->num_branches == 1 ? 1 : 0; } else { async_await_call(state, routing_filter_lookup_async, - &state->filter_state, + &state->work.bundle.filter_state, state->context->cc, state->context->cfg->filter_cfg, state->bndl->maplet, state->tgt, - &state->found_values, + &state->work.bundle.found_values, state->callback, state->callback_arg); - state->rc = async_result(&state->filter_state); + state->rc = async_result(&state->work.bundle.filter_state); if (!SUCCESS(state->rc)) { platform_error_log("ondisk_bundle_merge_lookup_async: " "routing_filter_lookup_async failed: %d\n", @@ -7353,7 +7350,7 @@ trunk_ondisk_bundle_merge_lookup_async(trunk_merge_lookup_async_state *state, async_return(state); } - if (state->found_values != 0 + if (state->work.bundle.found_values != 0 && !lookup_result_note_filter_hit(state->result) && state->log == NULL) { @@ -7365,37 +7362,39 @@ trunk_ondisk_bundle_merge_lookup_async(trunk_merge_lookup_async_state *state, } } - state->rc = trunk_ondisk_bundle_validate_found_values(state->bndl, - state->found_values); + state->rc = trunk_ondisk_bundle_validate_found_values( + state->bndl, state->work.bundle.found_values); if (!SUCCESS(state->rc)) { async_return(state); } if (state->log) { platform_log(state->log, "maplet: %lu\n", state->bndl->maplet.addr); - platform_log(state->log, "found_values: %lu\n", state->found_values); - state->found_values = (1ULL << state->bndl->num_branches) - 1; + platform_log( + state->log, "found_values: %lu\n", state->work.bundle.found_values); + state->work.bundle.found_values = (1ULL << state->bndl->num_branches) - 1; } - for (state->idx = routing_filter_get_next_value(state->found_values, - ROUTING_NOT_FOUND); - state->idx != ROUTING_NOT_FOUND; - state->idx = - routing_filter_get_next_value(state->found_values, state->idx)) + for (state->work.bundle.idx = routing_filter_get_next_value( + state->work.bundle.found_values, ROUTING_NOT_FOUND); + state->work.bundle.idx != ROUTING_NOT_FOUND; + state->work.bundle.idx = routing_filter_get_next_value( + state->work.bundle.found_values, state->work.bundle.idx)) { - async_await_call(state, - btree_lookup_and_merge_async, - &state->btree_state, - state->context->cc, - state->context->cfg->btree_cfg, - branch_ref_addr(state->bndl->branches[state->idx]), - state->bndl->branches[state->idx].format_version, - trunk_ondisk_bundle_branch_type(state->bndl), - state->tgt, - state->result, - state->callback, - state->callback_arg); - state->rc = async_result(&state->btree_state); + async_await_call( + state, + btree_lookup_and_merge_async, + &state->work.bundle.btree_state, + state->context->cc, + state->context->cfg->btree_cfg, + branch_ref_addr(state->bndl->branches[state->work.bundle.idx]), + state->bndl->branches[state->work.bundle.idx].format_version, + trunk_ondisk_bundle_branch_type(state->bndl), + state->tgt, + state->result, + state->callback, + state->callback_arg); + state->rc = async_result(&state->work.bundle.btree_state); if (!SUCCESS(state->rc)) { platform_error_log("ondisk_bundle_merge_lookup_async: " "btree_lookup_and_merge_async failed: %d\n", @@ -7405,7 +7404,7 @@ trunk_ondisk_bundle_merge_lookup_async(trunk_merge_lookup_async_state *state, if (state->context->stats) { state->context->stats[tid].branch_lookups[state->height]++; - if (key_is_null(state->btree_state.found_key)) { + if (key_is_null(state->work.bundle.btree_state.work.found_key)) { state->context->stats[tid].maplet_false_positives[state->height]++; } } @@ -7425,8 +7424,8 @@ trunk_ondisk_bundle_merge_lookup_async(trunk_merge_lookup_async_state *state, platform_status rc = btree_lookup_and_merge( state->context->cc, state->context->cfg->btree_cfg, - branch_ref_addr(state->bndl->branches[state->idx]), - state->bndl->branches[state->idx].format_version, + branch_ref_addr(state->bndl->branches[state->work.bundle.idx]), + state->bndl->branches[state->work.bundle.idx].format_version, trunk_ondisk_bundle_branch_type(state->bndl), state->tgt, &temp_result, @@ -7440,10 +7439,11 @@ trunk_ondisk_bundle_merge_lookup_async(trunk_merge_lookup_async_state *state, async_return(state); } bool32 found = lookup_result_found(&temp_result); - platform_log(state->log, - "branch: %lu found: %u\n", - branch_ref_addr(state->bndl->branches[state->idx]), - found); + platform_log( + state->log, + "branch: %lu found: %u\n", + branch_ref_addr(state->bndl->branches[state->work.bundle.idx]), + found); if (found) { message msg = merge_accumulator_to_message( lookup_result_accumulator(&temp_result)); @@ -7638,7 +7638,7 @@ trunk_merge_lookup_async(trunk_merge_lookup_async_state *state) if (state->log) { platform_log(state->log, "pivot_num: %lu pivot: %s\n", - state->min, + state->work.node.min, key_string(state->context->cfg->data_cfg, trunk_ondisk_pivot_key(state->pivot))); } @@ -7711,7 +7711,7 @@ trunk_merge_lookup_async(trunk_merge_lookup_async_state *state) if (state->handlep != state->inhandle) { trunk_ondisk_node_handle_deinit(state->handlep); } - state->handle = state->child_handle; + state->handle = state->work.node.child_handle; state->handlep = &state->handle; } else { if (state->handlep != state->inhandle) { diff --git a/src/trunk.h b/src/trunk.h index 275c167f8..f6669b019 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -374,6 +374,34 @@ trunk_collect_branches(const trunk_context *context, typedef struct trunk_ondisk_pivot trunk_ondisk_pivot; typedef struct trunk_ondisk_bundle trunk_ondisk_bundle; +/* Node navigation and bundle lookup occupy mutually exclusive phases. Within + * navigation, construct a child handle after all pivot/bundle page work is + * done. Within a bundle, consume the filter result before beginning B-tree + * lookup. */ +typedef union trunk_lookup_scratch { + struct { + union { + trunk_ondisk_node_handle child_handle; + struct { + uint64 offset; + page_handle *page; + uint64 pivot_num; + uint64 min; + uint64 max; + }; + }; + page_get_async_state_buffer cache_get_state; + } node; + struct { + uint64 found_values; + uint64 idx; + union { + routing_filter_lookup_async_state filter_state; + btree_lookup_async_state btree_state; + }; + } bundle; +} trunk_lookup_scratch; + // clang-format off DEFINE_ASYNC_STATE(trunk_merge_lookup_async_state, 4, param, trunk_context *, context, @@ -391,26 +419,7 @@ DEFINE_ASYNC_STATE(trunk_merge_lookup_async_state, 4, local, trunk_ondisk_pivot *, pivot, local, uint64, inflight_bundle_num, local, trunk_ondisk_bundle *, bndl, - local, trunk_ondisk_node_handle, child_handle, - // ondisk_node_handle_setup_content_page - // ondisk_node_get_pivot - // ondisk_node_bundle_at_offset - // ondisk_node_get_first_inflight_bundle - local, uint64, offset, - local, page_handle *, page, - local, uint64, pivot_num, - local, page_get_async_state_buffer, cache_get_state, - // ondisk_node_find_pivot - local, uint64, min, - local, uint64, max, - local, uint64, mid, - local, int, last_cmp, - local, trunk_ondisk_pivot *, min_pivot, - // ondisk_bundle_merge_lookup - local, uint64, found_values, - local, uint64, idx, - local, routing_filter_lookup_async_state, filter_state, - local, btree_lookup_async_state, btree_state, + local, trunk_lookup_scratch, work, ) // clang-format on diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index b69782b82..cd54ea9fe 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -2071,6 +2071,7 @@ delayed_prefetch_result(io_async_state *state) static platform_status delayed_prefetch_init(io_async_state *state, + uint64 buffer_size, io_handle *io, io_async_cmd cmd, uint64 addr, @@ -2080,7 +2081,7 @@ delayed_prefetch_init(io_async_state *state, platform_assert(cmd == io_async_preadv); platform_assert(delayed_prefetch.pending == NULL); platform_status rc = delayed_prefetch.saved_ops->async_state_init( - state, io, cmd, addr, callback, callback_arg); + state, buffer_size, io, cmd, addr, callback, callback_arg); if (SUCCESS(rc)) { delayed_prefetch.state_ops = *state->ops; delayed_prefetch.state_ops.run = delayed_prefetch_run; diff --git a/tests/functional/io_apis_test.c b/tests/functional/io_apis_test.c index d21dc6bb4..3ff3f7edc 100644 --- a/tests/functional/io_apis_test.c +++ b/tests/functional/io_apis_test.c @@ -556,8 +556,8 @@ test_readable_ranges(platform_heap_id hid, master_config *master_cfg) /* io_event.res is positive for a short read. The async result API must * nevertheless reject it because the untouched suffix of buf is stale. */ - io_async_state_buffer short_read_state; - rc = io_async_state_init( + io_async_state_page_buffer short_read_state; + rc = io_async_state_init_page( short_read_state, io, io_async_preadv, partial_page, NULL, NULL); platform_assert_status_ok(rc); rc = io_async_state_append_page(short_read_state, buf); @@ -587,9 +587,9 @@ test_readable_ranges(platform_heap_id hid, master_config *master_cfg) platform_assert_status_ok(rc); platform_assert(!readable); // primes the pre-extension cached size again - io_async_state_buffer state; - rc = - io_async_state_init(state, io, io_async_pwritev, async_addr, NULL, NULL); + io_async_state_page_buffer state; + rc = io_async_state_init_page( + state, io, io_async_pwritev, async_addr, NULL, NULL); platform_assert_status_ok(rc); rc = io_async_state_append_page(state, buf); platform_assert_status_ok(rc); @@ -946,10 +946,10 @@ load_thread_params(io_test_fn_args *io_test_param, */ typedef struct async_read_state { - platform_heap_id hid; - uint64 lock; - char *expected; - io_async_state_buffer iostate; + platform_heap_id hid; + uint64 lock; + char *expected; + io_async_state_page_buffer iostate; } async_read_state; static void @@ -1055,12 +1055,12 @@ test_async_reads(platform_heap_id hid, state->lock = 0; state->expected = exp; state->hid = hid; - io_async_state_init(state->iostate, - ioh, - io_async_preadv, - this_addr, - read_async_callback, - state); + io_async_state_init_page(state->iostate, + ioh, + io_async_preadv, + this_addr, + read_async_callback, + state); io_async_state_append_page(state->iostate, buf_addr); async_read_state_lock(state); diff --git a/tests/unit/platform_apis_test.c b/tests/unit/platform_apis_test.c index f03d2a2fe..1528b3f9a 100644 --- a/tests/unit/platform_apis_test.c +++ b/tests/unit/platform_apis_test.c @@ -126,14 +126,106 @@ CTEST2(platform_api, test_platform_buffer_init_fails_for_very_large_length) set_log_streams_for_tests(MSG_LEVEL_INFO); } +/* Exercise both storage capacities with real vectored I/O. The canaries catch + * an append using the extent limit when the caller supplied a page buffer. */ +static void +test_async_io_capacity(platform_heap_id hid, bool32 extent) +{ + char filename[] = "/tmp/splinterdb-async-capacity-XXXXXX"; + int fd = mkstemp(filename); + ASSERT_TRUE(fd >= 0); + ASSERT_EQUAL(0, close(fd)); + io_config cfg; + io_config_init(&cfg, + IO_DEFAULT_PAGE_SIZE, + IO_DEFAULT_EXTENT_SIZE, + IO_DEFAULT_FLAGS, + IO_DEFAULT_PERMS, + IO_DEFAULT_KERNEL_QUEUE_SIZE, + filename); + io_handle *io = io_handle_create(&cfg, hid); + ASSERT_NOT_NULL(io); + ASSERT_EQUAL(0, unlink(filename)); + + struct { + char prefix; + io_async_state_page_buffer state; + unsigned char canary[32]; + } page; + struct { + char prefix; + io_async_state_extent_buffer state; + unsigned char canary[32]; + } full; + memset(page.canary, 0xa5, sizeof(page.canary)); + memset(full.canary, 0xa5, sizeof(full.canary)); + void *state = extent ? (void *)full.state : (void *)page.state; + unsigned char *canary = extent ? full.canary : page.canary; + const uint64 pages = extent ? IO_DEFAULT_PAGES_PER_EXTENT : 1; + const uint64 bytes = pages * IO_DEFAULT_PAGE_SIZE; + buffer_handle buffer; + ASSERT_TRUE(SUCCESS(platform_buffer_init(&buffer, bytes))); + char *buf = platform_buffer_getaddr(&buffer); + + /* An invalid capacity must fail before writing even the backend header. */ + memset(state, 0xa5, extent ? sizeof(full.state) : sizeof(page.state)); + ASSERT_TRUE(STATUS_IS_EQ( + io->ops->async_state_init(state, 0, io, io_async_preadv, 0, NULL, NULL), + STATUS_BAD_PARAM)); + ASSERT_EQUAL(0xa5, ((unsigned char *)state)[0]); + + for (unsigned reading = 0; reading < 2; reading++) { + io_async_cmd cmd = reading ? io_async_preadv : io_async_pwritev; + platform_status rc = + extent ? io_async_state_init_extent(full.state, io, cmd, 0, NULL, NULL) + : io_async_state_init_page(page.state, io, cmd, 0, NULL, NULL); + ASSERT_TRUE(SUCCESS(rc)); + for (uint64 i = 0; i < pages; i++) { + memset(buf + i * IO_DEFAULT_PAGE_SIZE, + reading ? 0 : (int)(i + 1), + IO_DEFAULT_PAGE_SIZE); + ASSERT_TRUE(SUCCESS( + io_async_state_append_page(state, buf + i * IO_DEFAULT_PAGE_SIZE))); + } + ASSERT_TRUE(STATUS_IS_EQ(io_async_state_append_page(state, buf), + STATUS_LIMIT_EXCEEDED)); + uint64 count; + io_async_state_get_iovec(state, &count); + ASSERT_EQUAL(pages, count); + io_async_run(state); + io_wait_all(io); + ASSERT_EQUAL(ASYNC_STATUS_DONE, io_async_run(state)); + ASSERT_TRUE(SUCCESS(io_async_state_get_result(state))); + io_async_state_deinit(state); + for (unsigned i = 0; i < 32; i++) { + ASSERT_EQUAL(0xa5, canary[i]); + } + } + for (uint64 i = 0; i < bytes; i++) { + ASSERT_EQUAL(i / IO_DEFAULT_PAGE_SIZE + 1, (unsigned char)buf[i]); + } + ASSERT_TRUE(SUCCESS(platform_buffer_deinit(&buffer))); + io_handle_destroy(io); +} + +CTEST2(platform_api, test_async_page_io_capacity) +{ + test_async_io_capacity(data->hid, FALSE); +} + +CTEST2(platform_api, test_async_extent_io_capacity) +{ + test_async_io_capacity(data->hid, TRUE); +} + typedef struct async_read_shadow_probe { - io_async_state_buffer state; - char *buffer; - uint64 length; - uint64 initialized_bytes; - bool32 shadow_ok; - bool32 data_ok; - uint64 callbacks; + io_async_state_extent_buffer state; + char *buffer; + uint64 length; + uint64 initialized_bytes; + bool32 shadow_ok; + bool32 data_ok; + uint64 callbacks; } async_read_shadow_probe; /* Inspect shadow before touching the data so the unfixed implementation @@ -215,7 +307,7 @@ test_async_io_shadow(platform_heap_id hid, .length = 3 * page_size, .initialized_bytes = initialized_bytes, }; - ASSERT_TRUE(SUCCESS(io_async_state_init( + ASSERT_TRUE(SUCCESS(io_async_state_init_extent( probe.state, io, cmd, addr, async_read_shadow_callback, &probe))); for (uint64 i = 0; i < 3; i++) { ASSERT_TRUE(SUCCESS( diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 694f55a42..b09e0ca61 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -5891,6 +5891,147 @@ test_splinter_bsearch(trunk_shadow *shadow, key needle) return lo; } +static const cache_ops *async_scratch_saved_ops; +static bool32 async_scratch_yield; +static uint64 async_scratch_gets[NUM_PAGE_TYPES]; + +static void +async_scratch_get_init(void *payload, + cache *cc, + uint64 addr, + page_type type, + async_callback_fn callback, + void *callback_arg) +{ + async_scratch_saved_ops->page_get_async_state_init( + payload, cc, addr, type, callback, callback_arg); + async_scratch_gets[type]++; + async_scratch_yield = TRUE; +} + +static async_status +async_scratch_get(void *payload) +{ + if (async_scratch_yield) { + async_scratch_yield = FALSE; + return ASYNC_STATUS_RUNNING; + } + return async_scratch_saved_ops->page_get_async(payload); +} + +static void +async_scratch_ready(void *arg) +{ +} + +static void +async_scratch_resume(void *arg) +{ + core_lookup_async_state *state = arg; + timestamp start = platform_get_timestamp(); + do { + cache_cleanup(state->spl->cc); + platform_assert(platform_timestamp_elapsed(start) < SEC_TO_NSEC(10)); + } while (core_lookup_async(state) != ASYNC_STATUS_DONE); +} + +/* Force every async page get to yield, including resident hits, and hand the + * suspended lookup to another thread. A multi-level tree exercises navigation, + * filter, B-tree traversal, leaf decoding, and child-handle union transitions. + */ +CTEST2(splinter, test_async_lookup_scratch_transitions) +{ + system_config *cfg = data->system_cfg; + cfg->trunk_node_cfg.incorporation_size_kv_bytes = 64 * KiB; + cfg->trunk_node_cfg.target_fanout = 4; + cfg->trunk_node_cfg.use_stats = TRUE; + cache *cc = (cache *)data->clock_cache; + core_handle spl; + ASSERT_TRUE(SUCCESS(core_mkfs(&spl, + &cfg->splinter_cfg, + (allocator *)&data->al, + cc, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid))); + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + const uint64 per_round = 2048; + const uint64 rounds = 8; + for (uint64 round = 0; round < rounds; round++) { + for (uint64 i = round * per_round; i < (round + 1) * per_round; i++) { + test_key( + &keybuf, TEST_RANDOM, i, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, i, &msg); + ASSERT_TRUE(SUCCESS(core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL))); + } + ASSERT_TRUE(SUCCESS(core_checkpoint(&spl, 0))); + } + merge_accumulator_deinit(&msg); + ASSERT_TRUE(SUCCESS(task_perform_until_quiescent(&data->tasks))); + uint64 height; + ASSERT_TRUE(SUCCESS(trunk_get_height(&spl.trunk_context, &height))); + ASSERT_TRUE(height > 0); + + async_scratch_saved_ops = cc->ops; + cache_ops ops = *cc->ops; + ops.page_get_async_state_init = async_scratch_get_init; + ops.page_get_async = async_scratch_get; + memset(async_scratch_gets, 0, sizeof(async_scratch_gets)); + lookup_result result; + lookup_result_init( + &result, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + core_lookup_async_state state; + for (unsigned cold = 0; cold < 2; cold++) { + if (cold) { + cache_flush(cc); + // The empty active memtable retains its pinned pages. + ASSERT_TRUE(SUCCESS(cache_evict(cc, TRUE))); + } + for (uint64 sample = 0; sample < 64; sample++) { + bool32 present = sample % 2 == 0; + uint64 i = (sample / 2) * 257 + (present ? 0 : per_round * rounds); + test_key( + &keybuf, TEST_RANDOM, i, 0, 0, data->workload_cfg->key_size, 0); + memset(&state, 0xa5, sizeof(state)); + core_lookup_async_state_init(&state, + &spl, + key_buffer_key(&keybuf), + &result, + async_scratch_ready, + NULL); + cc->ops = &ops; + ASSERT_EQUAL(ASYNC_STATUS_RUNNING, core_lookup_async(&state)); + platform_thread thread; + ASSERT_TRUE(SUCCESS(platform_thread_create( + &thread, FALSE, async_scratch_resume, &state, data->hid))); + ASSERT_TRUE(SUCCESS(platform_thread_join(&thread))); + cc->ops = async_scratch_saved_ops; + ASSERT_TRUE(SUCCESS(async_result(&state))); + ASSERT_EQUAL(present, lookup_result_found(&result)); + if (present) { + verify_tuple( + &spl, + &data->gen, + i, + key_buffer_key(&keybuf), + merge_accumulator_to_message(lookup_result_accumulator(&result)), + TRUE); + } + } + } + ASSERT_TRUE(async_scratch_gets[PAGE_TYPE_TRUNK] > 0); + ASSERT_TRUE(async_scratch_gets[PAGE_TYPE_FILTER] > 0); + ASSERT_TRUE(async_scratch_gets[PAGE_TYPE_BRANCH] > 0); + lookup_result_deinit(&result); + core_destroy(&spl); +} + /* * ************************************************************************** * Test case to run a bunch of inserts into Splinter, and then perform diff --git a/tests/unit/writeback_set_test.c b/tests/unit/writeback_set_test.c index 61c6aac2d..b113e11e5 100644 --- a/tests/unit/writeback_set_test.c +++ b/tests/unit/writeback_set_test.c @@ -159,6 +159,7 @@ rejected_submission_result(io_async_state *state) static platform_status reject_write_init(io_async_state *state, + uint64 buffer_size, io_handle *io, io_async_cmd cmd, uint64 addr, @@ -166,7 +167,7 @@ reject_write_init(io_async_state *state, void *callback_arg) { platform_status rc = submission_saved_ops->async_state_init( - state, io, cmd, addr, callback, callback_arg); + state, buffer_size, io, cmd, addr, callback, callback_arg); if (SUCCESS(rc) && cmd == io_async_pwritev) { submission_state_ops = *state->ops; submission_state_ops.run = reject_submission; @@ -475,6 +476,7 @@ read_fault_append(io_async_state *state, void *buf) static platform_status read_fault_init(io_async_state *state, + uint64 buffer_size, io_handle *io, io_async_cmd cmd, uint64 addr, @@ -488,7 +490,7 @@ read_fault_init(io_async_state *state, return STATUS_NO_MEMORY; } platform_status rc = read_saved_ops->async_state_init( - state, io, cmd, addr, callback, callback_arg); + state, buffer_size, io, cmd, addr, callback, callback_arg); if (SUCCESS(rc) && cmd == io_async_preadv) { read_fault_state_ops = *state->ops; read_fault_state_ops.run = read_fault_run;