From 4733b339e99a5c6518dcfef6730750a2af407f75 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 07:22:41 -0400 Subject: [PATCH] llama: on HRX keep the file mapping only for models up to GGML_HRX_MODEL_MAP_MAX_MIB (default 4096) (engine #2163) #304 makes HRX copy every weight from the mapping through its host staging path. That is free on a 4B and slow on a large model: Qwen3.8-27B dense loads in 42.6 s instead of 14.9 s, and its first long prompt runs at half speed. Above the limit the model takes the upload again, which reads the file into device buffers. Co-Authored-By: Claude Opus 5.5 --- src/llama-model.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 9a8caf2525f3..bad8670a0ec0 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1598,9 +1598,20 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { hrx_maps_model = hrx_maps_model && props.caps.buffer_from_host_ptr; } } + // engine #2163: HRX copies a mapped weight through its host staging path, which is slow for a large model (a 27B + // dense model loads in 42.6 s instead of 14.9 s). Above the limit the model takes the upload below. + size_t map_max_mib = 4096; + if (const char * env = std::getenv("GGML_HRX_MODEL_MAP_MAX_MIB")) { + map_max_mib = std::strtoull(env, nullptr, 10); + } + const bool under_map_limit = ml.n_bytes <= map_max_mib * 1024 * 1024; + if (uses_hrx && hrx_maps_model && ml.use_mmap && !under_map_limit) { + LLAMA_LOG_INFO("%s: HRX: the model (%zu MiB) is over GGML_HRX_MODEL_MAP_MAX_MIB=%zu, so it is read into device buffers (engine #2163)\n", + __func__, ml.n_bytes >> 20, map_max_mib); + } // engine #856: an HRX device that wraps the mapping as host memory (no import) packs each weight from the // file pages, so the device holds one copy and the mapping stays. - if (uses_hrx && hrx_maps_model && ml.use_mmap) { + if (uses_hrx && hrx_maps_model && ml.use_mmap && under_map_limit) { LLAMA_LOG_INFO("%s: HRX: the weights stay in the file mapping and are packed from it (engine #856)\n", __func__); } else if (uses_hrx) { if (ml.use_mmap) {