hrx: keep the model weights in the loader's file mapping instead of a device buffer (engine #856)

ggml/src/ggml-hrx/CMakeLists.txt

@@ -209,9 +209,11 @@ ggml_add_backend_library(ggml-hrx
     backend-buffer-binding.cpp
     hrx-1bit-safety.cpp
     host-weight-placement.cpp
+    model-buffer-from-mmap.cpp
     backend-buffer-binding.h
     backend-context.h
     host-weight-placement.h
+    model-buffer-from-mmap.h
     dispatch/command-plan-metadata.cpp
     dispatch/command-plan-metadata.h
     dispatch/command-plan.h

ggml/src/ggml-hrx/backend-context.h

@@ -57,6 +57,8 @@ struct ggml_backend_hrx_device_context {
     ggml_backend_hrx_buffer_type_context           buft_context       = {};
     ggml_backend_buffer_type                       host_buft          = {};
     ggml_backend_hrx_buffer_type_context           host_buft_context  = {};
+    ggml_backend_buffer_type                       mapped_buft        = {};
+    ggml_backend_hrx_buffer_type_context           mapped_buft_context = {};
     ggml::hrx::HostBufferRegistry                  host_buffers;
     std::atomic<uint64_t>                          synchronous_upload_fallbacks{ 0 };
     std::atomic<uint64_t>                          synchronous_download_fallbacks{ 0 };

ggml/src/ggml-hrx/ggml-hrx.cpp

@@ -9,6 +9,7 @@
 #include "backend-buffer-binding.h"
 #include "backend-context.h"
 #include "fused-context-claim.h"
+#include "model-buffer-from-mmap.h"
 #include "host-weight-placement.h"
 #include "ggml-backend-impl.h"
 #include "ggml-impl.h"
@@ -961,7 +962,7 @@ static void device_props(ggml_backend_dev_t device, ggml_backend_dev_props * pro
     device_memory(device, &props->memory_free, &props->memory_total);
     props->type      = GGML_BACKEND_DEVICE_TYPE_GPU;
     props->device_id = nullptr;
-    props->caps      = { true, true, false, false };
+    props->caps      = { true, true, ggml::hrx::model_buffer_from_mmap_enabled(), false };
 }
 
 static ggml_backend_t device_init(ggml_backend_dev_t device, const char * parameters) {
@@ -1513,9 +1514,15 @@ static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op
             ggml::hrx::can_execute_standalone_op_as_graph(op, device_context(device)->architecture));
 }
 
+static ggml_backend_buffer_t device_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size,
+                                                         size_t max_tensor_size) {
+    GGML_UNUSED(max_tensor_size);
+    return ggml::hrx::model_buffer_from_mmap(device_context(device), ptr, size);
+}
+
 static bool device_supports_buffer_type(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) {
     auto * context = device_context(device);
-    if (buft == &context->buft || buft == &context->host_buft) {
+    if (buft == &context->buft || buft == &context->host_buft || buft == &context->mapped_buft) {
         return true;
     }
     // Claiming another backend's host buffer makes the scheduler place host-resident weights and every
@@ -1533,7 +1540,7 @@ static const ggml_backend_device_i device_i = {
     device_init,
     device_buffer_type,
     device_host_buffer_type,
-    nullptr,
+    device_buffer_from_host_ptr,
     device_supports_op,
     device_supports_buffer_type,
     nullptr,
@@ -1681,10 +1688,12 @@ static std::unique_ptr<ggml_backend_hrx_reg_context> create_registry_context() {
         device_ctx->buft              = { buffer_type_i, nullptr, &device_ctx->buft_context };
         device_ctx->host_buft_context = { device_ctx.get(), device_ctx->name + "_HOST", true };
         device_ctx->host_buft         = { buffer_type_i, nullptr, &device_ctx->host_buft_context };
+        ggml::hrx::model_buffer_from_mmap_init_type(device_ctx.get());
         context->device_contexts.emplace_back(std::move(device_ctx));
         context->devices.push_back({ device_i, nullptr, context->device_contexts.back().get() });
         context->device_contexts.back()->buft.device      = &context->devices.back();
         context->device_contexts.back()->host_buft.device = &context->devices.back();
+        context->device_contexts.back()->mapped_buft.device = &context->devices.back();
     }
     return context;
 }

ggml/src/ggml-hrx/model-buffer-from-mmap.cpp

@@ -0,0 +1,84 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "model-buffer-from-mmap.h"
+
+#include "backend-context.h"
+#include "ggml-backend-impl.h"
+#include "runtime/host-memory.h"
+
+#include <cstdlib>
+#include <cstring>
+
+namespace ggml::hrx {
+
+namespace {
+
+const char * mapped_type_name(ggml_backend_buffer_type_t buft) {
+    return static_cast<ggml_backend_hrx_buffer_type_context *>(buft->context)->name.c_str();
+}
+
+// Only the loader's mmap path makes these buffers. Anything else that asks this type for memory gets plain host memory
+// with the same type, so it is still host data to the executor.
+ggml_backend_buffer_t mapped_type_alloc(ggml_backend_buffer_type_t buft, size_t size) {
+    ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);
+    if (buffer != nullptr) {
+        buffer->buft = buft;
+    }
+    return buffer;
+}
+
+size_t mapped_type_alignment(ggml_backend_buffer_type_t buft) {
+    return ggml_backend_buft_get_alignment(ggml_backend_cpu_buffer_type());
+    GGML_UNUSED(buft);
+}
+
+bool mapped_type_is_host(ggml_backend_buffer_type_t buft) {
+    return true;
+    GGML_UNUSED(buft);
+}
+
+}  // namespace
+
+bool model_buffer_from_mmap_enabled() {
+    const char * copy = std::getenv("GGML_HRX_MODEL_BUFFER_COPY");
+    if (copy != nullptr && copy[0] != '\0' && std::strcmp(copy, "0") != 0) {
+        return false;
+    }
+    // Under GGML_HRX_HOST_WEIGHT_LIMIT_MB a host weight over the budget runs on the CPU. Keep the upload there, so a
+    // limit does not move device weights off the device.
+    return HostWeightCache::residency_limit() == 0;
+}
+
+void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device) {
+    device->mapped_buft_context = { device, device->name + "_Mapped", true };
+    device->mapped_buft         = {
+        { mapped_type_name, mapped_type_alloc, mapped_type_alignment, nullptr, nullptr, mapped_type_is_host },
+        nullptr,
+        &device->mapped_buft_context,
+    };
+}
+
+ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * device, void * ptr, size_t size) {
+    // The CPU from-pointer buffer neither copies nor frees: the loader owns the mapping. Retag it so the scheduler places
+    // these weights on this device; the binding resolver then reads them as host data and packs them once.
+    ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
+    if (buffer != nullptr) {
+        buffer->buft = &device->mapped_buft;
+    }
+    return buffer;
+}
+
+}  // namespace ggml::hrx

ggml/src/ggml-hrx/model-buffer-from-mmap.h

@@ -0,0 +1,37 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "ggml-backend.h"
+
+#include <cstddef>
+
+struct ggml_backend_hrx_device_context;
+
+namespace ggml::hrx {
+
+// engine #856: the model weights stay in the loader's mmap and are not uploaded to a device buffer. The host-weight cache
+// packs each weight from these pages, so GTT holds one copy of the model instead of two. GGML_HRX_MODEL_BUFFER_COPY=1
+// restores the old upload, and so does a residency limit (GGML_HRX_HOST_WEIGHT_LIMIT_MB).
+bool model_buffer_from_mmap_enabled();
+
+// The buffer type of these buffers ("<device>_Mapped"). It is host memory; only this device claims it.
+void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device);
+
+// ggml_backend_dev_buffer_from_host_ptr for HRX: wrap [ptr, ptr + size) without a copy or a device allocation.
+ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * device, void * ptr, size_t size);
+
+}  // namespace ggml::hrx

src/llama-model.cpp

@@ -1588,14 +1588,21 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
     // An explicit mlock mode keeps the mapping.
     if (n_gpu_layers > 0 && params.load_mode != LLAMA_LOAD_MODE_MMAP_MLOCK) {
         bool uses_hrx = false;
+        bool hrx_maps_model = true;
         for (const auto & d : devices) {
             if (!d.is_meta && d.dev != nullptr &&
                 std::strcmp(ggml_backend_reg_name(ggml_backend_dev_backend_reg(d.dev)), "HRX") == 0) {
                 uses_hrx = true;
-                break;
+                ggml_backend_dev_props props;
+                ggml_backend_dev_get_props(d.dev, &props);
+                hrx_maps_model = hrx_maps_model && props.caps.buffer_from_host_ptr;
             }
         }
-        if (uses_hrx) {
+        // engine #856: an HRX device that wraps the mapping as host memory (no import) packs each weight from the
+        // file pages, so the device holds one copy and the mapping stays.
+        if (uses_hrx && hrx_maps_model && ml.use_mmap) {
+            LLAMA_LOG_INFO("%s: HRX: the weights stay in the file mapping and are packed from it (engine #856)\n", __func__);
+        } else if (uses_hrx) {
             if (ml.use_mmap) {
                 LLAMA_LOG_INFO("%s: HRX: reading the weights into their buffers instead of mapping the file\n", __func__);
             }