From b23e1149af40d9ae072356e7eef052cabbbc90b3 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 02:00:37 -0400 Subject: [PATCH] hrx: keep the model weights in the loader's file mapping instead of a device buffer (engine #856) HRX makes its own device copy of every weight it runs (HostWeightCache). The loader also uploaded the whole model into an HRX0 device buffer, and on this APU device memory is GTT, so a model could be in GTT twice. HRX now reports buffer_from_host_ptr. The loader keeps its file mapping on HRX (the #1339 override steps aside when every HRX device reports it), wraps the mapping as an HRX0_Mapped host buffer (no copy, no device allocation, no import), and HRX copies each weight from it into a device copy. GGML_HRX_MODEL_BUFFER_COPY=1, or a residency limit (GGML_HRX_HOST_WEIGHT_LIMIT_MB), keeps the old upload. This removes the loader's copy. It does not make GTT one copy per weight after prefill: a weight that prefill reads in one layout and decode in another gets a device copy for each layout. Co-Authored-By: Claude Opus 5.5 --- ggml/src/ggml-hrx/CMakeLists.txt | 2 + ggml/src/ggml-hrx/backend-context.h | 2 + ggml/src/ggml-hrx/ggml-hrx.cpp | 15 +++- ggml/src/ggml-hrx/model-buffer-from-mmap.cpp | 84 ++++++++++++++++++++ ggml/src/ggml-hrx/model-buffer-from-mmap.h | 37 +++++++++ src/llama-model.cpp | 11 ++- 6 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 ggml/src/ggml-hrx/model-buffer-from-mmap.cpp create mode 100644 ggml/src/ggml-hrx/model-buffer-from-mmap.h diff --git a/ggml/src/ggml-hrx/CMakeLists.txt b/ggml/src/ggml-hrx/CMakeLists.txt index 68b538b3805e..d086ef757aef 100644 --- a/ggml/src/ggml-hrx/CMakeLists.txt +++ b/ggml/src/ggml-hrx/CMakeLists.txt @@ -209,9 +209,11 @@ ggml_add_backend_library(ggml-hrx backend-buffer-binding.cpp hrx-1bit-safety.cpp host-weight-placement.cpp + model-buffer-from-mmap.cpp backend-buffer-binding.h backend-context.h host-weight-placement.h + model-buffer-from-mmap.h dispatch/command-plan-metadata.cpp dispatch/command-plan-metadata.h dispatch/command-plan.h diff --git a/ggml/src/ggml-hrx/backend-context.h b/ggml/src/ggml-hrx/backend-context.h index 5af84f611d43..68b6dd8911a4 100644 --- a/ggml/src/ggml-hrx/backend-context.h +++ b/ggml/src/ggml-hrx/backend-context.h @@ -57,6 +57,8 @@ struct ggml_backend_hrx_device_context { ggml_backend_hrx_buffer_type_context buft_context = {}; ggml_backend_buffer_type host_buft = {}; ggml_backend_hrx_buffer_type_context host_buft_context = {}; + ggml_backend_buffer_type mapped_buft = {}; + ggml_backend_hrx_buffer_type_context mapped_buft_context = {}; ggml::hrx::HostBufferRegistry host_buffers; std::atomic synchronous_upload_fallbacks{ 0 }; std::atomic synchronous_download_fallbacks{ 0 }; diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 4a99f3b8dc38..771e30122d38 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -9,6 +9,7 @@ #include "backend-buffer-binding.h" #include "backend-context.h" #include "fused-context-claim.h" +#include "model-buffer-from-mmap.h" #include "host-weight-placement.h" #include "ggml-backend-impl.h" #include "ggml-impl.h" @@ -961,7 +962,7 @@ static void device_props(ggml_backend_dev_t device, ggml_backend_dev_props * pro device_memory(device, &props->memory_free, &props->memory_total); props->type = GGML_BACKEND_DEVICE_TYPE_GPU; props->device_id = nullptr; - props->caps = { true, true, false, false }; + props->caps = { true, true, ggml::hrx::model_buffer_from_mmap_enabled(), false }; } static ggml_backend_t device_init(ggml_backend_dev_t device, const char * parameters) { @@ -1513,9 +1514,15 @@ static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op ggml::hrx::can_execute_standalone_op_as_graph(op, device_context(device)->architecture)); } +static ggml_backend_buffer_t device_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, + size_t max_tensor_size) { + GGML_UNUSED(max_tensor_size); + return ggml::hrx::model_buffer_from_mmap(device_context(device), ptr, size); +} + static bool device_supports_buffer_type(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) { auto * context = device_context(device); - if (buft == &context->buft || buft == &context->host_buft) { + if (buft == &context->buft || buft == &context->host_buft || buft == &context->mapped_buft) { return true; } // Claiming another backend's host buffer makes the scheduler place host-resident weights and every @@ -1533,7 +1540,7 @@ static const ggml_backend_device_i device_i = { device_init, device_buffer_type, device_host_buffer_type, - nullptr, + device_buffer_from_host_ptr, device_supports_op, device_supports_buffer_type, nullptr, @@ -1681,10 +1688,12 @@ static std::unique_ptr create_registry_context() { device_ctx->buft = { buffer_type_i, nullptr, &device_ctx->buft_context }; device_ctx->host_buft_context = { device_ctx.get(), device_ctx->name + "_HOST", true }; device_ctx->host_buft = { buffer_type_i, nullptr, &device_ctx->host_buft_context }; + ggml::hrx::model_buffer_from_mmap_init_type(device_ctx.get()); context->device_contexts.emplace_back(std::move(device_ctx)); context->devices.push_back({ device_i, nullptr, context->device_contexts.back().get() }); context->device_contexts.back()->buft.device = &context->devices.back(); context->device_contexts.back()->host_buft.device = &context->devices.back(); + context->device_contexts.back()->mapped_buft.device = &context->devices.back(); } return context; } diff --git a/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp b/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp new file mode 100644 index 000000000000..b2d535e6fd35 --- /dev/null +++ b/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp @@ -0,0 +1,84 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "model-buffer-from-mmap.h" + +#include "backend-context.h" +#include "ggml-backend-impl.h" +#include "runtime/host-memory.h" + +#include +#include + +namespace ggml::hrx { + +namespace { + +const char * mapped_type_name(ggml_backend_buffer_type_t buft) { + return static_cast(buft->context)->name.c_str(); +} + +// Only the loader's mmap path makes these buffers. Anything else that asks this type for memory gets plain host memory +// with the same type, so it is still host data to the executor. +ggml_backend_buffer_t mapped_type_alloc(ggml_backend_buffer_type_t buft, size_t size) { + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); + if (buffer != nullptr) { + buffer->buft = buft; + } + return buffer; +} + +size_t mapped_type_alignment(ggml_backend_buffer_type_t buft) { + return ggml_backend_buft_get_alignment(ggml_backend_cpu_buffer_type()); + GGML_UNUSED(buft); +} + +bool mapped_type_is_host(ggml_backend_buffer_type_t buft) { + return true; + GGML_UNUSED(buft); +} + +} // namespace + +bool model_buffer_from_mmap_enabled() { + const char * copy = std::getenv("GGML_HRX_MODEL_BUFFER_COPY"); + if (copy != nullptr && copy[0] != '\0' && std::strcmp(copy, "0") != 0) { + return false; + } + // Under GGML_HRX_HOST_WEIGHT_LIMIT_MB a host weight over the budget runs on the CPU. Keep the upload there, so a + // limit does not move device weights off the device. + return HostWeightCache::residency_limit() == 0; +} + +void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device) { + device->mapped_buft_context = { device, device->name + "_Mapped", true }; + device->mapped_buft = { + { mapped_type_name, mapped_type_alloc, mapped_type_alignment, nullptr, nullptr, mapped_type_is_host }, + nullptr, + &device->mapped_buft_context, + }; +} + +ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * device, void * ptr, size_t size) { + // The CPU from-pointer buffer neither copies nor frees: the loader owns the mapping. Retag it so the scheduler places + // these weights on this device; the binding resolver then reads them as host data and packs them once. + ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); + if (buffer != nullptr) { + buffer->buft = &device->mapped_buft; + } + return buffer; +} + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/model-buffer-from-mmap.h b/ggml/src/ggml-hrx/model-buffer-from-mmap.h new file mode 100644 index 000000000000..3e075be1e091 --- /dev/null +++ b/ggml/src/ggml-hrx/model-buffer-from-mmap.h @@ -0,0 +1,37 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "ggml-backend.h" + +#include + +struct ggml_backend_hrx_device_context; + +namespace ggml::hrx { + +// engine #856: the model weights stay in the loader's mmap and are not uploaded to a device buffer. The host-weight cache +// packs each weight from these pages, so GTT holds one copy of the model instead of two. GGML_HRX_MODEL_BUFFER_COPY=1 +// restores the old upload, and so does a residency limit (GGML_HRX_HOST_WEIGHT_LIMIT_MB). +bool model_buffer_from_mmap_enabled(); + +// The buffer type of these buffers ("_Mapped"). It is host memory; only this device claims it. +void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device); + +// ggml_backend_dev_buffer_from_host_ptr for HRX: wrap [ptr, ptr + size) without a copy or a device allocation. +ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * device, void * ptr, size_t size); + +} // namespace ggml::hrx diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 3e5a80fcd752..9a8caf2525f3 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1588,14 +1588,21 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { // An explicit mlock mode keeps the mapping. if (n_gpu_layers > 0 && params.load_mode != LLAMA_LOAD_MODE_MMAP_MLOCK) { bool uses_hrx = false; + bool hrx_maps_model = true; for (const auto & d : devices) { if (!d.is_meta && d.dev != nullptr && std::strcmp(ggml_backend_reg_name(ggml_backend_dev_backend_reg(d.dev)), "HRX") == 0) { uses_hrx = true; - break; + ggml_backend_dev_props props; + ggml_backend_dev_get_props(d.dev, &props); + hrx_maps_model = hrx_maps_model && props.caps.buffer_from_host_ptr; } } - if (uses_hrx) { + // engine #856: an HRX device that wraps the mapping as host memory (no import) packs each weight from the + // file pages, so the device holds one copy and the mapping stays. + if (uses_hrx && hrx_maps_model && ml.use_mmap) { + LLAMA_LOG_INFO("%s: HRX: the weights stay in the file mapping and are packed from it (engine #856)\n", __func__); + } else if (uses_hrx) { if (ml.use_mmap) { LLAMA_LOG_INFO("%s: HRX: reading the weights into their buffers instead of mapping the file\n", __func__); }