diff --git a/ggml/src/ggml-hrx/CMakeLists.txt b/ggml/src/ggml-hrx/CMakeLists.txt index 68b538b3805e..d086ef757aef 100644 --- a/ggml/src/ggml-hrx/CMakeLists.txt +++ b/ggml/src/ggml-hrx/CMakeLists.txt @@ -209,9 +209,11 @@ ggml_add_backend_library(ggml-hrx backend-buffer-binding.cpp hrx-1bit-safety.cpp host-weight-placement.cpp + model-buffer-from-mmap.cpp backend-buffer-binding.h backend-context.h host-weight-placement.h + model-buffer-from-mmap.h dispatch/command-plan-metadata.cpp dispatch/command-plan-metadata.h dispatch/command-plan.h diff --git a/ggml/src/ggml-hrx/backend-context.h b/ggml/src/ggml-hrx/backend-context.h index 5af84f611d43..68b6dd8911a4 100644 --- a/ggml/src/ggml-hrx/backend-context.h +++ b/ggml/src/ggml-hrx/backend-context.h @@ -57,6 +57,8 @@ struct ggml_backend_hrx_device_context { ggml_backend_hrx_buffer_type_context buft_context = {}; ggml_backend_buffer_type host_buft = {}; ggml_backend_hrx_buffer_type_context host_buft_context = {}; + ggml_backend_buffer_type mapped_buft = {}; + ggml_backend_hrx_buffer_type_context mapped_buft_context = {}; ggml::hrx::HostBufferRegistry host_buffers; std::atomic synchronous_upload_fallbacks{ 0 }; std::atomic synchronous_download_fallbacks{ 0 }; diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 4a99f3b8dc38..771e30122d38 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -9,6 +9,7 @@ #include "backend-buffer-binding.h" #include "backend-context.h" #include "fused-context-claim.h" +#include "model-buffer-from-mmap.h" #include "host-weight-placement.h" #include "ggml-backend-impl.h" #include "ggml-impl.h" @@ -961,7 +962,7 @@ static void device_props(ggml_backend_dev_t device, ggml_backend_dev_props * pro device_memory(device, &props->memory_free, &props->memory_total); props->type = GGML_BACKEND_DEVICE_TYPE_GPU; props->device_id = nullptr; - props->caps = { true, true, false, false }; + props->caps = { true, true, ggml::hrx::model_buffer_from_mmap_enabled(), false }; } static ggml_backend_t device_init(ggml_backend_dev_t device, const char * parameters) { @@ -1513,9 +1514,15 @@ static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op ggml::hrx::can_execute_standalone_op_as_graph(op, device_context(device)->architecture)); } +static ggml_backend_buffer_t device_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, + size_t max_tensor_size) { + GGML_UNUSED(max_tensor_size); + return ggml::hrx::model_buffer_from_mmap(device_context(device), ptr, size); +} + static bool device_supports_buffer_type(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) { auto * context = device_context(device); - if (buft == &context->buft || buft == &context->host_buft) { + if (buft == &context->buft || buft == &context->host_buft || buft == &context->mapped_buft) { return true; } // Claiming another backend's host buffer makes the scheduler place host-resident weights and every @@ -1533,7 +1540,7 @@ static const ggml_backend_device_i device_i = { device_init, device_buffer_type, device_host_buffer_type, - nullptr, + device_buffer_from_host_ptr, device_supports_op, device_supports_buffer_type, nullptr, @@ -1681,10 +1688,12 @@ static std::unique_ptr create_registry_context() { device_ctx->buft = { buffer_type_i, nullptr, &device_ctx->buft_context }; device_ctx->host_buft_context = { device_ctx.get(), device_ctx->name + "_HOST", true }; device_ctx->host_buft = { buffer_type_i, nullptr, &device_ctx->host_buft_context }; + ggml::hrx::model_buffer_from_mmap_init_type(device_ctx.get()); context->device_contexts.emplace_back(std::move(device_ctx)); context->devices.push_back({ device_i, nullptr, context->device_contexts.back().get() }); context->device_contexts.back()->buft.device = &context->devices.back(); context->device_contexts.back()->host_buft.device = &context->devices.back(); + context->device_contexts.back()->mapped_buft.device = &context->devices.back(); } return context; } diff --git a/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp b/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp new file mode 100644 index 000000000000..b2d535e6fd35 --- /dev/null +++ b/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp @@ -0,0 +1,84 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "model-buffer-from-mmap.h" + +#include "backend-context.h" +#include "ggml-backend-impl.h" +#include "runtime/host-memory.h" + +#include +#include + +namespace ggml::hrx { + +namespace { + +const char * mapped_type_name(ggml_backend_buffer_type_t buft) { + return static_cast(buft->context)->name.c_str(); +} + +// Only the loader's mmap path makes these buffers. Anything else that asks this type for memory gets plain host memory +// with the same type, so it is still host data to the executor. +ggml_backend_buffer_t mapped_type_alloc(ggml_backend_buffer_type_t buft, size_t size) { + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); + if (buffer != nullptr) { + buffer->buft = buft; + } + return buffer; +} + +size_t mapped_type_alignment(ggml_backend_buffer_type_t buft) { + return ggml_backend_buft_get_alignment(ggml_backend_cpu_buffer_type()); + GGML_UNUSED(buft); +} + +bool mapped_type_is_host(ggml_backend_buffer_type_t buft) { + return true; + GGML_UNUSED(buft); +} + +} // namespace + +bool model_buffer_from_mmap_enabled() { + const char * copy = std::getenv("GGML_HRX_MODEL_BUFFER_COPY"); + if (copy != nullptr && copy[0] != '\0' && std::strcmp(copy, "0") != 0) { + return false; + } + // Under GGML_HRX_HOST_WEIGHT_LIMIT_MB a host weight over the budget runs on the CPU. Keep the upload there, so a + // limit does not move device weights off the device. + return HostWeightCache::residency_limit() == 0; +} + +void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device) { + device->mapped_buft_context = { device, device->name + "_Mapped", true }; + device->mapped_buft = { + { mapped_type_name, mapped_type_alloc, mapped_type_alignment, nullptr, nullptr, mapped_type_is_host }, + nullptr, + &device->mapped_buft_context, + }; +} + +ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * device, void * ptr, size_t size) { + // The CPU from-pointer buffer neither copies nor frees: the loader owns the mapping. Retag it so the scheduler places + // these weights on this device; the binding resolver then reads them as host data and packs them once. + ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); + if (buffer != nullptr) { + buffer->buft = &device->mapped_buft; + } + return buffer; +} + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/model-buffer-from-mmap.h b/ggml/src/ggml-hrx/model-buffer-from-mmap.h new file mode 100644 index 000000000000..3e075be1e091 --- /dev/null +++ b/ggml/src/ggml-hrx/model-buffer-from-mmap.h @@ -0,0 +1,37 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "ggml-backend.h" + +#include + +struct ggml_backend_hrx_device_context; + +namespace ggml::hrx { + +// engine #856: the model weights stay in the loader's mmap and are not uploaded to a device buffer. The host-weight cache +// packs each weight from these pages, so GTT holds one copy of the model instead of two. GGML_HRX_MODEL_BUFFER_COPY=1 +// restores the old upload, and so does a residency limit (GGML_HRX_HOST_WEIGHT_LIMIT_MB). +bool model_buffer_from_mmap_enabled(); + +// The buffer type of these buffers ("_Mapped"). It is host memory; only this device claims it. +void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device); + +// ggml_backend_dev_buffer_from_host_ptr for HRX: wrap [ptr, ptr + size) without a copy or a device allocation. +ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * device, void * ptr, size_t size); + +} // namespace ggml::hrx diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 3e5a80fcd752..9a8caf2525f3 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1588,14 +1588,21 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { // An explicit mlock mode keeps the mapping. if (n_gpu_layers > 0 && params.load_mode != LLAMA_LOAD_MODE_MMAP_MLOCK) { bool uses_hrx = false; + bool hrx_maps_model = true; for (const auto & d : devices) { if (!d.is_meta && d.dev != nullptr && std::strcmp(ggml_backend_reg_name(ggml_backend_dev_backend_reg(d.dev)), "HRX") == 0) { uses_hrx = true; - break; + ggml_backend_dev_props props; + ggml_backend_dev_get_props(d.dev, &props); + hrx_maps_model = hrx_maps_model && props.caps.buffer_from_host_ptr; } } - if (uses_hrx) { + // engine #856: an HRX device that wraps the mapping as host memory (no import) packs each weight from the + // file pages, so the device holds one copy and the mapping stays. + if (uses_hrx && hrx_maps_model && ml.use_mmap) { + LLAMA_LOG_INFO("%s: HRX: the weights stay in the file mapping and are packed from it (engine #856)\n", __func__); + } else if (uses_hrx) { if (ml.use_mmap) { LLAMA_LOG_INFO("%s: HRX: reading the weights into their buffers instead of mapping the file\n", __func__); }