From d80aa3c5434f8ee89168a9a27d914fd4d40e72e6 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Wed, 23 Sep 2026 11:05:05 -0500 Subject: [PATCH 1/5] fix(barrier): resolve cuDSS mtlayer path at runtime, drop scary wording The compiled-in absolute path to cuDSS's prebuilt threading-layer plugin is resolved at CMake configure time and can go stale once installed elsewhere (e.g. an unrelocated conda/rattler-build sandbox path), causing dlopen to fail. Re-derive it from wherever libcudss.so.0 actually loaded from at runtime instead. Also drops "to avoid a possible crash during teardown" from the fallback warning. Co-Authored-By: Claude Sonnet 5 --- cpp/src/barrier/sparse_cholesky.cuh | 33 +++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cpp/src/barrier/sparse_cholesky.cuh b/cpp/src/barrier/sparse_cholesky.cuh index 71ba3c0e01..72c3b4237a 100644 --- a/cpp/src/barrier/sparse_cholesky.cuh +++ b/cpp/src/barrier/sparse_cholesky.cuh @@ -23,6 +23,8 @@ #include +#include + namespace cuopt::mathematical_optimization::barrier { namespace detail { @@ -47,6 +49,19 @@ inline bool pin_cudss_threading_layer(const char* lib_file) return true; } +// Directory libcudss.so.0 actually loaded from, via dladdr on a cuDSS symbol. cuDSS ships its +// threading-layer plugin alongside it, so this finds it regardless of install prefix. +inline std::string cudss_library_dir() +{ + Dl_info info{}; + if (dladdr(reinterpret_cast(&cudssCreateMg), &info) == 0 || info.dli_fname == nullptr) { + return {}; + } + std::string path(info.dli_fname); + auto slash = path.find_last_of('/'); + return slash == std::string::npos ? std::string{} : path.substr(0, slash); +} + } // namespace detail template @@ -278,20 +293,30 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { cudssSetDeviceMemHandler(handle, &mem_handler), status, "cudssSetDeviceMemHandler"); const char* cudss_mt_lib_file = nullptr; - char* env_value = std::getenv("CUDSS_THREADING_LIB"); + std::string resolved_lib_file; + char* env_value = std::getenv("CUDSS_THREADING_LIB"); if (env_value != nullptr) { cudss_mt_lib_file = env_value; } else if (CUDSS_MT_LIB_FILE_NAME != nullptr) { cudss_mt_lib_file = CUDSS_MT_LIB_FILE_NAME; + // An absolute path is cuDSS's own prebuilt mtlayer, baked in at configure time and prone + // to going stale (e.g. an unrelocated build sandbox path); re-derive it at runtime. A bare + // filename (cuOpt's own mtlayer) already resolves fine via $ORIGIN rpath as-is. + if (cudss_mt_lib_file[0] == '/') { + auto loaded_dir = detail::cudss_library_dir(); + if (!loaded_dir.empty()) { + resolved_lib_file = loaded_dir + "/libcudss_mtlayer_gomp.so.0"; + cudss_mt_lib_file = resolved_lib_file.c_str(); + } + } } if (cudss_mt_lib_file != nullptr) { if (!detail::pin_cudss_threading_layer(cudss_mt_lib_file)) { settings.log.printf( "cuDSS Threading layer : could not pin '%s'; falling back to single-threaded " - "cuDSS to avoid a possible crash during teardown. Set the CUDSS_THREADING_LIB " - "environment variable to an absolute path, or ensure the host provides " - "libgomp.so.1, to enable multi-threaded cuDSS.\n", + "cuDSS. Set the CUDSS_THREADING_LIB environment variable to an absolute path, or " + "ensure the host provides libgomp.so.1, to enable multi-threaded cuDSS.\n", cudss_mt_lib_file); } else { cudssStatus_t threading_status = cudssSetThreadingLayer(handle, cudss_mt_lib_file); From 0058b8bdd307d7aaa3549ab72512824911cfee62 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Wed, 23 Sep 2026 11:17:48 -0500 Subject: [PATCH 2/5] fix review: resolve via dlsym before dladdr to avoid PLT stub &cudssCreateMg can be our own PLT stub rather than the real symbol, which dladdr would attribute to cuOpt's own module instead of libcudss.so.0. Co-Authored-By: Claude Sonnet 5 --- cpp/src/barrier/sparse_cholesky.cuh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cpp/src/barrier/sparse_cholesky.cuh b/cpp/src/barrier/sparse_cholesky.cuh index 72c3b4237a..6a0e8411f2 100644 --- a/cpp/src/barrier/sparse_cholesky.cuh +++ b/cpp/src/barrier/sparse_cholesky.cuh @@ -49,14 +49,16 @@ inline bool pin_cudss_threading_layer(const char* lib_file) return true; } -// Directory libcudss.so.0 actually loaded from, via dladdr on a cuDSS symbol. cuDSS ships its -// threading-layer plugin alongside it, so this finds it regardless of install prefix. +// Directory libcudss.so.0 actually loaded from. cuDSS ships its threading-layer plugin +// alongside it, so this finds it regardless of install prefix. Resolves the symbol via dlsym +// first -- &cudssCreateMg can be our own PLT stub, which dladdr would attribute to us instead +// of libcudss.so.0. inline std::string cudss_library_dir() { + void* sym = dlsym(RTLD_DEFAULT, "cudssCreateMg"); + if (sym == nullptr) return {}; Dl_info info{}; - if (dladdr(reinterpret_cast(&cudssCreateMg), &info) == 0 || info.dli_fname == nullptr) { - return {}; - } + if (dladdr(sym, &info) == 0 || info.dli_fname == nullptr) return {}; std::string path(info.dli_fname); auto slash = path.find_last_of('/'); return slash == std::string::npos ? std::string{} : path.substr(0, slash); From 2f44c48e8c1b4c193fcb92a590bd1e97a8465333 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Wed, 23 Sep 2026 11:19:09 -0500 Subject: [PATCH 3/5] trim comments to minimum Co-Authored-By: Claude Sonnet 5 --- cpp/src/barrier/sparse_cholesky.cuh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cpp/src/barrier/sparse_cholesky.cuh b/cpp/src/barrier/sparse_cholesky.cuh index 6a0e8411f2..d41f45164b 100644 --- a/cpp/src/barrier/sparse_cholesky.cuh +++ b/cpp/src/barrier/sparse_cholesky.cuh @@ -49,10 +49,7 @@ inline bool pin_cudss_threading_layer(const char* lib_file) return true; } -// Directory libcudss.so.0 actually loaded from. cuDSS ships its threading-layer plugin -// alongside it, so this finds it regardless of install prefix. Resolves the symbol via dlsym -// first -- &cudssCreateMg can be our own PLT stub, which dladdr would attribute to us instead -// of libcudss.so.0. +// Directory libcudss.so.0 loaded from; cuDSS ships its mtlayer plugin alongside it. inline std::string cudss_library_dir() { void* sym = dlsym(RTLD_DEFAULT, "cudssCreateMg"); @@ -301,9 +298,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { cudss_mt_lib_file = env_value; } else if (CUDSS_MT_LIB_FILE_NAME != nullptr) { cudss_mt_lib_file = CUDSS_MT_LIB_FILE_NAME; - // An absolute path is cuDSS's own prebuilt mtlayer, baked in at configure time and prone - // to going stale (e.g. an unrelocated build sandbox path); re-derive it at runtime. A bare - // filename (cuOpt's own mtlayer) already resolves fine via $ORIGIN rpath as-is. + // Absolute path (cuDSS's own mtlayer) can go stale; re-derive at runtime. if (cudss_mt_lib_file[0] == '/') { auto loaded_dir = detail::cudss_library_dir(); if (!loaded_dir.empty()) { From 0cbb7f96b434f65be290a01be46dc8a3d176ad53 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Wed, 23 Sep 2026 13:05:55 -0500 Subject: [PATCH 4/5] address review: single std::string instead of raw-pointer dance, clarify comment Co-Authored-By: Claude Sonnet 5 --- cpp/src/barrier/sparse_cholesky.cuh | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/cpp/src/barrier/sparse_cholesky.cuh b/cpp/src/barrier/sparse_cholesky.cuh index d41f45164b..b24111505f 100644 --- a/cpp/src/barrier/sparse_cholesky.cuh +++ b/cpp/src/barrier/sparse_cholesky.cuh @@ -291,41 +291,39 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { CUDSS_CALL_AND_CHECK_EXIT( cudssSetDeviceMemHandler(handle, &mem_handler), status, "cudssSetDeviceMemHandler"); - const char* cudss_mt_lib_file = nullptr; - std::string resolved_lib_file; + std::string cudss_mt_lib_file; char* env_value = std::getenv("CUDSS_THREADING_LIB"); if (env_value != nullptr) { cudss_mt_lib_file = env_value; } else if (CUDSS_MT_LIB_FILE_NAME != nullptr) { cudss_mt_lib_file = CUDSS_MT_LIB_FILE_NAME; - // Absolute path (cuDSS's own mtlayer) can go stale; re-derive at runtime. + // Absolute path means CUDSS_MT_LIB_FILE_NAME was baked in at configure time from cuDSS's + // build-time install location, which can differ from (or no longer exist at) its install + // location here; re-derive it from where libcudss.so.0 actually loaded from. if (cudss_mt_lib_file[0] == '/') { auto loaded_dir = detail::cudss_library_dir(); - if (!loaded_dir.empty()) { - resolved_lib_file = loaded_dir + "/libcudss_mtlayer_gomp.so.0"; - cudss_mt_lib_file = resolved_lib_file.c_str(); - } + if (!loaded_dir.empty()) { cudss_mt_lib_file = loaded_dir + "/libcudss_mtlayer_gomp.so.0"; } } } - if (cudss_mt_lib_file != nullptr) { - if (!detail::pin_cudss_threading_layer(cudss_mt_lib_file)) { + if (!cudss_mt_lib_file.empty()) { + if (!detail::pin_cudss_threading_layer(cudss_mt_lib_file.c_str())) { settings.log.printf( "cuDSS Threading layer : could not pin '%s'; falling back to single-threaded " "cuDSS. Set the CUDSS_THREADING_LIB environment variable to an absolute path, or " "ensure the host provides libgomp.so.1, to enable multi-threaded cuDSS.\n", - cudss_mt_lib_file); + cudss_mt_lib_file.c_str()); } else { - cudssStatus_t threading_status = cudssSetThreadingLayer(handle, cudss_mt_lib_file); + cudssStatus_t threading_status = cudssSetThreadingLayer(handle, cudss_mt_lib_file.c_str()); if (threading_status == CUDSS_STATUS_SUCCESS) { - settings.log.printf("cuDSS Threading layer : %s\n", cudss_mt_lib_file); + settings.log.printf("cuDSS Threading layer : %s\n", cudss_mt_lib_file.c_str()); } else { settings.log.printf( "cuDSS Threading layer : could not load '%s' (status = %d); falling back " "to single-threaded cuDSS. Set the CUDSS_THREADING_LIB environment variable to " "an absolute path, or ensure the host provides libgomp.so.1, to enable " "multi-threaded cuDSS.\n", - cudss_mt_lib_file, + cudss_mt_lib_file.c_str(), threading_status); } } From 18f2437697cef019ff2952c4f821bf5053d7bb1a Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Wed, 23 Sep 2026 17:02:57 -0500 Subject: [PATCH 5/5] simplify per review: use the mtlayer basename, drop runtime path discovery glibc's dlopen matches an already-loaded library by name before searching paths, so a bare filename resolves the same mapping as the initial pin without needing a build-time-resolved absolute path that can go stale. Co-Authored-By: Claude Sonnet 5 --- cpp/CMakeLists.txt | 8 +++---- cpp/src/barrier/sparse_cholesky.cuh | 37 +++++++---------------------- 2 files changed, 12 insertions(+), 33 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cab33c9e9d..b9504f576a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -875,11 +875,11 @@ if (CUOPT_BUILD_CUSTOM_CUDSS_MTLAYER) INSTALL_RPATH "\$ORIGIN" ) set(CUDSS_MT_LIB_FILE_NAME "libcudss_mtlayer_cuopt.so") -else () - # Full path, not just the basename, so the runtime dlopen pin (sparse_cholesky.cuh) and - # cudssSetThreadingLayer() resolve the exact same file CMake found (#1883). - set(CUDSS_MT_LIB_FILE_NAME "${CUDSS_MT_LIB_FILE}") endif () +# else: keep the basename already computed above (line 740) -- glibc's dlopen matches an +# already-loaded library by name before searching paths, so the runtime pin (sparse_cholesky.cuh) +# and cudssSetThreadingLayer()'s own dlopen still resolve to the same mapping. A full, +# build-time-resolved path isn't needed for that, and can go stale after install. # cuopt_routing: VRP / routing engine (omitted when SKIP_ROUTING_BUILD=ON) # diff --git a/cpp/src/barrier/sparse_cholesky.cuh b/cpp/src/barrier/sparse_cholesky.cuh index b24111505f..d842a778fc 100644 --- a/cpp/src/barrier/sparse_cholesky.cuh +++ b/cpp/src/barrier/sparse_cholesky.cuh @@ -23,8 +23,6 @@ #include -#include - namespace cuopt::mathematical_optimization::barrier { namespace detail { @@ -49,18 +47,6 @@ inline bool pin_cudss_threading_layer(const char* lib_file) return true; } -// Directory libcudss.so.0 loaded from; cuDSS ships its mtlayer plugin alongside it. -inline std::string cudss_library_dir() -{ - void* sym = dlsym(RTLD_DEFAULT, "cudssCreateMg"); - if (sym == nullptr) return {}; - Dl_info info{}; - if (dladdr(sym, &info) == 0 || info.dli_fname == nullptr) return {}; - std::string path(info.dli_fname); - auto slash = path.find_last_of('/'); - return slash == std::string::npos ? std::string{} : path.substr(0, slash); -} - } // namespace detail template @@ -291,39 +277,32 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { CUDSS_CALL_AND_CHECK_EXIT( cudssSetDeviceMemHandler(handle, &mem_handler), status, "cudssSetDeviceMemHandler"); - std::string cudss_mt_lib_file; - char* env_value = std::getenv("CUDSS_THREADING_LIB"); + const char* cudss_mt_lib_file = nullptr; + char* env_value = std::getenv("CUDSS_THREADING_LIB"); if (env_value != nullptr) { cudss_mt_lib_file = env_value; } else if (CUDSS_MT_LIB_FILE_NAME != nullptr) { cudss_mt_lib_file = CUDSS_MT_LIB_FILE_NAME; - // Absolute path means CUDSS_MT_LIB_FILE_NAME was baked in at configure time from cuDSS's - // build-time install location, which can differ from (or no longer exist at) its install - // location here; re-derive it from where libcudss.so.0 actually loaded from. - if (cudss_mt_lib_file[0] == '/') { - auto loaded_dir = detail::cudss_library_dir(); - if (!loaded_dir.empty()) { cudss_mt_lib_file = loaded_dir + "/libcudss_mtlayer_gomp.so.0"; } - } } - if (!cudss_mt_lib_file.empty()) { - if (!detail::pin_cudss_threading_layer(cudss_mt_lib_file.c_str())) { + if (cudss_mt_lib_file != nullptr) { + if (!detail::pin_cudss_threading_layer(cudss_mt_lib_file)) { settings.log.printf( "cuDSS Threading layer : could not pin '%s'; falling back to single-threaded " "cuDSS. Set the CUDSS_THREADING_LIB environment variable to an absolute path, or " "ensure the host provides libgomp.so.1, to enable multi-threaded cuDSS.\n", - cudss_mt_lib_file.c_str()); + cudss_mt_lib_file); } else { - cudssStatus_t threading_status = cudssSetThreadingLayer(handle, cudss_mt_lib_file.c_str()); + cudssStatus_t threading_status = cudssSetThreadingLayer(handle, cudss_mt_lib_file); if (threading_status == CUDSS_STATUS_SUCCESS) { - settings.log.printf("cuDSS Threading layer : %s\n", cudss_mt_lib_file.c_str()); + settings.log.printf("cuDSS Threading layer : %s\n", cudss_mt_lib_file); } else { settings.log.printf( "cuDSS Threading layer : could not load '%s' (status = %d); falling back " "to single-threaded cuDSS. Set the CUDSS_THREADING_LIB environment variable to " "an absolute path, or ensure the host provides libgomp.so.1, to enable " "multi-threaded cuDSS.\n", - cudss_mt_lib_file.c_str(), + cudss_mt_lib_file, threading_status); } }