From aaf2d08622ad1022351d0f20fc0d53bda35caf2b Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 2 Mar 2026 11:55:47 +0100 Subject: [PATCH 01/29] feat: configure leading dimensions explicitly --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index cbf18e0..7c9b328 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -118,10 +118,10 @@ class BlasCuda { } } - void AddLayoutConfig(std::size_t m, std::size_t n, std::size_t k) { - CheckAndAddLayout(k, m); - CheckAndAddLayout(k, n); - CheckAndAddLayout(m, n); + void AddLayoutConfig(std::size_t m, std::size_t n, std::size_t k, std::size_t lda, std::size_t ldb, std::size_t ldc) { + CheckAndAddLayout(k, m, lda); + CheckAndAddLayout(k, n, ldb); + CheckAndAddLayout(m, n, ldc); } template @@ -313,11 +313,10 @@ gemmrelu(char transa, char transb, const unsigned int m, private: alpaka::QueueCudaRtNonBlocking m_queue; - void CheckAndAddLayout(size_t rows, size_t cols) { + void CheckAndAddLayout(size_t rows, size_t cols, size_t ld) { auto key = std::make_pair(rows, cols); if (LayoutStore.find(key) == LayoutStore.end()) { cublasLtMatrixLayout_t temp = nullptr; - size_t ld = rows; CHECK_CUBLAS( cublasLtMatrixLayoutCreate(&temp, CUDA_R_32F, rows, cols, ld)); LayoutStore.emplace(key, temp); From 9aa4b884854ed82985667e8481710f3124d7dd48 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 2 Mar 2026 14:31:55 +0100 Subject: [PATCH 02/29] fix: set stream while initializing handle --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 7c9b328..58c0f27 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -49,7 +49,6 @@ struct PairEq { class BlasCuda { cublasLtHandle_t ltHandle = nullptr; - cublasHandle_t handle = nullptr; cublasLtMatmulDesc_t operationDesc = nullptr; cublasLtMatmulPreference_t preference = nullptr; void *d_workspace = nullptr; @@ -72,7 +71,7 @@ class BlasCuda { BlasCuda(alpaka::QueueCudaRtNonBlocking &queue) : m_queue{queue} { stream = static_cast(m_queue.getNativeHandle()); CHECK_CUBLAS(cublasLtCreate(<Handle)); - CHECK_CUBLAS(cublasCreate(&handle)); + CHECK_CUBLAS(cublasSetStream(ltHandle, stream)); heuristic = {}; CHECK_CUBLAS(cublasLtMatmulDescCreate(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); From 2498197aec00d4c65f4c46eec23d75e253b468b6 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 2 Mar 2026 14:52:00 +0100 Subject: [PATCH 03/29] feat: print requested workspace --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 58c0f27..1c9dd24 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -71,7 +71,6 @@ class BlasCuda { BlasCuda(alpaka::QueueCudaRtNonBlocking &queue) : m_queue{queue} { stream = static_cast(m_queue.getNativeHandle()); CHECK_CUBLAS(cublasLtCreate(<Handle)); - CHECK_CUBLAS(cublasSetStream(ltHandle, stream)); heuristic = {}; CHECK_CUBLAS(cublasLtMatmulDescCreate(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); @@ -170,7 +169,8 @@ gemm(char transa, char transb, const unsigned int m, 1, &localHeuristic, &returnedResults)); - + std::cout << "Requested workspace: " + << localHeuristic.workspaceSize << std::endl; if (returnedResults == 0) { cublasLtMatmulDescDestroy(localDesc); std::cerr << "No suitable cuBLASLt algorithm found!\n"; @@ -237,7 +237,8 @@ gemmrelu(char transa, char transb, const unsigned int m, 1, &localHeuristic, &error_flag)); - + std::cout << "Requested workspace: " + << localHeuristic.workspaceSize << std::endl; if (error_flag == 0) { cublasLtMatmulDescDestroy(localDesc); std::cerr << "No suitable cuBLASLt algorithm found!\n"; From 901c5ae33fd687b7c3ad0b01babf0b0008846ca7 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 2 Mar 2026 17:31:13 +0100 Subject: [PATCH 04/29] fix: provide layout order during adding --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 1c9dd24..e921ebb 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -319,6 +319,8 @@ gemmrelu(char transa, char transb, const unsigned int m, cublasLtMatrixLayout_t temp = nullptr; CHECK_CUBLAS( cublasLtMatrixLayoutCreate(&temp, CUDA_R_32F, rows, cols, ld)); + cublasLtMatrixLayoutSetAttribute( + temp, CUBLASLT_MATRIX_LAYOUT_ORDER, CUBLASLT_ORDER_COL, sizeof(int)); LayoutStore.emplace(key, temp); } } From 8a27ffe044d87c4f87269bd3e5d8772769edae7d Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Tue, 3 Mar 2026 10:38:08 +0100 Subject: [PATCH 05/29] fix: layout order configuration --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index e921ebb..1c9dd24 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -319,8 +319,6 @@ gemmrelu(char transa, char transb, const unsigned int m, cublasLtMatrixLayout_t temp = nullptr; CHECK_CUBLAS( cublasLtMatrixLayoutCreate(&temp, CUDA_R_32F, rows, cols, ld)); - cublasLtMatrixLayoutSetAttribute( - temp, CUBLASLT_MATRIX_LAYOUT_ORDER, CUBLASLT_ORDER_COL, sizeof(int)); LayoutStore.emplace(key, temp); } } From 54dce6320accc9aea63bb3401cb19da36dbef779 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Tue, 3 Mar 2026 10:46:56 +0100 Subject: [PATCH 06/29] fix: remove look for requested workspace --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 1c9dd24..47d4b98 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -169,8 +169,6 @@ gemm(char transa, char transb, const unsigned int m, 1, &localHeuristic, &returnedResults)); - std::cout << "Requested workspace: " - << localHeuristic.workspaceSize << std::endl; if (returnedResults == 0) { cublasLtMatmulDescDestroy(localDesc); std::cerr << "No suitable cuBLASLt algorithm found!\n"; From c5128e65d644826766fdaba0417432765d4d357b Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Fri, 17 Apr 2026 11:31:30 +0200 Subject: [PATCH 07/29] feat: matmul method for without bias matrix multiplication condition --- .../backends/cuda/sofieBLAS_cublas.hpp | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 47d4b98..831d0cb 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -308,6 +308,66 @@ gemmrelu(char transa, char transb, const unsigned int m, workspaceSize, stream)); } + // matmul without bias + template + inline void + matmul(char transa, char transb, const unsigned int m, + const unsigned int n, const unsigned int k, + const float alpha, + alpaka::BufCudaRt, TIdx> const &A, + alpaka::BufCudaRt, TIdx> const &B, + const float beta, + alpaka::BufCudaRt, TIdx> &C) + { + cublasLtMatmulDesc_t localDesc = nullptr; + CHECK_CUBLAS(cublasLtMatmulDescCreate(&localDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); + + cublasOperation_t transB_op = charToCuBlasTranspose(transb); + CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( + localDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transB_op, sizeof(transB_op))); + + cublasOperation_t transA_op = charToCuBlasTranspose(transa); + CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( + localDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transA_op, sizeof(transA_op))); + + + cublasLtMatmulHeuristicResult_t localHeuristic{}; + int returnedResults = 0; + CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( + ltHandle, + localDesc, + LayoutStore.at({k, m}), + LayoutStore.at({k, n}), + LayoutStore.at({m, n}), + LayoutStore.at({m, n}), + preference, + 1, + &localHeuristic, + &returnedResults)); + if (returnedResults == 0) { + cublasLtMatmulDescDestroy(localDesc); + std::cerr << "No suitable cuBLASLt algorithm found!\n"; + exit(EXIT_FAILURE); + } + + CHECK_CUBLAS(cublasLtMatmul( + ltHandle, + localDesc, + &alpha, + alpaka::getPtrNative(A), LayoutStore.at({k, m}), + alpaka::getPtrNative(B), LayoutStore.at({k, n}), + &beta, + alpaka::getPtrNative(C), LayoutStore.at({m, n}), + alpaka::getPtrNative(C), LayoutStore.at({m, n}), + &(localHeuristic.algo), + d_workspace, + workspaceSize, + stream)); + + cudaDeviceSynchronize(); + CHECK_CUBLAS(cublasLtMatmulDescDestroy(localDesc)); + } + private: alpaka::QueueCudaRtNonBlocking m_queue; From 9282ba9770d9064f6d77e8571081957b4051c754 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Fri, 17 Apr 2026 11:42:30 +0200 Subject: [PATCH 08/29] feat: option to pass gpu pointers directly --- .../backends/cuda/sofieBLAS_cublas.hpp | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 831d0cb..39d3b5b 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -308,6 +308,7 @@ gemmrelu(char transa, char transb, const unsigned int m, workspaceSize, stream)); } + // matmul without bias template inline void @@ -319,6 +320,24 @@ gemmrelu(char transa, char transb, const unsigned int m, const float beta, alpaka::BufCudaRt, TIdx> &C) { + + matmul(transa, transb, m, n, k, alpha, + reinterpret_cast(alpaka::getPtrNative(A)), + reinterpret_cast(alpaka::getPtrNative(B)), + beta, + reinterpret_cast(alpaka::getPtrNative(C))); + } + + template + inline void + matmul(char transa, char transb, const unsigned int m, + const unsigned int n, const unsigned int k, + const float alpha, + void const &A, + void const &B, + const float beta, + void &C) + { cublasLtMatmulDesc_t localDesc = nullptr; CHECK_CUBLAS(cublasLtMatmulDescCreate(&localDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); @@ -354,11 +373,11 @@ gemmrelu(char transa, char transb, const unsigned int m, ltHandle, localDesc, &alpha, - alpaka::getPtrNative(A), LayoutStore.at({k, m}), - alpaka::getPtrNative(B), LayoutStore.at({k, n}), + A, LayoutStore.at({k, m}), + B, LayoutStore.at({k, n}), &beta, - alpaka::getPtrNative(C), LayoutStore.at({m, n}), - alpaka::getPtrNative(C), LayoutStore.at({m, n}), + C, LayoutStore.at({m, n}), + C, LayoutStore.at({m, n}), &(localHeuristic.algo), d_workspace, workspaceSize, From fc9073798360fabe61c6de31dc4e3d8c2da76049 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Fri, 17 Apr 2026 12:08:02 +0200 Subject: [PATCH 09/29] fix: function signature for matmul method with void pointers --- .../sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 39d3b5b..c174ad3 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -322,21 +322,21 @@ gemmrelu(char transa, char transb, const unsigned int m, { matmul(transa, transb, m, n, k, alpha, - reinterpret_cast(alpaka::getPtrNative(A)), - reinterpret_cast(alpaka::getPtrNative(B)), + alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, - reinterpret_cast(alpaka::getPtrNative(C))); + alpaka::getPtrNative(C)); } - + template inline void matmul(char transa, char transb, const unsigned int m, const unsigned int n, const unsigned int k, const float alpha, - void const &A, - void const &B, + void const * A, + void const * B, const float beta, - void &C) + void * C) { cublasLtMatmulDesc_t localDesc = nullptr; CHECK_CUBLAS(cublasLtMatmulDescCreate(&localDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); From b591c4a11085a480821077713205a8da06675fb3 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Fri, 17 Apr 2026 12:16:20 +0200 Subject: [PATCH 10/29] fix: template types for arguments to function with pointer signatures --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index c174ad3..b8abe23 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -333,10 +333,10 @@ gemmrelu(char transa, char transb, const unsigned int m, matmul(char transa, char transb, const unsigned int m, const unsigned int n, const unsigned int k, const float alpha, - void const * A, - void const * B, + T const * A, + T const * B, const float beta, - void * C) + T * C) { cublasLtMatmulDesc_t localDesc = nullptr; CHECK_CUBLAS(cublasLtMatmulDescCreate(&localDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); From 19b5b3f4951f8caf46b742d965612f13d58696af Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Fri, 17 Apr 2026 12:40:23 +0200 Subject: [PATCH 11/29] fix: use float signatures for pointer arguments for blascuda --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index b8abe23..9b9c5e4 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -333,10 +333,10 @@ gemmrelu(char transa, char transb, const unsigned int m, matmul(char transa, char transb, const unsigned int m, const unsigned int n, const unsigned int k, const float alpha, - T const * A, - T const * B, + float * A, + float * B, const float beta, - T * C) + float * C) { cublasLtMatmulDesc_t localDesc = nullptr; CHECK_CUBLAS(cublasLtMatmulDescCreate(&localDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); From c9954648aec4460d4a193296954df8201dbab3e7 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Fri, 17 Apr 2026 12:50:40 +0200 Subject: [PATCH 12/29] fix: use explicit data type for signatures with pointer arguments --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 9b9c5e4..57da5e7 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -328,7 +328,6 @@ gemmrelu(char transa, char transb, const unsigned int m, alpaka::getPtrNative(C)); } - template inline void matmul(char transa, char transb, const unsigned int m, const unsigned int n, const unsigned int k, From 75f75a4ef66a0654ff4e8380f0ecee19c208e62b Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Fri, 17 Apr 2026 12:58:37 +0200 Subject: [PATCH 13/29] fix: non transpose axis for matmul method --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 57da5e7..21269fc 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -354,7 +354,7 @@ gemmrelu(char transa, char transb, const unsigned int m, CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( ltHandle, localDesc, - LayoutStore.at({k, m}), + LayoutStore.at({m, k}), LayoutStore.at({k, n}), LayoutStore.at({m, n}), LayoutStore.at({m, n}), @@ -372,7 +372,7 @@ gemmrelu(char transa, char transb, const unsigned int m, ltHandle, localDesc, &alpha, - A, LayoutStore.at({k, m}), + A, LayoutStore.at({m, k}), B, LayoutStore.at({k, n}), &beta, C, LayoutStore.at({m, n}), From ed303fbc7edef361332afbc857f65f1e9e8c26b1 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Fri, 17 Apr 2026 13:38:18 +0200 Subject: [PATCH 14/29] fix (experimental): layout shape order --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 21269fc..de36507 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -117,7 +117,7 @@ class BlasCuda { } void AddLayoutConfig(std::size_t m, std::size_t n, std::size_t k, std::size_t lda, std::size_t ldb, std::size_t ldc) { - CheckAndAddLayout(k, m, lda); + CheckAndAddLayout(m, k, lda); CheckAndAddLayout(k, n, ldb); CheckAndAddLayout(m, n, ldc); } From 54a94013c8203a1b3ac88dd365623dde0a957da6 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 4 May 2026 16:55:12 +0200 Subject: [PATCH 15/29] feat: cuda cleanup and cpu blas api --- .gitignore | 5 +- include/.vscode/settings.json | 8 - .../sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp | 86 ++- .../backends/cuda/sofieBLAS_cublas.hpp | 461 +++++-------- tests/CMakeLists.txt | 14 +- tests/test.cc | 634 +++++++++++++++--- 6 files changed, 796 insertions(+), 412 deletions(-) delete mode 100644 include/.vscode/settings.json diff --git a/.gitignore b/.gitignore index 475c359..2f9ffe1 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,7 @@ *.dwo # build files -**/build/ \ No newline at end of file +**/build/ + +# vscode settings +.vscode/ \ No newline at end of file diff --git a/include/.vscode/settings.json b/include/.vscode/settings.json deleted file mode 100644 index 64f45f4..0000000 --- a/include/.vscode/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "files.associations": { - "array": "cpp", - "string": "cpp", - "string_view": "cpp", - "span": "cpp" - } -} diff --git a/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp b/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp index 263703c..23b205f 100644 --- a/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp +++ b/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp @@ -15,6 +15,8 @@ #endif #include +#include +#include class BlasCpu { public: @@ -36,18 +38,82 @@ class BlasCpu { } } + // C = alpha * op(A) * op(B) + beta * C (no bias, leading dims inferred) + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::BufCpu, TIdx> const &A, + alpaka::BufCpu, TIdx> const &B, + float beta, + alpaka::BufCpu, TIdx> &C) { + int lda = (transa == 'N' || transa == 'n') ? static_cast(m) + : static_cast(k); + int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) + : static_cast(n); + cblas_sgemm(CblasColMajor, charToTranspose(transa), charToTranspose(transb), + static_cast(m), static_cast(n), static_cast(k), + alpha, alpaka::getPtrNative(A), lda, alpaka::getPtrNative(B), + ldb, beta, alpaka::getPtrNative(C), static_cast(m)); + } + + // C = alpha * op(A) * op(B) + beta * bias + bias_vec (bias_vec broadcast per + // row) Matches the cuBLASLt EPILOGUE_BIAS semantics: the bias buffer serves + // as both the beta-scaled accumulator and provides the per-row bias vector + // (first m elements). template inline void - gemm(char transa, char transb, const unsigned int m, const unsigned int n, - const unsigned int k, const float alpha, - alpaka::BufCpu, TIdx> const &A, const int lda, - alpaka::BufCpu, TIdx> const &B, const int ldb, - const float beta, alpaka::BufCpu, TIdx> &C, - const int ldc) { - CBLAS_TRANSPOSE TransA = charToTranspose(transa); - CBLAS_TRANSPOSE TransB = charToTranspose(transb); - cblas_sgemm(CblasColMajor, TransA, TransB, m, n, k, alpha, A.data(), lda, - B.data(), ldb, beta, C.data(), ldc); + gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, alpaka::BufCpu, TIdx> const &A, + alpaka::BufCpu, TIdx> const &B, float beta, + alpaka::BufCpu, TIdx> &bias, + alpaka::BufCpu, TIdx> &C) { + int lda = (transa == 'N' || transa == 'n') ? static_cast(m) + : static_cast(k); + int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) + : static_cast(n); + // Step 1: C = alpha * op(A) * op(B) (beta=0 so C is fully overwritten) + cblas_sgemm(CblasColMajor, charToTranspose(transa), charToTranspose(transb), + static_cast(m), static_cast(n), static_cast(k), + alpha, alpaka::getPtrNative(A), lda, alpaka::getPtrNative(B), + ldb, 0.0f, alpaka::getPtrNative(C), static_cast(m)); + // Step 2: C += beta * bias_matrix + bias_vec (per-row broadcast) + float *c = alpaka::getPtrNative(C); + const float *b = alpaka::getPtrNative(bias); + for (unsigned int j = 0; j < n; ++j) + for (unsigned int i = 0; i < m; ++i) + c[j * m + i] += beta * b[j * m + i] + b[i]; + } + + // C = relu(alpha * op(A) * op(B) + beta * bias + bias_vec) + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::BufCpu, TIdx> const &A, + alpaka::BufCpu, TIdx> const &B, + float beta, + alpaka::BufCpu, TIdx> &bias, + alpaka::BufCpu, TIdx> &C) { + gemm(transa, transb, m, n, k, alpha, A, B, beta, bias, C); + float *c = alpaka::getPtrNative(C); + for (unsigned int i = 0; i < m * n; ++i) + c[i] = c[i] > 0.0f ? c[i] : 0.0f; + } + + // C = gelu(alpha * op(A) * op(B) + beta * bias + bias_vec) + // Uses the standard GELU: x * 0.5 * (1 + erf(x / sqrt(2))) + template + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::BufCpu, TIdx> const &A, + alpaka::BufCpu, TIdx> const &B, + float beta, + alpaka::BufCpu, TIdx> &bias, + alpaka::BufCpu, TIdx> &C) { + gemm(transa, transb, m, n, k, alpha, A, B, beta, bias, C); + float *c = alpaka::getPtrNative(C); + constexpr float kInvSqrt2 = 0.7071067811865476f; + for (unsigned int i = 0; i < m * n; ++i) + c[i] *= 0.5f * (1.0f + std::erff(c[i] * kInvSqrt2)); } }; diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index de36507..3c4e93c 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -15,7 +15,7 @@ #include #define CHECK_CUDA(err) \ - if (err != cudaSuccess) { \ + if ((err) != cudaSuccess) { \ std::cerr << "CUDA error: " << cudaGetErrorString(err) << " at line " \ << __LINE__ << "\n"; \ exit(EXIT_FAILURE); \ @@ -23,10 +23,9 @@ #define CHECK_CUBLAS(status) \ do { \ - cublasStatus_t s = (status); \ - if (s != CUBLAS_STATUS_SUCCESS) { \ - std::cerr << "cuBLAS error " << s << " at line " << __LINE__ \ - << std::endl; \ + cublasStatus_t _s = (status); \ + if (_s != CUBLAS_STATUS_SUCCESS) { \ + std::cerr << "cuBLAS error " << _s << " at line " << __LINE__ << "\n"; \ exit(EXIT_FAILURE); \ } \ } while (0) @@ -49,31 +48,23 @@ struct PairEq { class BlasCuda { cublasLtHandle_t ltHandle = nullptr; - cublasLtMatmulDesc_t operationDesc = nullptr; cublasLtMatmulPreference_t preference = nullptr; void *d_workspace = nullptr; - size_t workspaceSize = 1 << 22; // 4MB + size_t workspaceSize = 1 << 22; // 4 MB cudaStream_t stream = nullptr; - cublasLtMatmulHeuristicResult_t heuristic; - cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DEFAULT; - int error_flag = 0; - std::unordered_map, cublasLtMatrixLayout_t, PairHash, PairEq> - LayoutStore; + layoutStore; public: - BlasCuda(const BlasCuda&) = delete; - BlasCuda& operator=(const BlasCuda&) = delete; - BlasCuda(BlasCuda&&) = delete; - BlasCuda& operator=(BlasCuda&&) = delete; + BlasCuda(const BlasCuda &) = delete; + BlasCuda &operator=(const BlasCuda &) = delete; + BlasCuda(BlasCuda &&) = delete; + BlasCuda &operator=(BlasCuda &&) = delete; BlasCuda(alpaka::QueueCudaRtNonBlocking &queue) : m_queue{queue} { stream = static_cast(m_queue.getNativeHandle()); CHECK_CUBLAS(cublasLtCreate(<Handle)); - heuristic = {}; - CHECK_CUBLAS(cublasLtMatmulDescCreate(&operationDesc, CUBLAS_COMPUTE_32F, - CUDA_R_32F)); CHECK_CUBLAS(cublasLtMatmulPreferenceCreate(&preference)); CHECK_CUDA(cudaMalloc(&d_workspace, workspaceSize)); CHECK_CUBLAS(cublasLtMatmulPreferenceSetAttribute( @@ -82,22 +73,15 @@ class BlasCuda { } ~BlasCuda() { - for (auto& [key, layout] : LayoutStore) { - if (layout) { + for (auto &[key, layout] : layoutStore) + if (layout) cublasLtMatrixLayoutDestroy(layout); - } - } - LayoutStore.clear(); - if (preference) cublasLtMatmulPreferenceDestroy(preference); - if (operationDesc) - cublasLtMatmulDescDestroy(operationDesc); if (ltHandle) cublasLtDestroy(ltHandle); if (d_workspace) cudaFree(d_workspace); - } inline cublasOperation_t charToCuBlasTranspose(char trans) { @@ -116,308 +100,175 @@ class BlasCuda { } } - void AddLayoutConfig(std::size_t m, std::size_t n, std::size_t k, std::size_t lda, std::size_t ldb, std::size_t ldc) { - CheckAndAddLayout(m, k, lda); - CheckAndAddLayout(k, n, ldb); - CheckAndAddLayout(m, n, ldc); + // Register matrix layouts for a given (m, n, k, lda, ldb, ldc, transa, + // transb). Must be called before gemm/gemmrelu/gemmgelu/matmul for each + // unique combination. + void addLayoutConfig(std::size_t m, std::size_t n, std::size_t k, + std::size_t lda, std::size_t ldb, std::size_t ldc, + char transa, char transb) { + // Physical A: (m×k) if NoTrans, (k×m) if Trans + if (transa == 'N' || transa == 'n') + checkAndAddLayout(m, k, lda); + else + checkAndAddLayout(k, m, lda); + // Physical B: (k×n) if NoTrans, (n×k) if Trans + if (transb == 'N' || transb == 'n') + checkAndAddLayout(k, n, ldb); + else + checkAndAddLayout(n, k, ldb); + // C is always (m×n) + checkAndAddLayout(m, n, ldc); } -template -inline void -gemm(char transa, char transb, const unsigned int m, - const unsigned int n, const unsigned int k, - const float alpha, - alpaka::BufCudaRt, TIdx> const &A, - alpaka::BufCudaRt, TIdx> const &B, - const float beta, - alpaka::BufCudaRt, TIdx> &bias, - alpaka::BufCudaRt, TIdx> &C) -{ - cublasLtMatmulDesc_t localDesc = nullptr; - CHECK_CUBLAS(cublasLtMatmulDescCreate(&localDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); - - cublasOperation_t transB_op = charToCuBlasTranspose(transb); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transB_op, sizeof(transB_op))); - - cublasOperation_t transA_op = charToCuBlasTranspose(transa); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transA_op, sizeof(transA_op))); - - void *bias_ptr = reinterpret_cast(alpaka::getPtrNative(bias)); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_ptr, sizeof(bias_ptr))); - - cublasLtEpilogue_t ep = CUBLASLT_EPILOGUE_BIAS; - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, - CUBLASLT_MATMUL_DESC_EPILOGUE, - &ep, - sizeof(ep))); - - - cublasLtMatmulHeuristicResult_t localHeuristic{}; - int returnedResults = 0; - CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( - ltHandle, - localDesc, - LayoutStore.at({k, m}), - LayoutStore.at({k, n}), - LayoutStore.at({m, n}), - LayoutStore.at({m, n}), - preference, - 1, - &localHeuristic, - &returnedResults)); - if (returnedResults == 0) { - cublasLtMatmulDescDestroy(localDesc); - std::cerr << "No suitable cuBLASLt algorithm found!\n"; - exit(EXIT_FAILURE); - } - - CHECK_CUBLAS(cublasLtMatmul( - ltHandle, - localDesc, - &alpha, - alpaka::getPtrNative(A), LayoutStore.at({k, m}), - alpaka::getPtrNative(B), LayoutStore.at({k, n}), - &beta, - alpaka::getPtrNative(bias), LayoutStore.at({m, n}), - alpaka::getPtrNative(C), LayoutStore.at({m, n}), - &(localHeuristic.algo), - d_workspace, - workspaceSize, - stream)); - - cudaDeviceSynchronize(); - CHECK_CUBLAS(cublasLtMatmulDescDestroy(localDesc)); -} - -template -inline void -gemmrelu(char transa, char transb, const unsigned int m, - const unsigned int n, const unsigned int k, - const float alpha, - alpaka::BufCudaRt, TIdx> const &A, - alpaka::BufCudaRt, TIdx> const &B, - const float beta, - alpaka::BufCudaRt, TIdx> &bias, - alpaka::BufCudaRt, TIdx> &C) -{ - cublasLtMatmulDesc_t localDesc = nullptr; - CHECK_CUBLAS(cublasLtMatmulDescCreate(&localDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); - - cublasOperation_t transB_op = charToCuBlasTranspose(transb); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transB_op, sizeof(transB_op))); - - cublasOperation_t transA_op = charToCuBlasTranspose(transa); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transA_op, sizeof(transA_op))); - - void *bias_ptr = reinterpret_cast(alpaka::getPtrNative(bias)); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_ptr, sizeof(bias_ptr))); - - cublasLtEpilogue_t ep = CUBLASLT_EPILOGUE_RELU_BIAS; - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &ep, sizeof(ep))); - - cublasLtMatmulHeuristicResult_t localHeuristic{}; - CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( - ltHandle, - localDesc, - LayoutStore.at({k, m}), - LayoutStore.at({k, n}), - LayoutStore.at({m, n}), - LayoutStore.at({m, n}), - preference, - 1, - &localHeuristic, - &error_flag)); - std::cout << "Requested workspace: " - << localHeuristic.workspaceSize << std::endl; - if (error_flag == 0) { - cublasLtMatmulDescDestroy(localDesc); - std::cerr << "No suitable cuBLASLt algorithm found!\n"; - exit(EXIT_FAILURE); - } - - CHECK_CUBLAS(cublasLtMatmul( - ltHandle, - localDesc, - &alpha, - alpaka::getPtrNative(A), LayoutStore.at({k, m}), - alpaka::getPtrNative(B), LayoutStore.at({k, n}), - &beta, - alpaka::getPtrNative(bias), LayoutStore.at({m, n}), - alpaka::getPtrNative(C), LayoutStore.at({m, n}), - &(localHeuristic.algo), - d_workspace, - workspaceSize, - stream)); - - cudaDeviceSynchronize(); - CHECK_CUBLAS(cublasLtMatmulDescDestroy(localDesc)); -} + // C = alpha * op(A) * op(B) + beta * bias + bias_vec (bias_vec broadcast per + // row) + template + inline void + gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, alpaka::BufCudaRt, TIdx> const &A, + alpaka::BufCudaRt, TIdx> const &B, float beta, + alpaka::BufCudaRt, TIdx> &bias, + alpaka::BufCudaRt, TIdx> &C) { + const void *bptr = alpaka::getPtrNative(bias); + auto desc = + makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_BIAS, bptr); + executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), + beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + // C = relu(alpha * op(A) * op(B) + beta * bias + bias_vec) template - inline void gemmgelu(char transa, char transb, const unsigned int m, - const unsigned int n, const unsigned int k, - const float alpha, + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, alpaka::BufCudaRt, TIdx> const &A, alpaka::BufCudaRt, TIdx> const &B, - const float beta, + float beta, alpaka::BufCudaRt, TIdx> &bias, alpaka::BufCudaRt, TIdx> &C) { - - cublasLtMatmulDesc_t localDesc = nullptr; - CHECK_CUBLAS(cublasLtMatmulDescCreate(&localDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); - - void *bias_ptr = reinterpret_cast(alpaka::getPtrNative(bias)); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_ptr, - sizeof(bias_ptr))); - - cublasOperation_t transB = charToCuBlasTranspose(transb); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transB, sizeof(transB))); - - cublasOperation_t transA = charToCuBlasTranspose(transa); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transA, sizeof(transA))); - SetGeluActivation(); - - cublasLtMatmulHeuristicResult_t localHeuristic{}; - CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( - ltHandle, localDesc, - LayoutStore.at({k, m}), - LayoutStore.at({k, n}), - LayoutStore.at({m, n}), - LayoutStore.at({m, n}), - preference, 1, &localHeuristic, &error_flag)); - if (error_flag == 0) { - std::cerr << "No suitable cuBLASLt algorithm found!\n"; - exit(EXIT_FAILURE); - } - - CHECK_CUBLAS(cublasLtMatmul( - ltHandle, localDesc, &alpha, alpaka::getPtrNative(A), LayoutStore.at({k, m}), - alpaka::getPtrNative(B), LayoutStore.at({k, n}), &beta, alpaka::getPtrNative(bias), LayoutStore.at({m, n}), - alpaka::getPtrNative(C), LayoutStore.at({m, n}), &(localHeuristic.algo), d_workspace, - workspaceSize, stream)); + const void *bptr = alpaka::getPtrNative(bias); + auto desc = + makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_RELU_BIAS, bptr); + executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), + beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } - - // matmul without bias + // C = gelu(alpha * op(A) * op(B) + beta * bias + bias_vec) template - inline void - matmul(char transa, char transb, const unsigned int m, - const unsigned int n, const unsigned int k, - const float alpha, - alpaka::BufCudaRt, TIdx> const &A, - alpaka::BufCudaRt, TIdx> const &B, - const float beta, - alpaka::BufCudaRt, TIdx> &C) - { - - matmul(transa, transb, m, n, k, alpha, - alpaka::getPtrNative(A), - alpaka::getPtrNative(B), - beta, - alpaka::getPtrNative(C)); + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::BufCudaRt, TIdx> const &A, + alpaka::BufCudaRt, TIdx> const &B, + float beta, + alpaka::BufCudaRt, TIdx> &bias, + alpaka::BufCudaRt, TIdx> &C) { + const void *bptr = alpaka::getPtrNative(bias); + auto desc = + makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_GELU_BIAS, bptr); + executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), + beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } - inline void - matmul(char transa, char transb, const unsigned int m, - const unsigned int n, const unsigned int k, - const float alpha, - float * A, - float * B, - const float beta, - float * C) - { - cublasLtMatmulDesc_t localDesc = nullptr; - CHECK_CUBLAS(cublasLtMatmulDescCreate(&localDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); - - cublasOperation_t transB_op = charToCuBlasTranspose(transb); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transB_op, sizeof(transB_op))); - - cublasOperation_t transA_op = charToCuBlasTranspose(transa); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - localDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transA_op, sizeof(transA_op))); - - - cublasLtMatmulHeuristicResult_t localHeuristic{}; - int returnedResults = 0; - CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( - ltHandle, - localDesc, - LayoutStore.at({m, k}), - LayoutStore.at({k, n}), - LayoutStore.at({m, n}), - LayoutStore.at({m, n}), - preference, - 1, - &localHeuristic, - &returnedResults)); - if (returnedResults == 0) { - cublasLtMatmulDescDestroy(localDesc); - std::cerr << "No suitable cuBLASLt algorithm found!\n"; - exit(EXIT_FAILURE); - } - - CHECK_CUBLAS(cublasLtMatmul( - ltHandle, - localDesc, - &alpha, - A, LayoutStore.at({m, k}), - B, LayoutStore.at({k, n}), - &beta, - C, LayoutStore.at({m, n}), - C, LayoutStore.at({m, n}), - &(localHeuristic.algo), - d_workspace, - workspaceSize, - stream)); - - cudaDeviceSynchronize(); - CHECK_CUBLAS(cublasLtMatmulDescDestroy(localDesc)); + // C = alpha * op(A) * op(B) + beta * C (no bias) + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::BufCudaRt, TIdx> const &A, + alpaka::BufCudaRt, TIdx> const &B, + float beta, + alpaka::BufCudaRt, TIdx> &C) { + auto desc = + makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_DEFAULT); + float *c = alpaka::getPtrNative(C); + executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), + beta, c, c, layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); } private: alpaka::QueueCudaRtNonBlocking m_queue; - void CheckAndAddLayout(size_t rows, size_t cols, size_t ld) { + // Returns the layout map key for matrix A based on its transpose flag. + // Physical dimensions: NoTrans → (m×k), Trans → (k×m). + static std::pair + layoutKeyA(char trans, std::size_t m, std::size_t k) { + return (trans == 'N' || trans == 'n') ? std::make_pair(m, k) + : std::make_pair(k, m); + } + + // Returns the layout map key for matrix B based on its transpose flag. + // Physical dimensions: NoTrans → (k×n), Trans → (n×k). + static std::pair + layoutKeyB(char trans, std::size_t k, std::size_t n) { + return (trans == 'N' || trans == 'n') ? std::make_pair(k, n) + : std::make_pair(n, k); + } + + void checkAndAddLayout(std::size_t rows, std::size_t cols, std::size_t ld) { auto key = std::make_pair(rows, cols); - if (LayoutStore.find(key) == LayoutStore.end()) { - cublasLtMatrixLayout_t temp = nullptr; + if (layoutStore.find(key) == layoutStore.end()) { + cublasLtMatrixLayout_t layout = nullptr; CHECK_CUBLAS( - cublasLtMatrixLayoutCreate(&temp, CUDA_R_32F, rows, cols, ld)); - LayoutStore.emplace(key, temp); + cublasLtMatrixLayoutCreate(&layout, CUDA_R_32F, rows, cols, ld)); + layoutStore.emplace(key, layout); } } - void ResetActivation() { - epilogue = CUBLASLT_EPILOGUE_BIAS; - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute(operationDesc, - CUBLASLT_MATMUL_DESC_EPILOGUE, - &epilogue, sizeof(epilogue))); - } - - void SetReluActivation() { - epilogue = CUBLASLT_EPILOGUE_RELU_BIAS; - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute(operationDesc, - CUBLASLT_MATMUL_DESC_EPILOGUE, - &epilogue, sizeof(epilogue))); + // Creates a per-call matmul descriptor with transpose ops, epilogue, and + // optional bias pointer. Caller owns the returned descriptor. + cublasLtMatmulDesc_t makeDesc(cublasOperation_t transA, + cublasOperation_t transB, + cublasLtEpilogue_t epilogue, + const void *bias_ptr = nullptr) { + cublasLtMatmulDesc_t desc = nullptr; + CHECK_CUBLAS( + cublasLtMatmulDescCreate(&desc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); + CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( + desc, CUBLASLT_MATMUL_DESC_TRANSA, &transA, sizeof(transA))); + CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( + desc, CUBLASLT_MATMUL_DESC_TRANSB, &transB, sizeof(transB))); + CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( + desc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue))); + if (bias_ptr) { + CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( + desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_ptr, + sizeof(bias_ptr))); + } + return desc; } - void SetGeluActivation() { - epilogue = CUBLASLT_EPILOGUE_GELU; - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute(operationDesc, - CUBLASLT_MATMUL_DESC_EPILOGUE, - &epilogue, sizeof(epilogue))); + // Runs heuristic selection, executes cublasLtMatmul, syncs stream, and + // destroys desc. D_in is the matrix scaled by beta (may equal C_out for + // in-place accumulation). + void executeMatmul(cublasLtMatmulDesc_t desc, float alpha, const float *A, + const float *B, float beta, const float *D_in, + float *C_out, + const std::pair &kA, + const std::pair &kB, + const std::pair &kC) { + cublasLtMatmulHeuristicResult_t h{}; + int returnedResults = 0; + CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( + ltHandle, desc, layoutStore.at(kA), layoutStore.at(kB), + layoutStore.at(kC), layoutStore.at(kC), preference, 1, &h, + &returnedResults)); + if (returnedResults == 0) { + cublasLtMatmulDescDestroy(desc); + std::cerr << "No suitable cuBLASLt algorithm found!\n"; + exit(EXIT_FAILURE); + } + CHECK_CUBLAS(cublasLtMatmul(ltHandle, desc, &alpha, A, layoutStore.at(kA), + B, layoutStore.at(kB), &beta, D_in, + layoutStore.at(kC), C_out, layoutStore.at(kC), + &h.algo, d_workspace, workspaceSize, stream)); + CHECK_CUDA(cudaStreamSynchronize(stream)); + CHECK_CUBLAS(cublasLtMatmulDescDestroy(desc)); } }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2ebded2..0a4aa01 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,10 +19,15 @@ endif() # --- Compiler flags --- set(CXXFLAGS -O2 -g -DALPAKA_HAS_STD_ATOMIC_REF) set(CXX_HOST_FLAGS -fPIC -pthread) -set(CUDA_ARCH "sm_86") -set(CXX_CUDA_FLAGS -arch=${CUDA_ARCH} -Wno-deprecated-gpu-targets --extended-lambda --expt-relaxed-constexpr) +set(CXX_CUDA_FLAGS -Wno-deprecated-gpu-targets --extended-lambda --expt-relaxed-constexpr) set(XCOMPILER_FLAGS -Xcompiler=-fPIC,-pthread) +# --- CUDA architecture: must be set before enable_language(CUDA) so all +# targets inherit a valid default (CMake 3.18+ requires CUDA_ARCHITECTURES +# to be non-empty on every target once CUDA is enabled globally). +set(CMAKE_CUDA_ARCHITECTURES 86) +enable_language(CUDA) + # --- Include directories --- include_directories(${ALPAKA_BASE}/include "../include") @@ -86,12 +91,9 @@ target_include_directories(test_cpu PRIVATE "../sofieBLAS/include" ${ALPAKA_BASE target_link_libraries(test_cpu PRIVATE ${BLAS_LIBS}) -set(TEST_SRC test.cc) - add_executable(test_cuda) target_sources(test_cuda PRIVATE test.cc) set_source_files_properties(test.cc PROPERTIES LANGUAGE CUDA) -enable_language(CUDA) set_target_properties(test_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON) target_compile_features(test_cuda PUBLIC cxx_std_20) @@ -104,7 +106,7 @@ target_compile_options(test_cuda PRIVATE target_compile_definitions(test_cuda PRIVATE ALPAKA_ACC_GPU_CUDA_ENABLED) target_include_directories(test_cuda PRIVATE ${ALPAKA_BASE}/include "../sofieBLAS/include" ${CUDA_BASE}/include) target_link_directories(test_cuda PRIVATE ${CUDA_BASE}/lib64) -target_link_libraries(test_cuda PRIVATE cublas cudart) +target_link_libraries(test_cuda PRIVATE cublasLt cublas cudart) # --- clean target equivalent --- add_custom_target(clean-all diff --git a/tests/test.cc b/tests/test.cc index 5812bc9..e106275 100644 --- a/tests/test.cc +++ b/tests/test.cc @@ -1,121 +1,591 @@ #include "sofieBLAS/sofieBLAS.hpp" #include +#include +#include #include #include -#include +#include +#include -// index and size type using Idx = uint32_t; - -// dimensions -using Dim0D = alpaka::DimInt<0u>; using Dim1D = alpaka::DimInt<1u>; -using Dim2D = alpaka::DimInt<2u>; -using Dim3D = alpaka::DimInt<3u>; - -// Print a column-major matrix -template -void print(alpaka::BufCpu const &M, TIdx size) { - assert(alpaka::getExtentProduct(M) == size * size); - - for (TIdx row = 0; row < size; ++row) { - for (TIdx col = 0; col < size; ++col) { - std::cout << std::fixed << std::setprecision(2) << std::setw(7) - << M[col * size + row] << " "; + +// --------------------------------------------------------------------------- +// Reference implementations (column-major, float) +// --------------------------------------------------------------------------- + +// Access element (row, col) of a column-major matrix with leading dim `ld`. +static inline float cm(const float *M, int row, int col, int ld) { + return M[col * ld + row]; +} + +// C = alpha * op(A) * op(B) + beta * C (in-place, column-major) +static void refMatmul(float *C, const float *A, const float *B, int m, int n, + int k, float alpha, float beta, bool transA, + bool transB) { + int lda = transA ? k : m; + int ldb = transB ? n : k; + for (int j = 0; j < n; ++j) { + for (int i = 0; i < m; ++i) { + float sum = 0.f; + for (int p = 0; p < k; ++p) { + float a = transA ? cm(A, p, i, lda) : cm(A, i, p, lda); + float b = transB ? cm(B, j, p, ldb) : cm(B, p, j, ldb); + sum += a * b; + } + C[j * m + i] = alpha * sum + beta * C[j * m + i]; } - std::cout << "\n"; } } -int main() { - constexpr Idx size = 4; +// C = alpha * op(A) * op(B) + beta * bias_matrix + bias_vec (per-row broadcast) +static void refGemm(float *C, const float *A, const float *B, const float *bias, + int m, int n, int k, float alpha, float beta, bool transA, + bool transB) { + int lda = transA ? k : m; + int ldb = transB ? n : k; + for (int j = 0; j < n; ++j) { + for (int i = 0; i < m; ++i) { + float sum = 0.f; + for (int p = 0; p < k; ++p) { + float a = transA ? cm(A, p, i, lda) : cm(A, i, p, lda); + float b = transB ? cm(B, j, p, ldb) : cm(B, p, j, ldb); + sum += a * b; + } + C[j * m + i] = alpha * sum + beta * bias[j * m + i] + bias[i]; + } + } +} - // Host platform and device - alpaka::PlatformCpu host_platform{}; - auto host = alpaka::getDevByIdx(host_platform, 0u); +static void refGemmRelu(float *C, const float *A, const float *B, + const float *bias, int m, int n, int k, float alpha, + float beta, bool transA, bool transB) { + refGemm(C, A, B, bias, m, n, k, alpha, beta, transA, transB); + for (int i = 0; i < m * n; ++i) + C[i] = C[i] > 0.f ? C[i] : 0.f; +} - // Allocate matrices (column-major) - auto A = alpaka::allocBuf(host, size * size); - auto B = alpaka::allocBuf(host, size * size); - auto C = alpaka::allocBuf(host, size * size); +static void refGemmGelu(float *C, const float *A, const float *B, + const float *bias, int m, int n, int k, float alpha, + float beta, bool transA, bool transB) { + refGemm(C, A, B, bias, m, n, k, alpha, beta, transA, transB); + constexpr float kInvSqrt2 = 0.7071067811865476f; + for (int i = 0; i < m * n; ++i) + C[i] *= 0.5f * (1.f + std::erff(C[i] * kInvSqrt2)); +} - // Fill A and B with random floats centered around 0 - std::random_device rd; - std::mt19937 gen(rd()); - std::normal_distribution dist(0.0f, 10.0f); +static int gFailures = 0; - for (int i = 0; i < size * size; ++i) { - A[i] = dist(gen); - B[i] = dist(gen); +static void checkClose(const float *got, const float *expected, int n, + const std::string &name, float rtol = 1e-4f, + float atol = 1e-4f) { + bool pass = true; + for (int i = 0; i < n; ++i) { + float diff = std::abs(got[i] - expected[i]); + float thr = atol + rtol * std::abs(expected[i]); + if (diff > thr) { + std::cerr << " FAIL [" << name << "] idx=" << i << " got=" << got[i] + << " expected=" << expected[i] << " diff=" << diff << "\n"; + pass = false; + } } - std::cout << "Matrix A:\n"; - print(A, size); - std::cout << '\n'; - std::cout << "Matrix B:\n"; - print(B, size); - std::cout << '\n'; + if (pass) + std::cout << " PASS " << name << "\n"; + else + ++gFailures; +} -#ifdef ALPAKA_ACC_GPU_CUDA_ENABLED +// --------------------------------------------------------------------------- +// Helpers to fill test matrices +// --------------------------------------------------------------------------- + +static void fillSeq(float *M, int n, float start = 1.f, float step = 1.f) { + for (int i = 0; i < n; ++i) + M[i] = start + static_cast(i) * step; +} + +static void fillVal(float *M, int n, float v) { + for (int i = 0; i < n; ++i) + M[i] = v; +} + +// --------------------------------------------------------------------------- +// CPU tests +// --------------------------------------------------------------------------- + +#ifdef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED + +static void runCpuTests() { + std::cout << "\n=== CPU Tests ===\n"; + + alpaka::PlatformCpu platform{}; + auto dev = alpaka::getDevByIdx(platform, 0u); + alpaka::Queue queue{dev}; + sofieBLAS blas(queue); + + constexpr int M = 4, N = 3, K = 5; + + // Allocate host buffers + auto hA = alpaka::allocBuf(dev, static_cast(M * K)); + auto hB = alpaka::allocBuf(dev, static_cast(K * N)); + auto hC = alpaka::allocBuf(dev, static_cast(M * N)); + auto hBias = alpaka::allocBuf(dev, static_cast(M * N)); + + float *A = alpaka::getPtrNative(hA); + float *B = alpaka::getPtrNative(hB); + float *C = alpaka::getPtrNative(hC); + float *bias = alpaka::getPtrNative(hBias); + + fillSeq(A, M * K); + fillSeq(B, K * N, 1.f, 0.5f); + fillSeq(bias, M * N, 0.1f, 0.1f); + + std::vector ref(M * N); + + // --- matmul NN --- + fillVal(C, M * N, 0.f); + blas.matmul('N', 'N', M, N, K, 1.f, hA, hB, 0.f, hC); + std::copy(C, C + M * N, ref.data()); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), A, B, M, N, K, 1.f, 0.f, false, false); + checkClose(C, ref.data(), M * N, "cpu::matmul NN"); + + // --- matmul TN (A^T: K×M physical → M×K logical) --- + // For TN we need A to be K×M so we create a separate buffer + { + auto hAt = alpaka::allocBuf(dev, static_cast(K * M)); + float *At = alpaka::getPtrNative(hAt); + fillSeq(At, K * M); + fillVal(C, M * N, 0.f); + blas.matmul('T', 'N', M, N, K, 1.f, hAt, hB, 0.f, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), At, B, M, N, K, 1.f, 0.f, true, false); + checkClose(C, ref.data(), M * N, "cpu::matmul TN"); + } + + // --- matmul NT --- + { + auto hBt = alpaka::allocBuf(dev, static_cast(N * K)); + float *Bt = alpaka::getPtrNative(hBt); + fillSeq(Bt, N * K, 1.f, 0.5f); + fillVal(C, M * N, 0.f); + blas.matmul('N', 'T', M, N, K, 1.f, hA, hBt, 0.f, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), A, Bt, M, N, K, 1.f, 0.f, false, true); + checkClose(C, ref.data(), M * N, "cpu::matmul NT"); + } + + // --- matmul TT --- { - alpaka::PlatformCudaRt platform; - alpaka::DevCudaRt device = alpaka::getDevByIdx(platform, 0u); - alpaka::Queue queue{device}; + auto hAt = alpaka::allocBuf(dev, static_cast(K * M)); + auto hBt = alpaka::allocBuf(dev, static_cast(N * K)); + float *At = alpaka::getPtrNative(hAt); + float *Bt = alpaka::getPtrNative(hBt); + fillSeq(At, K * M); + fillSeq(Bt, N * K, 1.f, 0.5f); + fillVal(C, M * N, 0.f); + blas.matmul('T', 'T', M, N, K, 1.f, hAt, hBt, 0.f, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), At, Bt, M, N, K, 1.f, 0.f, true, true); + checkClose(C, ref.data(), M * N, "cpu::matmul TT"); + } - const Idx m = size; // rows of A and C - const Idx n = size; // columns of B and C - const Idx k = size; // columns of A and rows of B + // --- matmul: alpha scaling --- + fillVal(C, M * N, 0.f); + blas.matmul('N', 'N', M, N, K, 2.5f, hA, hB, 0.f, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), A, B, M, N, K, 2.5f, 0.f, false, false); + checkClose(C, ref.data(), M * N, "cpu::matmul alpha=2.5"); - const float alpha = 1.0f; - const float beta = 0.0f; + // --- matmul: beta accumulation --- + fillSeq(C, M * N, 10.f); // pre-fill C + blas.matmul('N', 'N', M, N, K, 1.f, hA, hB, 0.5f, hC); + { + std::vector C0(M * N); + fillSeq(C0.data(), M * N, 10.f); + refMatmul(ref.data(), A, B, M, N, K, 1.f, 0.5f, false, false); + // ref already has the pre-filled values baked in via refMatmul beta path + // but refMatmul reads C in-place, so we need to re-run with correct init + std::copy(C0.begin(), C0.end(), ref.data()); + refMatmul(ref.data(), A, B, M, N, K, 1.f, 0.5f, false, false); + } + checkClose(C, ref.data(), M * N, "cpu::matmul beta=0.5"); - const Idx lda = size; // leading dimension of A - const Idx ldb = size; // leading dimension of B - const Idx ldc = size; + // --- gemm NN (beta=0, no prior accumulation) --- + fillVal(C, M * N, 0.f); + fillSeq(bias, M * N, 0.1f, 0.1f); + blas.gemm('N', 'N', M, N, K, 1.f, hA, hB, 0.f, hBias, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refGemm(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + checkClose(C, ref.data(), M * N, "cpu::gemm NN beta=0"); - auto A_d = alpaka::allocAsyncBuf(queue, size * size); - auto B_d = alpaka::allocAsyncBuf(queue, size * size); - auto C_d = alpaka::allocAsyncBuf(queue, size * size); - alpaka::memcpy(queue, A_d, A); - alpaka::memcpy(queue, B_d, B); + // --- gemm NN (beta=1 accumulation) --- + fillVal(C, M * N, 0.f); + blas.gemm('N', 'N', M, N, K, 1.f, hA, hB, 1.f, hBias, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refGemm(ref.data(), A, B, bias, M, N, K, 1.f, 1.f, false, false); + checkClose(C, ref.data(), M * N, "cpu::gemm NN beta=1"); - sofieBLAS blas(queue); - blas.gemm('n', 'n', m, n, k, alpha, A_d, lda, B_d, ldb, beta, C_d, ldc); - alpaka::memcpy(queue, C, C_d); + // --- gemm TN --- + { + auto hAt = alpaka::allocBuf(dev, static_cast(K * M)); + float *At = alpaka::getPtrNative(hAt); + fillSeq(At, K * M); + fillVal(C, M * N, 0.f); + blas.gemm('T', 'N', M, N, K, 1.f, hAt, hB, 0.f, hBias, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refGemm(ref.data(), At, B, bias, M, N, K, 1.f, 0.f, true, false); + checkClose(C, ref.data(), M * N, "cpu::gemm TN"); + } + // --- gemmrelu: all-positive matmul result stays unchanged --- + { + // A and B with positive values ensure result is positive before bias + auto hAp = alpaka::allocBuf(dev, static_cast(M * K)); + auto hBp = alpaka::allocBuf(dev, static_cast(K * N)); + auto hBiasp = alpaka::allocBuf(dev, static_cast(M * N)); + float *Ap = alpaka::getPtrNative(hAp); + float *Bp = alpaka::getPtrNative(hBp); + float *biasp = alpaka::getPtrNative(hBiasp); + fillSeq(Ap, M * K, 0.1f, 0.1f); + fillSeq(Bp, K * N, 0.1f, 0.1f); + fillVal(biasp, M * N, 0.f); + fillVal(C, M * N, 0.f); + blas.gemmrelu('N', 'N', M, N, K, 1.f, hAp, hBp, 0.f, hBiasp, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmRelu(ref.data(), Ap, Bp, biasp, M, N, K, 1.f, 0.f, false, false); + checkClose(C, ref.data(), M * N, "cpu::gemmrelu all-positive"); + } + + // --- gemmrelu: negative values clamped to zero --- + { + // Use alpha=-1 to force negative results + auto hBiasz = alpaka::allocBuf(dev, static_cast(M * N)); + fillVal(alpaka::getPtrNative(hBiasz), M * N, 0.f); + fillVal(C, M * N, 0.f); + blas.gemmrelu('N', 'N', M, N, K, -1.f, hA, hB, 0.f, hBiasz, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmRelu(ref.data(), A, B, alpaka::getPtrNative(hBiasz), M, N, K, -1.f, + 0.f, false, false); + checkClose(C, ref.data(), M * N, "cpu::gemmrelu alpha=-1 (clamped)"); + } + + // --- gemmrelu with bias --- + fillVal(C, M * N, 0.f); + fillSeq(bias, M * N, -5.f, 2.f); // mixed sign bias + blas.gemmrelu('N', 'N', M, N, K, 1.f, hA, hB, 0.f, hBias, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmRelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + checkClose(C, ref.data(), M * N, "cpu::gemmrelu with mixed bias"); + + // --- gemmgelu NN --- + fillVal(C, M * N, 0.f); + fillVal(bias, M * N, 0.f); + blas.gemmgelu('N', 'N', M, N, K, 1.f, hA, hB, 0.f, hBias, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmGelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + checkClose(C, ref.data(), M * N, "cpu::gemmgelu NN"); + + // --- gemmgelu with bias --- + fillVal(C, M * N, 0.f); + fillSeq(bias, M * N, -2.f, 0.5f); + blas.gemmgelu('N', 'N', M, N, K, 1.f, hA, hB, 0.f, hBias, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmGelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + checkClose(C, ref.data(), M * N, "cpu::gemmgelu with bias"); + + // --- gemmgelu TN --- + { + auto hAt = alpaka::allocBuf(dev, static_cast(K * M)); + float *At = alpaka::getPtrNative(hAt); + fillSeq(At, K * M); + fillVal(C, M * N, 0.f); + fillVal(bias, M * N, 0.f); + blas.gemmgelu('T', 'N', M, N, K, 1.f, hAt, hB, 0.f, hBias, hC); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmGelu(ref.data(), At, B, bias, M, N, K, 1.f, 0.f, true, false); + checkClose(C, ref.data(), M * N, "cpu::gemmgelu TN"); + } + + // --- edge: zero matrix --- + { + auto hZ = alpaka::allocBuf(dev, static_cast(M * K)); + fillVal(alpaka::getPtrNative(hZ), M * K, 0.f); + fillVal(C, M * N, 99.f); + fillVal(bias, M * N, 0.f); + blas.matmul('N', 'N', M, N, K, 1.f, hZ, hB, 0.f, hC); + std::fill(ref.begin(), ref.end(), 0.f); + checkClose(C, ref.data(), M * N, "cpu::matmul zero-A"); + } + + // --- edge: identity-like (square, known result) --- + { + constexpr int S = 3; + auto hI = alpaka::allocBuf(dev, static_cast(S * S)); + auto hX = alpaka::allocBuf(dev, static_cast(S * S)); + auto hY = alpaka::allocBuf(dev, static_cast(S * S)); + float *I = alpaka::getPtrNative(hI); + float *X = alpaka::getPtrNative(hX); + float *Y = alpaka::getPtrNative(hY); + fillVal(I, S * S, 0.f); + for (int i = 0; i < S; ++i) + I[i * S + i] = 1.f; + fillSeq(X, S * S); + fillVal(Y, S * S, 0.f); + blas.matmul('N', 'N', S, S, S, 1.f, hI, hX, 0.f, hY); + checkClose(Y, X, S * S, "cpu::matmul identity×X=X"); + } +} + +#endif // ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED + +// --------------------------------------------------------------------------- +// CUDA tests +// --------------------------------------------------------------------------- + +#ifdef ALPAKA_ACC_GPU_CUDA_ENABLED + +// Helper: infer packed column-major leading dimensions +static int ldaFor(char trans, int m, int k) { + return (trans == 'N' || trans == 'n') ? m : k; +} +static int ldbFor(char trans, int k, int n) { + return (trans == 'N' || trans == 'n') ? k : n; +} + +static void runCudaTests() { + std::cout << "\n=== CUDA Tests ===\n"; + + alpaka::PlatformCudaRt platform{}; + auto dev = alpaka::getDevByIdx(platform, 0u); + alpaka::Queue queue{dev}; + sofieBLAS blas(queue); + + alpaka::PlatformCpu hostPlatform{}; + auto hostDev = alpaka::getDevByIdx(hostPlatform, 0u); + + constexpr int M = 4, N = 3, K = 5; + + auto hA = alpaka::allocBuf(hostDev, static_cast(M * K)); + auto hB = alpaka::allocBuf(hostDev, static_cast(K * N)); + auto hC = alpaka::allocBuf(hostDev, static_cast(M * N)); + auto hBias = alpaka::allocBuf(hostDev, static_cast(M * N)); + + float *A = alpaka::getPtrNative(hA); + float *B = alpaka::getPtrNative(hB); + float *bias = alpaka::getPtrNative(hBias); + + fillSeq(A, M * K); + fillSeq(B, K * N, 1.f, 0.5f); + fillVal(bias, M * N, 0.f); + + auto dA = alpaka::allocAsyncBuf(queue, static_cast(M * K)); + auto dB = alpaka::allocAsyncBuf(queue, static_cast(K * N)); + auto dC = alpaka::allocAsyncBuf(queue, static_cast(M * N)); + auto dBias = + alpaka::allocAsyncBuf(queue, static_cast(M * N)); + + alpaka::memcpy(queue, dA, hA); + alpaka::memcpy(queue, dB, hB); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + + std::vector ref(M * N); + float *C = alpaka::getPtrNative(hC); + + auto verify = [&](const std::string &name) { + alpaka::memcpy(queue, hC, dC); + alpaka::wait(queue); + checkClose(C, ref.data(), M * N, name); + }; + + // ---- matmul NN ---- + blas.addLayoutConfig(M, N, K, ldaFor('N', M, K), ldbFor('N', K, N), M, 'N', + 'N'); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), A, B, M, N, K, 1.f, 0.f, false, false); + blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); + verify("cuda::matmul NN"); + + // ---- matmul TN ---- + { + auto hAt = alpaka::allocBuf(hostDev, static_cast(K * M)); + float *At = alpaka::getPtrNative(hAt); + fillSeq(At, K * M); + auto dAt = + alpaka::allocAsyncBuf(queue, static_cast(K * M)); + alpaka::memcpy(queue, dAt, hAt); alpaka::wait(queue); - std::cout << "CUDA Matrix C = A × B:\n"; - print(C, size); - std::cout << '\n'; + blas.addLayoutConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, 'T', + 'N'); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), At, B, M, N, K, 1.f, 0.f, true, false); + blas.matmul('T', 'N', M, N, K, 1.f, dAt, dB, 0.f, dC); + verify("cuda::matmul TN"); } -#endif -#ifdef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED + // ---- matmul NT ---- { - alpaka::PlatformCpu platform; - alpaka::DevCpu device = alpaka::getDevByIdx(platform, 0u); - alpaka::Queue queue{device}; + auto hBt = alpaka::allocBuf(hostDev, static_cast(N * K)); + float *Bt = alpaka::getPtrNative(hBt); + fillSeq(Bt, N * K, 1.f, 0.5f); + auto dBt = + alpaka::allocAsyncBuf(queue, static_cast(N * K)); + alpaka::memcpy(queue, dBt, hBt); + alpaka::wait(queue); + blas.addLayoutConfig(M, N, K, ldaFor('N', M, K), ldbFor('T', K, N), M, 'N', + 'T'); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), A, Bt, M, N, K, 1.f, 0.f, false, true); + blas.matmul('N', 'T', M, N, K, 1.f, dA, dBt, 0.f, dC); + verify("cuda::matmul NT"); + } + + // ---- matmul alpha=2.5 ---- + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), A, B, M, N, K, 2.5f, 0.f, false, false); + blas.matmul('N', 'N', M, N, K, 2.5f, dA, dB, 0.f, dC); + verify("cuda::matmul alpha=2.5"); - const Idx m = size; // rows of A and C - const Idx n = size; // columns of B and C - const Idx k = size; // columns of A and rows of B + // ---- gemm NN beta=0 ---- + fillSeq(bias, M * N, 0.1f, 0.1f); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + refGemm(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + blas.gemm('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dBias, dC); + verify("cuda::gemm NN beta=0"); - const float alpha = 1.0f; - const float beta = 0.0f; + // ---- gemm NN beta=1 ---- + // D_in = bias, so result = A*B + 1*bias_matrix + bias_vec + std::fill(ref.begin(), ref.end(), 0.f); + refGemm(ref.data(), A, B, bias, M, N, K, 1.f, 1.f, false, false); + blas.gemm('N', 'N', M, N, K, 1.f, dA, dB, 1.f, dBias, dC); + verify("cuda::gemm NN beta=1"); - const Idx lda = size; // leading dimension of A - const Idx ldb = size; // leading dimension of B - const Idx ldc = size; // leading dimension of C + // ---- gemm TN ---- + { + auto hAt = alpaka::allocBuf(hostDev, static_cast(K * M)); + float *At = alpaka::getPtrNative(hAt); + fillSeq(At, K * M); + auto dAt = + alpaka::allocAsyncBuf(queue, static_cast(K * M)); + alpaka::memcpy(queue, dAt, hAt); + alpaka::wait(queue); + blas.addLayoutConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, 'T', + 'N'); + std::fill(ref.begin(), ref.end(), 0.f); + refGemm(ref.data(), At, B, bias, M, N, K, 1.f, 0.f, true, false); + blas.gemm('T', 'N', M, N, K, 1.f, dAt, dB, 0.f, dBias, dC); + verify("cuda::gemm TN"); + } - sofieBLAS blas(queue); - blas.gemm('n', 'n', m, n, k, alpha, A, lda, B, ldb, beta, C, ldc); + // ---- gemmrelu: all-positive (relu is identity) ---- + { + auto hAp = alpaka::allocBuf(hostDev, static_cast(M * K)); + auto hBp = alpaka::allocBuf(hostDev, static_cast(K * N)); + auto hBiasz = + alpaka::allocBuf(hostDev, static_cast(M * N)); + float *Ap = alpaka::getPtrNative(hAp); + float *Bp = alpaka::getPtrNative(hBp); + fillSeq(Ap, M * K, 0.1f, 0.1f); + fillSeq(Bp, K * N, 0.1f, 0.1f); + fillVal(alpaka::getPtrNative(hBiasz), M * N, 0.f); + auto dAp = + alpaka::allocAsyncBuf(queue, static_cast(M * K)); + auto dBp = + alpaka::allocAsyncBuf(queue, static_cast(K * N)); + auto dBiasz = + alpaka::allocAsyncBuf(queue, static_cast(M * N)); + alpaka::memcpy(queue, dAp, hAp); + alpaka::memcpy(queue, dBp, hBp); + alpaka::memcpy(queue, dBiasz, hBiasz); + alpaka::wait(queue); + blas.addLayoutConfig(M, N, K, M, K, M, 'N', 'N'); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmRelu(ref.data(), Ap, Bp, alpaka::getPtrNative(hBiasz), M, N, K, 1.f, + 0.f, false, false); + blas.gemmrelu('N', 'N', M, N, K, 1.f, dAp, dBp, 0.f, dBiasz, dC); + verify("cuda::gemmrelu all-positive"); + } + // ---- gemmrelu: alpha=-1 forces negatives -> clamped to zero ---- + { + auto hBiasz = + alpaka::allocBuf(hostDev, static_cast(M * N)); + fillVal(alpaka::getPtrNative(hBiasz), M * N, 0.f); + auto dBiasz = + alpaka::allocAsyncBuf(queue, static_cast(M * N)); + alpaka::memcpy(queue, dBiasz, hBiasz); alpaka::wait(queue); - std::cout << "CPU Matrix C = A × B:\n"; - print(C, size); - std::cout << '\n'; + std::fill(ref.begin(), ref.end(), 0.f); + refGemmRelu(ref.data(), A, B, alpaka::getPtrNative(hBiasz), M, N, K, -1.f, + 0.f, false, false); + blas.gemmrelu('N', 'N', M, N, K, -1.f, dA, dB, 0.f, dBiasz, dC); + verify("cuda::gemmrelu alpha=-1 (clamped)"); } + + // ---- gemmrelu with mixed bias ---- + fillSeq(bias, M * N, -5.f, 2.f); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmRelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + blas.gemmrelu('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dBias, dC); + verify("cuda::gemmrelu with mixed bias"); + + // ---- gemmgelu NN ---- + fillVal(bias, M * N, 0.f); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmGelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + blas.gemmgelu('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dBias, dC); + verify("cuda::gemmgelu NN"); + + // ---- gemmgelu with bias ---- + fillSeq(bias, M * N, -2.f, 0.5f); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmGelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + blas.gemmgelu('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dBias, dC); + verify("cuda::gemmgelu with bias"); + + // ---- edge: zero A ---- + { + auto hZero = alpaka::allocBuf(hostDev, static_cast(M * K)); + fillVal(alpaka::getPtrNative(hZero), M * K, 0.f); + auto dZero = + alpaka::allocAsyncBuf(queue, static_cast(M * K)); + alpaka::memcpy(queue, dZero, hZero); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + blas.matmul('N', 'N', M, N, K, 1.f, dZero, dB, 0.f, dC); + verify("cuda::matmul zero-A"); + } +} + +#endif // ALPAKA_ACC_GPU_CUDA_ENABLED + +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- + +int main() { +#ifdef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED + runCpuTests(); #endif +#ifdef ALPAKA_ACC_GPU_CUDA_ENABLED + runCudaTests(); +#endif + + std::cout << "\n"; + if (gFailures == 0) + std::cout << "All tests passed.\n"; + else + std::cout << gFailures << " test(s) FAILED.\n"; - return 0; + return gFailures > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From 066fdaef7ae0704baab7787f212255c7b71e252f Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 4 May 2026 17:15:24 +0200 Subject: [PATCH 16/29] feat: matmul method on raw cuda pointers --- .../sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 3c4e93c..8631751 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -191,6 +191,19 @@ class BlasCuda { layoutKeyB(transb, k, n), {m, n}); } + // matmul on raw pointers directly (no layout caching, caller must ensure + // correct leading dims and layout) + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const &A, T const &B, + float beta, T &C) { + auto desc = + makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_DEFAULT); + executeMatmul(desc, alpha, A, B, beta, C, C, layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); + } + private: alpaka::QueueCudaRtNonBlocking m_queue; From 1f1c4c72d5e88c31b956d19b1f0f86e72870d182 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 4 May 2026 17:28:39 +0200 Subject: [PATCH 17/29] fix: remove extra template parameter in cuda matmul method on raw pointers --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 8631751..189d821 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -193,7 +193,7 @@ class BlasCuda { // matmul on raw pointers directly (no layout caching, caller must ensure // correct leading dims and layout) - template + template inline void matmul(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, float alpha, T const &A, T const &B, float beta, T &C) { From a8679396411d1cc033ad8dab668d0f874eacb162 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 4 May 2026 17:35:29 +0200 Subject: [PATCH 18/29] fix: method signature for matmul on cuda raw pointers --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 189d821..a89368f 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -195,8 +195,8 @@ class BlasCuda { // correct leading dims and layout) template inline void matmul(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const &A, T const &B, - float beta, T &C) { + unsigned int k, float alpha, T const * A, T const * B, + float beta, T * C) { auto desc = makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), CUBLASLT_EPILOGUE_DEFAULT); From ad41e993137b9d2d529f6963db2d3a8db3134d68 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 4 May 2026 18:00:02 +0200 Subject: [PATCH 19/29] fix: dimension order for checkAddLayout method in cuda --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index a89368f..5edf813 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -113,9 +113,9 @@ class BlasCuda { checkAndAddLayout(k, m, lda); // Physical B: (k×n) if NoTrans, (n×k) if Trans if (transb == 'N' || transb == 'n') - checkAndAddLayout(k, n, ldb); - else checkAndAddLayout(n, k, ldb); + else + checkAndAddLayout(k, n, ldb); // C is always (m×n) checkAndAddLayout(m, n, ldc); } From f86d299c33f9598f204e31447fe24872057787ad Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 4 May 2026 18:26:46 +0200 Subject: [PATCH 20/29] fix: correct transpose values while adding layouts for cuda gemm methods --- include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 5edf813..a89368f 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -113,9 +113,9 @@ class BlasCuda { checkAndAddLayout(k, m, lda); // Physical B: (k×n) if NoTrans, (n×k) if Trans if (transb == 'N' || transb == 'n') - checkAndAddLayout(n, k, ldb); - else checkAndAddLayout(k, n, ldb); + else + checkAndAddLayout(n, k, ldb); // C is always (m×n) checkAndAddLayout(m, n, ldc); } From 7b2133c9d7d1d81c81a71a2f8ceb9c94f8248390 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 11 May 2026 10:22:45 +0200 Subject: [PATCH 21/29] feat: gemm apis for alpaka views --- .../sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp | 70 +++++++++++++++++++ .../backends/cuda/sofieBLAS_cublas.hpp | 68 ++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp b/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp index 23b205f..e2fe0b2 100644 --- a/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp +++ b/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp @@ -56,6 +56,23 @@ class BlasCpu { ldb, beta, alpaka::getPtrNative(C), static_cast(m)); } + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::ViewPlainPtr, TIdx> const &A, + alpaka::ViewPlainPtr, TIdx> const &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &C) { + int lda = (transa == 'N' || transa == 'n') ? static_cast(m) + : static_cast(k); + int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) + : static_cast(n); + cblas_sgemm(CblasColMajor, charToTranspose(transa), charToTranspose(transb), + static_cast(m), static_cast(n), static_cast(k), + alpha, alpaka::getPtrNative(A), lda, alpaka::getPtrNative(B), + ldb, beta, alpaka::getPtrNative(C), static_cast(m)); + } + // C = alpha * op(A) * op(B) + beta * bias + bias_vec (bias_vec broadcast per // row) Matches the cuBLASLt EPILOGUE_BIAS semantics: the bias buffer serves // as both the beta-scaled accumulator and provides the per-row bias vector @@ -84,6 +101,30 @@ class BlasCpu { c[j * m + i] += beta * b[j * m + i] + b[i]; } + template + inline void + gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const &A, + alpaka::ViewPlainPtr, TIdx> const &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { + int lda = (transa == 'N' || transa == 'n') ? static_cast(m) + : static_cast(k); + int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) + : static_cast(n); + cblas_sgemm(CblasColMajor, charToTranspose(transa), charToTranspose(transb), + static_cast(m), static_cast(n), static_cast(k), + alpha, alpaka::getPtrNative(A), lda, alpaka::getPtrNative(B), + ldb, 0.0f, alpaka::getPtrNative(C), static_cast(m)); + float *c = alpaka::getPtrNative(C); + const float *b = alpaka::getPtrNative(bias); + for (unsigned int j = 0; j < n; ++j) + for (unsigned int i = 0; i < m; ++i) + c[j * m + i] += beta * b[j * m + i] + b[i]; + } + // C = relu(alpha * op(A) * op(B) + beta * bias + bias_vec) template inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, @@ -99,6 +140,20 @@ class BlasCpu { c[i] = c[i] > 0.0f ? c[i] : 0.0f; } + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::ViewPlainPtr, TIdx> const &A, + alpaka::ViewPlainPtr, TIdx> const &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { + gemm(transa, transb, m, n, k, alpha, A, B, beta, bias, C); + float *c = alpaka::getPtrNative(C); + for (unsigned int i = 0; i < m * n; ++i) + c[i] = c[i] > 0.0f ? c[i] : 0.0f; + } + // C = gelu(alpha * op(A) * op(B) + beta * bias + bias_vec) // Uses the standard GELU: x * 0.5 * (1 + erf(x / sqrt(2))) template @@ -115,6 +170,21 @@ class BlasCpu { for (unsigned int i = 0; i < m * n; ++i) c[i] *= 0.5f * (1.0f + std::erff(c[i] * kInvSqrt2)); } + + template + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::ViewPlainPtr, TIdx> const &A, + alpaka::ViewPlainPtr, TIdx> const &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { + gemm(transa, transb, m, n, k, alpha, A, B, beta, bias, C); + float *c = alpaka::getPtrNative(C); + constexpr float kInvSqrt2 = 0.7071067811865476f; + for (unsigned int i = 0; i < m * n; ++i) + c[i] *= 0.5f * (1.0f + std::erff(c[i] * kInvSqrt2)); + } }; namespace traits { diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index a89368f..dc502f6 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -138,6 +138,24 @@ class BlasCuda { layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } + template + inline void + gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const &A, + alpaka::ViewPlainPtr, TIdx> const &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { + const void *bptr = alpaka::getPtrNative(bias); + auto desc = + makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_BIAS, bptr); + executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), + beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + // C = relu(alpha * op(A) * op(B) + beta * bias + bias_vec) template inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, @@ -156,6 +174,23 @@ class BlasCuda { layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::ViewPlainPtr, TIdx> const &A, + alpaka::ViewPlainPtr, TIdx> const &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { + const void *bptr = alpaka::getPtrNative(bias); + auto desc = + makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_RELU_BIAS, bptr); + executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), + beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + // C = gelu(alpha * op(A) * op(B) + beta * bias + bias_vec) template inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, @@ -174,6 +209,23 @@ class BlasCuda { layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } + template + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::ViewPlainPtr, TIdx> const &A, + alpaka::ViewPlainPtr, TIdx> const &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { + const void *bptr = alpaka::getPtrNative(bias); + auto desc = + makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_GELU_BIAS, bptr); + executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), + beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + // C = alpha * op(A) * op(B) + beta * C (no bias) template inline void matmul(char transa, char transb, unsigned int m, unsigned int n, @@ -191,6 +243,22 @@ class BlasCuda { layoutKeyB(transb, k, n), {m, n}); } + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::ViewPlainPtr, TIdx> const &A, + alpaka::ViewPlainPtr, TIdx> const &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &C) { + auto desc = + makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_DEFAULT); + T *c = alpaka::getPtrNative(C); + executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), + beta, c, c, layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); + } + // matmul on raw pointers directly (no layout caching, caller must ensure // correct leading dims and layout) template From 6f2a9e16e3d4daabafdb46e1e771b67921b0a64b Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 11 May 2026 10:43:26 +0200 Subject: [PATCH 22/29] feat: gemm functions for cublas using raw pointers --- .../sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp | 34 +++++++++++++++++++ .../backends/cuda/sofieBLAS_cublas.hpp | 34 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp b/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp index e2fe0b2..6cfeb1f 100644 --- a/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp +++ b/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp @@ -185,6 +185,40 @@ class BlasCpu { for (unsigned int i = 0; i < m * n; ++i) c[i] *= 0.5f * (1.0f + std::erff(c[i] * kInvSqrt2)); } + + // Raw-pointer overloads: accept T const*/T* from any BufXxx or ViewPlainPtr via getPtrNative() + template + inline void gemm(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + int lda = (transa == 'N' || transa == 'n') ? static_cast(m) : static_cast(k); + int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) : static_cast(n); + cblas_sgemm(CblasColMajor, charToTranspose(transa), charToTranspose(transb), + static_cast(m), static_cast(n), static_cast(k), + alpha, A, lda, B, ldb, 0.0f, C, static_cast(m)); + for (unsigned int j = 0; j < n; ++j) + for (unsigned int i = 0; i < m; ++i) + C[j * m + i] += beta * bias[j * m + i] + bias[i]; + } + + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + gemm(transa, transb, m, n, k, alpha, A, B, beta, bias, C); + for (unsigned int i = 0; i < m * n; ++i) + C[i] = C[i] > 0.0f ? C[i] : 0.0f; + } + + template + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + gemm(transa, transb, m, n, k, alpha, A, B, beta, bias, C); + constexpr float kInvSqrt2 = 0.7071067811865476f; + for (unsigned int i = 0; i < m * n; ++i) + C[i] *= 0.5f * (1.0f + std::erff(C[i] * kInvSqrt2)); + } }; namespace traits { diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index dc502f6..33baadd 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -226,6 +226,40 @@ class BlasCuda { layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } + // Raw-pointer overloads: accept T const*/T* from any BufXxx or ViewPlainPtr via getPtrNative() + template + inline void gemm(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + const void *bptr = bias; + auto desc = makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_BIAS, bptr); + executeMatmul(desc, alpha, A, B, beta, bias, C, + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + const void *bptr = bias; + auto desc = makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_RELU_BIAS, bptr); + executeMatmul(desc, alpha, A, B, beta, bias, C, + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + const void *bptr = bias; + auto desc = makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_GELU_BIAS, bptr); + executeMatmul(desc, alpha, A, B, beta, bias, C, + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + // C = alpha * op(A) * op(B) + beta * C (no bias) template inline void matmul(char transa, char transb, unsigned int m, unsigned int n, From fa108fb4e6e3c896bfd6bb83dceafee240161474 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 25 May 2026 12:36:32 +0200 Subject: [PATCH 23/29] feat: strided batch computation for gemm in cublaslt --- .../backends/cuda/sofieBLAS_cublas.hpp | 374 +++++++++++------- 1 file changed, 228 insertions(+), 146 deletions(-) diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index 33baadd..f0b9e57 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -30,6 +30,7 @@ } \ } while (0) + struct PairHash { std::size_t operator()(const std::pair &p) const noexcept { @@ -46,16 +47,66 @@ struct PairEq { } }; +struct DescKey { + int transA; // CUBLAS_OP_N / CUBLAS_OP_T encoded as int + int transB; + int epilogue; // cublasLtEpilogue_t encoded as int + bool operator==(const DescKey &o) const noexcept { + return transA == o.transA && transB == o.transB && epilogue == o.epilogue; + } +}; + +struct DescKeyHash { + std::size_t operator()(const DescKey &k) const noexcept { + // Small values: simple polynomial hash + std::size_t h = static_cast(k.transA) * 97u + + static_cast(k.transB) * 31u + + static_cast(k.epilogue); + return h ^ (h >> 16); + } +}; + + +struct AlgoKey { + DescKey dk; + std::size_t rowsA, colsA; // physical dimensions of A in layoutStore + std::size_t rowsB, colsB; // physical dimensions of B in layoutStore + bool operator==(const AlgoKey &o) const noexcept { + return dk == o.dk + && rowsA == o.rowsA && colsA == o.colsA + && rowsB == o.rowsB && colsB == o.colsB; + } +}; + +struct AlgoKeyHash { + std::size_t operator()(const AlgoKey &k) const noexcept { + std::size_t h = DescKeyHash{}(k.dk); + auto mix = [&](std::size_t v) { + h ^= std::hash{}(v) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); + }; + mix(k.rowsA); mix(k.colsA); + mix(k.rowsB); mix(k.colsB); + return h; + } +}; + class BlasCuda { - cublasLtHandle_t ltHandle = nullptr; + cublasLtHandle_t ltHandle = nullptr; + cublasHandle_t handle = nullptr; // legacy cuBLAS for batched ops cublasLtMatmulPreference_t preference = nullptr; - void *d_workspace = nullptr; - size_t workspaceSize = 1 << 22; // 4 MB + void *d_workspace = nullptr; + size_t workspaceSize = 1u << 25; // 32 MB (was 4 MB) cudaStream_t stream = nullptr; + std::unordered_map, cublasLtMatrixLayout_t, PairHash, PairEq> layoutStore; + std::unordered_map descStore; + + std::unordered_map + algoCache; + public: BlasCuda(const BlasCuda &) = delete; BlasCuda &operator=(const BlasCuda &) = delete; @@ -64,7 +115,12 @@ class BlasCuda { BlasCuda(alpaka::QueueCudaRtNonBlocking &queue) : m_queue{queue} { stream = static_cast(m_queue.getNativeHandle()); + CHECK_CUBLAS(cublasLtCreate(<Handle)); + + CHECK_CUBLAS(cublasCreate(&handle)); + CHECK_CUBLAS(cublasSetStream(handle, stream)); + CHECK_CUBLAS(cublasLtMatmulPreferenceCreate(&preference)); CHECK_CUDA(cudaMalloc(&d_workspace, workspaceSize)); CHECK_CUBLAS(cublasLtMatmulPreferenceSetAttribute( @@ -74,35 +130,25 @@ class BlasCuda { ~BlasCuda() { for (auto &[key, layout] : layoutStore) - if (layout) - cublasLtMatrixLayoutDestroy(layout); - if (preference) - cublasLtMatmulPreferenceDestroy(preference); - if (ltHandle) - cublasLtDestroy(ltHandle); - if (d_workspace) - cudaFree(d_workspace); + if (layout) cublasLtMatrixLayoutDestroy(layout); + for (auto &[key, desc] : descStore) + if (desc) cublasLtMatmulDescDestroy(desc); + if (preference) cublasLtMatmulPreferenceDestroy(preference); + if (ltHandle) cublasLtDestroy(ltHandle); + if (handle) cublasDestroy(handle); + if (d_workspace) cudaFree(d_workspace); } inline cublasOperation_t charToCuBlasTranspose(char trans) { switch (trans) { - case 'N': - case 'n': - return CUBLAS_OP_N; - case 'T': - case 't': - return CUBLAS_OP_T; - case 'C': - case 'c': - return CUBLAS_OP_C; + case 'N': case 'n': return CUBLAS_OP_N; + case 'T': case 't': return CUBLAS_OP_T; + case 'C': case 'c': return CUBLAS_OP_C; default: throw std::invalid_argument("Invalid transpose character for cuBLAS."); } } - // Register matrix layouts for a given (m, n, k, lda, ldb, ldc, transa, - // transb). Must be called before gemm/gemmrelu/gemmgelu/matmul for each - // unique combination. void addLayoutConfig(std::size_t m, std::size_t n, std::size_t k, std::size_t lda, std::size_t ldb, std::size_t ldc, char transa, char transb) { @@ -120,8 +166,6 @@ class BlasCuda { checkAndAddLayout(m, n, ldc); } - // C = alpha * op(A) * op(B) + beta * bias + bias_vec (bias_vec broadcast per - // row) template inline void gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, @@ -129,12 +173,11 @@ class BlasCuda { alpaka::BufCudaRt, TIdx> const &B, float beta, alpaka::BufCudaRt, TIdx> &bias, alpaka::BufCudaRt, TIdx> &C) { - const void *bptr = alpaka::getPtrNative(bias); - auto desc = - makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_BIAS, bptr); - executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), - beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_BIAS, alpha, + alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } @@ -147,16 +190,24 @@ class BlasCuda { float beta, alpaka::ViewPlainPtr, TIdx> &bias, alpaka::ViewPlainPtr, TIdx> &C) { - const void *bptr = alpaka::getPtrNative(bias); - auto desc = - makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_BIAS, bptr); - executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), - beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_BIAS, alpha, + alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemm(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_BIAS, alpha, A, B, beta, bias, C, + static_cast(bias), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } - // C = relu(alpha * op(A) * op(B) + beta * bias + bias_vec) template inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, float alpha, @@ -165,12 +216,11 @@ class BlasCuda { float beta, alpaka::BufCudaRt, TIdx> &bias, alpaka::BufCudaRt, TIdx> &C) { - const void *bptr = alpaka::getPtrNative(bias); - auto desc = - makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_RELU_BIAS, bptr); - executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), - beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_RELU_BIAS, alpha, + alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } @@ -182,16 +232,24 @@ class BlasCuda { float beta, alpaka::ViewPlainPtr, TIdx> &bias, alpaka::ViewPlainPtr, TIdx> &C) { - const void *bptr = alpaka::getPtrNative(bias); - auto desc = - makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_RELU_BIAS, bptr); - executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), - beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_RELU_BIAS, alpha, + alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_RELU_BIAS, alpha, A, B, beta, bias, C, + static_cast(bias), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } - // C = gelu(alpha * op(A) * op(B) + beta * bias + bias_vec) template inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, float alpha, @@ -200,12 +258,11 @@ class BlasCuda { float beta, alpaka::BufCudaRt, TIdx> &bias, alpaka::BufCudaRt, TIdx> &C) { - const void *bptr = alpaka::getPtrNative(bias); - auto desc = - makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_GELU_BIAS, bptr); - executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), - beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_GELU_BIAS, alpha, + alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } @@ -217,35 +274,11 @@ class BlasCuda { float beta, alpaka::ViewPlainPtr, TIdx> &bias, alpaka::ViewPlainPtr, TIdx> &C) { - const void *bptr = alpaka::getPtrNative(bias); - auto desc = - makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_GELU_BIAS, bptr); - executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), - beta, alpaka::getPtrNative(bias), alpaka::getPtrNative(C), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - // Raw-pointer overloads: accept T const*/T* from any BufXxx or ViewPlainPtr via getPtrNative() - template - inline void gemm(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *bias, T *C) { - const void *bptr = bias; - auto desc = makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_BIAS, bptr); - executeMatmul(desc, alpha, A, B, beta, bias, C, - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *bias, T *C) { - const void *bptr = bias; - auto desc = makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_RELU_BIAS, bptr); - executeMatmul(desc, alpha, A, B, beta, bias, C, + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_GELU_BIAS, alpha, + alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } @@ -253,14 +286,12 @@ class BlasCuda { inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, float alpha, T const *A, T const *B, float beta, T *bias, T *C) { - const void *bptr = bias; - auto desc = makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_GELU_BIAS, bptr); - executeMatmul(desc, alpha, A, B, beta, bias, C, + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_GELU_BIAS, alpha, A, B, beta, bias, C, + static_cast(bias), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } - // C = alpha * op(A) * op(B) + beta * C (no bias) template inline void matmul(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, float alpha, @@ -268,13 +299,12 @@ class BlasCuda { alpaka::BufCudaRt, TIdx> const &B, float beta, alpaka::BufCudaRt, TIdx> &C) { - auto desc = - makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_DEFAULT); float *c = alpaka::getPtrNative(C); - executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), - beta, c, c, layoutKeyA(transa, m, k), - layoutKeyB(transb, k, n), {m, n}); + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_DEFAULT, alpha, + alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + c, c, nullptr, + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } template @@ -284,41 +314,56 @@ class BlasCuda { alpaka::ViewPlainPtr, TIdx> const &B, float beta, alpaka::ViewPlainPtr, TIdx> &C) { - auto desc = - makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_DEFAULT); T *c = alpaka::getPtrNative(C); - executeMatmul(desc, alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), - beta, c, c, layoutKeyA(transa, m, k), - layoutKeyB(transb, k, n), {m, n}); + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_DEFAULT, alpha, + alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + c, c, nullptr, + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } - // matmul on raw pointers directly (no layout caching, caller must ensure - // correct leading dims and layout) + // Raw-pointer overload template inline void matmul(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const * A, T const * B, - float beta, T * C) { - auto desc = - makeDesc(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_DEFAULT); - executeMatmul(desc, alpha, A, B, beta, C, C, layoutKeyA(transa, m, k), - layoutKeyB(transb, k, n), {m, n}); + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *C) { + executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + CUBLASLT_EPILOGUE_DEFAULT, alpha, A, B, beta, C, C, nullptr, + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + inline void gemmStridedBatched( + char transa, char transb, + int m, int n, int k, float alpha, + const float *A, int lda, long long strideA, + const float *B, int ldb, long long strideB, + float beta, + float *C, int ldc, long long strideC, + int batchCount) + { + CHECK_CUBLAS(cublasSgemmStridedBatched( + handle, + charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), + m, n, k, + &alpha, + A, lda, strideA, + B, ldb, strideB, + &beta, + C, ldc, strideC, + batchCount)); + // No cudaStreamSynchronize — operations remain asynchronous on the stream } private: alpaka::QueueCudaRtNonBlocking m_queue; - // Returns the layout map key for matrix A based on its transpose flag. - // Physical dimensions: NoTrans → (m×k), Trans → (k×m). + static std::pair layoutKeyA(char trans, std::size_t m, std::size_t k) { return (trans == 'N' || trans == 'n') ? std::make_pair(m, k) : std::make_pair(k, m); } - // Returns the layout map key for matrix B based on its transpose flag. - // Physical dimensions: NoTrans → (k×n), Trans → (n×k). static std::pair layoutKeyB(char trans, std::size_t k, std::size_t n) { return (trans == 'N' || trans == 'n') ? std::make_pair(k, n) @@ -335,12 +380,14 @@ class BlasCuda { } } - // Creates a per-call matmul descriptor with transpose ops, epilogue, and - // optional bias pointer. Caller owns the returned descriptor. - cublasLtMatmulDesc_t makeDesc(cublasOperation_t transA, - cublasOperation_t transB, - cublasLtEpilogue_t epilogue, - const void *bias_ptr = nullptr) { + cublasLtMatmulDesc_t &getOrCreateDesc(cublasOperation_t transA, + cublasOperation_t transB, + cublasLtEpilogue_t epilogue) { + DescKey key{(int)transA, (int)transB, (int)epilogue}; + auto it = descStore.find(key); + if (it != descStore.end()) + return it->second; + cublasLtMatmulDesc_t desc = nullptr; CHECK_CUBLAS( cublasLtMatmulDescCreate(&desc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); @@ -350,40 +397,75 @@ class BlasCuda { desc, CUBLASLT_MATMUL_DESC_TRANSB, &transB, sizeof(transB))); CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( desc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue))); - if (bias_ptr) { + // For bias epilogues: set a non-null dummy pointer so the descriptor is + // valid for cublasLtMatmulAlgoGetHeuristic (real pointer patched per call). + if (epilogue != CUBLASLT_EPILOGUE_DEFAULT) { + const void *dummy = d_workspace; CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_ptr, - sizeof(bias_ptr))); + desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &dummy, sizeof(dummy))); } - return desc; + descStore.emplace(key, desc); + return descStore.at(key); } - // Runs heuristic selection, executes cublasLtMatmul, syncs stream, and - // destroys desc. D_in is the matrix scaled by beta (may equal C_out for - // in-place accumulation). - void executeMatmul(cublasLtMatmulDesc_t desc, float alpha, const float *A, - const float *B, float beta, const float *D_in, - float *C_out, - const std::pair &kA, - const std::pair &kB, - const std::pair &kC) { + cublasLtMatmulHeuristicResult_t & + getOrComputeAlgo(cublasOperation_t transA, cublasOperation_t transB, + cublasLtEpilogue_t epilogue, + const std::pair &kA, + const std::pair &kB, + const std::pair &kC) { + AlgoKey key{{(int)transA, (int)transB, (int)epilogue}, + kA.first, kA.second, kB.first, kB.second}; + auto it = algoCache.find(key); + if (it != algoCache.end()) + return it->second; + + auto &desc = getOrCreateDesc(transA, transB, epilogue); cublasLtMatmulHeuristicResult_t h{}; int returnedResults = 0; CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( - ltHandle, desc, layoutStore.at(kA), layoutStore.at(kB), - layoutStore.at(kC), layoutStore.at(kC), preference, 1, &h, - &returnedResults)); + ltHandle, desc, + layoutStore.at(kA), layoutStore.at(kB), + layoutStore.at(kC), layoutStore.at(kC), + preference, 1, &h, &returnedResults)); if (returnedResults == 0) { - cublasLtMatmulDescDestroy(desc); - std::cerr << "No suitable cuBLASLt algorithm found!\n"; + std::cerr << "[sofieBLAS] No suitable cuBLASLt algorithm found for " + << "transA=" << transA << " transB=" << transB + << " epilogue=" << epilogue + << " A=[" << kA.first << "x" << kA.second << "]" + << " B=[" << kB.first << "x" << kB.second << "]\n"; exit(EXIT_FAILURE); } - CHECK_CUBLAS(cublasLtMatmul(ltHandle, desc, &alpha, A, layoutStore.at(kA), - B, layoutStore.at(kB), &beta, D_in, - layoutStore.at(kC), C_out, layoutStore.at(kC), - &h.algo, d_workspace, workspaceSize, stream)); - CHECK_CUDA(cudaStreamSynchronize(stream)); - CHECK_CUBLAS(cublasLtMatmulDescDestroy(desc)); + algoCache.emplace(key, h); + return algoCache.at(key); + } + + void executeMatmul(cublasOperation_t transA, cublasOperation_t transB, + cublasLtEpilogue_t epilogue, + float alpha, const float *A, const float *B, + float beta, const float *D_in, float *C_out, + const void *bias_ptr, + const std::pair &kA, + const std::pair &kB, + const std::pair &kC) { + // Retrieve (or lazily compute) the cached algorithm for this shape + auto &h = getOrComputeAlgo(transA, transB, epilogue, kA, kB, kC); + + // Retrieve the cached descriptor and patch the real bias pointer in-place + auto &desc = getOrCreateDesc(transA, transB, epilogue); + if (bias_ptr) { + CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( + desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, + &bias_ptr, sizeof(bias_ptr))); + } + + CHECK_CUBLAS(cublasLtMatmul( + ltHandle, desc, + &alpha, A, layoutStore.at(kA), + B, layoutStore.at(kB), + &beta, D_in, layoutStore.at(kC), + C_out, layoutStore.at(kC), + &h.algo, d_workspace, workspaceSize, stream)); } }; From 97c12bc997ad3ab0a0925b1c807c6df0c0e7c11f Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Tue, 14 Jul 2026 12:19:12 +0200 Subject: [PATCH 24/29] feat: support for gemm functions on AMD hardware using hipBLASLt and cpu blas methods from different libraries --- README.md | 47 +- .../sofieBLAS_cblas_common.hpp} | 97 ++-- .../backends/cpu/sofieBLAS_accelerate.hpp | 11 + .../sofieBLAS/backends/cpu/sofieBLAS_blis.hpp | 11 + .../sofieBLAS/backends/cpu/sofieBLAS_mkl.hpp | 9 + .../backends/cpu/sofieBLAS_openblas.hpp | 10 + .../backends/cuda/sofieBLAS_cublas.hpp | 267 +++++----- .../backends/hip/sofieBLAS_hipblaslt.hpp | 483 ++++++++++++++++++ include/sofieBLAS/sofieBLAS.hpp | 21 +- tests/CMakeLists.txt | 114 ++++- tests/test.cc | 246 ++++++++- 11 files changed, 1108 insertions(+), 208 deletions(-) rename include/sofieBLAS/backends/cpu/{sofieBLAS_cpu.hpp => detail/sofieBLAS_cblas_common.hpp} (78%) create mode 100644 include/sofieBLAS/backends/cpu/sofieBLAS_accelerate.hpp create mode 100644 include/sofieBLAS/backends/cpu/sofieBLAS_blis.hpp create mode 100644 include/sofieBLAS/backends/cpu/sofieBLAS_mkl.hpp create mode 100644 include/sofieBLAS/backends/cpu/sofieBLAS_openblas.hpp create mode 100644 include/sofieBLAS/backends/hip/sofieBLAS_hipblaslt.hpp diff --git a/README.md b/README.md index 93bfdf6..3f1608f 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,36 @@ # sofieBLAS -sofieBLAS is an abstract C++ (header-only) interface for BLAS operations targeting heterogeneous architectures. It currently supports only ALPAKA buffers and the GEMM operation, acting as a thin, efficient wrapper over existing BLAS libraries such as OpenBLAS, MKL, cuBLAS, and others- allowing the actual backend to be selected through template-based dispatching using traits. +sofieBLAS is an abstract C++ (header-only) interface for BLAS operations targeting heterogeneous architectures. It currently supports only ALPAKA buffers and the GEMM operation, acting as a thin, efficient wrapper over existing BLAS libraries such as OpenBLAS, MKL, BLIS, Apple Accelerate, cuBLASLt, and hipBLASLt - allowing the actual backend to be selected through template-based dispatching using traits. We plan to extend support to more BLAS routines and buffer types in future releases. ## Features - Unified Interface: Common C++ API over multiple BLAS backends. -- Heterogeneous Support: CPU (OpenBLAS, MKL) and GPU (cuBLAS) support. -- Template-Based Dispatching: Backend selection via traits at compile-time. +- Heterogeneous Support: + - CPU: OpenBLAS, MKL, BLIS, Apple Accelerate (any CBLAS-compatible library). + - GPU: NVIDIA (cuBLASLt) and AMD (hipBLASLt). +- Template-Based Dispatching: Backend selection via traits at compile-time, keyed on the Alpaka accelerator tag (e.g. `alpaka::TagCpuSerial`, `alpaka::TagGpuCudaRt`, `alpaka::TagGpuHipRt`). - Header-Only: Lightweight, easy to integrate- no separate compilation required. - Minimal Dependency Overhead: Only depends on the backend BLAS libraries of choice. +- One File Per Backend: Each vendor library (CPU or GPU) lives in its own header under `include/sofieBLAS/backends//`, selected at compile time via a preprocessor macro (`ALPAKA_ACC_GPU_CUDA_ENABLED`, `ALPAKA_ACC_GPU_HIP_ENABLED`, `SOFIEBLAS_USE_OPENBLAS`, `SOFIEBLAS_USE_MKL`, `SOFIEBLAS_USE_BLIS`, `SOFIEBLAS_USE_ACCELERATE`) so only the code for the library you actually build against gets compiled. + +## Selecting a backend + +Backend selection happens entirely at compile time, in two steps: + +1. **Compiler-flag macros decide the backend implementations.** A macro (or macro pair) must be defined to compile in a given backend's header - see the table below. For CPU, if `ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED` is defined but none of the `SOFIEBLAS_USE_*` macros are, sofieBLAS defaults to OpenBLAS; the GPU backends have no such default (each is tied 1:1 to its `ALPAKA_ACC_GPU_*_ENABLED` macro, so there's nothing to default between). +2. **The Alpaka tag you instantiate `sofieBLAS` with decides which compiled-in backend a given call site actually uses**, via the `traits::sofieBLAS` specialization (e.g. `sofieBLAS` resolves to the hipBLASLt backend). There is no runtime dispatch - if the macro for that tag's backend wasn't defined, the code simply won't compile. + +| Accelerator | Macro(s) | Alpaka tag | +| --- | --- | --- | +| CPU (OpenBLAS) | `ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED` + `SOFIEBLAS_USE_OPENBLAS` | `alpaka::TagCpuSerial` (or other `TagCpu*`) | +| CPU (Intel MKL) | `ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED` + `SOFIEBLAS_USE_MKL` | `alpaka::TagCpuSerial` (or other `TagCpu*`) | +| CPU (BLIS) | `ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED` + `SOFIEBLAS_USE_BLIS` | `alpaka::TagCpuSerial` (or other `TagCpu*`) | +| CPU (Apple Accelerate) | `ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED` + `SOFIEBLAS_USE_ACCELERATE` | `alpaka::TagCpuSerial` (or other `TagCpu*`) | +| NVIDIA GPU (cuBLASLt) | `ALPAKA_ACC_GPU_CUDA_ENABLED` | `alpaka::TagGpuCudaRt` | +| AMD GPU (hipBLASLt) | `ALPAKA_ACC_GPU_HIP_ENABLED` | `alpaka::TagGpuHipRt` | + +`sofieBLAS/sofieBLAS.hpp` includes the matching backend header(s) for you based on these macros; nothing else needs to change in your source beyond picking the right tag. See `tests/CMakeLists.txt` for a working example that detects the available libraries and wires up a `test_cpu`, `test_cuda`, and `test_hip` target, each built against a different backend. ## Usage example @@ -32,11 +53,12 @@ int main() { auto C = alpaka::allocBuf(device, size * size); // (Initialize A and B here...) - // Create sofieBLAS instance for CPU backend + // Create sofieBLAS instance for the CPU backend selected via + // SOFIEBLAS_USE_OPENBLAS / SOFIEBLAS_USE_MKL / SOFIEBLAS_USE_BLIS / SOFIEBLAS_USE_ACCELERATE sofieBLAS blas(queue); - // Perform GEMM: C = alpha * A * B + beta * C - blas.gemm('n', 'n', size, size, size, 1.0f, A, size, B, size, 0.0f, C, size); + // C = alpha * op(A) * op(B) + beta * C (leading dimensions inferred from m, n, k) + blas.matmul('N', 'N', size, size, size, 1.0f, A, B, 0.0f, C); alpaka::wait(queue); std::cout << "GEMM completed on CPU backend.\n"; @@ -45,6 +67,19 @@ int main() { } ``` +Switching to a GPU is the same code shape, just a different Alpaka tag, queue/device type, and build-time macro. For example, on AMD (built with `-DALPAKA_ACC_GPU_HIP_ENABLED`): + +```cpp +alpaka::PlatformHipRt platform; +auto device = alpaka::getDevByIdx(platform, 0u); +alpaka::Queue queue{device}; +sofieBLAS blas(queue); + +blas.matmul('N', 'N', size, size, size, 1.0f, dA, dB, 0.0f, dC); +``` + +The GPU backends (`BlasCuda`, `BlasHip`) additionally expose `gemmrelu`/`gemmgelu` (fused bias + activation via cuBLASLt/hipBLASLt epilogues), `gemmStridedBatched`, and `addLayoutConfig` (used to pre-register cuBLASLt/hipBLASLt matrix layouts for a given shape before the first `matmul`/`gemm` call on that shape). + ## Contributing diff --git a/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp b/include/sofieBLAS/backends/cpu/detail/sofieBLAS_cblas_common.hpp similarity index 78% rename from include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp rename to include/sofieBLAS/backends/cpu/detail/sofieBLAS_cblas_common.hpp index 6cfeb1f..ac2d1a2 100644 --- a/include/sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp +++ b/include/sofieBLAS/backends/cpu/detail/sofieBLAS_cblas_common.hpp @@ -1,19 +1,15 @@ #pragma once +// Shared implementation for every CPU backend that exposes a standard CBLAS +// API. The vendor-specific header (cblas.h / mkl.h / blis/cblas.h / +// Accelerate.h) must already be included by the backend wrapper before this +// file, since they all provide the same CBLAS symbols. + #ifdef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED #include "sofieBLAS/core.hpp" #include -#if defined(SOFIEBLAS_USE_MKL) -#include -#elif defined(SOFIEBLAS_USE_OPENBLAS) -#include -#else -#error \ - "No CPU BLAS backend selected. Define SOFIEBLAS_USE_MKL or SOFIEBLAS_USE_OPENBLAS." -#endif - #include #include #include @@ -57,12 +53,15 @@ class BlasCpu { } template - inline void matmul(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::ViewPlainPtr, TIdx> const &A, - alpaka::ViewPlainPtr, TIdx> const &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &C) { + inline void + matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &C) { int lda = (transa == 'N' || transa == 'n') ? static_cast(m) : static_cast(k); int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) @@ -74,9 +73,7 @@ class BlasCpu { } // C = alpha * op(A) * op(B) + beta * bias + bias_vec (bias_vec broadcast per - // row) Matches the cuBLASLt EPILOGUE_BIAS semantics: the bias buffer serves - // as both the beta-scaled accumulator and provides the per-row bias vector - // (first m elements). + // row) template inline void gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, @@ -88,12 +85,10 @@ class BlasCpu { : static_cast(k); int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) : static_cast(n); - // Step 1: C = alpha * op(A) * op(B) (beta=0 so C is fully overwritten) cblas_sgemm(CblasColMajor, charToTranspose(transa), charToTranspose(transb), static_cast(m), static_cast(n), static_cast(k), alpha, alpaka::getPtrNative(A), lda, alpaka::getPtrNative(B), ldb, 0.0f, alpaka::getPtrNative(C), static_cast(m)); - // Step 2: C += beta * bias_matrix + bias_vec (per-row broadcast) float *c = alpaka::getPtrNative(C); const float *b = alpaka::getPtrNative(bias); for (unsigned int j = 0; j < n; ++j) @@ -105,8 +100,10 @@ class BlasCpu { inline void gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, float alpha, - alpaka::ViewPlainPtr, TIdx> const &A, - alpaka::ViewPlainPtr, TIdx> const &B, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, float beta, alpaka::ViewPlainPtr, TIdx> &bias, alpaka::ViewPlainPtr, TIdx> &C) { @@ -141,13 +138,16 @@ class BlasCpu { } template - inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::ViewPlainPtr, TIdx> const &A, - alpaka::ViewPlainPtr, TIdx> const &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &bias, - alpaka::ViewPlainPtr, TIdx> &C) { + inline void gemmrelu( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { gemm(transa, transb, m, n, k, alpha, A, B, beta, bias, C); float *c = alpaka::getPtrNative(C); for (unsigned int i = 0; i < m * n; ++i) @@ -172,13 +172,16 @@ class BlasCpu { } template - inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::ViewPlainPtr, TIdx> const &A, - alpaka::ViewPlainPtr, TIdx> const &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &bias, - alpaka::ViewPlainPtr, TIdx> &C) { + inline void gemmgelu( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { gemm(transa, transb, m, n, k, alpha, A, B, beta, bias, C); float *c = alpaka::getPtrNative(C); constexpr float kInvSqrt2 = 0.7071067811865476f; @@ -186,13 +189,29 @@ class BlasCpu { c[i] *= 0.5f * (1.0f + std::erff(c[i] * kInvSqrt2)); } - // Raw-pointer overloads: accept T const*/T* from any BufXxx or ViewPlainPtr via getPtrNative() + // Raw-pointer overloads: accept T const*/T* from any BufXxx or ViewPlainPtr + // via getPtrNative() + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *C) { + int lda = (transa == 'N' || transa == 'n') ? static_cast(m) + : static_cast(k); + int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) + : static_cast(n); + cblas_sgemm(CblasColMajor, charToTranspose(transa), charToTranspose(transb), + static_cast(m), static_cast(n), static_cast(k), + alpha, A, lda, B, ldb, beta, C, static_cast(m)); + } + template inline void gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, float alpha, T const *A, T const *B, float beta, T *bias, T *C) { - int lda = (transa == 'N' || transa == 'n') ? static_cast(m) : static_cast(k); - int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) : static_cast(n); + int lda = (transa == 'N' || transa == 'n') ? static_cast(m) + : static_cast(k); + int ldb = (transb == 'N' || transb == 'n') ? static_cast(k) + : static_cast(n); cblas_sgemm(CblasColMajor, charToTranspose(transa), charToTranspose(transb), static_cast(m), static_cast(n), static_cast(k), alpha, A, lda, B, ldb, 0.0f, C, static_cast(m)); diff --git a/include/sofieBLAS/backends/cpu/sofieBLAS_accelerate.hpp b/include/sofieBLAS/backends/cpu/sofieBLAS_accelerate.hpp new file mode 100644 index 0000000..9ba7b6b --- /dev/null +++ b/include/sofieBLAS/backends/cpu/sofieBLAS_accelerate.hpp @@ -0,0 +1,11 @@ +#pragma once + +#if defined(ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED) && \ + defined(SOFIEBLAS_USE_ACCELERATE) + +// Apple's Accelerate framework (macOS only) exposes the same CBLAS API. +#include + +#include "sofieBLAS/backends/cpu/detail/sofieBLAS_cblas_common.hpp" + +#endif // ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED && SOFIEBLAS_USE_ACCELERATE diff --git a/include/sofieBLAS/backends/cpu/sofieBLAS_blis.hpp b/include/sofieBLAS/backends/cpu/sofieBLAS_blis.hpp new file mode 100644 index 0000000..952bb51 --- /dev/null +++ b/include/sofieBLAS/backends/cpu/sofieBLAS_blis.hpp @@ -0,0 +1,11 @@ +#pragma once + +#if defined(ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED) && defined(SOFIEBLAS_USE_BLIS) + +// BLIS installs its CBLAS-compatible header under blis/cblas.h +// when configured with --enable-cblas. +#include + +#include "sofieBLAS/backends/cpu/detail/sofieBLAS_cblas_common.hpp" + +#endif // ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED && SOFIEBLAS_USE_BLIS diff --git a/include/sofieBLAS/backends/cpu/sofieBLAS_mkl.hpp b/include/sofieBLAS/backends/cpu/sofieBLAS_mkl.hpp new file mode 100644 index 0000000..66e7d1f --- /dev/null +++ b/include/sofieBLAS/backends/cpu/sofieBLAS_mkl.hpp @@ -0,0 +1,9 @@ +#pragma once + +#if defined(ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED) && defined(SOFIEBLAS_USE_MKL) + +#include + +#include "sofieBLAS/backends/cpu/detail/sofieBLAS_cblas_common.hpp" + +#endif // ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED && SOFIEBLAS_USE_MKL diff --git a/include/sofieBLAS/backends/cpu/sofieBLAS_openblas.hpp b/include/sofieBLAS/backends/cpu/sofieBLAS_openblas.hpp new file mode 100644 index 0000000..3f32cb9 --- /dev/null +++ b/include/sofieBLAS/backends/cpu/sofieBLAS_openblas.hpp @@ -0,0 +1,10 @@ +#pragma once + +#if defined(ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED) && \ + defined(SOFIEBLAS_USE_OPENBLAS) + +#include + +#include "sofieBLAS/backends/cpu/detail/sofieBLAS_cblas_common.hpp" + +#endif // ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED && SOFIEBLAS_USE_OPENBLAS diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index f0b9e57..ded6ba8 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -30,7 +30,6 @@ } \ } while (0) - struct PairHash { std::size_t operator()(const std::pair &p) const noexcept { @@ -48,7 +47,7 @@ struct PairEq { }; struct DescKey { - int transA; // CUBLAS_OP_N / CUBLAS_OP_T encoded as int + int transA; // CUBLAS_OP_N / CUBLAS_OP_T encoded as int int transB; int epilogue; // cublasLtEpilogue_t encoded as int bool operator==(const DescKey &o) const noexcept { @@ -58,23 +57,20 @@ struct DescKey { struct DescKeyHash { std::size_t operator()(const DescKey &k) const noexcept { - // Small values: simple polynomial hash - std::size_t h = static_cast(k.transA) * 97u - + static_cast(k.transB) * 31u - + static_cast(k.epilogue); + std::size_t h = static_cast(k.transA) * 97u + + static_cast(k.transB) * 31u + + static_cast(k.epilogue); return h ^ (h >> 16); } }; - struct AlgoKey { - DescKey dk; - std::size_t rowsA, colsA; // physical dimensions of A in layoutStore - std::size_t rowsB, colsB; // physical dimensions of B in layoutStore + DescKey dk; + std::size_t rowsA, colsA; // physical dimensions of A in layoutStore + std::size_t rowsB, colsB; // physical dimensions of B in layoutStore bool operator==(const AlgoKey &o) const noexcept { - return dk == o.dk - && rowsA == o.rowsA && colsA == o.colsA - && rowsB == o.rowsB && colsB == o.colsB; + return dk == o.dk && rowsA == o.rowsA && colsA == o.colsA && + rowsB == o.rowsB && colsB == o.colsB; } }; @@ -82,20 +78,23 @@ struct AlgoKeyHash { std::size_t operator()(const AlgoKey &k) const noexcept { std::size_t h = DescKeyHash{}(k.dk); auto mix = [&](std::size_t v) { - h ^= std::hash{}(v) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); + h ^= std::hash{}(v) + 0x9e3779b97f4a7c15ULL + (h << 6) + + (h >> 2); }; - mix(k.rowsA); mix(k.colsA); - mix(k.rowsB); mix(k.colsB); + mix(k.rowsA); + mix(k.colsA); + mix(k.rowsB); + mix(k.colsB); return h; } }; class BlasCuda { - cublasLtHandle_t ltHandle = nullptr; - cublasHandle_t handle = nullptr; // legacy cuBLAS for batched ops + cublasLtHandle_t ltHandle = nullptr; + cublasHandle_t handle = nullptr; cublasLtMatmulPreference_t preference = nullptr; - void *d_workspace = nullptr; - size_t workspaceSize = 1u << 25; // 32 MB (was 4 MB) + void *d_workspace = nullptr; + size_t workspaceSize = 1u << 25; // 32 MB cudaStream_t stream = nullptr; std::unordered_map, @@ -130,20 +129,32 @@ class BlasCuda { ~BlasCuda() { for (auto &[key, layout] : layoutStore) - if (layout) cublasLtMatrixLayoutDestroy(layout); + if (layout) + cublasLtMatrixLayoutDestroy(layout); for (auto &[key, desc] : descStore) - if (desc) cublasLtMatmulDescDestroy(desc); - if (preference) cublasLtMatmulPreferenceDestroy(preference); - if (ltHandle) cublasLtDestroy(ltHandle); - if (handle) cublasDestroy(handle); - if (d_workspace) cudaFree(d_workspace); + if (desc) + cublasLtMatmulDescDestroy(desc); + if (preference) + cublasLtMatmulPreferenceDestroy(preference); + if (ltHandle) + cublasLtDestroy(ltHandle); + if (handle) + cublasDestroy(handle); + if (d_workspace) + cudaFree(d_workspace); } inline cublasOperation_t charToCuBlasTranspose(char trans) { switch (trans) { - case 'N': case 'n': return CUBLAS_OP_N; - case 'T': case 't': return CUBLAS_OP_T; - case 'C': case 'c': return CUBLAS_OP_C; + case 'N': + case 'n': + return CUBLAS_OP_N; + case 'T': + case 't': + return CUBLAS_OP_T; + case 'C': + case 'c': + return CUBLAS_OP_C; default: throw std::invalid_argument("Invalid transpose character for cuBLAS."); } @@ -174,26 +185,29 @@ class BlasCuda { alpaka::BufCudaRt, TIdx> &bias, alpaka::BufCudaRt, TIdx> &C) { executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_BIAS, alpha, - alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + CUBLASLT_EPILOGUE_BIAS, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), + alpaka::getPtrNative(C), static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } template - inline void - gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, - alpaka::ViewPlainPtr, TIdx> const &A, - alpaka::ViewPlainPtr, TIdx> const &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &bias, - alpaka::ViewPlainPtr, TIdx> &C) { + inline void gemm( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> + &bias, + alpaka::ViewPlainPtr, TIdx> &C) { executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_BIAS, alpha, - alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + CUBLASLT_EPILOGUE_BIAS, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), + alpaka::getPtrNative(C), static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } @@ -204,8 +218,8 @@ class BlasCuda { float beta, T *bias, T *C) { executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), CUBLASLT_EPILOGUE_BIAS, alpha, A, B, beta, bias, C, - static_cast(bias), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + static_cast(bias), layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); } template @@ -217,25 +231,29 @@ class BlasCuda { alpaka::BufCudaRt, TIdx> &bias, alpaka::BufCudaRt, TIdx> &C) { executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_RELU_BIAS, alpha, - alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + CUBLASLT_EPILOGUE_RELU_BIAS, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), + alpaka::getPtrNative(C), static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } template - inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::ViewPlainPtr, TIdx> const &A, - alpaka::ViewPlainPtr, TIdx> const &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &bias, - alpaka::ViewPlainPtr, TIdx> &C) { + inline void gemmrelu( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> + &bias, + alpaka::ViewPlainPtr, TIdx> &C) { executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_RELU_BIAS, alpha, - alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + CUBLASLT_EPILOGUE_RELU_BIAS, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), + alpaka::getPtrNative(C), static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } @@ -246,8 +264,8 @@ class BlasCuda { float beta, T *bias, T *C) { executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), CUBLASLT_EPILOGUE_RELU_BIAS, alpha, A, B, beta, bias, C, - static_cast(bias), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + static_cast(bias), layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); } template @@ -259,25 +277,29 @@ class BlasCuda { alpaka::BufCudaRt, TIdx> &bias, alpaka::BufCudaRt, TIdx> &C) { executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_GELU_BIAS, alpha, - alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + CUBLASLT_EPILOGUE_GELU_BIAS, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), + alpaka::getPtrNative(C), static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } template - inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::ViewPlainPtr, TIdx> const &A, - alpaka::ViewPlainPtr, TIdx> const &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &bias, - alpaka::ViewPlainPtr, TIdx> &C) { + inline void gemmgelu( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> + &bias, + alpaka::ViewPlainPtr, TIdx> &C) { executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_GELU_BIAS, alpha, - alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + CUBLASLT_EPILOGUE_GELU_BIAS, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), + alpaka::getPtrNative(C), static_cast(alpaka::getPtrNative(bias)), layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } @@ -288,8 +310,8 @@ class BlasCuda { float beta, T *bias, T *C) { executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), CUBLASLT_EPILOGUE_GELU_BIAS, alpha, A, B, beta, bias, C, - static_cast(bias), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + static_cast(bias), layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); } template @@ -301,24 +323,25 @@ class BlasCuda { alpaka::BufCudaRt, TIdx> &C) { float *c = alpaka::getPtrNative(C); executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_DEFAULT, alpha, - alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - c, c, nullptr, + CUBLASLT_EPILOGUE_DEFAULT, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, c, c, nullptr, layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } template - inline void matmul(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::ViewPlainPtr, TIdx> const &A, - alpaka::ViewPlainPtr, TIdx> const &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &C) { + inline void matmul( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &C) { T *c = alpaka::getPtrNative(C); executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_DEFAULT, alpha, - alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - c, c, nullptr, + CUBLASLT_EPILOGUE_DEFAULT, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, c, c, nullptr, layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } @@ -332,32 +355,20 @@ class BlasCuda { layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); } - inline void gemmStridedBatched( - char transa, char transb, - int m, int n, int k, float alpha, - const float *A, int lda, long long strideA, - const float *B, int ldb, long long strideB, - float beta, - float *C, int ldc, long long strideC, - int batchCount) - { + inline void gemmStridedBatched(char transa, char transb, int m, int n, int k, + float alpha, const float *A, int lda, + long long strideA, const float *B, int ldb, + long long strideB, float beta, float *C, + int ldc, long long strideC, int batchCount) { CHECK_CUBLAS(cublasSgemmStridedBatched( - handle, - charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - m, n, k, - &alpha, - A, lda, strideA, - B, ldb, strideB, - &beta, - C, ldc, strideC, + handle, charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), m, + n, k, &alpha, A, lda, strideA, B, ldb, strideB, &beta, C, ldc, strideC, batchCount)); - // No cudaStreamSynchronize — operations remain asynchronous on the stream } private: alpaka::QueueCudaRtNonBlocking m_queue; - static std::pair layoutKeyA(char trans, std::size_t m, std::size_t k) { return (trans == 'N' || trans == 'n') ? std::make_pair(m, k) @@ -381,8 +392,8 @@ class BlasCuda { } cublasLtMatmulDesc_t &getOrCreateDesc(cublasOperation_t transA, - cublasOperation_t transB, - cublasLtEpilogue_t epilogue) { + cublasOperation_t transB, + cublasLtEpilogue_t epilogue) { DescKey key{(int)transA, (int)transB, (int)epilogue}; auto it = descStore.find(key); if (it != descStore.end()) @@ -398,7 +409,7 @@ class BlasCuda { CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( desc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue))); // For bias epilogues: set a non-null dummy pointer so the descriptor is - // valid for cublasLtMatmulAlgoGetHeuristic (real pointer patched per call). + // valid for cublasLtMatmulAlgoGetHeuristic. if (epilogue != CUBLASLT_EPILOGUE_DEFAULT) { const void *dummy = d_workspace; CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( @@ -415,7 +426,10 @@ class BlasCuda { const std::pair &kB, const std::pair &kC) { AlgoKey key{{(int)transA, (int)transB, (int)epilogue}, - kA.first, kA.second, kB.first, kB.second}; + kA.first, + kA.second, + kB.first, + kB.second}; auto it = algoCache.find(key); if (it != algoCache.end()) return it->second; @@ -424,15 +438,14 @@ class BlasCuda { cublasLtMatmulHeuristicResult_t h{}; int returnedResults = 0; CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( - ltHandle, desc, - layoutStore.at(kA), layoutStore.at(kB), - layoutStore.at(kC), layoutStore.at(kC), - preference, 1, &h, &returnedResults)); + ltHandle, desc, layoutStore.at(kA), layoutStore.at(kB), + layoutStore.at(kC), layoutStore.at(kC), preference, 1, &h, + &returnedResults)); if (returnedResults == 0) { std::cerr << "[sofieBLAS] No suitable cuBLASLt algorithm found for " << "transA=" << transA << " transB=" << transB - << " epilogue=" << epilogue - << " A=[" << kA.first << "x" << kA.second << "]" + << " epilogue=" << epilogue << " A=[" << kA.first << "x" + << kA.second << "]" << " B=[" << kB.first << "x" << kB.second << "]\n"; exit(EXIT_FAILURE); } @@ -441,10 +454,9 @@ class BlasCuda { } void executeMatmul(cublasOperation_t transA, cublasOperation_t transB, - cublasLtEpilogue_t epilogue, - float alpha, const float *A, const float *B, - float beta, const float *D_in, float *C_out, - const void *bias_ptr, + cublasLtEpilogue_t epilogue, float alpha, const float *A, + const float *B, float beta, const float *D_in, + float *C_out, const void *bias_ptr, const std::pair &kA, const std::pair &kB, const std::pair &kC) { @@ -455,17 +467,14 @@ class BlasCuda { auto &desc = getOrCreateDesc(transA, transB, epilogue); if (bias_ptr) { CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, - &bias_ptr, sizeof(bias_ptr))); + desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_ptr, + sizeof(bias_ptr))); } - CHECK_CUBLAS(cublasLtMatmul( - ltHandle, desc, - &alpha, A, layoutStore.at(kA), - B, layoutStore.at(kB), - &beta, D_in, layoutStore.at(kC), - C_out, layoutStore.at(kC), - &h.algo, d_workspace, workspaceSize, stream)); + CHECK_CUBLAS(cublasLtMatmul(ltHandle, desc, &alpha, A, layoutStore.at(kA), + B, layoutStore.at(kB), &beta, D_in, + layoutStore.at(kC), C_out, layoutStore.at(kC), + &h.algo, d_workspace, workspaceSize, stream)); } }; diff --git a/include/sofieBLAS/backends/hip/sofieBLAS_hipblaslt.hpp b/include/sofieBLAS/backends/hip/sofieBLAS_hipblaslt.hpp new file mode 100644 index 0000000..0e41e70 --- /dev/null +++ b/include/sofieBLAS/backends/hip/sofieBLAS_hipblaslt.hpp @@ -0,0 +1,483 @@ +#pragma once + +#ifdef ALPAKA_ACC_GPU_HIP_ENABLED + +#include +#include +#include +#include +#include +#include + +#include "sofieBLAS/core.hpp" +#include +#include +#include + +#define CHECK_HIP(err) \ + if ((err) != hipSuccess) { \ + std::cerr << "HIP error: " << hipGetErrorString(err) << " at line " \ + << __LINE__ << "\n"; \ + exit(EXIT_FAILURE); \ + } + +#define CHECK_HIPBLAS(status) \ + do { \ + hipblasStatus_t _s = (status); \ + if (_s != HIPBLAS_STATUS_SUCCESS) { \ + std::cerr << "hipBLAS error " << _s << " at line " << __LINE__ << "\n"; \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +struct PairHash { + std::size_t + operator()(const std::pair &p) const noexcept { + std::size_t h1 = std::hash{}(p.first); + std::size_t h2 = std::hash{}(p.second); + return h1 ^ (h2 + 0x9e3779b97f4a7c15ULL + (h1 << 6) + (h1 >> 2)); + } +}; + +struct PairEq { + bool operator()(const std::pair &a, + const std::pair &b) const noexcept { + return a.first == b.first && a.second == b.second; + } +}; + +struct DescKey { + int transA; // HIPBLAS_OP_N / HIPBLAS_OP_T encoded as int + int transB; + int epilogue; // hipblasLtEpilogue_t encoded as int + bool operator==(const DescKey &o) const noexcept { + return transA == o.transA && transB == o.transB && epilogue == o.epilogue; + } +}; + +struct DescKeyHash { + std::size_t operator()(const DescKey &k) const noexcept { + std::size_t h = static_cast(k.transA) * 97u + + static_cast(k.transB) * 31u + + static_cast(k.epilogue); + return h ^ (h >> 16); + } +}; + +struct AlgoKey { + DescKey dk; + std::size_t rowsA, colsA; // physical dimensions of A in layoutStore + std::size_t rowsB, colsB; // physical dimensions of B in layoutStore + bool operator==(const AlgoKey &o) const noexcept { + return dk == o.dk && rowsA == o.rowsA && colsA == o.colsA && + rowsB == o.rowsB && colsB == o.colsB; + } +}; + +struct AlgoKeyHash { + std::size_t operator()(const AlgoKey &k) const noexcept { + std::size_t h = DescKeyHash{}(k.dk); + auto mix = [&](std::size_t v) { + h ^= std::hash{}(v) + 0x9e3779b97f4a7c15ULL + (h << 6) + + (h >> 2); + }; + mix(k.rowsA); + mix(k.colsA); + mix(k.rowsB); + mix(k.colsB); + return h; + } +}; + +class BlasHip { + hipblasLtHandle_t ltHandle = nullptr; + hipblasHandle_t handle = nullptr; + hipblasLtMatmulPreference_t preference = nullptr; + void *d_workspace = nullptr; + size_t workspaceSize = 1u << 25; // 32 MB + hipStream_t stream = nullptr; + + std::unordered_map, + hipblasLtMatrixLayout_t, PairHash, PairEq> + layoutStore; + + std::unordered_map descStore; + + std::unordered_map + algoCache; + +public: + BlasHip(const BlasHip &) = delete; + BlasHip &operator=(const BlasHip &) = delete; + BlasHip(BlasHip &&) = delete; + BlasHip &operator=(BlasHip &&) = delete; + + BlasHip(alpaka::QueueHipRtNonBlocking &queue) : m_queue{queue} { + stream = static_cast(m_queue.getNativeHandle()); + + CHECK_HIPBLAS(hipblasLtCreate(<Handle)); + + CHECK_HIPBLAS(hipblasCreate(&handle)); + CHECK_HIPBLAS(hipblasSetStream(handle, stream)); + + CHECK_HIPBLAS(hipblasLtMatmulPreferenceCreate(&preference)); + CHECK_HIP(hipMalloc(&d_workspace, workspaceSize)); + CHECK_HIPBLAS(hipblasLtMatmulPreferenceSetAttribute( + preference, HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, + sizeof(workspaceSize))); + } + + ~BlasHip() { + for (auto &[key, layout] : layoutStore) + if (layout) + hipblasLtMatrixLayoutDestroy(layout); + for (auto &[key, desc] : descStore) + if (desc) + hipblasLtMatmulDescDestroy(desc); + if (preference) + hipblasLtMatmulPreferenceDestroy(preference); + if (ltHandle) + hipblasLtDestroy(ltHandle); + if (handle) + hipblasDestroy(handle); + if (d_workspace) + hipFree(d_workspace); + } + + inline hipblasOperation_t charToHipBlasTranspose(char trans) { + switch (trans) { + case 'N': + case 'n': + return HIPBLAS_OP_N; + case 'T': + case 't': + return HIPBLAS_OP_T; + case 'C': + case 'c': + return HIPBLAS_OP_C; + default: + throw std::invalid_argument("Invalid transpose character for hipBLAS."); + } + } + + void addLayoutConfig(std::size_t m, std::size_t n, std::size_t k, + std::size_t lda, std::size_t ldb, std::size_t ldc, + char transa, char transb) { + if (transa == 'N' || transa == 'n') + checkAndAddLayout(m, k, lda); + else + checkAndAddLayout(k, m, lda); + if (transb == 'N' || transb == 'n') + checkAndAddLayout(k, n, ldb); + else + checkAndAddLayout(n, k, ldb); + checkAndAddLayout(m, n, ldc); + } + + template + inline void + gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, alpaka::BufHipRt, TIdx> const &A, + alpaka::BufHipRt, TIdx> const &B, float beta, + alpaka::BufHipRt, TIdx> &bias, + alpaka::BufHipRt, TIdx> &C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_BIAS, + alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemm( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_BIAS, + alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemm(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_BIAS, + alpha, A, B, beta, bias, C, static_cast(bias), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::BufHipRt, TIdx> const &A, + alpaka::BufHipRt, TIdx> const &B, + float beta, + alpaka::BufHipRt, TIdx> &bias, + alpaka::BufHipRt, TIdx> &C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_RELU_BIAS, + alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmrelu( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_RELU_BIAS, + alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_RELU_BIAS, + alpha, A, B, beta, bias, C, static_cast(bias), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::BufHipRt, TIdx> const &A, + alpaka::BufHipRt, TIdx> const &B, + float beta, + alpaka::BufHipRt, TIdx> &bias, + alpaka::BufHipRt, TIdx> &C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_GELU_BIAS, + alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmgelu( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &bias, + alpaka::ViewPlainPtr, TIdx> &C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_GELU_BIAS, + alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + alpaka::getPtrNative(bias), alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_GELU_BIAS, + alpha, A, B, beta, bias, C, static_cast(bias), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, + alpaka::BufHipRt, TIdx> const &A, + alpaka::BufHipRt, TIdx> const &B, + float beta, + alpaka::BufHipRt, TIdx> &C) { + float *c = alpaka::getPtrNative(C); + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_DEFAULT, + alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + c, c, nullptr, layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void matmul( + char transa, char transb, unsigned int m, unsigned int n, unsigned int k, + float alpha, + alpaka::ViewPlainPtr, TIdx> const + &A, + alpaka::ViewPlainPtr, TIdx> const + &B, + float beta, + alpaka::ViewPlainPtr, TIdx> &C) { + T *c = alpaka::getPtrNative(C); + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_DEFAULT, + alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, + c, c, nullptr, layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *C) { + executeMatmul(charToHipBlasTranspose(transa), + charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_DEFAULT, + alpha, A, B, beta, C, C, nullptr, layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); + } + + inline void gemmStridedBatched(char transa, char transb, int m, int n, int k, + float alpha, const float *A, int lda, + long long strideA, const float *B, int ldb, + long long strideB, float beta, float *C, + int ldc, long long strideC, int batchCount) { + CHECK_HIPBLAS(hipblasSgemmStridedBatched( + handle, charToHipBlasTranspose(transa), charToHipBlasTranspose(transb), + m, n, k, &alpha, A, lda, strideA, B, ldb, strideB, &beta, C, ldc, + strideC, batchCount)); + } + +private: + alpaka::QueueHipRtNonBlocking m_queue; + + static std::pair + layoutKeyA(char trans, std::size_t m, std::size_t k) { + return (trans == 'N' || trans == 'n') ? std::make_pair(m, k) + : std::make_pair(k, m); + } + + static std::pair + layoutKeyB(char trans, std::size_t k, std::size_t n) { + return (trans == 'N' || trans == 'n') ? std::make_pair(k, n) + : std::make_pair(n, k); + } + + void checkAndAddLayout(std::size_t rows, std::size_t cols, std::size_t ld) { + auto key = std::make_pair(rows, cols); + if (layoutStore.find(key) == layoutStore.end()) { + hipblasLtMatrixLayout_t layout = nullptr; + CHECK_HIPBLAS( + hipblasLtMatrixLayoutCreate(&layout, HIP_R_32F, rows, cols, ld)); + layoutStore.emplace(key, layout); + } + } + + hipblasLtMatmulDesc_t &getOrCreateDesc(hipblasOperation_t transA, + hipblasOperation_t transB, + hipblasLtEpilogue_t epilogue) { + DescKey key{(int)transA, (int)transB, (int)epilogue}; + auto it = descStore.find(key); + if (it != descStore.end()) + return it->second; + + hipblasLtMatmulDesc_t desc = nullptr; + CHECK_HIPBLAS( + hipblasLtMatmulDescCreate(&desc, HIPBLAS_COMPUTE_32F, HIP_R_32F)); + CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( + desc, HIPBLASLT_MATMUL_DESC_TRANSA, &transA, sizeof(transA))); + CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( + desc, HIPBLASLT_MATMUL_DESC_TRANSB, &transB, sizeof(transB))); + CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( + desc, HIPBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue))); + + if (epilogue != HIPBLASLT_EPILOGUE_DEFAULT) { + const void *dummy = d_workspace; + CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( + desc, HIPBLASLT_MATMUL_DESC_BIAS_POINTER, &dummy, sizeof(dummy))); + } + descStore.emplace(key, desc); + return descStore.at(key); + } + + hipblasLtMatmulHeuristicResult_t & + getOrComputeAlgo(hipblasOperation_t transA, hipblasOperation_t transB, + hipblasLtEpilogue_t epilogue, + const std::pair &kA, + const std::pair &kB, + const std::pair &kC) { + AlgoKey key{{(int)transA, (int)transB, (int)epilogue}, + kA.first, + kA.second, + kB.first, + kB.second}; + auto it = algoCache.find(key); + if (it != algoCache.end()) + return it->second; + + auto &desc = getOrCreateDesc(transA, transB, epilogue); + hipblasLtMatmulHeuristicResult_t h{}; + int returnedResults = 0; + CHECK_HIPBLAS(hipblasLtMatmulAlgoGetHeuristic( + ltHandle, desc, layoutStore.at(kA), layoutStore.at(kB), + layoutStore.at(kC), layoutStore.at(kC), preference, 1, &h, + &returnedResults)); + if (returnedResults == 0) { + std::cerr << "[sofieBLAS] No suitable hipBLASLt algorithm found for " + << "transA=" << transA << " transB=" << transB + << " epilogue=" << epilogue << " A=[" << kA.first << "x" + << kA.second << "]" + << " B=[" << kB.first << "x" << kB.second << "]\n"; + exit(EXIT_FAILURE); + } + algoCache.emplace(key, h); + return algoCache.at(key); + } + + void executeMatmul(hipblasOperation_t transA, hipblasOperation_t transB, + hipblasLtEpilogue_t epilogue, float alpha, const float *A, + const float *B, float beta, const float *D_in, + float *C_out, const void *bias_ptr, + const std::pair &kA, + const std::pair &kB, + const std::pair &kC) { + auto &h = getOrComputeAlgo(transA, transB, epilogue, kA, kB, kC); + + auto &desc = getOrCreateDesc(transA, transB, epilogue); + if (bias_ptr) { + CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( + desc, HIPBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_ptr, + sizeof(bias_ptr))); + } + + CHECK_HIPBLAS(hipblasLtMatmul(ltHandle, desc, &alpha, A, layoutStore.at(kA), + B, layoutStore.at(kB), &beta, D_in, + layoutStore.at(kC), C_out, layoutStore.at(kC), + &h.algo, d_workspace, workspaceSize, stream)); + } +}; + +namespace traits { + +template <> class sofieBLAS { +public: + using Impl = BlasHip; +}; + +} // namespace traits + +#endif // ALPAKA_ACC_GPU_HIP_ENABLED diff --git a/include/sofieBLAS/sofieBLAS.hpp b/include/sofieBLAS/sofieBLAS.hpp index 0d23c33..2ad40f8 100644 --- a/include/sofieBLAS/sofieBLAS.hpp +++ b/include/sofieBLAS/sofieBLAS.hpp @@ -3,9 +3,28 @@ #include "sofieBLAS/core.hpp" #if defined(ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED) -#include "sofieBLAS/backends/cpu/sofieBLAS_cpu.hpp" +// If no CPU BLAS backend was selected on the compiler command line, +// default to OpenBLAS +#if !defined(SOFIEBLAS_USE_OPENBLAS) && !defined(SOFIEBLAS_USE_MKL) && \ + !defined(SOFIEBLAS_USE_BLIS) && !defined(SOFIEBLAS_USE_ACCELERATE) +#define SOFIEBLAS_USE_OPENBLAS +#endif + +#if defined(SOFIEBLAS_USE_OPENBLAS) +#include "sofieBLAS/backends/cpu/sofieBLAS_openblas.hpp" +#elif defined(SOFIEBLAS_USE_MKL) +#include "sofieBLAS/backends/cpu/sofieBLAS_mkl.hpp" +#elif defined(SOFIEBLAS_USE_BLIS) +#include "sofieBLAS/backends/cpu/sofieBLAS_blis.hpp" +#elif defined(SOFIEBLAS_USE_ACCELERATE) +#include "sofieBLAS/backends/cpu/sofieBLAS_accelerate.hpp" +#endif #endif #if defined(ALPAKA_ACC_GPU_CUDA_ENABLED) #include "sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp" #endif + +#if defined(ALPAKA_ACC_GPU_HIP_ENABLED) +#include "sofieBLAS/backends/hip/sofieBLAS_hipblaslt.hpp" +#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0a4aa01..1247ec8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -9,13 +9,15 @@ set(TBB_BASE "/usr" CACHE PATH "TBB base path") set(ALPAKA_BASE "$ENV{HOME}/src/alpaka-group/alpaka" CACHE PATH "Alpaka base path") # --- CPU BLAS selection --- -set(AVAILABLE_BLAS_LIBS OpenBLAS MKL CACHE STRING "Choose CPU BLAS library") -set_property(CACHE AVAILABLE_BLAS_LIBS PROPERTY STRINGS OpenBLAS MKL) +set(AVAILABLE_BLAS_LIBS OpenBLAS MKL BLIS Accelerate CACHE STRING "Choose CPU BLAS library") +set_property(CACHE AVAILABLE_BLAS_LIBS PROPERTY STRINGS OpenBLAS MKL BLIS Accelerate) if(NOT DEFINED CPU_BLAS_LIB) set(CPU_BLAS_LIB "OpenBLAS" CACHE STRING "CPU BLAS library to use") endif() +set(BLIS_BASE "/usr" CACHE PATH "BLIS base path") + # --- Compiler flags --- set(CXXFLAGS -O2 -g -DALPAKA_HAS_STD_ATOMIC_REF) set(CXX_HOST_FLAGS -fPIC -pthread) @@ -28,58 +30,108 @@ set(XCOMPILER_FLAGS -Xcompiler=-fPIC,-pthread) set(CMAKE_CUDA_ARCHITECTURES 86) enable_language(CUDA) +# --- HIP architecture: must be set before enable_language(HIP) for the same +# reason as CUDA_ARCHITECTURES above. +set(CMAKE_HIP_ARCHITECTURES gfx1100) +include(CheckLanguage) +check_language(HIP) +if(CMAKE_HIP_COMPILER) + enable_language(HIP) +else() + message(STATUS "No HIP compiler found, test_hip target will not be built") +endif() + # --- Include directories --- include_directories(${ALPAKA_BASE}/include "../include") # --- Functions to find BLAS libraries --- -function(find_openblas blas_lib blas_define) +function(find_openblas blas_lib blas_define blas_include) find_path(OPENBLAS_PATH NAMES libopenblas.a PATHS /usr/lib/x86_64-linux-gnu/openblas-serial /usr/lib/x86_64-linux-gnu) if(OPENBLAS_PATH) find_library(OPENBLAS_LIB openblas PATHS ${OPENBLAS_PATH}) + find_path(OPENBLAS_INCLUDE_DIR NAMES cblas.h + PATHS /usr/include /usr/include/openblas /usr/include/cblas) set(${blas_lib} ${OPENBLAS_LIB} PARENT_SCOPE) set(${blas_define} SOFIEBLAS_USE_OPENBLAS PARENT_SCOPE) + set(${blas_include} ${OPENBLAS_INCLUDE_DIR} PARENT_SCOPE) else() set(${blas_lib} "" PARENT_SCOPE) set(${blas_define} "" PARENT_SCOPE) + set(${blas_include} "" PARENT_SCOPE) endif() endfunction() -function(find_mkl blas_lib blas_define) +function(find_mkl blas_lib blas_define blas_include) find_path(MKL_PATH NAMES libmkl_rt.a PATHS ${ONEAPI_BASE}/mkl/latest/lib/intel64) if(MKL_PATH) find_library(MKL_LIB mkl_rt PATHS ${MKL_PATH}) set(${blas_lib} ${MKL_LIB} PARENT_SCOPE) set(${blas_define} SOFIEBLAS_USE_MKL PARENT_SCOPE) + set(${blas_include} ${ONEAPI_BASE}/mkl/latest/include PARENT_SCOPE) else() set(${blas_lib} "" PARENT_SCOPE) set(${blas_define} "" PARENT_SCOPE) + set(${blas_include} "" PARENT_SCOPE) endif() endfunction() -# --- BLAS detection and selection --- -if(CPU_BLAS_LIB STREQUAL "OpenBLAS") - find_openblas(BLAS_LIBS SOFIEBLAS_DEFINES) - if(NOT BLAS_LIBS) - message(WARNING "OpenBLAS not found, trying MKL...") - find_mkl(BLAS_LIBS SOFIEBLAS_DEFINES) +function(find_blis blas_lib blas_define blas_include) + find_path(BLIS_PATH NAMES libblis.a libblis.so + PATHS ${BLIS_BASE}/lib ${BLIS_BASE}/lib64) + if(BLIS_PATH) + find_library(BLIS_LIB blis PATHS ${BLIS_PATH}) + find_path(BLIS_INCLUDE_DIR NAMES blis/cblas.h PATHS ${BLIS_BASE}/include) + set(${blas_lib} ${BLIS_LIB} PARENT_SCOPE) + set(${blas_define} SOFIEBLAS_USE_BLIS PARENT_SCOPE) + set(${blas_include} ${BLIS_INCLUDE_DIR} PARENT_SCOPE) + else() + set(${blas_lib} "" PARENT_SCOPE) + set(${blas_define} "" PARENT_SCOPE) + set(${blas_include} "" PARENT_SCOPE) endif() -elseif(CPU_BLAS_LIB STREQUAL "MKL") - find_mkl(BLAS_LIBS SOFIEBLAS_DEFINES) - if(NOT BLAS_LIBS) - message(WARNING "MKL not found, trying OpenBLAS...") - find_openblas(BLAS_LIBS SOFIEBLAS_DEFINES) +endfunction() + +function(find_accelerate blas_lib blas_define blas_include) + set(${blas_lib} "" PARENT_SCOPE) + set(${blas_define} "" PARENT_SCOPE) + set(${blas_include} "" PARENT_SCOPE) + if(APPLE) + find_library(ACCELERATE_LIB Accelerate) + if(ACCELERATE_LIB) + set(${blas_lib} ${ACCELERATE_LIB} PARENT_SCOPE) + set(${blas_define} SOFIEBLAS_USE_ACCELERATE PARENT_SCOPE) + endif() endif() -else() +endfunction() + +# --- BLAS detection and selection --- +set(SOFIEBLAS_BLAS_SEARCH_ORDER OpenBLAS MKL BLIS Accelerate) +if(NOT CPU_BLAS_LIB IN_LIST SOFIEBLAS_BLAS_SEARCH_ORDER) message(FATAL_ERROR "Unknown CPU_BLAS_LIB option: ${CPU_BLAS_LIB}") endif() +list(REMOVE_ITEM SOFIEBLAS_BLAS_SEARCH_ORDER ${CPU_BLAS_LIB}) +list(PREPEND SOFIEBLAS_BLAS_SEARCH_ORDER ${CPU_BLAS_LIB}) + +foreach(lib IN LISTS SOFIEBLAS_BLAS_SEARCH_ORDER) + if(NOT BLAS_LIBS) + string(TOLOWER ${lib} lib_lower) + cmake_language(CALL find_${lib_lower} BLAS_LIBS SOFIEBLAS_DEFINES BLAS_INCLUDE_DIR) + if(BLAS_LIBS) + set(FOUND_BLAS_LIB ${lib}) + else() + message(WARNING "${lib} not found, trying next option...") + endif() + endif() +endforeach() if(NOT BLAS_LIBS) - message(FATAL_ERROR "No suitable CPU BLAS library found!") + message(FATAL_ERROR "No suitable CPU BLAS library found! Tried: ${SOFIEBLAS_BLAS_SEARCH_ORDER}") endif() -message(STATUS "Using CPU BLAS library: ${CPU_BLAS_LIB}") +message(STATUS "Using CPU BLAS library: ${FOUND_BLAS_LIB}") message(STATUS "BLAS libraries: ${BLAS_LIBS}") +message(STATUS "BLAS include dir: ${BLAS_INCLUDE_DIR}") message(STATUS "Compile defines: ${SOFIEBLAS_DEFINES}") # --- test_cpu target --- @@ -87,7 +139,7 @@ add_executable(test_cpu test.cc) target_compile_features(test_cpu PUBLIC cxx_std_20) target_compile_options(test_cpu PRIVATE ${CXXFLAGS} ${CXX_HOST_FLAGS}) target_compile_definitions(test_cpu PRIVATE ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED ${SOFIEBLAS_DEFINES}) -target_include_directories(test_cpu PRIVATE "../sofieBLAS/include" ${ALPAKA_BASE}/include) +target_include_directories(test_cpu PRIVATE "../sofieBLAS/include" ${ALPAKA_BASE}/include ${BLAS_INCLUDE_DIR}) target_link_libraries(test_cpu PRIVATE ${BLAS_LIBS}) @@ -108,8 +160,30 @@ target_include_directories(test_cuda PRIVATE ${ALPAKA_BASE}/include "../sofieBLA target_link_directories(test_cuda PRIVATE ${CUDA_BASE}/lib64) target_link_libraries(test_cuda PRIVATE cublasLt cublas cudart) +# --- test_hip target --- +if(CMAKE_HIP_COMPILER) + # Use a private copy of the source: set_source_files_properties(LANGUAGE ...) + # is directory-scoped, so reusing test.cc directly here would clobber the + # LANGUAGE CUDA property already set on it for test_cuda above. + configure_file(test.cc test_hip_gen.cc COPYONLY) + add_executable(test_hip) + target_sources(test_hip PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/test_hip_gen.cc) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/test_hip_gen.cc PROPERTIES LANGUAGE HIP) + target_compile_features(test_hip PUBLIC cxx_std_20) + + target_compile_options(test_hip PRIVATE + ${CXXFLAGS} + ${CXX_HOST_FLAGS} + ) + + target_compile_definitions(test_hip PRIVATE ALPAKA_ACC_GPU_HIP_ENABLED) + target_include_directories(test_hip PRIVATE ${ALPAKA_BASE}/include "../sofieBLAS/include" ${ROCM_BASE}/include) + target_link_directories(test_hip PRIVATE ${ROCM_BASE}/lib) + target_link_libraries(test_hip PRIVATE hipblaslt hipblas amdhip64) +endif() + # --- clean target equivalent --- add_custom_target(clean-all - COMMAND ${CMAKE_COMMAND} -E rm -f test_cpu test_cuda *.d *.o *.so + COMMAND ${CMAKE_COMMAND} -E rm -f test_cpu test_cuda test_hip *.d *.o *.so COMMENT "Cleaning all generated files" ) diff --git a/tests/test.cc b/tests/test.cc index e106275..4b47b67 100644 --- a/tests/test.cc +++ b/tests/test.cc @@ -14,8 +14,6 @@ using Dim1D = alpaka::DimInt<1u>; // --------------------------------------------------------------------------- // Reference implementations (column-major, float) // --------------------------------------------------------------------------- - -// Access element (row, col) of a column-major matrix with leading dim `ld`. static inline float cm(const float *M, int row, int col, int ld) { return M[col * ld + row]; } @@ -152,7 +150,6 @@ static void runCpuTests() { checkClose(C, ref.data(), M * N, "cpu::matmul NN"); // --- matmul TN (A^T: K×M physical → M×K logical) --- - // For TN we need A to be K×M so we create a separate buffer { auto hAt = alpaka::allocBuf(dev, static_cast(K * M)); float *At = alpaka::getPtrNative(hAt); @@ -205,8 +202,6 @@ static void runCpuTests() { std::vector C0(M * N); fillSeq(C0.data(), M * N, 10.f); refMatmul(ref.data(), A, B, M, N, K, 1.f, 0.5f, false, false); - // ref already has the pre-filled values baked in via refMatmul beta path - // but refMatmul reads C in-place, so we need to re-run with correct init std::copy(C0.begin(), C0.end(), ref.data()); refMatmul(ref.data(), A, B, M, N, K, 1.f, 0.5f, false, false); } @@ -273,7 +268,7 @@ static void runCpuTests() { // --- gemmrelu with bias --- fillVal(C, M * N, 0.f); - fillSeq(bias, M * N, -5.f, 2.f); // mixed sign bias + fillSeq(bias, M * N, -5.f, 2.f); blas.gemmrelu('N', 'N', M, N, K, 1.f, hA, hB, 0.f, hBias, hC); std::fill(ref.begin(), ref.end(), 0.f); refGemmRelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); @@ -340,19 +335,20 @@ static void runCpuTests() { #endif // ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED -// --------------------------------------------------------------------------- -// CUDA tests -// --------------------------------------------------------------------------- - -#ifdef ALPAKA_ACC_GPU_CUDA_ENABLED - -// Helper: infer packed column-major leading dimensions +#if defined(ALPAKA_ACC_GPU_CUDA_ENABLED) || defined(ALPAKA_ACC_GPU_HIP_ENABLED) static int ldaFor(char trans, int m, int k) { return (trans == 'N' || trans == 'n') ? m : k; } static int ldbFor(char trans, int k, int n) { return (trans == 'N' || trans == 'n') ? k : n; } +#endif + +// --------------------------------------------------------------------------- +// CUDA tests +// --------------------------------------------------------------------------- + +#ifdef ALPAKA_ACC_GPU_CUDA_ENABLED static void runCudaTests() { std::cout << "\n=== CUDA Tests ===\n"; @@ -569,6 +565,227 @@ static void runCudaTests() { #endif // ALPAKA_ACC_GPU_CUDA_ENABLED +// --------------------------------------------------------------------------- +// HIP tests +// --------------------------------------------------------------------------- + +#ifdef ALPAKA_ACC_GPU_HIP_ENABLED + +static void runHipTests() { + std::cout << "\n=== HIP Tests ===\n"; + + alpaka::PlatformHipRt platform{}; + auto dev = alpaka::getDevByIdx(platform, 0u); + alpaka::Queue queue{dev}; + sofieBLAS blas(queue); + + alpaka::PlatformCpu hostPlatform{}; + auto hostDev = alpaka::getDevByIdx(hostPlatform, 0u); + + constexpr int M = 4, N = 3, K = 5; + + auto hA = alpaka::allocBuf(hostDev, static_cast(M * K)); + auto hB = alpaka::allocBuf(hostDev, static_cast(K * N)); + auto hC = alpaka::allocBuf(hostDev, static_cast(M * N)); + auto hBias = alpaka::allocBuf(hostDev, static_cast(M * N)); + + float *A = alpaka::getPtrNative(hA); + float *B = alpaka::getPtrNative(hB); + float *bias = alpaka::getPtrNative(hBias); + + fillSeq(A, M * K); + fillSeq(B, K * N, 1.f, 0.5f); + fillVal(bias, M * N, 0.f); + + auto dA = alpaka::allocAsyncBuf(queue, static_cast(M * K)); + auto dB = alpaka::allocAsyncBuf(queue, static_cast(K * N)); + auto dC = alpaka::allocAsyncBuf(queue, static_cast(M * N)); + auto dBias = + alpaka::allocAsyncBuf(queue, static_cast(M * N)); + + alpaka::memcpy(queue, dA, hA); + alpaka::memcpy(queue, dB, hB); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + + std::vector ref(M * N); + float *C = alpaka::getPtrNative(hC); + + auto verify = [&](const std::string &name) { + alpaka::memcpy(queue, hC, dC); + alpaka::wait(queue); + checkClose(C, ref.data(), M * N, name); + }; + + // ---- matmul NN ---- + blas.addLayoutConfig(M, N, K, ldaFor('N', M, K), ldbFor('N', K, N), M, 'N', + 'N'); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), A, B, M, N, K, 1.f, 0.f, false, false); + blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); + verify("hip::matmul NN"); + + // ---- matmul TN ---- + { + auto hAt = alpaka::allocBuf(hostDev, static_cast(K * M)); + float *At = alpaka::getPtrNative(hAt); + fillSeq(At, K * M); + auto dAt = + alpaka::allocAsyncBuf(queue, static_cast(K * M)); + alpaka::memcpy(queue, dAt, hAt); + alpaka::wait(queue); + blas.addLayoutConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, 'T', + 'N'); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), At, B, M, N, K, 1.f, 0.f, true, false); + blas.matmul('T', 'N', M, N, K, 1.f, dAt, dB, 0.f, dC); + verify("hip::matmul TN"); + } + + // ---- matmul NT ---- + { + auto hBt = alpaka::allocBuf(hostDev, static_cast(N * K)); + float *Bt = alpaka::getPtrNative(hBt); + fillSeq(Bt, N * K, 1.f, 0.5f); + auto dBt = + alpaka::allocAsyncBuf(queue, static_cast(N * K)); + alpaka::memcpy(queue, dBt, hBt); + alpaka::wait(queue); + blas.addLayoutConfig(M, N, K, ldaFor('N', M, K), ldbFor('T', K, N), M, 'N', + 'T'); + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), A, Bt, M, N, K, 1.f, 0.f, false, true); + blas.matmul('N', 'T', M, N, K, 1.f, dA, dBt, 0.f, dC); + verify("hip::matmul NT"); + } + + // ---- matmul alpha=2.5 ---- + std::fill(ref.begin(), ref.end(), 0.f); + refMatmul(ref.data(), A, B, M, N, K, 2.5f, 0.f, false, false); + blas.matmul('N', 'N', M, N, K, 2.5f, dA, dB, 0.f, dC); + verify("hip::matmul alpha=2.5"); + + // ---- gemm NN beta=0 ---- + fillSeq(bias, M * N, 0.1f, 0.1f); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + refGemm(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + blas.gemm('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dBias, dC); + verify("hip::gemm NN beta=0"); + + // ---- gemm NN beta=1 ---- + // D_in = bias, so result = A*B + 1*bias_matrix + bias_vec + std::fill(ref.begin(), ref.end(), 0.f); + refGemm(ref.data(), A, B, bias, M, N, K, 1.f, 1.f, false, false); + blas.gemm('N', 'N', M, N, K, 1.f, dA, dB, 1.f, dBias, dC); + verify("hip::gemm NN beta=1"); + + // ---- gemm TN ---- + { + auto hAt = alpaka::allocBuf(hostDev, static_cast(K * M)); + float *At = alpaka::getPtrNative(hAt); + fillSeq(At, K * M); + auto dAt = + alpaka::allocAsyncBuf(queue, static_cast(K * M)); + alpaka::memcpy(queue, dAt, hAt); + alpaka::wait(queue); + blas.addLayoutConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, 'T', + 'N'); + std::fill(ref.begin(), ref.end(), 0.f); + refGemm(ref.data(), At, B, bias, M, N, K, 1.f, 0.f, true, false); + blas.gemm('T', 'N', M, N, K, 1.f, dAt, dB, 0.f, dBias, dC); + verify("hip::gemm TN"); + } + + // ---- gemmrelu: all-positive ---- + { + auto hAp = alpaka::allocBuf(hostDev, static_cast(M * K)); + auto hBp = alpaka::allocBuf(hostDev, static_cast(K * N)); + auto hBiasz = + alpaka::allocBuf(hostDev, static_cast(M * N)); + float *Ap = alpaka::getPtrNative(hAp); + float *Bp = alpaka::getPtrNative(hBp); + fillSeq(Ap, M * K, 0.1f, 0.1f); + fillSeq(Bp, K * N, 0.1f, 0.1f); + fillVal(alpaka::getPtrNative(hBiasz), M * N, 0.f); + auto dAp = + alpaka::allocAsyncBuf(queue, static_cast(M * K)); + auto dBp = + alpaka::allocAsyncBuf(queue, static_cast(K * N)); + auto dBiasz = + alpaka::allocAsyncBuf(queue, static_cast(M * N)); + alpaka::memcpy(queue, dAp, hAp); + alpaka::memcpy(queue, dBp, hBp); + alpaka::memcpy(queue, dBiasz, hBiasz); + alpaka::wait(queue); + blas.addLayoutConfig(M, N, K, M, K, M, 'N', 'N'); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmRelu(ref.data(), Ap, Bp, alpaka::getPtrNative(hBiasz), M, N, K, 1.f, + 0.f, false, false); + blas.gemmrelu('N', 'N', M, N, K, 1.f, dAp, dBp, 0.f, dBiasz, dC); + verify("hip::gemmrelu all-positive"); + } + + // ---- gemmrelu: alpha=-1 forces negatives ---- + { + auto hBiasz = + alpaka::allocBuf(hostDev, static_cast(M * N)); + fillVal(alpaka::getPtrNative(hBiasz), M * N, 0.f); + auto dBiasz = + alpaka::allocAsyncBuf(queue, static_cast(M * N)); + alpaka::memcpy(queue, dBiasz, hBiasz); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmRelu(ref.data(), A, B, alpaka::getPtrNative(hBiasz), M, N, K, -1.f, + 0.f, false, false); + blas.gemmrelu('N', 'N', M, N, K, -1.f, dA, dB, 0.f, dBiasz, dC); + verify("hip::gemmrelu alpha=-1 (clamped)"); + } + + // ---- gemmrelu with mixed bias ---- + fillSeq(bias, M * N, -5.f, 2.f); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmRelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + blas.gemmrelu('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dBias, dC); + verify("hip::gemmrelu with mixed bias"); + + // ---- gemmgelu NN ---- + fillVal(bias, M * N, 0.f); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmGelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + blas.gemmgelu('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dBias, dC); + verify("hip::gemmgelu NN"); + + // ---- gemmgelu with bias ---- + fillSeq(bias, M * N, -2.f, 0.5f); + alpaka::memcpy(queue, dBias, hBias); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + refGemmGelu(ref.data(), A, B, bias, M, N, K, 1.f, 0.f, false, false); + blas.gemmgelu('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dBias, dC); + verify("hip::gemmgelu with bias"); + + // ---- edge: zero A ---- + { + auto hZero = alpaka::allocBuf(hostDev, static_cast(M * K)); + fillVal(alpaka::getPtrNative(hZero), M * K, 0.f); + auto dZero = + alpaka::allocAsyncBuf(queue, static_cast(M * K)); + alpaka::memcpy(queue, dZero, hZero); + alpaka::wait(queue); + std::fill(ref.begin(), ref.end(), 0.f); + blas.matmul('N', 'N', M, N, K, 1.f, dZero, dB, 0.f, dC); + verify("hip::matmul zero-A"); + } +} + +#endif // ALPAKA_ACC_GPU_HIP_ENABLED + // --------------------------------------------------------------------------- // main // --------------------------------------------------------------------------- @@ -580,6 +797,9 @@ int main() { #ifdef ALPAKA_ACC_GPU_CUDA_ENABLED runCudaTests(); #endif +#ifdef ALPAKA_ACC_GPU_HIP_ENABLED + runHipTests(); +#endif std::cout << "\n"; if (gFailures == 0) From 6c2b9a00ab84ba0f2530c6f5f65ebe9eaccf872c Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 17 Aug 2026 12:08:12 +0200 Subject: [PATCH 25/29] feat: ci pipeline for tests and benchmarking --- .github/workflows/benchmark.yml | 280 ++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 212 ++++++++++++++++++++++++ CMakeLists.txt | 69 ++++++++ README.md | 20 ++- benchmark/CMakeLists.txt | 76 +++++++++ benchmark/bench.cc | 228 ++++++++++++++++++++++++++ check_style.sh | 5 +- cmake/SofieBLASBackends.cmake | 153 +++++++++++++++++ cmake/sofieBLASConfig.cmake.in | 5 + tests/CMakeLists.txt | 205 +++++------------------ tests/test.cc | 4 - 11 files changed, 1084 insertions(+), 173 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 .github/workflows/tests.yml create mode 100644 CMakeLists.txt create mode 100644 benchmark/CMakeLists.txt create mode 100644 benchmark/bench.cc create mode 100644 cmake/SofieBLASBackends.cmake create mode 100644 cmake/sofieBLASConfig.cmake.in diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..c1ffd3a --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,280 @@ +name: Benchmark + +on: + issue_comment: + types: [created] + workflow_dispatch: + inputs: + warmup: + description: "Warmup iterations" + default: "5" + iterations: + description: "Timed iterations" + default: "20" + sizes: + description: "Comma-separated square GEMM sizes (M=N=K)" + default: "256,512,1024,2048,4096" + +concurrency: + group: benchmark-${{ github.event.issue.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + issues: write + +env: + BUILD_TYPE: Release + BENCH_WARMUP: ${{ github.event.inputs.warmup || '5' }} + BENCH_ITERS: ${{ github.event.inputs.iterations || '20' }} + BENCH_SIZES: ${{ github.event.inputs.sizes || '256,512,1024,2048,4096' }} + DEPS_CACHE: /tmp/sofieblas-cmake-deps + +jobs: + parse-command: + name: Parse trigger + runs-on: ubuntu-latest + if: | + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'issue_comment' && + github.event.issue.pull_request != null && + contains(fromJSON('["sanjibansg","lmoneta"]'), github.event.comment.user.login) && + startsWith(github.event.comment.body, '/runbenchmark') + ) + outputs: + runner_label: ${{ steps.parse.outputs.runner_label }} + valid: ${{ steps.parse.outputs.valid }} + flavor: ${{ steps.parse.outputs.flavor }} + valid_flavors: ${{ steps.parse.outputs.valid_flavors }} + steps: + - name: React to trigger comment + if: github.event_name == 'issue_comment' + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'eyes' + }); + - name: Parse runner flavor + id: parse + uses: actions/github-script@v7 + with: + script: | + // default flavor keyed by name -> actual self-hosted runner label + const flavors = { + l40s: 'ml4ep-l40s', + h100: 'ml4ep-h100', + 'h100-47gb': 'ml4ep-h100-47gb', + }; + let flavor = 'l40s'; + let valid = true; + if (context.eventName === 'issue_comment') { + const m = context.payload.comment.body.trim().match(/^\/runbenchmark(?:\s+(\S+))?/i); + if (m && m[1]) { + flavor = m[1].toLowerCase(); + valid = Object.prototype.hasOwnProperty.call(flavors, flavor); + } + } + core.setOutput('flavor', flavor); + core.setOutput('valid', String(valid)); + core.setOutput('runner_label', valid ? flavors[flavor] : ''); + core.setOutput('valid_flavors', Object.keys(flavors).join(', ')); + - name: Post run link + if: github.event_name == 'issue_comment' + uses: actions/github-script@v7 + with: + script: | + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body: `\`/runbenchmark\` (${{ steps.parse.outputs.flavor }}): triggered - [view run](${runUrl})` + }); + + benchmark: + name: Benchmark Comparison + needs: parse-command + if: needs.parse-command.outputs.valid == 'true' + runs-on: ${{ needs.parse-command.outputs.runner_label }} + timeout-minutes: 60 + + steps: + - name: GPU check + run: nvidia-smi + + - name: Resolve PR info + id: pr + if: github.event_name == 'issue_comment' + uses: actions/github-script@v7 + with: + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.issue.number + }); + core.setOutput('number', pr.data.number); + core.setOutput('merge_ref', `refs/pull/${pr.data.number}/merge`); + core.setOutput('base_ref', pr.data.base.ref); + + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + path: sofieblas-pr + ref: ${{ github.event_name == 'issue_comment' && steps.pr.outputs.merge_ref || github.ref }} + + - name: Checkout base branch + if: github.event_name == 'issue_comment' + uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.base_ref }} + path: sofieblas-main + + - name: Cache FetchContent dependencies + uses: actions/cache@v4 + with: + path: ${{ env.DEPS_CACHE }} + key: cmake-deps-bench-${{ hashFiles('sofieblas-pr/cmake/SofieBLASBackends.cmake') }} + restore-keys: cmake-deps-bench- + + - name: Configure PR build + run: | + cmake -B sofieblas-pr/build -S sofieblas-pr \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DSOFIEBLAS_BUILD_BENCHMARKS=ON \ + -DCMAKE_CUDA_ARCHITECTURES=native \ + "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" + + - name: Build PR benchmark + run: cmake --build sofieblas-pr/build --target bench_cuda -j"$(nproc)" + + - name: Run PR benchmark + working-directory: sofieblas-pr/build/benchmark + run: | + ./bench_cuda \ + -w "${{ env.BENCH_WARMUP }}" \ + -n "${{ env.BENCH_ITERS }}" \ + --sizes "${{ env.BENCH_SIZES }}" \ + | tee benchmark_pr.txt + + - name: Check whether base branch supports the benchmark harness + id: base_support + if: github.event_name == 'issue_comment' + run: | + if grep -q "SOFIEBLAS_BUILD_BENCHMARKS" sofieblas-main/CMakeLists.txt 2>/dev/null; then + echo "supported=true" >> "$GITHUB_OUTPUT" + else + echo "supported=false" >> "$GITHUB_OUTPUT" + echo "::warning::${{ steps.pr.outputs.base_ref }} does not define the SOFIEBLAS_BUILD_BENCHMARKS option yet - skipping the base-branch comparison." + fi + + - name: Configure base build + if: github.event_name == 'issue_comment' && steps.base_support.outputs.supported == 'true' + run: | + cmake -B sofieblas-main/build -S sofieblas-main \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DSOFIEBLAS_BUILD_BENCHMARKS=ON \ + -DCMAKE_CUDA_ARCHITECTURES=native \ + "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" + + - name: Build base benchmark + if: github.event_name == 'issue_comment' && steps.base_support.outputs.supported == 'true' + run: cmake --build sofieblas-main/build --target bench_cuda -j"$(nproc)" + + - name: Run base benchmark + if: github.event_name == 'issue_comment' && steps.base_support.outputs.supported == 'true' + working-directory: sofieblas-main/build/benchmark + run: | + ./bench_cuda \ + -w "${{ env.BENCH_WARMUP }}" \ + -n "${{ env.BENCH_ITERS }}" \ + --sizes "${{ env.BENCH_SIZES }}" \ + | tee benchmark_main.txt + + - name: Build comparison summary + id: summary + if: always() + run: | + { + echo "summary</dev/null || echo "(no results)" + if [ -f sofieblas-main/build/benchmark/benchmark_main.txt ]; then + echo "" + echo "── ${{ steps.pr.outputs.base_ref }} ──────────────────────────────────────────────" + cat sofieblas-main/build/benchmark/benchmark_main.txt + fi + echo '```' + echo "BENCH_EOF" + } >> "$GITHUB_OUTPUT" + + - name: Post results as PR comment + if: always() && github.event_name == 'issue_comment' + uses: marocchino/sticky-pull-request-comment@v2 + with: + number: ${{ steps.pr.outputs.number }} + header: benchmark + message: ${{ steps.summary.outputs.summary }} + + - name: Write job summary + if: always() + run: echo "${{ steps.summary.outputs.summary }}" >> "$GITHUB_STEP_SUMMARY" + + - name: Notify benchmark completion + if: always() && github.event_name == 'issue_comment' + uses: actions/github-script@v7 + with: + script: | + const status = '${{ job.status }}' === 'success' ? '✅ completed' : '❌ failed'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body: `\`/runbenchmark\` (${{ needs.parse-command.outputs.flavor }}): Benchmark ${status} - [view results](${runUrl})` + }); + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ github.run_id }} + path: | + sofieblas-pr/build/benchmark/benchmark_pr.txt + sofieblas-main/build/benchmark/benchmark_main.txt + if-no-files-found: ignore + + report-invalid-flavor: + name: Report invalid runner flavor + needs: parse-command + runs-on: ubuntu-latest + if: >- + always() && github.event_name == 'issue_comment' && + needs.parse-command.result == 'success' && + needs.parse-command.outputs.valid == 'false' + steps: + - uses: actions/github-script@v7 + with: + script: | + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body: `\`/runbenchmark\`: unknown runner flavor \`${{ needs.parse-command.outputs.flavor }}\`. ` + + `Valid options: ${{ needs.parse-command.outputs.valid_flavors }} (or omit for the default, l40s).` + }); diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..bd94f52 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,212 @@ +name: Unit Tests + +on: + push: + branches: [main, dev] + pull_request: + issue_comment: + types: [created] + workflow_dispatch: + +concurrency: + group: tests-${{ github.event.issue.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + issues: write + +env: + BUILD_TYPE: Release + DEPS_CACHE: /tmp/sofieblas-cmake-deps + +jobs: + cpu-tests: + name: CPU Unit Tests + if: github.event_name != 'issue_comment' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install OpenBLAS + run: | + sudo apt-get update + sudo apt-get install -y libopenblas-dev + + - name: Cache FetchContent dependencies + uses: actions/cache@v4 + with: + path: ${{ env.DEPS_CACHE }} + key: cmake-deps-cpu-${{ hashFiles('cmake/SofieBLASBackends.cmake') }} + restore-keys: cmake-deps-cpu- + + - name: Configure + run: | + cmake -B build -S . \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DSOFIEBLAS_BUILD_TESTS=ON \ + -DSOFIEBLAS_BUILD_BENCHMARKS=ON \ + "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" + + - name: Build + run: cmake --build build -j"$(nproc)" + + - name: Run tests + working-directory: build + run: ctest --output-on-failure -j"$(nproc)" -R '\.cpu(\.|$)' + + - name: Upload test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: cpu-test-log-${{ github.run_id }} + path: build/Testing/Temporary/LastTest.log + if-no-files-found: ignore + + parse-command: + name: Parse trigger + runs-on: ubuntu-latest + if: | + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'issue_comment' && + github.event.issue.pull_request != null && + contains(fromJSON('["sanjibansg","lmoneta"]'), github.event.comment.user.login) && + startsWith(github.event.comment.body, '/runtest') + ) + outputs: + runner_label: ${{ steps.parse.outputs.runner_label }} + valid: ${{ steps.parse.outputs.valid }} + flavor: ${{ steps.parse.outputs.flavor }} + valid_flavors: ${{ steps.parse.outputs.valid_flavors }} + steps: + - name: React to trigger comment + if: github.event_name == 'issue_comment' + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'eyes' + }); + - name: Parse runner flavor + id: parse + uses: actions/github-script@v7 + with: + script: | + // default flavor keyed by name -> actual self-hosted runner label + const flavors = { + l40s: 'ml4ep-l40s', + h100: 'ml4ep-h100', + 'h100-47gb': 'ml4ep-h100-47gb', + }; + let flavor = 'l40s'; + let valid = true; + if (context.eventName === 'issue_comment') { + const m = context.payload.comment.body.trim().match(/^\/runtest(?:\s+(\S+))?/i); + if (m && m[1]) { + flavor = m[1].toLowerCase(); + valid = Object.prototype.hasOwnProperty.call(flavors, flavor); + } + } + core.setOutput('flavor', flavor); + core.setOutput('valid', String(valid)); + core.setOutput('runner_label', valid ? flavors[flavor] : ''); + core.setOutput('valid_flavors', Object.keys(flavors).join(', ')); + - name: Post run link + if: github.event_name == 'issue_comment' + uses: actions/github-script@v7 + with: + script: | + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body: `\`/runtest\` (${{ steps.parse.outputs.flavor }}): triggered - [view run](${runUrl})` + }); + + gpu-tests: + name: GPU Unit Tests + needs: parse-command + if: needs.parse-command.outputs.valid == 'true' + runs-on: ${{ needs.parse-command.outputs.runner_label }} + timeout-minutes: 45 + steps: + - name: GPU check + run: nvidia-smi + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'issue_comment' && format('refs/pull/{0}/merge', github.event.issue.number) || github.ref }} + + - name: Cache FetchContent dependencies + uses: actions/cache@v4 + with: + path: ${{ env.DEPS_CACHE }} + key: cmake-deps-gpu-${{ hashFiles('cmake/SofieBLASBackends.cmake') }} + restore-keys: cmake-deps-gpu- + + - name: Configure + run: | + cmake -B build -S . \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DSOFIEBLAS_BUILD_TESTS=ON \ + -DSOFIEBLAS_BUILD_BENCHMARKS=ON \ + -DCMAKE_CUDA_ARCHITECTURES=native \ + "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" + + - name: Build + run: cmake --build build -j"$(nproc)" + + - name: Run GPU tests + working-directory: build + run: ctest --output-on-failure -j"$(nproc)" -R '\.(cuda|hip)(\.|$)' + + - name: Upload test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpu-test-log-${{ github.run_id }} + path: build/Testing/Temporary/LastTest.log + if-no-files-found: ignore + + - name: Report result on PR + if: always() && github.event_name == 'issue_comment' + uses: actions/github-script@v7 + with: + script: | + const status = '${{ job.status }}' === 'success' ? '✅ passed' : '❌ failed'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body: `\`/runtest\` (${{ needs.parse-command.outputs.flavor }}): GPU Unit Tests ${status} - [view run](${runUrl})` + }); + + report-invalid-flavor: + name: Report invalid runner flavor + needs: parse-command + runs-on: ubuntu-latest + if: >- + always() && github.event_name == 'issue_comment' && + needs.parse-command.result == 'success' && + needs.parse-command.outputs.valid == 'false' + steps: + - uses: actions/github-script@v7 + with: + script: | + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body: `\`/runtest\`: unknown runner flavor \`${{ needs.parse-command.outputs.flavor }}\`. ` + + `Valid options: ${{ needs.parse-command.outputs.valid_flavors }} (or omit for the default, l40s).` + }); diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..368a522 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.21) +project(sofieBLAS + VERSION 0.1.0 + DESCRIPTION "Header-only C++ BLAS abstraction over heterogeneous backends" + LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) + +option(SOFIEBLAS_BUILD_TESTS "Build sofieBLAS unit tests" OFF) +option(SOFIEBLAS_BUILD_BENCHMARKS "Build sofieBLAS GEMM benchmarks" OFF) + +# --- Header-only interface library --- +add_library(sofieBLAS INTERFACE) +add_library(sofieBLAS::sofieBLAS ALIAS sofieBLAS) + +target_include_directories(sofieBLAS INTERFACE + $ + $) + +target_compile_features(sofieBLAS INTERFACE cxx_std_20) + +if(SOFIEBLAS_BUILD_TESTS OR SOFIEBLAS_BUILD_BENCHMARKS) + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + include(SofieBLASBackends) +endif() + +if(SOFIEBLAS_BUILD_TESTS) + enable_testing() + add_subdirectory(tests) +endif() + +if(SOFIEBLAS_BUILD_BENCHMARKS) + add_subdirectory(benchmark) +endif() + +# --- Install (header-only package) --- +install(TARGETS sofieBLAS EXPORT sofieBLASTargets) +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/sofieBLASConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/sofieBLASConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/sofieBLAS +) + +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/sofieBLASConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY AnyNewerVersion + ARCH_INDEPENDENT +) + +install( + EXPORT sofieBLASTargets + FILE sofieBLASTargets.cmake + NAMESPACE sofieBLAS:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/sofieBLAS +) + +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/sofieBLASConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/sofieBLASConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/sofieBLAS +) diff --git a/README.md b/README.md index 3f1608f..191fe2a 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,25 @@ Backend selection happens entirely at compile time, in two steps: | NVIDIA GPU (cuBLASLt) | `ALPAKA_ACC_GPU_CUDA_ENABLED` | `alpaka::TagGpuCudaRt` | | AMD GPU (hipBLASLt) | `ALPAKA_ACC_GPU_HIP_ENABLED` | `alpaka::TagGpuHipRt` | -`sofieBLAS/sofieBLAS.hpp` includes the matching backend header(s) for you based on these macros; nothing else needs to change in your source beyond picking the right tag. See `tests/CMakeLists.txt` for a working example that detects the available libraries and wires up a `test_cpu`, `test_cuda`, and `test_hip` target, each built against a different backend. +`sofieBLAS/sofieBLAS.hpp` includes the matching backend header(s) for you based on these macros; nothing else needs to change in your source beyond picking the right tag. + +## Building tests and benchmarks + +sofieBLAS itself is header-only (`add_subdirectory`/`find_package(sofieBLAS)` gives you an `INTERFACE` target with no build step). Tests and benchmarks are opt-in via CMake options and auto-detect whichever backends are available on the machine for each backend target (`test_cpu`/`test_cuda`/`test_hip`, `bench_cpu`/`bench_cuda`/`bench_hip`) and is skipped if its dependency isn't found: + +```bash +cmake -B build -S . -DSOFIEBLAS_BUILD_TESTS=ON -DSOFIEBLAS_BUILD_BENCHMARKS=ON +cmake --build build -j"$(nproc)" + +# Run the correctness tests (matmul/gemm/gemmrelu/gemmgelu vs. a reference +# implementation, per available backend) +ctest --test-dir build --output-on-failure + +# Run the GEMM throughput benchmark +./build/benchmark/bench_cuda -w 5 -n 20 --sizes 256,512,1024,2048,4096 +``` + +`CPU_BLAS_LIB` (`OpenBLAS`/`MKL`/`BLIS`/`Accelerate`) and `CUDA_BASE`/`ROCM_BASE`/`ONEAPI_BASE`/`BLIS_BASE` are available as `-D` cache variables to point at non-default install locations; alpaka is picked up via `find_package(alpaka)` if already installed, otherwise fetched automatically. See `tests/CMakeLists.txt` and `benchmark/CMakeLists.txt` for the target definitions, and `cmake/SofieBLASBackends.cmake` for the shared backend-detection logic. ## Usage example diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 0000000..dfb234c --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,76 @@ +# Built via the root CMakeLists.txt with -DSOFIEBLAS_BUILD_BENCHMARKS=ON, +# which already ran cmake/SofieBLASBackends.cmake to populate +# SOFIEBLAS_CPU_BLAS_*, SOFIEBLAS_CUDA_ENABLED, SOFIEBLAS_HIP_ENABLED and the +# alpaka::alpaka target. + +set(CXXFLAGS -O3 -DALPAKA_HAS_STD_ATOMIC_REF) +set(CXX_HOST_FLAGS -fPIC -pthread) +set(CXX_CUDA_FLAGS -Wno-deprecated-gpu-targets --extended-lambda --expt-relaxed-constexpr) +set(XCOMPILER_FLAGS -Xcompiler=-fPIC,-pthread) + +if(SOFIEBLAS_CPU_BLAS_LIBS) + add_executable(bench_cpu bench.cc) + target_compile_features(bench_cpu PUBLIC cxx_std_20) + target_compile_options(bench_cpu PRIVATE ${CXXFLAGS} ${CXX_HOST_FLAGS}) + target_compile_definitions(bench_cpu PRIVATE + ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED ${SOFIEBLAS_CPU_BLAS_DEFINE}) + target_include_directories(bench_cpu PRIVATE ${SOFIEBLAS_CPU_BLAS_INCLUDE_DIR}) + target_link_libraries(bench_cpu PRIVATE + sofieBLAS::sofieBLAS alpaka::alpaka ${SOFIEBLAS_CPU_BLAS_LIBS}) + + if(SOFIEBLAS_BUILD_TESTS) + add_test(NAME sofieBLAS.bench.cpu.smoke + COMMAND bench_cpu -w 1 -n 1 --sizes 64,128) + endif() +else() + message(STATUS "sofieBLAS benchmark: skipping bench_cpu (no CPU BLAS library found)") +endif() + +if(SOFIEBLAS_CUDA_ENABLED) + configure_file(bench.cc bench_cuda_gen.cc COPYONLY) + add_executable(bench_cuda) + target_sources(bench_cuda PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/bench_cuda_gen.cc) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/bench_cuda_gen.cc PROPERTIES LANGUAGE CUDA) + set_target_properties(bench_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON) + target_compile_features(bench_cuda PUBLIC cxx_std_20) + + target_compile_options(bench_cuda PRIVATE + ${CXXFLAGS} + ${CXX_CUDA_FLAGS} + ${XCOMPILER_FLAGS} + ) + + target_compile_definitions(bench_cuda PRIVATE ALPAKA_ACC_GPU_CUDA_ENABLED) + target_link_libraries(bench_cuda PRIVATE + sofieBLAS::sofieBLAS alpaka::alpaka CUDA::cudart CUDA::cublas CUDA::cublasLt) + + if(SOFIEBLAS_BUILD_TESTS) + add_test(NAME sofieBLAS.bench.cuda.smoke + COMMAND bench_cuda -w 1 -n 1 --sizes 64,128) + endif() +else() + message(STATUS "sofieBLAS benchmark: skipping bench_cuda (no CUDA compiler found)") +endif() + +if(SOFIEBLAS_HIP_ENABLED) + configure_file(bench.cc bench_hip_gen.cc COPYONLY) + add_executable(bench_hip) + target_sources(bench_hip PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/bench_hip_gen.cc) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/bench_hip_gen.cc PROPERTIES LANGUAGE HIP) + target_compile_features(bench_hip PUBLIC cxx_std_20) + + target_compile_options(bench_hip PRIVATE ${CXXFLAGS} ${CXX_HOST_FLAGS}) + + target_compile_definitions(bench_hip PRIVATE ALPAKA_ACC_GPU_HIP_ENABLED) + target_include_directories(bench_hip PRIVATE ${ROCM_BASE}/include) + target_link_directories(bench_hip PRIVATE ${ROCM_BASE}/lib) + target_link_libraries(bench_hip PRIVATE + sofieBLAS::sofieBLAS alpaka::alpaka hipblaslt hipblas amdhip64) + + if(SOFIEBLAS_BUILD_TESTS) + add_test(NAME sofieBLAS.bench.hip.smoke + COMMAND bench_hip -w 1 -n 1 --sizes 64,128) + endif() +else() + message(STATUS "sofieBLAS benchmark: skipping bench_hip (no HIP compiler found)") +endif() diff --git a/benchmark/bench.cc b/benchmark/bench.cc new file mode 100644 index 0000000..3263dab --- /dev/null +++ b/benchmark/bench.cc @@ -0,0 +1,228 @@ +#include "sofieBLAS/sofieBLAS.hpp" +#include + +#include +#include +#include +#include +#include +#include +#include + +using Idx = uint32_t; + +struct BenchOptions { + int warmup = 5; + int iterations = 20; + std::vector sizes = {256, 512, 1024, 2048, 4096}; +}; + +static BenchOptions parseArgs(int argc, char **argv) { + BenchOptions opt; + for (int i = 1; i < argc; ++i) { + std::string arg = argv[i]; + auto nextVal = [&](void) -> std::string { + if (i + 1 >= argc) { + std::cerr << "Missing value for " << arg << "\n"; + std::exit(EXIT_FAILURE); + } + return argv[++i]; + }; + if (arg == "-w" || arg == "--warmup") { + opt.warmup = std::stoi(nextVal()); + } else if (arg == "-n" || arg == "--iterations") { + opt.iterations = std::stoi(nextVal()); + } else if (arg == "--sizes") { + opt.sizes.clear(); + std::stringstream ss(nextVal()); + std::string tok; + while (std::getline(ss, tok, ',')) + opt.sizes.push_back(std::stoi(tok)); + } else if (arg == "-h" || arg == "--help") { + std::cout << "Usage: " << argv[0] + << " [-w warmup] [-n iterations] [--sizes s1,s2,...]\n"; + std::exit(EXIT_SUCCESS); + } else { + std::cerr << "Unknown argument: " << arg << "\n"; + std::exit(EXIT_FAILURE); + } + } + return opt; +} + +static void fillSeq(float *M, int n, float start = 0.01f, float step = 0.001f) { + for (int i = 0; i < n; ++i) + M[i] = start + static_cast(i % 997) * step; +} + +static void printHeader() { + std::cout << std::left << std::setw(10) << "Backend" << std::right + << std::setw(10) << "M=N=K" << std::setw(14) << "avg (ms)" + << std::setw(14) << "GFLOP/s" << "\n"; + std::cout << std::string(48, '-') << "\n"; +} + +static void printRow(const std::string &backend, int size, double avgMs, + double gflops) { + std::cout << std::left << std::setw(10) << backend << std::right + << std::setw(10) << size << std::setw(14) << std::fixed + << std::setprecision(3) << avgMs << std::setw(14) + << std::setprecision(2) << gflops << "\n"; +} + +static double gflopsFor(int size, double avgSeconds) { + const double flops = 2.0 * static_cast(size) * + static_cast(size) * static_cast(size); + return flops / avgSeconds / 1e9; +} + +// --------------------------------------------------------------------------- +// CPU benchmark +// --------------------------------------------------------------------------- +#ifdef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED + +static void runCpuBench(const BenchOptions &opt) { + alpaka::PlatformCpu platform{}; + auto dev = alpaka::getDevByIdx(platform, 0u); + alpaka::Queue queue{dev}; + sofieBLAS blas(queue); + + for (int size : opt.sizes) { + const int M = size, N = size, K = size; + auto hA = alpaka::allocBuf(dev, static_cast(M * K)); + auto hB = alpaka::allocBuf(dev, static_cast(K * N)); + auto hC = alpaka::allocBuf(dev, static_cast(M * N)); + fillSeq(alpaka::getPtrNative(hA), M * K); + fillSeq(alpaka::getPtrNative(hB), K * N, 0.02f, 0.0005f); + + for (int i = 0; i < opt.warmup; ++i) + blas.matmul('N', 'N', M, N, K, 1.f, hA, hB, 0.f, hC); + + auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < opt.iterations; ++i) + blas.matmul('N', 'N', M, N, K, 1.f, hA, hB, 0.f, hC); + auto t1 = std::chrono::steady_clock::now(); + + double avgSeconds = + std::chrono::duration(t1 - t0).count() / opt.iterations; + printRow("cpu", size, avgSeconds * 1e3, gflopsFor(size, avgSeconds)); + } +} + +#endif // ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED + +// --------------------------------------------------------------------------- +// CUDA benchmark +// --------------------------------------------------------------------------- +#ifdef ALPAKA_ACC_GPU_CUDA_ENABLED + +static void runCudaBench(const BenchOptions &opt) { + alpaka::PlatformCudaRt platform{}; + auto dev = alpaka::getDevByIdx(platform, 0u); + alpaka::Queue queue{dev}; + sofieBLAS blas(queue); + + alpaka::PlatformCpu hostPlatform{}; + auto hostDev = alpaka::getDevByIdx(hostPlatform, 0u); + + for (int size : opt.sizes) { + const int M = size, N = size, K = size; + auto hA = alpaka::allocBuf(hostDev, static_cast(M * K)); + auto hB = alpaka::allocBuf(hostDev, static_cast(K * N)); + fillSeq(alpaka::getPtrNative(hA), M * K); + fillSeq(alpaka::getPtrNative(hB), K * N, 0.02f, 0.0005f); + + auto dA = alpaka::allocAsyncBuf(queue, static_cast(M * K)); + auto dB = alpaka::allocAsyncBuf(queue, static_cast(K * N)); + auto dC = alpaka::allocAsyncBuf(queue, static_cast(M * N)); + alpaka::memcpy(queue, dA, hA); + alpaka::memcpy(queue, dB, hB); + alpaka::wait(queue); + + blas.addLayoutConfig(M, N, K, M, K, M, 'N', 'N'); + + for (int i = 0; i < opt.warmup; ++i) + blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); + alpaka::wait(queue); + + auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < opt.iterations; ++i) + blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); + alpaka::wait(queue); + auto t1 = std::chrono::steady_clock::now(); + + double avgSeconds = + std::chrono::duration(t1 - t0).count() / opt.iterations; + printRow("cuda", size, avgSeconds * 1e3, gflopsFor(size, avgSeconds)); + } +} + +#endif // ALPAKA_ACC_GPU_CUDA_ENABLED + +// --------------------------------------------------------------------------- +// HIP benchmark +// --------------------------------------------------------------------------- +#ifdef ALPAKA_ACC_GPU_HIP_ENABLED + +static void runHipBench(const BenchOptions &opt) { + alpaka::PlatformHipRt platform{}; + auto dev = alpaka::getDevByIdx(platform, 0u); + alpaka::Queue queue{dev}; + sofieBLAS blas(queue); + + alpaka::PlatformCpu hostPlatform{}; + auto hostDev = alpaka::getDevByIdx(hostPlatform, 0u); + + for (int size : opt.sizes) { + const int M = size, N = size, K = size; + auto hA = alpaka::allocBuf(hostDev, static_cast(M * K)); + auto hB = alpaka::allocBuf(hostDev, static_cast(K * N)); + fillSeq(alpaka::getPtrNative(hA), M * K); + fillSeq(alpaka::getPtrNative(hB), K * N, 0.02f, 0.0005f); + + auto dA = alpaka::allocAsyncBuf(queue, static_cast(M * K)); + auto dB = alpaka::allocAsyncBuf(queue, static_cast(K * N)); + auto dC = alpaka::allocAsyncBuf(queue, static_cast(M * N)); + alpaka::memcpy(queue, dA, hA); + alpaka::memcpy(queue, dB, hB); + alpaka::wait(queue); + + blas.addLayoutConfig(M, N, K, M, K, M, 'N', 'N'); + + for (int i = 0; i < opt.warmup; ++i) + blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); + alpaka::wait(queue); + + auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < opt.iterations; ++i) + blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); + alpaka::wait(queue); + auto t1 = std::chrono::steady_clock::now(); + + double avgSeconds = + std::chrono::duration(t1 - t0).count() / opt.iterations; + printRow("hip", size, avgSeconds * 1e3, gflopsFor(size, avgSeconds)); + } +} + +#endif // ALPAKA_ACC_GPU_HIP_ENABLED + +int main(int argc, char **argv) { + BenchOptions opt = parseArgs(argc, argv); + + std::cout << "sofieBLAS benchmark (warmup=" << opt.warmup + << ", iterations=" << opt.iterations << ")\n\n"; + printHeader(); + +#ifdef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED + runCpuBench(opt); +#endif +#ifdef ALPAKA_ACC_GPU_CUDA_ENABLED + runCudaBench(opt); +#endif +#ifdef ALPAKA_ACC_GPU_HIP_ENABLED + runHipBench(opt); +#endif + + return EXIT_SUCCESS; +} diff --git a/check_style.sh b/check_style.sh index 22a56e4..c937e86 100755 --- a/check_style.sh +++ b/check_style.sh @@ -4,11 +4,12 @@ set -e # Directories SRC_DIR="./include" TEST_DIR="./tests" +BENCH_DIR="./benchmark" echo "📝 Discovering source/header files..." -FILES=$(find "$SRC_DIR" "$TEST_DIR" \ - -path "$TEST_DIR/build" -prune -o \ +FILES=$(find "$SRC_DIR" "$TEST_DIR" "$BENCH_DIR" \ + -path "*/build" -prune -o \ -type f \( \ -name '*.cpp' -o -name '*.cc' -o -name '*.cxx' -o \ -name '*.h' -o -name '*.hpp' -o -name '*.hxx' -o -name '*.hh' \ diff --git a/cmake/SofieBLASBackends.cmake b/cmake/SofieBLASBackends.cmake new file mode 100644 index 0000000..4677516 --- /dev/null +++ b/cmake/SofieBLASBackends.cmake @@ -0,0 +1,153 @@ +# Shared backend discovery for sofieBLAS tests and benchmarks. + +include(CheckLanguage) +include(FetchContent) + +set(CUDA_BASE "/usr/local/cuda" CACHE PATH "CUDA base path") +set(ROCM_BASE "/opt/rocm" CACHE PATH "ROCm base path") +set(ONEAPI_BASE "/opt/intel/oneapi" CACHE PATH "Intel oneAPI base path") +set(TBB_BASE "/usr" CACHE PATH "TBB base path") +set(BLIS_BASE "/usr" CACHE PATH "BLIS base path") + +set(SOFIEBLAS_ALPAKA_GIT_TAG "2fa91a34ed11b2076e474c5507d920e85cf9b79d" CACHE STRING + "alpaka commit to fetch when no installed alpaka is found") + +find_package(alpaka CONFIG QUIET) +if(NOT alpaka_FOUND) + message(STATUS "sofieBLAS: alpaka not found via find_package, fetching ${SOFIEBLAS_ALPAKA_GIT_TAG}") + FetchContent_Declare( + alpaka + GIT_REPOSITORY https://github.com/alpaka-group/alpaka + GIT_TAG ${SOFIEBLAS_ALPAKA_GIT_TAG} + ) + FetchContent_MakeAvailable(alpaka) +else() + message(STATUS "sofieBLAS: using alpaka ${alpaka_VERSION} from ${alpaka_DIR}") +endif() + +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "") + set(CMAKE_CUDA_ARCHITECTURES native) +endif() +check_language(CUDA) +if(CMAKE_CUDA_COMPILER) + enable_language(CUDA) + find_package(CUDAToolkit QUIET) + if(CUDAToolkit_FOUND) + set(SOFIEBLAS_CUDA_ENABLED TRUE) + message(STATUS "sofieBLAS: CUDA toolkit ${CUDAToolkit_VERSION} found, CUDA targets enabled") + else() + message(STATUS "sofieBLAS: nvcc found but CUDAToolkit package not found, CUDA targets disabled") + endif() +else() + message(STATUS "sofieBLAS: no CUDA compiler found, CUDA targets disabled") +endif() + +if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "") + set(CMAKE_HIP_ARCHITECTURES gfx1100) +endif() +check_language(HIP) +if(CMAKE_HIP_COMPILER) + enable_language(HIP) + set(SOFIEBLAS_HIP_ENABLED TRUE) + message(STATUS "sofieBLAS: HIP compiler found, HIP targets enabled") +else() + message(STATUS "sofieBLAS: no HIP compiler found, HIP targets disabled") +endif() + +# --- CPU BLAS selection --- +set(AVAILABLE_BLAS_LIBS OpenBLAS MKL BLIS Accelerate CACHE STRING "Choose CPU BLAS library") +set_property(CACHE AVAILABLE_BLAS_LIBS PROPERTY STRINGS OpenBLAS MKL BLIS Accelerate) + +if(NOT DEFINED CPU_BLAS_LIB) + set(CPU_BLAS_LIB "OpenBLAS" CACHE STRING "CPU BLAS library to use") +endif() + +function(sofieblas_find_openblas out_lib out_define out_include) + find_library(OPENBLAS_LIB NAMES openblas + PATHS /usr/lib/x86_64-linux-gnu/openblas-pthread + /usr/lib/x86_64-linux-gnu/openblas-serial + /usr/lib/x86_64-linux-gnu + /usr/local/lib) + find_path(OPENBLAS_INCLUDE_DIR NAMES cblas.h + PATHS /usr/include + /usr/include/x86_64-linux-gnu/openblas-pthread + /usr/include/x86_64-linux-gnu/openblas-serial + /usr/include/openblas + /usr/local/include) + if(OPENBLAS_LIB AND OPENBLAS_INCLUDE_DIR) + set(${out_lib} ${OPENBLAS_LIB} PARENT_SCOPE) + set(${out_define} SOFIEBLAS_USE_OPENBLAS PARENT_SCOPE) + set(${out_include} ${OPENBLAS_INCLUDE_DIR} PARENT_SCOPE) + else() + set(${out_lib} "" PARENT_SCOPE) + set(${out_define} "" PARENT_SCOPE) + set(${out_include} "" PARENT_SCOPE) + endif() +endfunction() + +function(sofieblas_find_mkl out_lib out_define out_include) + find_library(MKL_LIB NAMES mkl_rt PATHS ${ONEAPI_BASE}/mkl/latest/lib/intel64 ${ONEAPI_BASE}/mkl/latest/lib) + find_path(MKL_INCLUDE_DIR NAMES mkl.h PATHS ${ONEAPI_BASE}/mkl/latest/include) + if(MKL_LIB AND MKL_INCLUDE_DIR) + set(${out_lib} ${MKL_LIB} PARENT_SCOPE) + set(${out_define} SOFIEBLAS_USE_MKL PARENT_SCOPE) + set(${out_include} ${MKL_INCLUDE_DIR} PARENT_SCOPE) + else() + set(${out_lib} "" PARENT_SCOPE) + set(${out_define} "" PARENT_SCOPE) + set(${out_include} "" PARENT_SCOPE) + endif() +endfunction() + +function(sofieblas_find_blis out_lib out_define out_include) + find_library(BLIS_LIB NAMES blis PATHS ${BLIS_BASE}/lib ${BLIS_BASE}/lib64) + find_path(BLIS_INCLUDE_DIR NAMES blis/cblas.h PATHS ${BLIS_BASE}/include) + if(BLIS_LIB AND BLIS_INCLUDE_DIR) + set(${out_lib} ${BLIS_LIB} PARENT_SCOPE) + set(${out_define} SOFIEBLAS_USE_BLIS PARENT_SCOPE) + set(${out_include} ${BLIS_INCLUDE_DIR} PARENT_SCOPE) + else() + set(${out_lib} "" PARENT_SCOPE) + set(${out_define} "" PARENT_SCOPE) + set(${out_include} "" PARENT_SCOPE) + endif() +endfunction() + +function(sofieblas_find_accelerate out_lib out_define out_include) + set(${out_lib} "" PARENT_SCOPE) + set(${out_define} "" PARENT_SCOPE) + set(${out_include} "" PARENT_SCOPE) + if(APPLE) + find_library(ACCELERATE_LIB Accelerate) + if(ACCELERATE_LIB) + set(${out_lib} ${ACCELERATE_LIB} PARENT_SCOPE) + set(${out_define} SOFIEBLAS_USE_ACCELERATE PARENT_SCOPE) + endif() + endif() +endfunction() + +set(_search_order OpenBLAS MKL BLIS Accelerate) +if(NOT CPU_BLAS_LIB IN_LIST _search_order) + message(FATAL_ERROR "Unknown CPU_BLAS_LIB option: ${CPU_BLAS_LIB}") +endif() +list(REMOVE_ITEM _search_order ${CPU_BLAS_LIB}) +list(PREPEND _search_order ${CPU_BLAS_LIB}) + +foreach(_lib IN LISTS _search_order) + if(NOT SOFIEBLAS_CPU_BLAS_LIBS) + string(TOLOWER ${_lib} _lib_lower) + cmake_language(CALL sofieblas_find_${_lib_lower} + SOFIEBLAS_CPU_BLAS_LIBS SOFIEBLAS_CPU_BLAS_DEFINE SOFIEBLAS_CPU_BLAS_INCLUDE_DIR) + if(SOFIEBLAS_CPU_BLAS_LIBS) + set(SOFIEBLAS_CPU_BLAS_FOUND ${_lib}) + else() + message(STATUS "sofieBLAS: ${_lib} not found, trying next CPU BLAS option...") + endif() + endif() +endforeach() + +if(NOT SOFIEBLAS_CPU_BLAS_LIBS) + message(WARNING "sofieBLAS: no suitable CPU BLAS library found (tried: ${_search_order}). CPU targets disabled.") +else() + message(STATUS "sofieBLAS: using CPU BLAS library ${SOFIEBLAS_CPU_BLAS_FOUND} (${SOFIEBLAS_CPU_BLAS_LIBS})") +endif() diff --git a/cmake/sofieBLASConfig.cmake.in b/cmake/sofieBLASConfig.cmake.in new file mode 100644 index 0000000..78a6644 --- /dev/null +++ b/cmake/sofieBLASConfig.cmake.in @@ -0,0 +1,5 @@ +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/sofieBLASTargets.cmake") + +check_required_components(sofieBLAS) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1247ec8..aff9f6c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,189 +1,62 @@ -cmake_minimum_required(VERSION 3.21) -project(AlpakaTest LANGUAGES CXX) - -# --- User-configurable options (override via -D) --- -set(CUDA_BASE "/usr/local/cuda" CACHE PATH "CUDA base path") -set(ROCM_BASE "/opt/rocm" CACHE PATH "ROCm base path") -set(ONEAPI_BASE "/opt/intel/oneapi" CACHE PATH "Intel oneAPI base path") -set(TBB_BASE "/usr" CACHE PATH "TBB base path") -set(ALPAKA_BASE "$ENV{HOME}/src/alpaka-group/alpaka" CACHE PATH "Alpaka base path") - -# --- CPU BLAS selection --- -set(AVAILABLE_BLAS_LIBS OpenBLAS MKL BLIS Accelerate CACHE STRING "Choose CPU BLAS library") -set_property(CACHE AVAILABLE_BLAS_LIBS PROPERTY STRINGS OpenBLAS MKL BLIS Accelerate) - -if(NOT DEFINED CPU_BLAS_LIB) - set(CPU_BLAS_LIB "OpenBLAS" CACHE STRING "CPU BLAS library to use") -endif() - -set(BLIS_BASE "/usr" CACHE PATH "BLIS base path") - -# --- Compiler flags --- set(CXXFLAGS -O2 -g -DALPAKA_HAS_STD_ATOMIC_REF) set(CXX_HOST_FLAGS -fPIC -pthread) set(CXX_CUDA_FLAGS -Wno-deprecated-gpu-targets --extended-lambda --expt-relaxed-constexpr) set(XCOMPILER_FLAGS -Xcompiler=-fPIC,-pthread) -# --- CUDA architecture: must be set before enable_language(CUDA) so all -# targets inherit a valid default (CMake 3.18+ requires CUDA_ARCHITECTURES -# to be non-empty on every target once CUDA is enabled globally). -set(CMAKE_CUDA_ARCHITECTURES 86) -enable_language(CUDA) - -# --- HIP architecture: must be set before enable_language(HIP) for the same -# reason as CUDA_ARCHITECTURES above. -set(CMAKE_HIP_ARCHITECTURES gfx1100) -include(CheckLanguage) -check_language(HIP) -if(CMAKE_HIP_COMPILER) - enable_language(HIP) +if(SOFIEBLAS_CPU_BLAS_LIBS) + add_executable(test_cpu test.cc) + target_compile_features(test_cpu PUBLIC cxx_std_20) + target_compile_options(test_cpu PRIVATE ${CXXFLAGS} ${CXX_HOST_FLAGS}) + target_compile_definitions(test_cpu PRIVATE + ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED ${SOFIEBLAS_CPU_BLAS_DEFINE}) + target_include_directories(test_cpu PRIVATE ${SOFIEBLAS_CPU_BLAS_INCLUDE_DIR}) + target_link_libraries(test_cpu PRIVATE + sofieBLAS::sofieBLAS alpaka::alpaka ${SOFIEBLAS_CPU_BLAS_LIBS}) + + add_test(NAME sofieBLAS.cpu COMMAND test_cpu) else() - message(STATUS "No HIP compiler found, test_hip target will not be built") + message(STATUS "sofieBLAS tests: skipping test_cpu (no CPU BLAS library found)") endif() -# --- Include directories --- -include_directories(${ALPAKA_BASE}/include "../include") +if(SOFIEBLAS_CUDA_ENABLED) + configure_file(test.cc test_cuda_gen.cc COPYONLY) + add_executable(test_cuda) + target_sources(test_cuda PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/test_cuda_gen.cc) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/test_cuda_gen.cc PROPERTIES LANGUAGE CUDA) + set_target_properties(test_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON) + target_compile_features(test_cuda PUBLIC cxx_std_20) -# --- Functions to find BLAS libraries --- -function(find_openblas blas_lib blas_define blas_include) - find_path(OPENBLAS_PATH NAMES libopenblas.a - PATHS /usr/lib/x86_64-linux-gnu/openblas-serial /usr/lib/x86_64-linux-gnu) - if(OPENBLAS_PATH) - find_library(OPENBLAS_LIB openblas PATHS ${OPENBLAS_PATH}) - find_path(OPENBLAS_INCLUDE_DIR NAMES cblas.h - PATHS /usr/include /usr/include/openblas /usr/include/cblas) - set(${blas_lib} ${OPENBLAS_LIB} PARENT_SCOPE) - set(${blas_define} SOFIEBLAS_USE_OPENBLAS PARENT_SCOPE) - set(${blas_include} ${OPENBLAS_INCLUDE_DIR} PARENT_SCOPE) - else() - set(${blas_lib} "" PARENT_SCOPE) - set(${blas_define} "" PARENT_SCOPE) - set(${blas_include} "" PARENT_SCOPE) - endif() -endfunction() - -function(find_mkl blas_lib blas_define blas_include) - find_path(MKL_PATH NAMES libmkl_rt.a PATHS ${ONEAPI_BASE}/mkl/latest/lib/intel64) - if(MKL_PATH) - find_library(MKL_LIB mkl_rt PATHS ${MKL_PATH}) - set(${blas_lib} ${MKL_LIB} PARENT_SCOPE) - set(${blas_define} SOFIEBLAS_USE_MKL PARENT_SCOPE) - set(${blas_include} ${ONEAPI_BASE}/mkl/latest/include PARENT_SCOPE) - else() - set(${blas_lib} "" PARENT_SCOPE) - set(${blas_define} "" PARENT_SCOPE) - set(${blas_include} "" PARENT_SCOPE) - endif() -endfunction() - -function(find_blis blas_lib blas_define blas_include) - find_path(BLIS_PATH NAMES libblis.a libblis.so - PATHS ${BLIS_BASE}/lib ${BLIS_BASE}/lib64) - if(BLIS_PATH) - find_library(BLIS_LIB blis PATHS ${BLIS_PATH}) - find_path(BLIS_INCLUDE_DIR NAMES blis/cblas.h PATHS ${BLIS_BASE}/include) - set(${blas_lib} ${BLIS_LIB} PARENT_SCOPE) - set(${blas_define} SOFIEBLAS_USE_BLIS PARENT_SCOPE) - set(${blas_include} ${BLIS_INCLUDE_DIR} PARENT_SCOPE) - else() - set(${blas_lib} "" PARENT_SCOPE) - set(${blas_define} "" PARENT_SCOPE) - set(${blas_include} "" PARENT_SCOPE) - endif() -endfunction() - -function(find_accelerate blas_lib blas_define blas_include) - set(${blas_lib} "" PARENT_SCOPE) - set(${blas_define} "" PARENT_SCOPE) - set(${blas_include} "" PARENT_SCOPE) - if(APPLE) - find_library(ACCELERATE_LIB Accelerate) - if(ACCELERATE_LIB) - set(${blas_lib} ${ACCELERATE_LIB} PARENT_SCOPE) - set(${blas_define} SOFIEBLAS_USE_ACCELERATE PARENT_SCOPE) - endif() - endif() -endfunction() - -# --- BLAS detection and selection --- -set(SOFIEBLAS_BLAS_SEARCH_ORDER OpenBLAS MKL BLIS Accelerate) -if(NOT CPU_BLAS_LIB IN_LIST SOFIEBLAS_BLAS_SEARCH_ORDER) - message(FATAL_ERROR "Unknown CPU_BLAS_LIB option: ${CPU_BLAS_LIB}") -endif() -list(REMOVE_ITEM SOFIEBLAS_BLAS_SEARCH_ORDER ${CPU_BLAS_LIB}) -list(PREPEND SOFIEBLAS_BLAS_SEARCH_ORDER ${CPU_BLAS_LIB}) + target_compile_options(test_cuda PRIVATE + ${CXXFLAGS} + ${CXX_CUDA_FLAGS} + ${XCOMPILER_FLAGS} + ) -foreach(lib IN LISTS SOFIEBLAS_BLAS_SEARCH_ORDER) - if(NOT BLAS_LIBS) - string(TOLOWER ${lib} lib_lower) - cmake_language(CALL find_${lib_lower} BLAS_LIBS SOFIEBLAS_DEFINES BLAS_INCLUDE_DIR) - if(BLAS_LIBS) - set(FOUND_BLAS_LIB ${lib}) - else() - message(WARNING "${lib} not found, trying next option...") - endif() - endif() -endforeach() + target_compile_definitions(test_cuda PRIVATE ALPAKA_ACC_GPU_CUDA_ENABLED) + target_link_libraries(test_cuda PRIVATE + sofieBLAS::sofieBLAS alpaka::alpaka CUDA::cudart CUDA::cublas CUDA::cublasLt) -if(NOT BLAS_LIBS) - message(FATAL_ERROR "No suitable CPU BLAS library found! Tried: ${SOFIEBLAS_BLAS_SEARCH_ORDER}") + add_test(NAME sofieBLAS.cuda COMMAND test_cuda) +else() + message(STATUS "sofieBLAS tests: skipping test_cuda (no CUDA compiler found)") endif() -message(STATUS "Using CPU BLAS library: ${FOUND_BLAS_LIB}") -message(STATUS "BLAS libraries: ${BLAS_LIBS}") -message(STATUS "BLAS include dir: ${BLAS_INCLUDE_DIR}") -message(STATUS "Compile defines: ${SOFIEBLAS_DEFINES}") - -# --- test_cpu target --- -add_executable(test_cpu test.cc) -target_compile_features(test_cpu PUBLIC cxx_std_20) -target_compile_options(test_cpu PRIVATE ${CXXFLAGS} ${CXX_HOST_FLAGS}) -target_compile_definitions(test_cpu PRIVATE ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED ${SOFIEBLAS_DEFINES}) -target_include_directories(test_cpu PRIVATE "../sofieBLAS/include" ${ALPAKA_BASE}/include ${BLAS_INCLUDE_DIR}) -target_link_libraries(test_cpu PRIVATE ${BLAS_LIBS}) - - -add_executable(test_cuda) -target_sources(test_cuda PRIVATE test.cc) -set_source_files_properties(test.cc PROPERTIES LANGUAGE CUDA) -set_target_properties(test_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON) -target_compile_features(test_cuda PUBLIC cxx_std_20) - -target_compile_options(test_cuda PRIVATE - ${CXXFLAGS} - ${CXX_CUDA_FLAGS} - ${XCOMPILER_FLAGS} -) - -target_compile_definitions(test_cuda PRIVATE ALPAKA_ACC_GPU_CUDA_ENABLED) -target_include_directories(test_cuda PRIVATE ${ALPAKA_BASE}/include "../sofieBLAS/include" ${CUDA_BASE}/include) -target_link_directories(test_cuda PRIVATE ${CUDA_BASE}/lib64) -target_link_libraries(test_cuda PRIVATE cublasLt cublas cudart) - -# --- test_hip target --- -if(CMAKE_HIP_COMPILER) - # Use a private copy of the source: set_source_files_properties(LANGUAGE ...) - # is directory-scoped, so reusing test.cc directly here would clobber the - # LANGUAGE CUDA property already set on it for test_cuda above. +if(SOFIEBLAS_HIP_ENABLED) configure_file(test.cc test_hip_gen.cc COPYONLY) add_executable(test_hip) target_sources(test_hip PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/test_hip_gen.cc) set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/test_hip_gen.cc PROPERTIES LANGUAGE HIP) target_compile_features(test_hip PUBLIC cxx_std_20) - target_compile_options(test_hip PRIVATE - ${CXXFLAGS} - ${CXX_HOST_FLAGS} - ) + target_compile_options(test_hip PRIVATE ${CXXFLAGS} ${CXX_HOST_FLAGS}) target_compile_definitions(test_hip PRIVATE ALPAKA_ACC_GPU_HIP_ENABLED) - target_include_directories(test_hip PRIVATE ${ALPAKA_BASE}/include "../sofieBLAS/include" ${ROCM_BASE}/include) + target_include_directories(test_hip PRIVATE ${ROCM_BASE}/include) target_link_directories(test_hip PRIVATE ${ROCM_BASE}/lib) - target_link_libraries(test_hip PRIVATE hipblaslt hipblas amdhip64) -endif() + target_link_libraries(test_hip PRIVATE + sofieBLAS::sofieBLAS alpaka::alpaka hipblaslt hipblas amdhip64) -# --- clean target equivalent --- -add_custom_target(clean-all - COMMAND ${CMAKE_COMMAND} -E rm -f test_cpu test_cuda test_hip *.d *.o *.so - COMMENT "Cleaning all generated files" -) + add_test(NAME sofieBLAS.hip COMMAND test_hip) +else() + message(STATUS "sofieBLAS tests: skipping test_hip (no HIP compiler found)") +endif() diff --git a/tests/test.cc b/tests/test.cc index 4b47b67..11d6e4c 100644 --- a/tests/test.cc +++ b/tests/test.cc @@ -94,10 +94,6 @@ static void checkClose(const float *got, const float *expected, int n, ++gFailures; } -// --------------------------------------------------------------------------- -// Helpers to fill test matrices -// --------------------------------------------------------------------------- - static void fillSeq(float *M, int n, float start = 1.f, float step = 1.f) { for (int i = 0; i < n; ++i) M[i] = start + static_cast(i) * step; From 6eb4c3b28433b0230b5c886ff65db7435fb25c77 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 17 Aug 2026 14:00:54 +0200 Subject: [PATCH 26/29] ci: add lxplus-like container from ngt for requirements setup --- .github/workflows/benchmark.yml | 1 + .github/workflows/tests.yml | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c1ffd3a..0e294f0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -102,6 +102,7 @@ jobs: needs: parse-command if: needs.parse-command.outputs.valid == 'true' runs-on: ${{ needs.parse-command.outputs.runner_label }} + container: registry.cern.ch/ngt/lxplus-like:9 timeout-minutes: 60 steps: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bd94f52..adeca04 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,12 +24,16 @@ env: jobs: cpu-tests: name: CPU Unit Tests - if: github.event_name != 'issue_comment' + if: | + github.event_name != 'issue_comment' || + github.event.issue.pull_request != null runs-on: ubuntu-latest timeout-minutes: 30 steps: - name: Checkout uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'issue_comment' && format('refs/pull/{0}/merge', github.event.issue.number) || github.ref }} - name: Install OpenBLAS run: | @@ -136,6 +140,9 @@ jobs: needs: parse-command if: needs.parse-command.outputs.valid == 'true' runs-on: ${{ needs.parse-command.outputs.runner_label }} + container: + image: registry.cern.ch/ngt/lxplus-like:9 + options: --user 0:0 timeout-minutes: 45 steps: - name: GPU check From 2aa7fc8af050cbd37b2eba82712cd7032ac650b4 Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 17 Aug 2026 14:12:22 +0200 Subject: [PATCH 27/29] ci: fix picking up tests based on the runner flavour --- .github/workflows/benchmark.yml | 43 +++++++++++++++++------- .github/workflows/tests.yml | 31 ++++++++++++----- cmake/SofieBLASBackends.cmake | 59 +++++++++++++++++++++------------ 3 files changed, 91 insertions(+), 42 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0e294f0..b483701 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -48,6 +48,7 @@ jobs: valid: ${{ steps.parse.outputs.valid }} flavor: ${{ steps.parse.outputs.flavor }} valid_flavors: ${{ steps.parse.outputs.valid_flavors }} + backend: ${{ steps.parse.outputs.backend }} steps: - name: React to trigger comment if: github.event_name == 'issue_comment' @@ -65,11 +66,13 @@ jobs: uses: actions/github-script@v7 with: script: | - // default flavor keyed by name -> actual self-hosted runner label + // default flavor keyed by name -> { self-hosted runner label, GPU backend } const flavors = { - l40s: 'ml4ep-l40s', - h100: 'ml4ep-h100', - 'h100-47gb': 'ml4ep-h100-47gb', + l40s: { runner: 'ml4ep-l40s', backend: 'cuda' }, + h100: { runner: 'ml4ep-h100', backend: 'cuda' }, + 'h100-47gb': { runner: 'ml4ep-h100-47gb', backend: 'cuda' }, + w7900: { runner: 'ml4ep-w7900', backend: 'hip' }, + mi300x: { runner: 'ml4ep-mi300x', backend: 'hip' }, }; let flavor = 'l40s'; let valid = true; @@ -82,7 +85,8 @@ jobs: } core.setOutput('flavor', flavor); core.setOutput('valid', String(valid)); - core.setOutput('runner_label', valid ? flavors[flavor] : ''); + core.setOutput('runner_label', valid ? flavors[flavor].runner : ''); + core.setOutput('backend', valid ? flavors[flavor].backend : ''); core.setOutput('valid_flavors', Object.keys(flavors).join(', ')); - name: Post run link if: github.event_name == 'issue_comment' @@ -106,9 +110,14 @@ jobs: timeout-minutes: 60 steps: - - name: GPU check + - name: GPU check (NVIDIA) + if: needs.parse-command.outputs.backend == 'cuda' run: nvidia-smi + - name: GPU check (AMD) + if: needs.parse-command.outputs.backend == 'hip' + run: rocm-smi + - name: Resolve PR info id: pr if: github.event_name == 'issue_comment' @@ -141,24 +150,29 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.DEPS_CACHE }} - key: cmake-deps-bench-${{ hashFiles('sofieblas-pr/cmake/SofieBLASBackends.cmake') }} - restore-keys: cmake-deps-bench- + key: cmake-deps-bench-${{ needs.parse-command.outputs.backend }}-${{ hashFiles('sofieblas-pr/cmake/SofieBLASBackends.cmake') }} + restore-keys: cmake-deps-bench-${{ needs.parse-command.outputs.backend }}- - name: Configure PR build + env: + BACKEND: ${{ needs.parse-command.outputs.backend }} run: | cmake -B sofieblas-pr/build -S sofieblas-pr \ -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ -DSOFIEBLAS_BUILD_BENCHMARKS=ON \ -DCMAKE_CUDA_ARCHITECTURES=native \ + -DCMAKE_HIP_ARCHITECTURES=native \ + "-DSOFIEBLAS_ENABLE_CUDA=$([ "$BACKEND" = "cuda" ] && echo ON || echo OFF)" \ + "-DSOFIEBLAS_ENABLE_HIP=$([ "$BACKEND" = "hip" ] && echo ON || echo OFF)" \ "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" - name: Build PR benchmark - run: cmake --build sofieblas-pr/build --target bench_cuda -j"$(nproc)" + run: cmake --build sofieblas-pr/build --target "bench_${{ needs.parse-command.outputs.backend }}" -j"$(nproc)" - name: Run PR benchmark working-directory: sofieblas-pr/build/benchmark run: | - ./bench_cuda \ + "./bench_${{ needs.parse-command.outputs.backend }}" \ -w "${{ env.BENCH_WARMUP }}" \ -n "${{ env.BENCH_ITERS }}" \ --sizes "${{ env.BENCH_SIZES }}" \ @@ -177,22 +191,27 @@ jobs: - name: Configure base build if: github.event_name == 'issue_comment' && steps.base_support.outputs.supported == 'true' + env: + BACKEND: ${{ needs.parse-command.outputs.backend }} run: | cmake -B sofieblas-main/build -S sofieblas-main \ -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ -DSOFIEBLAS_BUILD_BENCHMARKS=ON \ -DCMAKE_CUDA_ARCHITECTURES=native \ + -DCMAKE_HIP_ARCHITECTURES=native \ + "-DSOFIEBLAS_ENABLE_CUDA=$([ "$BACKEND" = "cuda" ] && echo ON || echo OFF)" \ + "-DSOFIEBLAS_ENABLE_HIP=$([ "$BACKEND" = "hip" ] && echo ON || echo OFF)" \ "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" - name: Build base benchmark if: github.event_name == 'issue_comment' && steps.base_support.outputs.supported == 'true' - run: cmake --build sofieblas-main/build --target bench_cuda -j"$(nproc)" + run: cmake --build sofieblas-main/build --target "bench_${{ needs.parse-command.outputs.backend }}" -j"$(nproc)" - name: Run base benchmark if: github.event_name == 'issue_comment' && steps.base_support.outputs.supported == 'true' working-directory: sofieblas-main/build/benchmark run: | - ./bench_cuda \ + "./bench_${{ needs.parse-command.outputs.backend }}" \ -w "${{ env.BENCH_WARMUP }}" \ -n "${{ env.BENCH_ITERS }}" \ --sizes "${{ env.BENCH_SIZES }}" \ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index adeca04..d73d617 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,6 +86,7 @@ jobs: valid: ${{ steps.parse.outputs.valid }} flavor: ${{ steps.parse.outputs.flavor }} valid_flavors: ${{ steps.parse.outputs.valid_flavors }} + backend: ${{ steps.parse.outputs.backend }} steps: - name: React to trigger comment if: github.event_name == 'issue_comment' @@ -103,11 +104,12 @@ jobs: uses: actions/github-script@v7 with: script: | - // default flavor keyed by name -> actual self-hosted runner label const flavors = { - l40s: 'ml4ep-l40s', - h100: 'ml4ep-h100', - 'h100-47gb': 'ml4ep-h100-47gb', + l40s: { runner: 'ml4ep-l40s', backend: 'cuda' }, + h100: { runner: 'ml4ep-h100', backend: 'cuda' }, + 'h100-47gb': { runner: 'ml4ep-h100-47gb', backend: 'cuda' }, + w7900: { runner: 'ml4ep-w7900', backend: 'hip' }, + mi300x: { runner: 'ml4ep-mi300x', backend: 'hip' }, }; let flavor = 'l40s'; let valid = true; @@ -120,7 +122,8 @@ jobs: } core.setOutput('flavor', flavor); core.setOutput('valid', String(valid)); - core.setOutput('runner_label', valid ? flavors[flavor] : ''); + core.setOutput('runner_label', valid ? flavors[flavor].runner : ''); + core.setOutput('backend', valid ? flavors[flavor].backend : ''); core.setOutput('valid_flavors', Object.keys(flavors).join(', ')); - name: Post run link if: github.event_name == 'issue_comment' @@ -145,9 +148,14 @@ jobs: options: --user 0:0 timeout-minutes: 45 steps: - - name: GPU check + - name: GPU check (NVIDIA) + if: needs.parse-command.outputs.backend == 'cuda' run: nvidia-smi + - name: GPU check (AMD) + if: needs.parse-command.outputs.backend == 'hip' + run: rocm-smi + - name: Checkout uses: actions/checkout@v4 with: @@ -157,16 +165,21 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.DEPS_CACHE }} - key: cmake-deps-gpu-${{ hashFiles('cmake/SofieBLASBackends.cmake') }} - restore-keys: cmake-deps-gpu- + key: cmake-deps-gpu-${{ needs.parse-command.outputs.backend }}-${{ hashFiles('cmake/SofieBLASBackends.cmake') }} + restore-keys: cmake-deps-gpu-${{ needs.parse-command.outputs.backend }}- - name: Configure + env: + BACKEND: ${{ needs.parse-command.outputs.backend }} run: | cmake -B build -S . \ -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ -DSOFIEBLAS_BUILD_TESTS=ON \ -DSOFIEBLAS_BUILD_BENCHMARKS=ON \ -DCMAKE_CUDA_ARCHITECTURES=native \ + -DCMAKE_HIP_ARCHITECTURES=native \ + "-DSOFIEBLAS_ENABLE_CUDA=$([ "$BACKEND" = "cuda" ] && echo ON || echo OFF)" \ + "-DSOFIEBLAS_ENABLE_HIP=$([ "$BACKEND" = "hip" ] && echo ON || echo OFF)" \ "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" - name: Build @@ -174,7 +187,7 @@ jobs: - name: Run GPU tests working-directory: build - run: ctest --output-on-failure -j"$(nproc)" -R '\.(cuda|hip)(\.|$)' + run: ctest --output-on-failure -j"$(nproc)" -R "\.${{ needs.parse-command.outputs.backend }}(\.|\$)" - name: Upload test log if: always() diff --git a/cmake/SofieBLASBackends.cmake b/cmake/SofieBLASBackends.cmake index 4677516..d137d2b 100644 --- a/cmake/SofieBLASBackends.cmake +++ b/cmake/SofieBLASBackends.cmake @@ -25,33 +25,50 @@ else() message(STATUS "sofieBLAS: using alpaka ${alpaka_VERSION} from ${alpaka_DIR}") endif() -if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "") - set(CMAKE_CUDA_ARCHITECTURES native) -endif() -check_language(CUDA) -if(CMAKE_CUDA_COMPILER) - enable_language(CUDA) - find_package(CUDAToolkit QUIET) - if(CUDAToolkit_FOUND) - set(SOFIEBLAS_CUDA_ENABLED TRUE) - message(STATUS "sofieBLAS: CUDA toolkit ${CUDAToolkit_VERSION} found, CUDA targets enabled") +set(SOFIEBLAS_ENABLE_CUDA "AUTO" CACHE STRING "ON/OFF/AUTO: whether to enable the CUDA backend") +set(SOFIEBLAS_ENABLE_HIP "AUTO" CACHE STRING "ON/OFF/AUTO: whether to enable the HIP backend") +set_property(CACHE SOFIEBLAS_ENABLE_CUDA PROPERTY STRINGS ON OFF AUTO) +set_property(CACHE SOFIEBLAS_ENABLE_HIP PROPERTY STRINGS ON OFF AUTO) + +if(NOT SOFIEBLAS_ENABLE_CUDA STREQUAL "OFF") + if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "") + set(CMAKE_CUDA_ARCHITECTURES native) + endif() + check_language(CUDA) + if(CMAKE_CUDA_COMPILER) + enable_language(CUDA) + find_package(CUDAToolkit QUIET) + if(CUDAToolkit_FOUND) + set(SOFIEBLAS_CUDA_ENABLED TRUE) + message(STATUS "sofieBLAS: CUDA toolkit ${CUDAToolkit_VERSION} found, CUDA targets enabled") + else() + message(STATUS "sofieBLAS: nvcc found but CUDAToolkit package not found, CUDA targets disabled") + endif() + elseif(SOFIEBLAS_ENABLE_CUDA STREQUAL "ON") + message(FATAL_ERROR "sofieBLAS: SOFIEBLAS_ENABLE_CUDA=ON but no CUDA compiler was found") else() - message(STATUS "sofieBLAS: nvcc found but CUDAToolkit package not found, CUDA targets disabled") + message(STATUS "sofieBLAS: no CUDA compiler found, CUDA targets disabled") endif() else() - message(STATUS "sofieBLAS: no CUDA compiler found, CUDA targets disabled") + message(STATUS "sofieBLAS: CUDA backend disabled (SOFIEBLAS_ENABLE_CUDA=OFF)") endif() -if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "") - set(CMAKE_HIP_ARCHITECTURES gfx1100) -endif() -check_language(HIP) -if(CMAKE_HIP_COMPILER) - enable_language(HIP) - set(SOFIEBLAS_HIP_ENABLED TRUE) - message(STATUS "sofieBLAS: HIP compiler found, HIP targets enabled") +if(NOT SOFIEBLAS_ENABLE_HIP STREQUAL "OFF") + if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "") + set(CMAKE_HIP_ARCHITECTURES gfx1100) + endif() + check_language(HIP) + if(CMAKE_HIP_COMPILER) + enable_language(HIP) + set(SOFIEBLAS_HIP_ENABLED TRUE) + message(STATUS "sofieBLAS: HIP compiler found, HIP targets enabled") + elseif(SOFIEBLAS_ENABLE_HIP STREQUAL "ON") + message(FATAL_ERROR "sofieBLAS: SOFIEBLAS_ENABLE_HIP=ON but no HIP compiler was found") + else() + message(STATUS "sofieBLAS: no HIP compiler found, HIP targets disabled") + endif() else() - message(STATUS "sofieBLAS: no HIP compiler found, HIP targets disabled") + message(STATUS "sofieBLAS: HIP backend disabled (SOFIEBLAS_ENABLE_HIP=OFF)") endif() # --- CPU BLAS selection --- From 38931d13b0d9820dac8fd05a00bf44d1272271cb Mon Sep 17 00:00:00 2001 From: Sanjiban Sengupta Date: Mon, 17 Aug 2026 14:30:07 +0200 Subject: [PATCH 28/29] ci: avoid triggering cpu tests with gpu ones --- .github/workflows/benchmark.yml | 7 +++---- .github/workflows/tests.yml | 18 +++++++++++++----- benchmark/CMakeLists.txt | 9 ++------- tests/CMakeLists.txt | 4 ++-- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b483701..8a5cdb0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -15,10 +15,6 @@ on: description: "Comma-separated square GEMM sizes (M=N=K)" default: "256,512,1024,2048,4096" -concurrency: - group: benchmark-${{ github.event.issue.number || github.ref }} - cancel-in-progress: true - permissions: contents: read pull-requests: write @@ -105,6 +101,9 @@ jobs: name: Benchmark Comparison needs: parse-command if: needs.parse-command.outputs.valid == 'true' + concurrency: + group: benchmark-${{ needs.parse-command.outputs.flavor }}-${{ github.event.issue.number || github.ref }} + cancel-in-progress: true runs-on: ${{ needs.parse-command.outputs.runner_label }} container: registry.cern.ch/ngt/lxplus-like:9 timeout-minutes: 60 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d73d617..e41ae81 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,10 +8,6 @@ on: types: [created] workflow_dispatch: -concurrency: - group: tests-${{ github.event.issue.number || github.ref }} - cancel-in-progress: true - permissions: contents: read pull-requests: write @@ -26,7 +22,14 @@ jobs: name: CPU Unit Tests if: | github.event_name != 'issue_comment' || - github.event.issue.pull_request != null + ( + github.event.issue.pull_request != null && + !startsWith(github.event.comment.body, '/runtest') && + !startsWith(github.event.comment.body, '/runbenchmark') + ) + concurrency: + group: cpu-tests-${{ github.event.issue.number || github.ref }} + cancel-in-progress: true runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -53,6 +56,8 @@ jobs: -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ -DSOFIEBLAS_BUILD_TESTS=ON \ -DSOFIEBLAS_BUILD_BENCHMARKS=ON \ + -DSOFIEBLAS_ENABLE_CUDA=OFF \ + -DSOFIEBLAS_ENABLE_HIP=OFF \ "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" - name: Build @@ -142,6 +147,9 @@ jobs: name: GPU Unit Tests needs: parse-command if: needs.parse-command.outputs.valid == 'true' + concurrency: + group: gpu-tests-${{ needs.parse-command.outputs.flavor }}-${{ github.event.issue.number || github.ref }} + cancel-in-progress: true runs-on: ${{ needs.parse-command.outputs.runner_label }} container: image: registry.cern.ch/ngt/lxplus-like:9 diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index dfb234c..a30efd1 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,8 +1,3 @@ -# Built via the root CMakeLists.txt with -DSOFIEBLAS_BUILD_BENCHMARKS=ON, -# which already ran cmake/SofieBLASBackends.cmake to populate -# SOFIEBLAS_CPU_BLAS_*, SOFIEBLAS_CUDA_ENABLED, SOFIEBLAS_HIP_ENABLED and the -# alpaka::alpaka target. - set(CXXFLAGS -O3 -DALPAKA_HAS_STD_ATOMIC_REF) set(CXX_HOST_FLAGS -fPIC -pthread) set(CXX_CUDA_FLAGS -Wno-deprecated-gpu-targets --extended-lambda --expt-relaxed-constexpr) @@ -49,7 +44,7 @@ if(SOFIEBLAS_CUDA_ENABLED) COMMAND bench_cuda -w 1 -n 1 --sizes 64,128) endif() else() - message(STATUS "sofieBLAS benchmark: skipping bench_cuda (no CUDA compiler found)") + message(VERBOSE "sofieBLAS benchmark: skipping bench_cuda (CUDA backend not enabled)") endif() if(SOFIEBLAS_HIP_ENABLED) @@ -72,5 +67,5 @@ if(SOFIEBLAS_HIP_ENABLED) COMMAND bench_hip -w 1 -n 1 --sizes 64,128) endif() else() - message(STATUS "sofieBLAS benchmark: skipping bench_hip (no HIP compiler found)") + message(VERBOSE "sofieBLAS benchmark: skipping bench_hip (HIP backend not enabled)") endif() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index aff9f6c..2c55d06 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -38,7 +38,7 @@ if(SOFIEBLAS_CUDA_ENABLED) add_test(NAME sofieBLAS.cuda COMMAND test_cuda) else() - message(STATUS "sofieBLAS tests: skipping test_cuda (no CUDA compiler found)") + message(VERBOSE "sofieBLAS tests: skipping test_cuda (CUDA backend not enabled)") endif() if(SOFIEBLAS_HIP_ENABLED) @@ -58,5 +58,5 @@ if(SOFIEBLAS_HIP_ENABLED) add_test(NAME sofieBLAS.hip COMMAND test_hip) else() - message(STATUS "sofieBLAS tests: skipping test_hip (no HIP compiler found)") + message(VERBOSE "sofieBLAS tests: skipping test_hip (HIP backend not enabled)") endif() From 7570dbf9a29b52ed02ee4869a2d927822dac0c60 Mon Sep 17 00:00:00 2001 From: Harsh Chauhan <73985490+harz05@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:44:48 +0530 Subject: [PATCH 29/29] Support runtime GEMM dimensions for dynamic shapes (#11) * fix: support runtime GEMM dimensions for dynamic shapes * feat: resolve GEMM algorithms per call site instead of per shape * test: cover multi-size use of one instance and bounded algorithm cache * feat: optional LRU limit on the algorithm cache * style: clang-format * chore: cleanup, cache limit via constructor * docs: layout and caching behavior * feat: dynamic gemm shapes and algorithm cache for hipBLASLt * docs: describe layout roles, envelope matching and shape naming * docs: explain addLayoutConfig, algo resolution and stats counters * fix: cache layouts and algorithms, one shared BLASLt backend * feat: addOperationConfig with a epilogue, tests for runtime sizes * docs: describe addOperationConfig, the epilogue enum and the cache limit * docs: address review comments * docs: README changes --- README.md | 40 +- benchmark/bench.cc | 4 +- .../backends/cuda/sofieBLAS_cublas.hpp | 502 ++---------------- .../gpu/detail/sofieBLAS_blaslt_common.tpp | 418 +++++++++++++++ .../backends/hip/sofieBLAS_hipblaslt.hpp | 496 +++-------------- include/sofieBLAS/core.hpp | 2 + tests/test.cc | 220 +++++++- 7 files changed, 782 insertions(+), 900 deletions(-) create mode 100644 include/sofieBLAS/backends/gpu/detail/sofieBLAS_blaslt_common.tpp diff --git a/README.md b/README.md index 191fe2a..bf3fa5d 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,45 @@ sofieBLAS blas(queue); blas.matmul('N', 'N', size, size, size, 1.0f, dA, dB, 0.0f, dC); ``` -The GPU backends (`BlasCuda`, `BlasHip`) additionally expose `gemmrelu`/`gemmgelu` (fused bias + activation via cuBLASLt/hipBLASLt epilogues), `gemmStridedBatched`, and `addLayoutConfig` (used to pre-register cuBLASLt/hipBLASLt matrix layouts for a given shape before the first `matmul`/`gemm` call on that shape). +The GPU backends (`BlasCuda`, `BlasHip`) additionally expose +- `gemmrelu`/`gemmgelu` (fused bias + activation via cuBLASLt/hipBLASLt epilogues) +- `gemmStridedBatched` for batched gemm operations through strides +- `addOperationConfig` that creates the matrix layouts and resolves the multiply algorithm for a call site's shape ahead of its first call (see below). + +## GEMM call instantiation and the algorithm cache + +A GEMM call computes `C = alpha * op(A) * op(B) + beta * C`, where A and B are the input matrices, C the output, and `op` an optional transpose. To run one, cuBLASLt and hipBLASLt need three kinds of objects besides the data: + +- a **matrix layout** per matrix: a descriptor holding its rows, columns and leading dimension; +- a **matmul descriptor**: the operation settings (the transposes and the epilogue); +- an **algorithm**: the concrete GEMM kernel the library selects for the given settings and dimensions, obtained by querying its heuristic (`cublasLtMatmulAlgoGetHeuristic` / `hipblasLtMatmulAlgoGetHeuristic`). The query runs on the host and is not free. + +The CUDA backend (`BlasCuda`, over cuBLASLt) and the HIP backend (`BlasHip`, over hipBLASLt) behave identically: all three objects are created the first time a combination appears and cached, keyed by the exact dimensions plus, for descriptors and algorithms, the transposes and the epilogue. One instance therefore serves GEMM calls at sizes that vary at runtime: a size seen for the first time creates and caches its objects, and a repeated size reuses them without another heuristic query. + +### addOperationConfig + +`addOperationConfig(m, n, k, lda, ldb, ldc, transa, transb, epilogue)` creates all three objects for one operation (the matrix layouts, the matmul descriptor and the algorithm) for the given dimensions, transposes and epilogue, before the corresponding call is made. It is optional: a combination that was never configured is created and cached on its first call. The `epilogue` argument is the `Epilogue` enum from `sofieBLAS/core.hpp` and names which call the site will make, because the fused epilogue is part of the selected kernel: + +| `Epilogue` value | call it configures | +| --- | --- | +| `Epilogue::Default` | `matmul` (no bias) | +| `Epilogue::Bias` | `gemm` (adds the bias vector) | +| `Epilogue::ReluBias` | `gemmrelu` (bias, then ReLU) | +| `Epilogue::GeluBias` | `gemmgelu` (bias, then GELU) | + +### Initializing the cache limit + +The algorithm cache is unbounded by default. Passing a limit as the second constructor argument caps the number of cached algorithms; when an insertion would exceed the limit, the least recently used entries are evicted. Choose a limit at least as large as the number of distinct shapes the workload uses regularly, or leave it unbounded. `algoCacheSize()` returns the current number of entries. + +```cpp +sofieBLAS blas(queue); // unbounded algorithm cache (default) +sofieBLAS capped(queue, 32); // at most 32 entries, LRU eviction + +blas.addOperationConfig(64, 3, 5, 64, 5, 64, 'N', 'N', Epilogue::Default); +blas.matmul('N', 'N', 64, 3, 5, 1.0f, dA, dB, 0.0f, dC); // created by addOperationConfig: cache hit +blas.matmul('N', 'N', 37, 3, 5, 1.0f, dA, dB, 0.0f, dC); // new size: created on first use +blas.gemmrelu('N', 'N', 64, 3, 5, 1.0f, dA, dB, 0.0f, dBias, dC); // same size, other epilogue: layouts reused, descriptor and algorithm created on first use +``` ## Contributing diff --git a/benchmark/bench.cc b/benchmark/bench.cc index 3263dab..687c312 100644 --- a/benchmark/bench.cc +++ b/benchmark/bench.cc @@ -139,7 +139,7 @@ static void runCudaBench(const BenchOptions &opt) { alpaka::memcpy(queue, dB, hB); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, M, K, M, 'N', 'N'); + blas.addOperationConfig(M, N, K, M, K, M, 'N', 'N', Epilogue::Default); for (int i = 0; i < opt.warmup; ++i) blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); @@ -187,7 +187,7 @@ static void runHipBench(const BenchOptions &opt) { alpaka::memcpy(queue, dB, hB); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, M, K, M, 'N', 'N'); + blas.addOperationConfig(M, N, K, M, K, M, 'N', 'N', Epilogue::Default); for (int i = 0; i < opt.warmup; ++i) blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); diff --git a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp index ded6ba8..5b3b92a 100644 --- a/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp +++ b/include/sofieBLAS/backends/cuda/sofieBLAS_cublas.hpp @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include #include @@ -30,453 +32,67 @@ } \ } while (0) -struct PairHash { - std::size_t - operator()(const std::pair &p) const noexcept { - std::size_t h1 = std::hash{}(p.first); - std::size_t h2 = std::hash{}(p.second); - return h1 ^ (h2 + 0x9e3779b97f4a7c15ULL + (h1 << 6) + (h1 >> 2)); - } -}; - -struct PairEq { - bool operator()(const std::pair &a, - const std::pair &b) const noexcept { - return a.first == b.first && a.second == b.second; - } -}; - -struct DescKey { - int transA; // CUBLAS_OP_N / CUBLAS_OP_T encoded as int - int transB; - int epilogue; // cublasLtEpilogue_t encoded as int - bool operator==(const DescKey &o) const noexcept { - return transA == o.transA && transB == o.transB && epilogue == o.epilogue; - } -}; - -struct DescKeyHash { - std::size_t operator()(const DescKey &k) const noexcept { - std::size_t h = static_cast(k.transA) * 97u + - static_cast(k.transB) * 31u + - static_cast(k.epilogue); - return h ^ (h >> 16); - } +// The cuBLASLt forwarding of the shared BlasLt implementation in +// backends/gpu/detail +struct CublasLtApi { + using Queue = alpaka::QueueCudaRtNonBlocking; + using Handle = cublasLtHandle_t; + using BlasHandle = cublasHandle_t; + using Preference = cublasLtMatmulPreference_t; + using Stream = cudaStream_t; + using Layout = cublasLtMatrixLayout_t; + using MatmulDesc = cublasLtMatmulDesc_t; + using HeuristicResult = cublasLtMatmulHeuristicResult_t; + using Operation = cublasOperation_t; + using Epilogue = cublasLtEpilogue_t; + + static constexpr auto OpN = CUBLAS_OP_N; + static constexpr auto OpT = CUBLAS_OP_T; + static constexpr auto OpC = CUBLAS_OP_C; + static constexpr auto EpilogueDefault = CUBLASLT_EPILOGUE_DEFAULT; + static constexpr auto EpilogueBias = CUBLASLT_EPILOGUE_BIAS; + static constexpr auto EpilogueReluBias = CUBLASLT_EPILOGUE_RELU_BIAS; + static constexpr auto EpilogueGeluBias = CUBLASLT_EPILOGUE_GELU_BIAS; + static constexpr auto ComputeF32 = CUBLAS_COMPUTE_32F; + static constexpr auto RealF32 = CUDA_R_32F; + static constexpr auto DescTransA = CUBLASLT_MATMUL_DESC_TRANSA; + static constexpr auto DescTransB = CUBLASLT_MATMUL_DESC_TRANSB; + static constexpr auto DescEpilogue = CUBLASLT_MATMUL_DESC_EPILOGUE; + static constexpr auto DescBiasPointer = CUBLASLT_MATMUL_DESC_BIAS_POINTER; + static constexpr auto PrefMaxWorkspace = + CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES; + static constexpr const char *name = "cuBLASLt"; + + static constexpr auto ltCreate = cublasLtCreate; + static constexpr auto ltDestroy = cublasLtDestroy; + static constexpr auto blasCreate = cublasCreate; + static constexpr auto blasDestroy = cublasDestroy; + static constexpr auto blasSetStream = cublasSetStream; + static constexpr auto prefCreate = cublasLtMatmulPreferenceCreate; + static constexpr auto prefDestroy = cublasLtMatmulPreferenceDestroy; + static constexpr auto prefSetAttribute = cublasLtMatmulPreferenceSetAttribute; + static constexpr auto layoutCreate = cublasLtMatrixLayoutCreate; + static constexpr auto layoutDestroy = cublasLtMatrixLayoutDestroy; + static constexpr auto descCreate = cublasLtMatmulDescCreate; + static constexpr auto descDestroy = cublasLtMatmulDescDestroy; + static constexpr auto descSetAttribute = cublasLtMatmulDescSetAttribute; + static constexpr auto getHeuristic = cublasLtMatmulAlgoGetHeuristic; + static constexpr auto matmul = cublasLtMatmul; + static constexpr auto sgemmStridedBatched = cublasSgemmStridedBatched; + + // cudaMalloc has a templated C++ overload, so a pointer to it is ambiguous + static cudaError_t rtMalloc(void **ptr, std::size_t size) { + return cudaMalloc(ptr, size); + } + static cudaError_t rtFree(void *ptr) { return cudaFree(ptr); } }; -struct AlgoKey { - DescKey dk; - std::size_t rowsA, colsA; // physical dimensions of A in layoutStore - std::size_t rowsB, colsB; // physical dimensions of B in layoutStore - bool operator==(const AlgoKey &o) const noexcept { - return dk == o.dk && rowsA == o.rowsA && colsA == o.colsA && - rowsB == o.rowsB && colsB == o.colsB; - } -}; - -struct AlgoKeyHash { - std::size_t operator()(const AlgoKey &k) const noexcept { - std::size_t h = DescKeyHash{}(k.dk); - auto mix = [&](std::size_t v) { - h ^= std::hash{}(v) + 0x9e3779b97f4a7c15ULL + (h << 6) + - (h >> 2); - }; - mix(k.rowsA); - mix(k.colsA); - mix(k.rowsB); - mix(k.colsB); - return h; - } -}; - -class BlasCuda { - cublasLtHandle_t ltHandle = nullptr; - cublasHandle_t handle = nullptr; - cublasLtMatmulPreference_t preference = nullptr; - void *d_workspace = nullptr; - size_t workspaceSize = 1u << 25; // 32 MB - cudaStream_t stream = nullptr; - - std::unordered_map, - cublasLtMatrixLayout_t, PairHash, PairEq> - layoutStore; - - std::unordered_map descStore; - - std::unordered_map - algoCache; - -public: - BlasCuda(const BlasCuda &) = delete; - BlasCuda &operator=(const BlasCuda &) = delete; - BlasCuda(BlasCuda &&) = delete; - BlasCuda &operator=(BlasCuda &&) = delete; - - BlasCuda(alpaka::QueueCudaRtNonBlocking &queue) : m_queue{queue} { - stream = static_cast(m_queue.getNativeHandle()); - - CHECK_CUBLAS(cublasLtCreate(<Handle)); - - CHECK_CUBLAS(cublasCreate(&handle)); - CHECK_CUBLAS(cublasSetStream(handle, stream)); - - CHECK_CUBLAS(cublasLtMatmulPreferenceCreate(&preference)); - CHECK_CUDA(cudaMalloc(&d_workspace, workspaceSize)); - CHECK_CUBLAS(cublasLtMatmulPreferenceSetAttribute( - preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, - sizeof(workspaceSize))); - } - - ~BlasCuda() { - for (auto &[key, layout] : layoutStore) - if (layout) - cublasLtMatrixLayoutDestroy(layout); - for (auto &[key, desc] : descStore) - if (desc) - cublasLtMatmulDescDestroy(desc); - if (preference) - cublasLtMatmulPreferenceDestroy(preference); - if (ltHandle) - cublasLtDestroy(ltHandle); - if (handle) - cublasDestroy(handle); - if (d_workspace) - cudaFree(d_workspace); - } - - inline cublasOperation_t charToCuBlasTranspose(char trans) { - switch (trans) { - case 'N': - case 'n': - return CUBLAS_OP_N; - case 'T': - case 't': - return CUBLAS_OP_T; - case 'C': - case 'c': - return CUBLAS_OP_C; - default: - throw std::invalid_argument("Invalid transpose character for cuBLAS."); - } - } - - void addLayoutConfig(std::size_t m, std::size_t n, std::size_t k, - std::size_t lda, std::size_t ldb, std::size_t ldc, - char transa, char transb) { - // Physical A: (m×k) if NoTrans, (k×m) if Trans - if (transa == 'N' || transa == 'n') - checkAndAddLayout(m, k, lda); - else - checkAndAddLayout(k, m, lda); - // Physical B: (k×n) if NoTrans, (n×k) if Trans - if (transb == 'N' || transb == 'n') - checkAndAddLayout(k, n, ldb); - else - checkAndAddLayout(n, k, ldb); - // C is always (m×n) - checkAndAddLayout(m, n, ldc); - } - - template - inline void - gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, alpaka::BufCudaRt, TIdx> const &A, - alpaka::BufCudaRt, TIdx> const &B, float beta, - alpaka::BufCudaRt, TIdx> &bias, - alpaka::BufCudaRt, TIdx> &C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_BIAS, alpha, alpaka::getPtrNative(A), - alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), - alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemm( - char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, - alpaka::ViewPlainPtr, TIdx> const - &A, - alpaka::ViewPlainPtr, TIdx> const - &B, - float beta, - alpaka::ViewPlainPtr, TIdx> - &bias, - alpaka::ViewPlainPtr, TIdx> &C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_BIAS, alpha, alpaka::getPtrNative(A), - alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), - alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemm(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *bias, T *C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_BIAS, alpha, A, B, beta, bias, C, - static_cast(bias), layoutKeyA(transa, m, k), - layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::BufCudaRt, TIdx> const &A, - alpaka::BufCudaRt, TIdx> const &B, - float beta, - alpaka::BufCudaRt, TIdx> &bias, - alpaka::BufCudaRt, TIdx> &C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_RELU_BIAS, alpha, alpaka::getPtrNative(A), - alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), - alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmrelu( - char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, - alpaka::ViewPlainPtr, TIdx> const - &A, - alpaka::ViewPlainPtr, TIdx> const - &B, - float beta, - alpaka::ViewPlainPtr, TIdx> - &bias, - alpaka::ViewPlainPtr, TIdx> &C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_RELU_BIAS, alpha, alpaka::getPtrNative(A), - alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), - alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } +#define SOFIEBLAS_CHECK_LT(status) CHECK_CUBLAS(status) +#define SOFIEBLAS_CHECK_RT(err) CHECK_CUDA(err) - template - inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *bias, T *C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_RELU_BIAS, alpha, A, B, beta, bias, C, - static_cast(bias), layoutKeyA(transa, m, k), - layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::BufCudaRt, TIdx> const &A, - alpaka::BufCudaRt, TIdx> const &B, - float beta, - alpaka::BufCudaRt, TIdx> &bias, - alpaka::BufCudaRt, TIdx> &C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_GELU_BIAS, alpha, alpaka::getPtrNative(A), - alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), - alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmgelu( - char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, - alpaka::ViewPlainPtr, TIdx> const - &A, - alpaka::ViewPlainPtr, TIdx> const - &B, - float beta, - alpaka::ViewPlainPtr, TIdx> - &bias, - alpaka::ViewPlainPtr, TIdx> &C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_GELU_BIAS, alpha, alpaka::getPtrNative(A), - alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), - alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } +#include "../gpu/detail/sofieBLAS_blaslt_common.tpp" - template - inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *bias, T *C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_GELU_BIAS, alpha, A, B, beta, bias, C, - static_cast(bias), layoutKeyA(transa, m, k), - layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void matmul(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::BufCudaRt, TIdx> const &A, - alpaka::BufCudaRt, TIdx> const &B, - float beta, - alpaka::BufCudaRt, TIdx> &C) { - float *c = alpaka::getPtrNative(C); - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_DEFAULT, alpha, alpaka::getPtrNative(A), - alpaka::getPtrNative(B), beta, c, c, nullptr, - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void matmul( - char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, - alpaka::ViewPlainPtr, TIdx> const - &A, - alpaka::ViewPlainPtr, TIdx> const - &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &C) { - T *c = alpaka::getPtrNative(C); - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_DEFAULT, alpha, alpaka::getPtrNative(A), - alpaka::getPtrNative(B), beta, c, c, nullptr, - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - // Raw-pointer overload - template - inline void matmul(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *C) { - executeMatmul(charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), - CUBLASLT_EPILOGUE_DEFAULT, alpha, A, B, beta, C, C, nullptr, - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - inline void gemmStridedBatched(char transa, char transb, int m, int n, int k, - float alpha, const float *A, int lda, - long long strideA, const float *B, int ldb, - long long strideB, float beta, float *C, - int ldc, long long strideC, int batchCount) { - CHECK_CUBLAS(cublasSgemmStridedBatched( - handle, charToCuBlasTranspose(transa), charToCuBlasTranspose(transb), m, - n, k, &alpha, A, lda, strideA, B, ldb, strideB, &beta, C, ldc, strideC, - batchCount)); - } - -private: - alpaka::QueueCudaRtNonBlocking m_queue; - - static std::pair - layoutKeyA(char trans, std::size_t m, std::size_t k) { - return (trans == 'N' || trans == 'n') ? std::make_pair(m, k) - : std::make_pair(k, m); - } - - static std::pair - layoutKeyB(char trans, std::size_t k, std::size_t n) { - return (trans == 'N' || trans == 'n') ? std::make_pair(k, n) - : std::make_pair(n, k); - } - - void checkAndAddLayout(std::size_t rows, std::size_t cols, std::size_t ld) { - auto key = std::make_pair(rows, cols); - if (layoutStore.find(key) == layoutStore.end()) { - cublasLtMatrixLayout_t layout = nullptr; - CHECK_CUBLAS( - cublasLtMatrixLayoutCreate(&layout, CUDA_R_32F, rows, cols, ld)); - layoutStore.emplace(key, layout); - } - } - - cublasLtMatmulDesc_t &getOrCreateDesc(cublasOperation_t transA, - cublasOperation_t transB, - cublasLtEpilogue_t epilogue) { - DescKey key{(int)transA, (int)transB, (int)epilogue}; - auto it = descStore.find(key); - if (it != descStore.end()) - return it->second; - - cublasLtMatmulDesc_t desc = nullptr; - CHECK_CUBLAS( - cublasLtMatmulDescCreate(&desc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - desc, CUBLASLT_MATMUL_DESC_TRANSA, &transA, sizeof(transA))); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - desc, CUBLASLT_MATMUL_DESC_TRANSB, &transB, sizeof(transB))); - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - desc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue))); - // For bias epilogues: set a non-null dummy pointer so the descriptor is - // valid for cublasLtMatmulAlgoGetHeuristic. - if (epilogue != CUBLASLT_EPILOGUE_DEFAULT) { - const void *dummy = d_workspace; - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &dummy, sizeof(dummy))); - } - descStore.emplace(key, desc); - return descStore.at(key); - } - - cublasLtMatmulHeuristicResult_t & - getOrComputeAlgo(cublasOperation_t transA, cublasOperation_t transB, - cublasLtEpilogue_t epilogue, - const std::pair &kA, - const std::pair &kB, - const std::pair &kC) { - AlgoKey key{{(int)transA, (int)transB, (int)epilogue}, - kA.first, - kA.second, - kB.first, - kB.second}; - auto it = algoCache.find(key); - if (it != algoCache.end()) - return it->second; - - auto &desc = getOrCreateDesc(transA, transB, epilogue); - cublasLtMatmulHeuristicResult_t h{}; - int returnedResults = 0; - CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic( - ltHandle, desc, layoutStore.at(kA), layoutStore.at(kB), - layoutStore.at(kC), layoutStore.at(kC), preference, 1, &h, - &returnedResults)); - if (returnedResults == 0) { - std::cerr << "[sofieBLAS] No suitable cuBLASLt algorithm found for " - << "transA=" << transA << " transB=" << transB - << " epilogue=" << epilogue << " A=[" << kA.first << "x" - << kA.second << "]" - << " B=[" << kB.first << "x" << kB.second << "]\n"; - exit(EXIT_FAILURE); - } - algoCache.emplace(key, h); - return algoCache.at(key); - } - - void executeMatmul(cublasOperation_t transA, cublasOperation_t transB, - cublasLtEpilogue_t epilogue, float alpha, const float *A, - const float *B, float beta, const float *D_in, - float *C_out, const void *bias_ptr, - const std::pair &kA, - const std::pair &kB, - const std::pair &kC) { - // Retrieve (or lazily compute) the cached algorithm for this shape - auto &h = getOrComputeAlgo(transA, transB, epilogue, kA, kB, kC); - - // Retrieve the cached descriptor and patch the real bias pointer in-place - auto &desc = getOrCreateDesc(transA, transB, epilogue); - if (bias_ptr) { - CHECK_CUBLAS(cublasLtMatmulDescSetAttribute( - desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_ptr, - sizeof(bias_ptr))); - } - - CHECK_CUBLAS(cublasLtMatmul(ltHandle, desc, &alpha, A, layoutStore.at(kA), - B, layoutStore.at(kB), &beta, D_in, - layoutStore.at(kC), C_out, layoutStore.at(kC), - &h.algo, d_workspace, workspaceSize, stream)); - } -}; +using BlasCuda = BlasLt; namespace traits { diff --git a/include/sofieBLAS/backends/gpu/detail/sofieBLAS_blaslt_common.tpp b/include/sofieBLAS/backends/gpu/detail/sofieBLAS_blaslt_common.tpp new file mode 100644 index 0000000..b688b76 --- /dev/null +++ b/include/sofieBLAS/backends/gpu/detail/sofieBLAS_blaslt_common.tpp @@ -0,0 +1,418 @@ +// Shared implementation of the cuBLASLt and hipBLASLt backends. The two +// vendor APIs have the same shape under different names, so the backend is +// written once against an Api table. A vendor header defines that table +// (the types, constants and functions of its library), defines the check +// macros SOFIEBLAS_CHECK_LT and SOFIEBLAS_CHECK_RT, includes the vendor and +// standard headers (, , , , +// , , , , alpaka), and then +// includes this file. + +struct PairHash { + std::size_t + operator()(const std::pair &p) const noexcept { + std::size_t h1 = std::hash{}(p.first); + std::size_t h2 = std::hash{}(p.second); + return h1 ^ (h2 + 0x9e3779b97f4a7c15ULL + (h1 << 6) + (h1 >> 2)); + } +}; + +struct PairEq { + bool operator()(const std::pair &a, + const std::pair &b) const noexcept { + return a.first == b.first && a.second == b.second; + } +}; + +struct DescKey { + int transA; // backend transpose enum encoded as int + int transB; + int epilogue; // backend epilogue enum encoded as int + bool operator==(const DescKey &o) const noexcept { + return transA == o.transA && transB == o.transB && epilogue == o.epilogue; + } +}; + +struct DescKeyHash { + std::size_t operator()(const DescKey &k) const noexcept { + std::size_t h = static_cast(k.transA) * 97u + + static_cast(k.transB) * 31u + + static_cast(k.epilogue); + return h ^ (h >> 16); + } +}; + +struct AlgoKey { + DescKey dk; + std::size_t rowsA, colsA; // physical dimensions of A in layoutStore + std::size_t rowsB, colsB; // physical dimensions of B in layoutStore + bool operator==(const AlgoKey &o) const noexcept { + return dk == o.dk && rowsA == o.rowsA && colsA == o.colsA && + rowsB == o.rowsB && colsB == o.colsB; + } +}; + +struct AlgoKeyHash { + std::size_t operator()(const AlgoKey &k) const noexcept { + std::size_t h = DescKeyHash{}(k.dk); + auto mix = [&](std::size_t v) { + h ^= std::hash{}(v) + 0x9e3779b97f4a7c15ULL + (h << 6) + + (h >> 2); + }; + mix(k.rowsA); + mix(k.colsA); + mix(k.rowsB); + mix(k.colsB); + return h; + } +}; + +template class BlasLt { + typename Api::Handle ltHandle = nullptr; + typename Api::BlasHandle handle = nullptr; + typename Api::Preference preference = nullptr; + void *d_workspace = nullptr; + size_t workspaceSize = 1u << 25; // 32 MB + typename Api::Stream stream = nullptr; + + std::unordered_map, typename Api::Layout, + PairHash, PairEq> + layoutStore; + + std::unordered_map descStore; + + // One cache entry per exact GEMM configuration: the heuristic result to + // reuse, plus this entry's position in the recency list so a hit can mark + // itself most-recently-used in O(1). The position is only maintained when a + // cache limit is set; with no limit the list stays empty. + struct CacheEntry { + typename Api::HeuristicResult h{}; + std::list::iterator lru{}; + }; + std::unordered_map algoCache; + // entries ordered most- to least-recently used; drives eviction + std::list lruOrder; + // 0 = unbounded + std::size_t algoCacheLimit = 0; + +public: + std::size_t algoCacheSize() const { return algoCache.size(); } + + BlasLt(const BlasLt &) = delete; + BlasLt &operator=(const BlasLt &) = delete; + BlasLt(BlasLt &&) = delete; + BlasLt &operator=(BlasLt &&) = delete; + + BlasLt(typename Api::Queue &queue, std::size_t cacheLimit = 0) + : algoCacheLimit{cacheLimit}, m_queue{queue} { + stream = static_cast(m_queue.getNativeHandle()); + + SOFIEBLAS_CHECK_LT(Api::ltCreate(<Handle)); + + SOFIEBLAS_CHECK_LT(Api::blasCreate(&handle)); + SOFIEBLAS_CHECK_LT(Api::blasSetStream(handle, stream)); + + SOFIEBLAS_CHECK_LT(Api::prefCreate(&preference)); + SOFIEBLAS_CHECK_RT(Api::rtMalloc(&d_workspace, workspaceSize)); + SOFIEBLAS_CHECK_LT(Api::prefSetAttribute(preference, Api::PrefMaxWorkspace, + &workspaceSize, + sizeof(workspaceSize))); + } + + ~BlasLt() { + for (auto &[key, layout] : layoutStore) + if (layout) + Api::layoutDestroy(layout); + for (auto &[key, desc] : descStore) + if (desc) + Api::descDestroy(desc); + if (preference) + Api::prefDestroy(preference); + if (ltHandle) + Api::ltDestroy(ltHandle); + if (handle) + Api::blasDestroy(handle); + if (d_workspace) + Api::rtFree(d_workspace); + } + + inline typename Api::Operation charToTranspose(char trans) { + switch (trans) { + case 'N': + case 'n': + return Api::OpN; + case 'T': + case 't': + return Api::OpT; + case 'C': + case 'c': + return Api::OpC; + default: + throw std::invalid_argument( + std::string("Invalid transpose character for ") + Api::name + "."); + } + } + + // Registers a call site's construction-time shape: creates the three matrix + // layouts and resolves the multiply algorithm for them up front, so the + // first call at this shape finds everything cached. + void addOperationConfig(std::size_t m, std::size_t n, std::size_t k, + std::size_t lda, std::size_t ldb, std::size_t ldc, + char transa, char transb, Epilogue epilogue) { + const auto shapeA = layoutKeyA(transa, m, k); + const auto shapeB = layoutKeyB(transb, k, n); + const std::pair shapeC{m, n}; + getOrCreateLayout(shapeA, lda); + getOrCreateLayout(shapeB, ldb); + getOrCreateLayout(shapeC, ldc); + + typename Api::Epilogue apiEpilogue = Api::EpilogueDefault; + switch (epilogue) { + case Epilogue::Bias: + apiEpilogue = Api::EpilogueBias; + break; + case Epilogue::ReluBias: + apiEpilogue = Api::EpilogueReluBias; + break; + case Epilogue::GeluBias: + apiEpilogue = Api::EpilogueGeluBias; + break; + case Epilogue::Default: + break; + } + getOrComputeAlgo(charToTranspose(transa), charToTranspose(transb), + apiEpilogue, shapeA, shapeB, shapeC); + } + + // Each multiply variant comes as one generic overload, where A, B, bias and + // C are any alpaka buffers or views (anything alpaka::getPtrNative + // accepts), and one raw device-pointer overload, which generated code + // calls. + template + inline void gemm(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, TA const &A, TB const &B, + float beta, TBias &bias, TC &C) { + executeMatmul(charToTranspose(transa), charToTranspose(transb), + Api::EpilogueBias, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), + alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemm(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + executeMatmul(charToTranspose(transa), charToTranspose(transb), + Api::EpilogueBias, alpha, A, B, beta, bias, C, + static_cast(bias), layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, TA const &A, TB const &B, + float beta, TBias &bias, TC &C) { + executeMatmul(charToTranspose(transa), charToTranspose(transb), + Api::EpilogueReluBias, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), + alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + executeMatmul(charToTranspose(transa), charToTranspose(transb), + Api::EpilogueReluBias, alpha, A, B, beta, bias, C, + static_cast(bias), layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, TA const &A, TB const &B, + float beta, TBias &bias, TC &C) { + executeMatmul(charToTranspose(transa), charToTranspose(transb), + Api::EpilogueGeluBias, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, alpaka::getPtrNative(bias), + alpaka::getPtrNative(C), + static_cast(alpaka::getPtrNative(bias)), + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *bias, T *C) { + executeMatmul(charToTranspose(transa), charToTranspose(transb), + Api::EpilogueGeluBias, alpha, A, B, beta, bias, C, + static_cast(bias), layoutKeyA(transa, m, k), + layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, TA const &A, TB const &B, + float beta, TC &C) { + auto *c = alpaka::getPtrNative(C); + executeMatmul(charToTranspose(transa), charToTranspose(transb), + Api::EpilogueDefault, alpha, alpaka::getPtrNative(A), + alpaka::getPtrNative(B), beta, c, c, nullptr, + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + template + inline void matmul(char transa, char transb, unsigned int m, unsigned int n, + unsigned int k, float alpha, T const *A, T const *B, + float beta, T *C) { + executeMatmul(charToTranspose(transa), charToTranspose(transb), + Api::EpilogueDefault, alpha, A, B, beta, C, C, nullptr, + layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); + } + + inline void gemmStridedBatched(char transa, char transb, int m, int n, int k, + float alpha, const float *A, int lda, + long long strideA, const float *B, int ldb, + long long strideB, float beta, float *C, + int ldc, long long strideC, int batchCount) { + SOFIEBLAS_CHECK_LT(Api::sgemmStridedBatched( + handle, charToTranspose(transa), charToTranspose(transb), m, n, k, + &alpha, A, lda, strideA, B, ldb, strideB, &beta, C, ldc, strideC, + batchCount)); + } + +private: + typename Api::Queue m_queue; + + static std::pair + layoutKeyA(char trans, std::size_t m, std::size_t k) { + return (trans == 'N' || trans == 'n') ? std::make_pair(m, k) + : std::make_pair(k, m); + } + + static std::pair + layoutKeyB(char trans, std::size_t k, std::size_t n) { + return (trans == 'N' || trans == 'n') ? std::make_pair(k, n) + : std::make_pair(n, k); + } + + // Returns the layout describing a (rows, cols) matrix, creating and caching + // it on first use. Every caller passes ld = rows (dense column-major). + typename Api::Layout + getOrCreateLayout(const std::pair &shape, + std::size_t ld) { + auto it = layoutStore.find(shape); + if (it != layoutStore.end()) + return it->second; + typename Api::Layout layout = nullptr; + SOFIEBLAS_CHECK_LT(Api::layoutCreate(&layout, Api::RealF32, shape.first, + shape.second, ld)); + layoutStore.emplace(shape, layout); + return layout; + } + + typename Api::MatmulDesc &getOrCreateDesc(typename Api::Operation transA, + typename Api::Operation transB, + typename Api::Epilogue epilogue) { + DescKey key{(int)transA, (int)transB, (int)epilogue}; + auto it = descStore.find(key); + if (it != descStore.end()) + return it->second; + + typename Api::MatmulDesc desc = nullptr; + SOFIEBLAS_CHECK_LT(Api::descCreate(&desc, Api::ComputeF32, Api::RealF32)); + SOFIEBLAS_CHECK_LT( + Api::descSetAttribute(desc, Api::DescTransA, &transA, sizeof(transA))); + SOFIEBLAS_CHECK_LT( + Api::descSetAttribute(desc, Api::DescTransB, &transB, sizeof(transB))); + SOFIEBLAS_CHECK_LT(Api::descSetAttribute(desc, Api::DescEpilogue, &epilogue, + sizeof(epilogue))); + // For bias epilogues: set a non-null dummy pointer so the descriptor is + // valid for the heuristic query. + if (epilogue != Api::EpilogueDefault) { + const void *dummy = d_workspace; + SOFIEBLAS_CHECK_LT(Api::descSetAttribute(desc, Api::DescBiasPointer, + &dummy, sizeof(dummy))); + } + descStore.emplace(key, desc); + return descStore.at(key); + } + + typename Api::HeuristicResult & + getOrComputeAlgo(typename Api::Operation transA, + typename Api::Operation transB, + typename Api::Epilogue epilogue, + const std::pair &shapeA, + const std::pair &shapeB, + const std::pair &shapeC) { + AlgoKey key{{(int)transA, (int)transB, (int)epilogue}, + shapeA.first, + shapeA.second, + shapeB.first, + shapeB.second}; + auto it = algoCache.find(key); + if (it != algoCache.end()) { + if (algoCacheLimit) + lruOrder.splice(lruOrder.begin(), lruOrder, it->second.lru); + return it->second.h; + } + + auto &desc = getOrCreateDesc(transA, transB, epilogue); + auto lA = getOrCreateLayout(shapeA, shapeA.first); + auto lB = getOrCreateLayout(shapeB, shapeB.first); + auto lC = getOrCreateLayout(shapeC, shapeC.first); + typename Api::HeuristicResult h{}; + int returnedResults = 0; + SOFIEBLAS_CHECK_LT(Api::getHeuristic(ltHandle, desc, lA, lB, lC, lC, + preference, 1, &h, &returnedResults)); + if (returnedResults == 0) { + std::cerr << "[sofieBLAS] No suitable " << Api::name + << " algorithm found for " + << "transA=" << transA << " transB=" << transB + << " epilogue=" << epilogue << " A=[" << shapeA.first << "x" + << shapeA.second << "]" + << " B=[" << shapeB.first << "x" << shapeB.second << "]\n"; + exit(EXIT_FAILURE); + } + auto ins = algoCache.emplace(key, CacheEntry{h, {}}).first; + if (algoCacheLimit) { + lruOrder.push_front(key); + ins->second.lru = lruOrder.begin(); + while (algoCache.size() > algoCacheLimit) { + algoCache.erase(lruOrder.back()); + lruOrder.pop_back(); + } + } + return ins->second.h; + } + + void executeMatmul(typename Api::Operation transA, + typename Api::Operation transB, + typename Api::Epilogue epilogue, float alpha, + const float *A, const float *B, float beta, + const float *D_in, float *C_out, const void *bias_ptr, + const std::pair &shapeA, + const std::pair &shapeB, + const std::pair &shapeC) { + // Retrieve (or lazily compute) the cached algorithm for this shape + auto &h = + getOrComputeAlgo(transA, transB, epilogue, shapeA, shapeB, shapeC); + + // Retrieve the cached descriptor and patch the real bias pointer in-place + auto &desc = getOrCreateDesc(transA, transB, epilogue); + if (bias_ptr) { + SOFIEBLAS_CHECK_LT(Api::descSetAttribute(desc, Api::DescBiasPointer, + &bias_ptr, sizeof(bias_ptr))); + } + + auto lA = getOrCreateLayout(shapeA, shapeA.first); + auto lB = getOrCreateLayout(shapeB, shapeB.first); + auto lC = getOrCreateLayout(shapeC, shapeC.first); + SOFIEBLAS_CHECK_LT(Api::matmul(ltHandle, desc, &alpha, A, lA, B, lB, &beta, + D_in, lC, C_out, lC, &h.algo, d_workspace, + workspaceSize, stream)); + } +}; diff --git a/include/sofieBLAS/backends/hip/sofieBLAS_hipblaslt.hpp b/include/sofieBLAS/backends/hip/sofieBLAS_hipblaslt.hpp index 0e41e70..4e178df 100644 --- a/include/sofieBLAS/backends/hip/sofieBLAS_hipblaslt.hpp +++ b/include/sofieBLAS/backends/hip/sofieBLAS_hipblaslt.hpp @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include #include @@ -30,446 +32,68 @@ } \ } while (0) -struct PairHash { - std::size_t - operator()(const std::pair &p) const noexcept { - std::size_t h1 = std::hash{}(p.first); - std::size_t h2 = std::hash{}(p.second); - return h1 ^ (h2 + 0x9e3779b97f4a7c15ULL + (h1 << 6) + (h1 >> 2)); - } -}; - -struct PairEq { - bool operator()(const std::pair &a, - const std::pair &b) const noexcept { - return a.first == b.first && a.second == b.second; - } -}; - -struct DescKey { - int transA; // HIPBLAS_OP_N / HIPBLAS_OP_T encoded as int - int transB; - int epilogue; // hipblasLtEpilogue_t encoded as int - bool operator==(const DescKey &o) const noexcept { - return transA == o.transA && transB == o.transB && epilogue == o.epilogue; - } -}; - -struct DescKeyHash { - std::size_t operator()(const DescKey &k) const noexcept { - std::size_t h = static_cast(k.transA) * 97u + - static_cast(k.transB) * 31u + - static_cast(k.epilogue); - return h ^ (h >> 16); - } +// The hipBLASLt forwarding of the shared BlasLt implementation in +// backends/gpu/detail +struct HipblasLtApi { + using Queue = alpaka::QueueHipRtNonBlocking; + using Handle = hipblasLtHandle_t; + using BlasHandle = hipblasHandle_t; + using Preference = hipblasLtMatmulPreference_t; + using Stream = hipStream_t; + using Layout = hipblasLtMatrixLayout_t; + using MatmulDesc = hipblasLtMatmulDesc_t; + using HeuristicResult = hipblasLtMatmulHeuristicResult_t; + using Operation = hipblasOperation_t; + using Epilogue = hipblasLtEpilogue_t; + + static constexpr auto OpN = HIPBLAS_OP_N; + static constexpr auto OpT = HIPBLAS_OP_T; + static constexpr auto OpC = HIPBLAS_OP_C; + static constexpr auto EpilogueDefault = HIPBLASLT_EPILOGUE_DEFAULT; + static constexpr auto EpilogueBias = HIPBLASLT_EPILOGUE_BIAS; + static constexpr auto EpilogueReluBias = HIPBLASLT_EPILOGUE_RELU_BIAS; + static constexpr auto EpilogueGeluBias = HIPBLASLT_EPILOGUE_GELU_BIAS; + static constexpr auto ComputeF32 = HIPBLAS_COMPUTE_32F; + static constexpr auto RealF32 = HIP_R_32F; + static constexpr auto DescTransA = HIPBLASLT_MATMUL_DESC_TRANSA; + static constexpr auto DescTransB = HIPBLASLT_MATMUL_DESC_TRANSB; + static constexpr auto DescEpilogue = HIPBLASLT_MATMUL_DESC_EPILOGUE; + static constexpr auto DescBiasPointer = HIPBLASLT_MATMUL_DESC_BIAS_POINTER; + static constexpr auto PrefMaxWorkspace = + HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES; + static constexpr const char *name = "hipBLASLt"; + + static constexpr auto ltCreate = hipblasLtCreate; + static constexpr auto ltDestroy = hipblasLtDestroy; + static constexpr auto blasCreate = hipblasCreate; + static constexpr auto blasDestroy = hipblasDestroy; + static constexpr auto blasSetStream = hipblasSetStream; + static constexpr auto prefCreate = hipblasLtMatmulPreferenceCreate; + static constexpr auto prefDestroy = hipblasLtMatmulPreferenceDestroy; + static constexpr auto prefSetAttribute = + hipblasLtMatmulPreferenceSetAttribute; + static constexpr auto layoutCreate = hipblasLtMatrixLayoutCreate; + static constexpr auto layoutDestroy = hipblasLtMatrixLayoutDestroy; + static constexpr auto descCreate = hipblasLtMatmulDescCreate; + static constexpr auto descDestroy = hipblasLtMatmulDescDestroy; + static constexpr auto descSetAttribute = hipblasLtMatmulDescSetAttribute; + static constexpr auto getHeuristic = hipblasLtMatmulAlgoGetHeuristic; + static constexpr auto matmul = hipblasLtMatmul; + static constexpr auto sgemmStridedBatched = hipblasSgemmStridedBatched; + + // hipMalloc has a templated C++ overload, so a pointer to it is ambiguous + static hipError_t rtMalloc(void **ptr, std::size_t size) { + return hipMalloc(ptr, size); + } + static hipError_t rtFree(void *ptr) { return hipFree(ptr); } }; -struct AlgoKey { - DescKey dk; - std::size_t rowsA, colsA; // physical dimensions of A in layoutStore - std::size_t rowsB, colsB; // physical dimensions of B in layoutStore - bool operator==(const AlgoKey &o) const noexcept { - return dk == o.dk && rowsA == o.rowsA && colsA == o.colsA && - rowsB == o.rowsB && colsB == o.colsB; - } -}; - -struct AlgoKeyHash { - std::size_t operator()(const AlgoKey &k) const noexcept { - std::size_t h = DescKeyHash{}(k.dk); - auto mix = [&](std::size_t v) { - h ^= std::hash{}(v) + 0x9e3779b97f4a7c15ULL + (h << 6) + - (h >> 2); - }; - mix(k.rowsA); - mix(k.colsA); - mix(k.rowsB); - mix(k.colsB); - return h; - } -}; - -class BlasHip { - hipblasLtHandle_t ltHandle = nullptr; - hipblasHandle_t handle = nullptr; - hipblasLtMatmulPreference_t preference = nullptr; - void *d_workspace = nullptr; - size_t workspaceSize = 1u << 25; // 32 MB - hipStream_t stream = nullptr; - - std::unordered_map, - hipblasLtMatrixLayout_t, PairHash, PairEq> - layoutStore; - - std::unordered_map descStore; - - std::unordered_map - algoCache; +#define SOFIEBLAS_CHECK_LT(status) CHECK_HIPBLAS(status) +#define SOFIEBLAS_CHECK_RT(err) CHECK_HIP(err) -public: - BlasHip(const BlasHip &) = delete; - BlasHip &operator=(const BlasHip &) = delete; - BlasHip(BlasHip &&) = delete; - BlasHip &operator=(BlasHip &&) = delete; - - BlasHip(alpaka::QueueHipRtNonBlocking &queue) : m_queue{queue} { - stream = static_cast(m_queue.getNativeHandle()); - - CHECK_HIPBLAS(hipblasLtCreate(<Handle)); - - CHECK_HIPBLAS(hipblasCreate(&handle)); - CHECK_HIPBLAS(hipblasSetStream(handle, stream)); - - CHECK_HIPBLAS(hipblasLtMatmulPreferenceCreate(&preference)); - CHECK_HIP(hipMalloc(&d_workspace, workspaceSize)); - CHECK_HIPBLAS(hipblasLtMatmulPreferenceSetAttribute( - preference, HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, - sizeof(workspaceSize))); - } - - ~BlasHip() { - for (auto &[key, layout] : layoutStore) - if (layout) - hipblasLtMatrixLayoutDestroy(layout); - for (auto &[key, desc] : descStore) - if (desc) - hipblasLtMatmulDescDestroy(desc); - if (preference) - hipblasLtMatmulPreferenceDestroy(preference); - if (ltHandle) - hipblasLtDestroy(ltHandle); - if (handle) - hipblasDestroy(handle); - if (d_workspace) - hipFree(d_workspace); - } - - inline hipblasOperation_t charToHipBlasTranspose(char trans) { - switch (trans) { - case 'N': - case 'n': - return HIPBLAS_OP_N; - case 'T': - case 't': - return HIPBLAS_OP_T; - case 'C': - case 'c': - return HIPBLAS_OP_C; - default: - throw std::invalid_argument("Invalid transpose character for hipBLAS."); - } - } +#include "../gpu/detail/sofieBLAS_blaslt_common.tpp" - void addLayoutConfig(std::size_t m, std::size_t n, std::size_t k, - std::size_t lda, std::size_t ldb, std::size_t ldc, - char transa, char transb) { - if (transa == 'N' || transa == 'n') - checkAndAddLayout(m, k, lda); - else - checkAndAddLayout(k, m, lda); - if (transb == 'N' || transb == 'n') - checkAndAddLayout(k, n, ldb); - else - checkAndAddLayout(n, k, ldb); - checkAndAddLayout(m, n, ldc); - } - - template - inline void - gemm(char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, alpaka::BufHipRt, TIdx> const &A, - alpaka::BufHipRt, TIdx> const &B, float beta, - alpaka::BufHipRt, TIdx> &bias, - alpaka::BufHipRt, TIdx> &C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_BIAS, - alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemm( - char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, - alpaka::ViewPlainPtr, TIdx> const - &A, - alpaka::ViewPlainPtr, TIdx> const - &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &bias, - alpaka::ViewPlainPtr, TIdx> &C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_BIAS, - alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemm(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *bias, T *C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_BIAS, - alpha, A, B, beta, bias, C, static_cast(bias), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::BufHipRt, TIdx> const &A, - alpaka::BufHipRt, TIdx> const &B, - float beta, - alpaka::BufHipRt, TIdx> &bias, - alpaka::BufHipRt, TIdx> &C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_RELU_BIAS, - alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmrelu( - char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, - alpaka::ViewPlainPtr, TIdx> const - &A, - alpaka::ViewPlainPtr, TIdx> const - &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &bias, - alpaka::ViewPlainPtr, TIdx> &C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_RELU_BIAS, - alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmrelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *bias, T *C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_RELU_BIAS, - alpha, A, B, beta, bias, C, static_cast(bias), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::BufHipRt, TIdx> const &A, - alpaka::BufHipRt, TIdx> const &B, - float beta, - alpaka::BufHipRt, TIdx> &bias, - alpaka::BufHipRt, TIdx> &C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_GELU_BIAS, - alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmgelu( - char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, - alpaka::ViewPlainPtr, TIdx> const - &A, - alpaka::ViewPlainPtr, TIdx> const - &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &bias, - alpaka::ViewPlainPtr, TIdx> &C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_GELU_BIAS, - alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - alpaka::getPtrNative(bias), alpaka::getPtrNative(C), - static_cast(alpaka::getPtrNative(bias)), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void gemmgelu(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *bias, T *C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_GELU_BIAS, - alpha, A, B, beta, bias, C, static_cast(bias), - layoutKeyA(transa, m, k), layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void matmul(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, - alpaka::BufHipRt, TIdx> const &A, - alpaka::BufHipRt, TIdx> const &B, - float beta, - alpaka::BufHipRt, TIdx> &C) { - float *c = alpaka::getPtrNative(C); - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_DEFAULT, - alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - c, c, nullptr, layoutKeyA(transa, m, k), - layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void matmul( - char transa, char transb, unsigned int m, unsigned int n, unsigned int k, - float alpha, - alpaka::ViewPlainPtr, TIdx> const - &A, - alpaka::ViewPlainPtr, TIdx> const - &B, - float beta, - alpaka::ViewPlainPtr, TIdx> &C) { - T *c = alpaka::getPtrNative(C); - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_DEFAULT, - alpha, alpaka::getPtrNative(A), alpaka::getPtrNative(B), beta, - c, c, nullptr, layoutKeyA(transa, m, k), - layoutKeyB(transb, k, n), {m, n}); - } - - template - inline void matmul(char transa, char transb, unsigned int m, unsigned int n, - unsigned int k, float alpha, T const *A, T const *B, - float beta, T *C) { - executeMatmul(charToHipBlasTranspose(transa), - charToHipBlasTranspose(transb), HIPBLASLT_EPILOGUE_DEFAULT, - alpha, A, B, beta, C, C, nullptr, layoutKeyA(transa, m, k), - layoutKeyB(transb, k, n), {m, n}); - } - - inline void gemmStridedBatched(char transa, char transb, int m, int n, int k, - float alpha, const float *A, int lda, - long long strideA, const float *B, int ldb, - long long strideB, float beta, float *C, - int ldc, long long strideC, int batchCount) { - CHECK_HIPBLAS(hipblasSgemmStridedBatched( - handle, charToHipBlasTranspose(transa), charToHipBlasTranspose(transb), - m, n, k, &alpha, A, lda, strideA, B, ldb, strideB, &beta, C, ldc, - strideC, batchCount)); - } - -private: - alpaka::QueueHipRtNonBlocking m_queue; - - static std::pair - layoutKeyA(char trans, std::size_t m, std::size_t k) { - return (trans == 'N' || trans == 'n') ? std::make_pair(m, k) - : std::make_pair(k, m); - } - - static std::pair - layoutKeyB(char trans, std::size_t k, std::size_t n) { - return (trans == 'N' || trans == 'n') ? std::make_pair(k, n) - : std::make_pair(n, k); - } - - void checkAndAddLayout(std::size_t rows, std::size_t cols, std::size_t ld) { - auto key = std::make_pair(rows, cols); - if (layoutStore.find(key) == layoutStore.end()) { - hipblasLtMatrixLayout_t layout = nullptr; - CHECK_HIPBLAS( - hipblasLtMatrixLayoutCreate(&layout, HIP_R_32F, rows, cols, ld)); - layoutStore.emplace(key, layout); - } - } - - hipblasLtMatmulDesc_t &getOrCreateDesc(hipblasOperation_t transA, - hipblasOperation_t transB, - hipblasLtEpilogue_t epilogue) { - DescKey key{(int)transA, (int)transB, (int)epilogue}; - auto it = descStore.find(key); - if (it != descStore.end()) - return it->second; - - hipblasLtMatmulDesc_t desc = nullptr; - CHECK_HIPBLAS( - hipblasLtMatmulDescCreate(&desc, HIPBLAS_COMPUTE_32F, HIP_R_32F)); - CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( - desc, HIPBLASLT_MATMUL_DESC_TRANSA, &transA, sizeof(transA))); - CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( - desc, HIPBLASLT_MATMUL_DESC_TRANSB, &transB, sizeof(transB))); - CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( - desc, HIPBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue))); - - if (epilogue != HIPBLASLT_EPILOGUE_DEFAULT) { - const void *dummy = d_workspace; - CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( - desc, HIPBLASLT_MATMUL_DESC_BIAS_POINTER, &dummy, sizeof(dummy))); - } - descStore.emplace(key, desc); - return descStore.at(key); - } - - hipblasLtMatmulHeuristicResult_t & - getOrComputeAlgo(hipblasOperation_t transA, hipblasOperation_t transB, - hipblasLtEpilogue_t epilogue, - const std::pair &kA, - const std::pair &kB, - const std::pair &kC) { - AlgoKey key{{(int)transA, (int)transB, (int)epilogue}, - kA.first, - kA.second, - kB.first, - kB.second}; - auto it = algoCache.find(key); - if (it != algoCache.end()) - return it->second; - - auto &desc = getOrCreateDesc(transA, transB, epilogue); - hipblasLtMatmulHeuristicResult_t h{}; - int returnedResults = 0; - CHECK_HIPBLAS(hipblasLtMatmulAlgoGetHeuristic( - ltHandle, desc, layoutStore.at(kA), layoutStore.at(kB), - layoutStore.at(kC), layoutStore.at(kC), preference, 1, &h, - &returnedResults)); - if (returnedResults == 0) { - std::cerr << "[sofieBLAS] No suitable hipBLASLt algorithm found for " - << "transA=" << transA << " transB=" << transB - << " epilogue=" << epilogue << " A=[" << kA.first << "x" - << kA.second << "]" - << " B=[" << kB.first << "x" << kB.second << "]\n"; - exit(EXIT_FAILURE); - } - algoCache.emplace(key, h); - return algoCache.at(key); - } - - void executeMatmul(hipblasOperation_t transA, hipblasOperation_t transB, - hipblasLtEpilogue_t epilogue, float alpha, const float *A, - const float *B, float beta, const float *D_in, - float *C_out, const void *bias_ptr, - const std::pair &kA, - const std::pair &kB, - const std::pair &kC) { - auto &h = getOrComputeAlgo(transA, transB, epilogue, kA, kB, kC); - - auto &desc = getOrCreateDesc(transA, transB, epilogue); - if (bias_ptr) { - CHECK_HIPBLAS(hipblasLtMatmulDescSetAttribute( - desc, HIPBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_ptr, - sizeof(bias_ptr))); - } - - CHECK_HIPBLAS(hipblasLtMatmul(ltHandle, desc, &alpha, A, layoutStore.at(kA), - B, layoutStore.at(kB), &beta, D_in, - layoutStore.at(kC), C_out, layoutStore.at(kC), - &h.algo, d_workspace, workspaceSize, stream)); - } -}; +using BlasHip = BlasLt; namespace traits { diff --git a/include/sofieBLAS/core.hpp b/include/sofieBLAS/core.hpp index f2f3da9..1fb8890 100644 --- a/include/sofieBLAS/core.hpp +++ b/include/sofieBLAS/core.hpp @@ -6,3 +6,5 @@ template class sofieBLAS; template using sofieBLAS = typename traits::sofieBLAS::Impl; + +enum class Epilogue { Default, Bias, ReluBias, GeluBias }; diff --git a/tests/test.cc b/tests/test.cc index 11d6e4c..512f59b 100644 --- a/tests/test.cc +++ b/tests/test.cc @@ -393,8 +393,8 @@ static void runCudaTests() { }; // ---- matmul NN ---- - blas.addLayoutConfig(M, N, K, ldaFor('N', M, K), ldbFor('N', K, N), M, 'N', - 'N'); + blas.addOperationConfig(M, N, K, ldaFor('N', M, K), ldbFor('N', K, N), M, 'N', + 'N', Epilogue::Default); std::fill(ref.begin(), ref.end(), 0.f); refMatmul(ref.data(), A, B, M, N, K, 1.f, 0.f, false, false); blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); @@ -409,8 +409,8 @@ static void runCudaTests() { alpaka::allocAsyncBuf(queue, static_cast(K * M)); alpaka::memcpy(queue, dAt, hAt); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, 'T', - 'N'); + blas.addOperationConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, + 'T', 'N', Epilogue::Default); std::fill(ref.begin(), ref.end(), 0.f); refMatmul(ref.data(), At, B, M, N, K, 1.f, 0.f, true, false); blas.matmul('T', 'N', M, N, K, 1.f, dAt, dB, 0.f, dC); @@ -426,8 +426,8 @@ static void runCudaTests() { alpaka::allocAsyncBuf(queue, static_cast(N * K)); alpaka::memcpy(queue, dBt, hBt); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, ldaFor('N', M, K), ldbFor('T', K, N), M, 'N', - 'T'); + blas.addOperationConfig(M, N, K, ldaFor('N', M, K), ldbFor('T', K, N), M, + 'N', 'T', Epilogue::Default); std::fill(ref.begin(), ref.end(), 0.f); refMatmul(ref.data(), A, Bt, M, N, K, 1.f, 0.f, false, true); blas.matmul('N', 'T', M, N, K, 1.f, dA, dBt, 0.f, dC); @@ -465,8 +465,8 @@ static void runCudaTests() { alpaka::allocAsyncBuf(queue, static_cast(K * M)); alpaka::memcpy(queue, dAt, hAt); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, 'T', - 'N'); + blas.addOperationConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, + 'T', 'N', Epilogue::Bias); std::fill(ref.begin(), ref.end(), 0.f); refGemm(ref.data(), At, B, bias, M, N, K, 1.f, 0.f, true, false); blas.gemm('T', 'N', M, N, K, 1.f, dAt, dB, 0.f, dBias, dC); @@ -494,7 +494,7 @@ static void runCudaTests() { alpaka::memcpy(queue, dBp, hBp); alpaka::memcpy(queue, dBiasz, hBiasz); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, M, K, M, 'N', 'N'); + blas.addOperationConfig(M, N, K, M, K, M, 'N', 'N', Epilogue::ReluBias); std::fill(ref.begin(), ref.end(), 0.f); refGemmRelu(ref.data(), Ap, Bp, alpaka::getPtrNative(hBiasz), M, N, K, 1.f, 0.f, false, false); @@ -559,6 +559,97 @@ static void runCudaTests() { } } +static void runDynamicShapeTests() { + std::cout << "\n=== CUDA Dynamic-Shape Tests ===\n"; + + alpaka::PlatformCudaRt platform{}; + auto dev = alpaka::getDevByIdx(platform, 0u); + alpaka::Queue queue{dev}; + + alpaka::PlatformCpu hostPlatform{}; + auto hostDev = alpaka::getDevByIdx(hostPlatform, 0u); + + // M0 is the construction-time size given to addOperationConfig; the buffers + // hold MCAP rows so sizes above M0 are exercised too. + constexpr int MCAP = 96, M0 = 64, N = 3, K = 5; + + auto hA = alpaka::allocBuf(hostDev, static_cast(MCAP * K)); + auto hB = alpaka::allocBuf(hostDev, static_cast(K * N)); + auto hC = alpaka::allocBuf(hostDev, static_cast(MCAP * N)); + float *A = alpaka::getPtrNative(hA); + float *B = alpaka::getPtrNative(hB); + float *C = alpaka::getPtrNative(hC); + fillSeq(A, MCAP * K, 0.5f, 0.25f); + fillSeq(B, K * N, 1.f, 0.5f); + + auto dA = + alpaka::allocAsyncBuf(queue, static_cast(MCAP * K)); + auto dB = alpaka::allocAsyncBuf(queue, static_cast(K * N)); + auto dC = + alpaka::allocAsyncBuf(queue, static_cast(MCAP * N)); + alpaka::memcpy(queue, dA, hA); + alpaka::memcpy(queue, dB, hB); + alpaka::wait(queue); + + // One instance serving sizes never passed to addOperationConfig (issue #10), + // including m=1 and a size above the construction-time one. + sofieBLAS blas(queue); + blas.addOperationConfig(M0, N, K, ldaFor('N', M0, K), ldbFor('N', K, N), M0, + 'N', 'N', Epilogue::Default); + + std::vector ref; + auto runAt = [&](int m, const std::string &name) { + ref.assign(static_cast(m) * N, 0.f); + refMatmul(ref.data(), A, B, m, N, K, 1.f, 0.f, false, false); + blas.matmul('N', 'N', static_cast(m), static_cast(N), + static_cast(K), 1.f, dA, dB, 0.f, dC); + alpaka::memcpy(queue, hC, dC); + alpaka::wait(queue); + checkClose(C, ref.data(), m * N, name); + }; + + for (int m : {M0, 37, 8, 51, 1, M0, MCAP}) + runAt(m, "cuda::dynamic m=" + std::to_string(m)); + + // Generated code calls the raw-pointer overloads; one call keeps them + // compiled and resolving to the right overload. + ref.assign(static_cast(45) * N, 0.f); + refMatmul(ref.data(), A, B, 45, N, K, 1.f, 0.f, false, false); + blas.matmul('N', 'N', 45u, static_cast(N), static_cast(K), + 1.f, alpaka::getPtrNative(dA), alpaka::getPtrNative(dB), 0.f, + alpaka::getPtrNative(dC)); + alpaka::memcpy(queue, hC, dC); + alpaka::wait(queue); + checkClose(C, ref.data(), 45 * N, "cuda::dynamic raw pointers m=45"); + + // 32 distinct sizes through a cache limited to 8 entries. + { + sofieBLAS capped(queue, 8); + capped.addOperationConfig(M0, N, K, ldaFor('N', M0, K), ldbFor('N', K, N), + M0, 'N', 'N', Epilogue::Default); + float worst = 0.f; + for (int m = M0 + 1; m <= MCAP; ++m) { + ref.assign(static_cast(m) * N, 0.f); + refMatmul(ref.data(), A, B, m, N, K, 1.f, 0.f, false, false); + capped.matmul('N', 'N', static_cast(m), + static_cast(N), static_cast(K), 1.f, dA, + dB, 0.f, dC); + alpaka::memcpy(queue, hC, dC); + alpaka::wait(queue); + for (std::size_t i = 0; i < ref.size(); ++i) + worst = std::max(worst, std::abs(C[i] - ref[i])); + } + if (capped.algoCacheSize() <= 8 && worst < 1e-3f) { + std::cout << " PASS cuda::cache limit honoured\n"; + } else { + std::cerr << " FAIL [cuda::cache limit honoured] " + << capped.algoCacheSize() << " entries, worst err " << worst + << "\n"; + ++gFailures; + } + } +} + #endif // ALPAKA_ACC_GPU_CUDA_ENABLED // --------------------------------------------------------------------------- @@ -614,8 +705,8 @@ static void runHipTests() { }; // ---- matmul NN ---- - blas.addLayoutConfig(M, N, K, ldaFor('N', M, K), ldbFor('N', K, N), M, 'N', - 'N'); + blas.addOperationConfig(M, N, K, ldaFor('N', M, K), ldbFor('N', K, N), M, 'N', + 'N', Epilogue::Default); std::fill(ref.begin(), ref.end(), 0.f); refMatmul(ref.data(), A, B, M, N, K, 1.f, 0.f, false, false); blas.matmul('N', 'N', M, N, K, 1.f, dA, dB, 0.f, dC); @@ -630,8 +721,8 @@ static void runHipTests() { alpaka::allocAsyncBuf(queue, static_cast(K * M)); alpaka::memcpy(queue, dAt, hAt); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, 'T', - 'N'); + blas.addOperationConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, + 'T', 'N', Epilogue::Default); std::fill(ref.begin(), ref.end(), 0.f); refMatmul(ref.data(), At, B, M, N, K, 1.f, 0.f, true, false); blas.matmul('T', 'N', M, N, K, 1.f, dAt, dB, 0.f, dC); @@ -647,8 +738,8 @@ static void runHipTests() { alpaka::allocAsyncBuf(queue, static_cast(N * K)); alpaka::memcpy(queue, dBt, hBt); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, ldaFor('N', M, K), ldbFor('T', K, N), M, 'N', - 'T'); + blas.addOperationConfig(M, N, K, ldaFor('N', M, K), ldbFor('T', K, N), M, + 'N', 'T', Epilogue::Default); std::fill(ref.begin(), ref.end(), 0.f); refMatmul(ref.data(), A, Bt, M, N, K, 1.f, 0.f, false, true); blas.matmul('N', 'T', M, N, K, 1.f, dA, dBt, 0.f, dC); @@ -686,8 +777,8 @@ static void runHipTests() { alpaka::allocAsyncBuf(queue, static_cast(K * M)); alpaka::memcpy(queue, dAt, hAt); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, 'T', - 'N'); + blas.addOperationConfig(M, N, K, ldaFor('T', M, K), ldbFor('N', K, N), M, + 'T', 'N', Epilogue::Bias); std::fill(ref.begin(), ref.end(), 0.f); refGemm(ref.data(), At, B, bias, M, N, K, 1.f, 0.f, true, false); blas.gemm('T', 'N', M, N, K, 1.f, dAt, dB, 0.f, dBias, dC); @@ -715,7 +806,7 @@ static void runHipTests() { alpaka::memcpy(queue, dBp, hBp); alpaka::memcpy(queue, dBiasz, hBiasz); alpaka::wait(queue); - blas.addLayoutConfig(M, N, K, M, K, M, 'N', 'N'); + blas.addOperationConfig(M, N, K, M, K, M, 'N', 'N', Epilogue::ReluBias); std::fill(ref.begin(), ref.end(), 0.f); refGemmRelu(ref.data(), Ap, Bp, alpaka::getPtrNative(hBiasz), M, N, K, 1.f, 0.f, false, false); @@ -780,6 +871,97 @@ static void runHipTests() { } } +static void runHipDynamicShapeTests() { + std::cout << "\n=== HIP Dynamic-Shape Tests ===\n"; + + alpaka::PlatformHipRt platform{}; + auto dev = alpaka::getDevByIdx(platform, 0u); + alpaka::Queue queue{dev}; + + alpaka::PlatformCpu hostPlatform{}; + auto hostDev = alpaka::getDevByIdx(hostPlatform, 0u); + + // M0 is the construction-time size given to addOperationConfig; the buffers + // hold MCAP rows so sizes above M0 are exercised too. + constexpr int MCAP = 96, M0 = 64, N = 3, K = 5; + + auto hA = alpaka::allocBuf(hostDev, static_cast(MCAP * K)); + auto hB = alpaka::allocBuf(hostDev, static_cast(K * N)); + auto hC = alpaka::allocBuf(hostDev, static_cast(MCAP * N)); + float *A = alpaka::getPtrNative(hA); + float *B = alpaka::getPtrNative(hB); + float *C = alpaka::getPtrNative(hC); + fillSeq(A, MCAP * K, 0.5f, 0.25f); + fillSeq(B, K * N, 1.f, 0.5f); + + auto dA = + alpaka::allocAsyncBuf(queue, static_cast(MCAP * K)); + auto dB = alpaka::allocAsyncBuf(queue, static_cast(K * N)); + auto dC = + alpaka::allocAsyncBuf(queue, static_cast(MCAP * N)); + alpaka::memcpy(queue, dA, hA); + alpaka::memcpy(queue, dB, hB); + alpaka::wait(queue); + + // One instance serving sizes never passed to addOperationConfig (issue #10), + // including m=1 and a size above the construction-time one. + sofieBLAS blas(queue); + blas.addOperationConfig(M0, N, K, ldaFor('N', M0, K), ldbFor('N', K, N), M0, + 'N', 'N', Epilogue::Default); + + std::vector ref; + auto runAt = [&](int m, const std::string &name) { + ref.assign(static_cast(m) * N, 0.f); + refMatmul(ref.data(), A, B, m, N, K, 1.f, 0.f, false, false); + blas.matmul('N', 'N', static_cast(m), static_cast(N), + static_cast(K), 1.f, dA, dB, 0.f, dC); + alpaka::memcpy(queue, hC, dC); + alpaka::wait(queue); + checkClose(C, ref.data(), m * N, name); + }; + + for (int m : {M0, 37, 8, 51, 1, M0, MCAP}) + runAt(m, "hip::dynamic m=" + std::to_string(m)); + + // Generated code calls the raw-pointer overloads; one call keeps them + // compiled and resolving to the right overload. + ref.assign(static_cast(45) * N, 0.f); + refMatmul(ref.data(), A, B, 45, N, K, 1.f, 0.f, false, false); + blas.matmul('N', 'N', 45u, static_cast(N), static_cast(K), + 1.f, alpaka::getPtrNative(dA), alpaka::getPtrNative(dB), 0.f, + alpaka::getPtrNative(dC)); + alpaka::memcpy(queue, hC, dC); + alpaka::wait(queue); + checkClose(C, ref.data(), 45 * N, "hip::dynamic raw pointers m=45"); + + // 32 distinct sizes through a cache limited to 8 entries. + { + sofieBLAS capped(queue, 8); + capped.addOperationConfig(M0, N, K, ldaFor('N', M0, K), ldbFor('N', K, N), + M0, 'N', 'N', Epilogue::Default); + float worst = 0.f; + for (int m = M0 + 1; m <= MCAP; ++m) { + ref.assign(static_cast(m) * N, 0.f); + refMatmul(ref.data(), A, B, m, N, K, 1.f, 0.f, false, false); + capped.matmul('N', 'N', static_cast(m), + static_cast(N), static_cast(K), 1.f, dA, + dB, 0.f, dC); + alpaka::memcpy(queue, hC, dC); + alpaka::wait(queue); + for (std::size_t i = 0; i < ref.size(); ++i) + worst = std::max(worst, std::abs(C[i] - ref[i])); + } + if (capped.algoCacheSize() <= 8 && worst < 1e-3f) { + std::cout << " PASS hip::cache limit honoured\n"; + } else { + std::cerr << " FAIL [hip::cache limit honoured] " + << capped.algoCacheSize() << " entries, worst err " << worst + << "\n"; + ++gFailures; + } + } +} + #endif // ALPAKA_ACC_GPU_HIP_ENABLED // --------------------------------------------------------------------------- @@ -792,9 +974,11 @@ int main() { #endif #ifdef ALPAKA_ACC_GPU_CUDA_ENABLED runCudaTests(); + runDynamicShapeTests(); #endif #ifdef ALPAKA_ACC_GPU_HIP_ENABLED runHipTests(); + runHipDynamicShapeTests(); #endif std::cout << "\n";