diff --git a/docs/installation.md b/docs/installation.md index 0ac6f3ff..12fe9fce 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -31,9 +31,12 @@ If you plan to use the hardware testing modules, you need to build the CUDA memo ```bash cd infinimetrics/hardware/cuda-memory-benchmark -bash build.sh +bash build.sh --platform cuda ``` +For MetaX, Iluvatar, Hygon, and Moore Threads build and runtime instructions, +see [Hardware Benchmarks](../infinimetrics/hardware/README.md). + **Note**: This requires: - CUDA toolkit (compatible with your GPU driver) - C++ compiler with CUDA support diff --git a/infinimetrics/common/constants.py b/infinimetrics/common/constants.py index 29991b0f..d3c5a4e4 100644 --- a/infinimetrics/common/constants.py +++ b/infinimetrics/common/constants.py @@ -19,6 +19,7 @@ class AcceleratorType(str, Enum): AMD = "amd" # ROCm ASCEND = "ascend" # Huawei NPU CAMBRICON = "cambricon" # Cambricon MLU + MTHREADS = "moore" # Moore Threads MUSA GENERIC = "generic" diff --git a/infinimetrics/common/hardware_info.py b/infinimetrics/common/hardware_info.py index 97e4b321..869b6bff 100644 --- a/infinimetrics/common/hardware_info.py +++ b/infinimetrics/common/hardware_info.py @@ -22,6 +22,11 @@ "default_name": "NVIDIA GPU", "parse_output": True, }, + "moore": { + "command": ["mthreads-gmi", "-q"], + "pattern": r"\bGPU\b|\bProduct\b", + "default_name": "Moore Threads GPU", + }, "amd": { "candidates": ["amd-smi", "rocm-smi"], "pattern": r"\bGPU\b", @@ -99,13 +104,17 @@ def collect(self, accel_type: str = "", device_ids: Any = None) -> Dict[str, Any hw["cuda_version"] = ( self._collect_cuda_version() or hw["cuda_version"] ) + elif probe_type == "moore": + musa_ver = self._collect_musa_version() + if musa_ver: + hw["cuda_version"] = f"MUSA {musa_ver}" return hw return hw def _get_probe_order(self, hint: str) -> List[str]: """Get probe order based on accelerator type hint.""" - order = ["nvidia", "amd", "ascend", "cambricon"] + order = ["nvidia", "moore", "amd", "ascend", "cambricon"] if hint in order: return [hint] + [p for p in order if p != hint] return order @@ -137,6 +146,7 @@ def _probe(self, probe_type: str, hw: Dict[str, Any]) -> ProbeResult: """Generic probe dispatcher.""" probe_methods = { "nvidia": self._probe_nvidia, + "moore": self._probe_mthreads, "amd": self._probe_amd, "ascend": self._probe_generic_command, "cambricon": self._probe_generic_command, @@ -182,6 +192,42 @@ def _probe_amd(self, probe_type: str, hw: Dict[str, Any]) -> ProbeResult: cmd, config["pattern"], config["default_name"], hw ) + def _probe_mthreads(self, probe_type: str, hw: Dict[str, Any]) -> ProbeResult: + """Probe Moore Threads GPU using mthreads-gmi.""" + config = PROBE_CONFIGS["moore"] + if not _which(config["command"][0]): + return ProbeResult(success=False) + + try: + r = subprocess.run( + config["command"], capture_output=True, text=True, timeout=5 + ) + if r.returncode != 0 or not r.stdout.strip(): + return ProbeResult(success=False) + + gpu_count = 0 + gpu_name = None + for line in r.stdout.splitlines(): + stripped = line.strip() + if re.match(r"^GPU\d+\s", stripped): + gpu_count += 1 + if stripped.startswith("Product Name") and ":" in stripped: + gpu_name = stripped.split(":", 1)[1].strip() + + if gpu_count > 0: + hw["gpu_count"] = max(hw["gpu_count"], gpu_count) + if hw["gpu_model"] == "Unknown": + hw["gpu_model"] = gpu_name or config["default_name"] + + # Try to get MUSA version + musa_ver = self._collect_musa_version() + if musa_ver: + hw["cuda_version"] = f"MUSA {musa_ver}" + + return ProbeResult(success=True, count=hw["gpu_count"]) + except Exception: + return ProbeResult(success=False) + def _probe_generic_command( self, probe_type: str, hw: Dict[str, Any] ) -> ProbeResult: @@ -228,6 +274,21 @@ def _collect_cuda_version(self) -> Optional[str]: logger.debug(f"Failed to collect CUDA version: {e}") return None + def _collect_musa_version(self) -> Optional[str]: + """Collect MUSA version using mcc.""" + try: + r = subprocess.run( + ["mcc", "--version"], capture_output=True, text=True, timeout=2 + ) + if r.returncode == 0: + for line in r.stdout.splitlines(): + match = re.search(r"(\d+\.\d+\.\d+)", line) + if match: + return match.group(1) + except Exception: + pass + return None + # Singleton instance for convenience _collector = HardwareCollector() diff --git a/infinimetrics/hardware/README.md b/infinimetrics/hardware/README.md new file mode 100644 index 00000000..27573cfe --- /dev/null +++ b/infinimetrics/hardware/README.md @@ -0,0 +1,81 @@ +# Hardware Benchmarks + +InfiniBench provides one hardware adapter for NVIDIA CUDA and four additional +CUDA-compatible accelerator platforms. Existing CUDA command shapes and metric +names are kept unchanged. + +## Platforms + +| Platform | `config.device` values | Build command | Binary | +| --- | --- | --- | --- | +| NVIDIA CUDA | `cuda`, `cudaUnified`, `nvidia` | `bash build.sh --platform cuda` | `cuda-memory-benchmark/build/cuda_perf_suite` | +| MetaX | `metax` | `bash build.sh --platform metax` | `cuda-memory-benchmark/build/cuda_perf_suite` | +| Iluvatar CoreX | `corex`, `iluvatar` | `bash build.sh --platform corex` | `cuda-memory-benchmark/build/cuda_perf_suite` | +| Hygon DCU | `hygon` | `bash build.sh --platform hygon` | `cuda-memory-benchmark/build/cuda_perf_suite` | +| Moore Threads | `moore` | `bash build.sh --platform moore` | `cuda-memory-benchmark/build/cuda_perf_suite` | + +Run each build command from its benchmark directory. All binaries use the same +test selectors and common arguments: + +```bash +./build/ --all +./build/ --memory +./build/ --stream --iterations 3 --array-size 1048576 +./build/ --cache --device 0 +``` + +## Platform Selection + +`HardwareTestAdapter` resolves a platform in this order: + +1. `config.device`, when supplied. +2. The locally installed accelerator toolchain. +3. CUDA as the compatibility fallback. + +The testcase framework remains `cudaUnified` for every hardware platform. For +example, Moore Threads STREAM uses: + +```json +{ + "testcase": "hardware.cudaUnified.Stream", + "config": { + "device": "moore" + } +} +``` + +The aliases `nvidia`, `musa`, and `mthreads` are also accepted as explicit +device values. A selected non-CUDA platform is recorded in the result +configuration as `platform`; metric names remain compatible with CUDA results. + +## Device Visibility + +Restrict the process to an idle physical device before starting a benchmark. +The selected physical device is renumbered to device 0 inside the process. + +| Platform | Visibility variable | +| --- | --- | +| NVIDIA, MetaX, Iluvatar | `CUDA_VISIBLE_DEVICES` | +| Hygon | `HIP_VISIBLE_DEVICES` and `ROCR_VISIBLE_DEVICES` | +| Moore Threads | `MUSA_VISIBLE_DEVICES` | + +For example: + +```bash +CUDA_VISIBLE_DEVICES=2 ./build/cuda_perf_suite --stream --device 0 +MUSA_VISIBLE_DEVICES=0 ./build/cuda_perf_suite --stream --device 0 +``` + +## Container Notes + +Hygon DTK containers may require the host driver libraries to be mounted at +the path expected by DTK: + +```bash +docker run --rm --privileged \ + --mount type=bind,source=/opt/hyhal,target=/opt/hyhal,readonly \ + bash +``` + +Without this mount, management tools can list DCUs while HIP applications fail +to load `libhsa-runtime64.so` or `libhydmi.so`. diff --git a/infinimetrics/hardware/constants.py b/infinimetrics/hardware/constants.py new file mode 100644 index 00000000..e2b117f8 --- /dev/null +++ b/infinimetrics/hardware/constants.py @@ -0,0 +1,47 @@ +"""Hardware platform aliases and benchmark configuration.""" + +PLATFORM_ALIASES = { + "cuda": "cuda", + "cudaunified": "cuda", + "nvidia": "cuda", + "metax": "metax", + "corex": "corex", + "iluvatar": "corex", + "hygon": "hygon", + "moore": "moore", + "mthreads": "moore", + "musa": "moore", +} + +PLATFORM_CONFIGS = { + "cuda": { + "binary_name": "cuda_perf_suite", + "benchmark_subdir": "cuda-memory-benchmark", + "build_platform": "cuda", + "cache_parser": "cuda", + }, + "metax": { + "binary_name": "cuda_perf_suite", + "benchmark_subdir": "cuda-memory-benchmark", + "build_platform": "metax", + "cache_parser": "cuda", + }, + "corex": { + "binary_name": "cuda_perf_suite", + "benchmark_subdir": "cuda-memory-benchmark", + "build_platform": "corex", + "cache_parser": "cuda", + }, + "hygon": { + "binary_name": "cuda_perf_suite", + "benchmark_subdir": "cuda-memory-benchmark", + "build_platform": "hygon", + "cache_parser": "cuda", + }, + "moore": { + "binary_name": "cuda_perf_suite", + "benchmark_subdir": "cuda-memory-benchmark", + "build_platform": "moore", + "cache_parser": "cuda", + }, +} diff --git a/infinimetrics/hardware/cuda-memory-benchmark/CMakeLists.txt b/infinimetrics/hardware/cuda-memory-benchmark/CMakeLists.txt index ae4703c1..223a6627 100644 --- a/infinimetrics/hardware/cuda-memory-benchmark/CMakeLists.txt +++ b/infinimetrics/hardware/cuda-memory-benchmark/CMakeLists.txt @@ -2,15 +2,19 @@ cmake_minimum_required(VERSION 3.18) # Platform detection if(NOT DEFINED PLATFORM) - message(FATAL_ERROR "PLATFORM is not set. Use -DPLATFORM=cuda or -DPLATFORM=metax") + message(FATAL_ERROR "PLATFORM is not set. Use -DPLATFORM=cuda, -DPLATFORM=metax, -DPLATFORM=corex, or -DPLATFORM=hygon") endif() if(PLATFORM STREQUAL "cuda") project(CudaPerfSuite LANGUAGES CXX CUDA VERSION 1.0.0) elseif(PLATFORM STREQUAL "metax") project(CudaPerfSuite LANGUAGES CXX VERSION 1.0.0) +elseif(PLATFORM STREQUAL "corex") + project(CudaPerfSuite LANGUAGES CXX CUDA VERSION 1.0.0) +elseif(PLATFORM STREQUAL "hygon") + project(CudaPerfSuite LANGUAGES CXX VERSION 1.0.0) else() - message(FATAL_ERROR "Unsupported PLATFORM: ${PLATFORM}. Use 'cuda' or 'metax'") + message(FATAL_ERROR "Unsupported PLATFORM: ${PLATFORM}. Use 'cuda', 'metax', 'corex', or 'hygon'") endif() # Set C++ standard @@ -53,6 +57,30 @@ elseif(PLATFORM STREQUAL "metax") # Treat .cu as CXX for cu-bridge set_source_files_properties(src/main.cu PROPERTIES LANGUAGE CXX) + add_executable(cuda_perf_suite ${CUDA_SOURCES} ${CXX_SOURCES}) + +elseif(PLATFORM STREQUAL "corex") + # CoreX adapted CMake handles CUDA compiler automatically (clang++) + set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CUDA_STANDARD_REQUIRED ON) + set(CMAKE_CUDA_ARCHITECTURES "ivcore20" CACHE STRING "CoreX GPU architectures") + + include_directories($ENV{COREX_PATH}/include) + link_directories($ENV{COREX_PATH}/lib64) + + add_executable(cuda_perf_suite ${CUDA_SOURCES} ${CXX_SOURCES}) + +elseif(PLATFORM STREQUAL "hygon") + # Hygon DCU: use hipcc from DTK + find_program(HIPCC hipcc REQUIRED) + set(CMAKE_CXX_COMPILER "${HIPCC}") + set(CMAKE_C_COMPILER "${HIPCC}") + + # Treat .cu as CXX so hipcc handles kernel launch syntax + set_source_files_properties(src/main.cu PROPERTIES LANGUAGE CXX) + + add_compile_definitions(GPU_PLATFORM_HIP) + add_executable(cuda_perf_suite ${CUDA_SOURCES} ${CXX_SOURCES}) endif() @@ -72,5 +100,7 @@ message(STATUS " Build type: ${CMAKE_BUILD_TYPE}") message(STATUS " C++ standard: ${CMAKE_CXX_STANDARD}") if(PLATFORM STREQUAL "cuda") message(STATUS " CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}") +elseif(PLATFORM STREQUAL "corex") + message(STATUS " CoreX arch: ${CMAKE_CUDA_ARCHITECTURES}") endif() message(STATUS "") diff --git a/infinimetrics/hardware/cuda-memory-benchmark/QUICKSTART.md b/infinimetrics/hardware/cuda-memory-benchmark/QUICKSTART.md index c18c803f..862b1a06 100644 --- a/infinimetrics/hardware/cuda-memory-benchmark/QUICKSTART.md +++ b/infinimetrics/hardware/cuda-memory-benchmark/QUICKSTART.md @@ -1,16 +1,40 @@ # Quick Start Guide -## 1. Build the Project +## 1. Build ```bash -cd benchmarks/hardware/cuda-memory-benchmark -./build.sh +cd cuda-memory-benchmark + +# NVIDIA GPU +bash build.sh --platform cuda + +# MetaX +bash build.sh --platform metax + +# Iluvatar CoreX +bash build.sh --platform corex + +# Hygon DCU +bash build.sh --platform hygon + +# Moore Threads +bash build.sh --platform moore ``` ## 2. Run All Tests ```bash +# NVIDIA ./build/cuda_perf_suite --all + +# Moore Threads (specify GPU device) +MUSA_VISIBLE_DEVICES=0 ./build/cuda_perf_suite --all + +# MetaX +MACA_VISIBLE_DEVICES=0 ./build/cuda_perf_suite --all + +# Hygon +HIP_VISIBLE_DEVICES=0 ./build/cuda_perf_suite --all ``` ## 3. Run Individual Test Suites @@ -33,15 +57,14 @@ Standard STREAM benchmark measuring sustainable memory bandwidth. ``` Tests L1 and L2 cache performance with varying working set sizes. - ## 4. Common Usage Patterns -### Quick Performance Check (Default Settings) +### Quick Performance Check ```bash ./build/cuda_perf_suite --all ``` -### Detailed STREAM Benchmark (More Iterations) +### Detailed STREAM Benchmark ```bash ./build/cuda_perf_suite --stream --iterations 50 ``` @@ -51,7 +74,7 @@ Tests L1 and L2 cache performance with varying working set sizes. ./build/cuda_perf_suite --all --device 1 ``` -### Quiet Mode (Less Output) +### Quiet Mode ```bash ./build/cuda_perf_suite --all --quiet ``` @@ -65,42 +88,8 @@ The tests report: Lower CV = more consistent results. -## 6. Example Output - -``` -╔═══════════════════════════════════════════════════════════════╗ -║ ║ -║ CUDA Performance Benchmark Suite v1.0 ║ -║ ║ -║ Comprehensive GPU Memory & Cache Testing ║ -║ ║ -╚═══════════════════════════════════════════════════════════════╝ - -=== System Information === -CUDA Devices: 1 - -Device 0: NVIDIA A100-SXM4-40GB - Compute Capability: 8.0 - Total Global Memory: 39.25 GB - L2 Cache Size: 40960 KB - Multiprocessors: 108 - Max Threads per Block: 1024 - -... - -Memory Copy Bandwidth Sweep Test -Direction: Host to Device -Memory Type: Pinned - -Size (MB) Time (ms) Bandwidth (GB/s) CV (%) --------------------------------------------------------------- - 0.06 1.234 25.60 2.30 - 0.13 2.456 25.80 1.90 - ... -``` - -## 7. Next Steps +## 6. Next Steps -- Read [README.md](README.md) for detailed documentation +- Read [README.md](README.md) for detailed documentation and platform notes - Adjust test parameters for your specific use case - Integrate into your performance testing workflow diff --git a/infinimetrics/hardware/cuda-memory-benchmark/README.md b/infinimetrics/hardware/cuda-memory-benchmark/README.md index 93e8650b..89bc3b01 100644 --- a/infinimetrics/hardware/cuda-memory-benchmark/README.md +++ b/infinimetrics/hardware/cuda-memory-benchmark/README.md @@ -1,37 +1,63 @@ -# CUDA Performance Benchmark Suite +# GPU Performance Benchmark Suite -A comprehensive, modern CUDA performance testing suite written in C++17 with CMake build system. +A comprehensive GPU performance testing suite for memory bandwidth, STREAM benchmark, and cache analysis. Supports NVIDIA CUDA and domestic GPU platforms via a unified build system. + +## Supported Platforms + +| Platform | Flag | Compiler | Notes | +|----------|------|----------|-------| +| NVIDIA GPU | `--platform cuda` | nvcc | Native CUDA | +| MetaX | `--platform metax` | cucc (cu-bridge) | CUDA-compatible via cu-bridge | +| Iluvatar CoreX | `--platform corex` | nvcc (CoreX SDK) | CUDA-compatible via CoreX | +| Hygon DCU | `--platform hygon` | hipcc (DTK) | HIP backend | +| Moore Threads | `--platform moore` | mcc (MUSA SDK) | MUSA backend via `-mtgpu` | ## Features - **Memory Bandwidth Tests**: Host-to-Device, Device-to-Host, Device-to-Device transfers - **STREAM Benchmark**: Standard memory bandwidth benchmark (Copy, Scale, Add, Triad operations) - **Cache Performance Tests**: L1 and L2 cache bandwidth analysis +- **Multi-Platform**: Unified source code, per-platform build via `--platform` flag - **Modern C++ Design**: RAII patterns, smart pointers, exception safety -- **CMake Build System**: Cross-platform, easy to configure -- **Comprehensive Metrics**: Statistical analysis with trimmed mean, coefficient of variation ## Requirements -- CUDA Toolkit 11.0 or higher +Common: - CMake 3.18 or higher - C++17 compatible compiler -- NVIDIA GPU with compute capability 8.0+ (configurable) -## Building +Platform-specific: +- **NVIDIA / MetaX / CoreX**: CUDA Toolkit 11.0+ +- **Hygon**: DTK (HIP) +- **Moore Threads**: MUSA SDK (mcc) -### Quick Start +## Building ```bash -cd benchmarks/hardware/cuda-memory-benchmark -./build.sh +cd cuda-memory-benchmark + +# NVIDIA +bash build.sh --platform cuda + +# MetaX +bash build.sh --platform metax + +# Iluvatar CoreX +bash build.sh --platform corex + +# Hygon DCU +bash build.sh --platform hygon + +# Moore Threads +bash build.sh --platform moore ``` -### Manual Build +All platforms produce the same output binary: `build/cuda_perf_suite` + +### Manual Build (NVIDIA CUDA only) ```bash -mkdir build -cd build +mkdir build && cd build cmake .. -DCMAKE_BUILD_TYPE=Release make -j$(nproc) ``` @@ -52,15 +78,12 @@ Common architectures: ## Usage -### Run All Tests +All platforms share the same CLI interface: ```bash +# Run all tests ./build/cuda_perf_suite --all -``` - -### Run Specific Tests -```bash # Memory bandwidth tests only ./build/cuda_perf_suite --memory @@ -69,8 +92,28 @@ Common architectures: # Cache performance tests only ./build/cuda_perf_suite --cache + +# Specify GPU device +./build/cuda_perf_suite --all --device 1 + +# More iterations +./build/cuda_perf_suite --all --iterations 20 + +# Quiet mode +./build/cuda_perf_suite --all --quiet ``` +### Environment Variables + +Some platforms use environment variables to select GPU devices: + +| Platform | Environment Variable | Example | +|----------|---------------------|---------| +| NVIDIA | `CUDA_VISIBLE_DEVICES` | `CUDA_VISIBLE_DEVICES=0 ./build/cuda_perf_suite --all` | +| Moore Threads | `MUSA_VISIBLE_DEVICES` | `MUSA_VISIBLE_DEVICES=1 ./build/cuda_perf_suite --all` | +| Hygon | `HIP_VISIBLE_DEVICES` | `HIP_VISIBLE_DEVICES=0 ./build/cuda_perf_suite --all` | +| MetaX | `MACA_VISIBLE_DEVICES` | `MACA_VISIBLE_DEVICES=0 ./build/cuda_perf_suite --all` | + ### Command Line Options ``` @@ -79,29 +122,13 @@ Options: --memory Run memory bandwidth tests only --stream Run STREAM benchmark only --cache Run cache benchmarks only - --device Specify CUDA device ID (default: 0) + --device Specify GPU device ID (default: 0) --iterations Number of measurement iterations (default: 10) --array-size Array size for STREAM test (default: 67108864) --quiet Reduce output verbosity --help Show help message ``` -### Examples - -```bash -# Run STREAM benchmark with 128M elements -./build/cuda_perf_suite --stream --array-size 134217728 - -# Run memory tests with 512MB buffer -./build/cuda_perf_suite --memory --buffer-size 512 - -# Run all tests with 20 iterations -./build/cuda_perf_suite --all --iterations 20 - -# Run tests on specific GPU -./build/cuda_perf_suite --all --device 1 -``` - ## Test Descriptions ### 1. Memory Bandwidth Tests @@ -144,14 +171,16 @@ Tests L1 and L2 cache bandwidth by varying working set sizes ``` cuda-memory-benchmark/ ├── include/ # Header files -│ ├── performance_test.h # Base testing framework +│ ├── gpu_runtime.h # Cross-platform GPU API abstraction │ ├── cuda_utils.h # CUDA utilities (RAII wrappers) +│ ├── performance_test.h # Base testing framework │ ├── memory_bandwidth_test.h # Memory copy tests │ ├── stream_benchmark.h # STREAM benchmark │ └── cache_benchmark.h # Cache tests ├── src/ │ └── main.cu # Main program entry ├── CMakeLists.txt # CMake build configuration -├── build.sh # Build script -└── README.md # This file +├── build.sh # Unified build script (all platforms) +├── README.md # This file +└── QUICKSTART.md # Quick start guide ``` diff --git a/infinimetrics/hardware/cuda-memory-benchmark/build.sh b/infinimetrics/hardware/cuda-memory-benchmark/build.sh index fedce824..723f3551 100644 --- a/infinimetrics/hardware/cuda-memory-benchmark/build.sh +++ b/infinimetrics/hardware/cuda-memory-benchmark/build.sh @@ -2,8 +2,11 @@ # Build script for CUDA Performance Suite # Usage: -# bash build.sh --platform cuda # Build with native CUDA (NVIDIA GPU) -# bash build.sh --platform metax # Build with cu-bridge (MetaX GPU) +# bash build.sh --platform cuda # Build with native CUDA (NVIDIA GPU) +# bash build.sh --platform metax # Build with cu-bridge (MetaX GPU) +# bash build.sh --platform corex # Build with CoreX SDK (Iluvatar GPU) +# bash build.sh --platform hygon # Build with DTK/HIP (Hygon DCU) +# bash build.sh --platform moore # Build with mcc -mtgpu (Moore Threads) set -e # Exit on error @@ -23,7 +26,7 @@ while [[ $# -gt 0 ]]; do ;; *) echo -e "${RED}Unknown option: $1${NC}" - echo "Usage: bash build.sh --platform " + echo "Usage: bash build.sh --platform " exit 1 ;; esac @@ -31,7 +34,7 @@ done if [[ -z "$PLATFORM" ]]; then echo -e "${RED}ERROR: --platform is required.${NC}" - echo "Usage: bash build.sh --platform " + echo "Usage: bash build.sh --platform " exit 1 fi @@ -62,14 +65,102 @@ if [[ "$PLATFORM" == "metax" ]]; then echo -e "${YELLOW}[MetaX] Using cu-bridge: ${CUCC_PATH}${NC}" + mkdir -p build + + if command -v cmake &> /dev/null && \ + command -v cmake_maca &> /dev/null && \ + command -v make_maca &> /dev/null; then + cd build + + echo -e "${YELLOW}Configuring with cmake_maca...${NC}" + cmake_maca .. -DCMAKE_BUILD_TYPE=Release -DPLATFORM=metax + + echo -e "${YELLOW}Building with make_maca...${NC}" + make_maca -j$(nproc) + else + echo -e "${YELLOW}CMake unavailable; compiling directly with cucc...${NC}" + cucc -O3 -std=c++17 \ + -I./include \ + -I"${CUCC_PATH}/include" \ + ./src/main.cu \ + -o build/cuda_perf_suite + fi + +elif [[ "$PLATFORM" == "corex" ]]; then + # ---- CoreX (Iluvatar) platform ---- + + export COREX_PATH=${COREX_PATH:-/usr/local/corex} + export LD_LIBRARY_PATH=${COREX_PATH}/lib64:${LD_LIBRARY_PATH:-} + + if [ ! -d "${COREX_PATH}" ]; then + echo -e "${RED}ERROR: CoreX SDK not found at ${COREX_PATH}${NC}" + exit 1 + fi + + echo -e "${YELLOW}[CoreX] Using CoreX SDK: ${COREX_PATH}${NC}" + echo -e "${YELLOW}[CoreX] CMake: $(cmake --version | head -1)${NC}" + + # Clean CMake cache per CoreX migration guide requirement + if [ -d "build" ]; then + rm -rf build/CMakeCache.txt build/CMakeFiles build/Makefile + fi + + mkdir -p build + cd build + + echo -e "${YELLOW}Configuring with CoreX CMake...${NC}" + cmake .. -DCMAKE_BUILD_TYPE=Release -DPLATFORM=corex \ + -DCMAKE_CUDA_ARCHITECTURES=ivcore20 + + echo -e "${YELLOW}Building...${NC}" + make -j$(nproc) + +elif [[ "$PLATFORM" == "hygon" ]]; then + # ---- Hygon DCU platform: using DTK (HIP) ---- + + export DTK_PATH=${DTK_PATH:-/opt/dtk} + export PATH=$DTK_PATH/bin:$PATH + export LD_LIBRARY_PATH=$DTK_PATH/lib64:${LD_LIBRARY_PATH:-} + + if ! command -v hipcc &> /dev/null; then + echo -e "${RED}ERROR: hipcc not found. Please install DTK at ${DTK_PATH}${NC}" + exit 1 + fi + + echo -e "${YELLOW}[Hygon DCU] Using DTK: ${DTK_PATH}${NC}" + echo -e "${YELLOW}[Hygon DCU] hipcc: $(which hipcc)${NC}" + mkdir -p build cd build - echo -e "${YELLOW}Configuring with cmake_maca...${NC}" - cmake_maca .. -DCMAKE_BUILD_TYPE=Release -DPLATFORM=metax + echo -e "${YELLOW}Configuring with CMake + HIP...${NC}" + cmake .. -DCMAKE_BUILD_TYPE=Release -DPLATFORM=hygon + + echo -e "${YELLOW}Building...${NC}" + make -j$(nproc) + +elif [[ "$PLATFORM" == "moore" ]]; then + # ---- Moore Threads platform: using mcc -mtgpu ---- + + export MUSA_HOME=${MUSA_HOME:-/usr/local/musa} + + if ! command -v mcc &> /dev/null; then + echo -e "${RED}ERROR: mcc not found. Please install MUSA SDK at ${MUSA_HOME}${NC}" + exit 1 + fi + + echo -e "${YELLOW}[Moore Threads] Using mcc: $(which mcc)${NC}" + echo -e "${YELLOW}[Moore Threads] MUSA_HOME: ${MUSA_HOME}${NC}" + + mkdir -p build - echo -e "${YELLOW}Building with make_maca...${NC}" - make_maca -j$(nproc) + echo -e "${YELLOW}Building with mcc -mtgpu (MUSA)...${NC}" + mcc -O3 -DGPU_PLATFORM_MUSA -mtgpu \ + --musa-path="$MUSA_HOME" \ + -I./include \ + ./src/main.cu \ + -o build/cuda_perf_suite \ + -L"$MUSA_HOME/lib" -lmusart elif [[ "$PLATFORM" == "cuda" ]]; then # ---- NVIDIA CUDA platform ---- @@ -91,7 +182,7 @@ elif [[ "$PLATFORM" == "cuda" ]]; then make -j$(nproc) else - echo -e "${RED}ERROR: Unsupported platform '${PLATFORM}'. Use 'cuda' or 'metax'.${NC}" + echo -e "${RED}ERROR: Unsupported platform '${PLATFORM}'. Use 'cuda', 'metax', 'corex', 'hygon', or 'moore'.${NC}" exit 1 fi diff --git a/infinimetrics/hardware/cuda-memory-benchmark/include/cache_benchmark.h b/infinimetrics/hardware/cuda-memory-benchmark/include/cache_benchmark.h index d4ffca1b..d3ceb57c 100644 --- a/infinimetrics/hardware/cuda-memory-benchmark/include/cache_benchmark.h +++ b/infinimetrics/hardware/cuda-memory-benchmark/include/cache_benchmark.h @@ -2,7 +2,6 @@ #include "cuda_utils.h" #include "performance_test.h" -#include #include #include #include diff --git a/infinimetrics/hardware/cuda-memory-benchmark/include/cuda_utils.h b/infinimetrics/hardware/cuda-memory-benchmark/include/cuda_utils.h index db5db05b..b3a32a78 100644 --- a/infinimetrics/hardware/cuda-memory-benchmark/include/cuda_utils.h +++ b/infinimetrics/hardware/cuda-memory-benchmark/include/cuda_utils.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "gpu_runtime.h" #include #include #include @@ -241,9 +241,9 @@ struct CudaDeviceInfo { info.shared_mem_per_block = prop.sharedMemPerBlock; info.max_threads_per_block = prop.maxThreadsPerBlock; info.multi_processor_count = prop.multiProcessorCount; -#if CUDART_VERSION >= 11000 - // clockRate was removed in CUDA 11.0+ - // Set to 0 as it's no longer available +#if defined(GPU_PLATFORM_MUSA) + info.clock_rate = 0; +#elif CUDART_VERSION >= 11000 info.clock_rate = 0; #else info.clock_rate = prop.clockRate; diff --git a/infinimetrics/hardware/cuda-memory-benchmark/include/gpu_runtime.h b/infinimetrics/hardware/cuda-memory-benchmark/include/gpu_runtime.h new file mode 100644 index 00000000..0ceb3441 --- /dev/null +++ b/infinimetrics/hardware/cuda-memory-benchmark/include/gpu_runtime.h @@ -0,0 +1,120 @@ +#pragma once + +/// @file gpu_runtime.h +/// Unified GPU runtime header for cross-platform GPU computing. +/// +/// - GPU_PLATFORM_MUSA: Moore Threads MUSA (maps cuda* -> musa*) +/// - GPU_PLATFORM_HIP: Hygon DCU / AMD ROCm (maps cuda* -> hip*) +/// - Otherwise: Native CUDA (NVIDIA, MetaX cu-bridge, CoreX) + +#ifdef GPU_PLATFORM_MUSA +// ---- MUSA backend (Moore Threads) ---- +#include + +// --- Error types --- +#define cudaError_t musaError_t +#define cudaSuccess musaSuccess +#define cudaGetErrorString musaGetErrorString +#define cudaGetLastError musaGetLastError + +// --- Device management --- +#define cudaSetDevice musaSetDevice +#define cudaGetDevice musaGetDevice +#define cudaGetDeviceCount musaGetDeviceCount +#define cudaGetDeviceProperties musaGetDeviceProperties +#define cudaDeviceSynchronize musaDeviceSynchronize +#define cudaDeviceProp musaDeviceProp + +// --- Memory management --- +#define cudaMalloc musaMalloc +#define cudaFree musaFree +#define cudaMallocHost musaMallocHost +#define cudaFreeHost musaFreeHost +#define cudaMemcpy musaMemcpy +#define cudaMemcpyAsync musaMemcpyAsync +#define cudaMemset musaMemset + +// --- Stream --- +#define cudaStream_t musaStream_t +#define cudaStreamCreate musaStreamCreate +#define cudaStreamDestroy musaStreamDestroy +#define cudaStreamSynchronize musaStreamSynchronize + +// --- Event --- +#define cudaEvent_t musaEvent_t +#define cudaEventCreate musaEventCreate +#define cudaEventDestroy musaEventDestroy +#define cudaEventRecord musaEventRecord +#define cudaEventSynchronize musaEventSynchronize +#define cudaEventElapsedTime musaEventElapsedTime + +// --- Memory copy constants --- +#define cudaMemcpyHostToDevice musaMemcpyHostToDevice +#define cudaMemcpyDeviceToHost musaMemcpyDeviceToHost +#define cudaMemcpyDeviceToDevice musaMemcpyDeviceToDevice + +// --- Version --- +#define cudaRuntimeGetVersion musaRuntimeGetVersion +#define cudaDriverGetVersion musaDriverGetVersion + +#elif defined(GPU_PLATFORM_HIP) +// ---- HIP backend (Hygon DCU, AMD ROCm) ---- +#include + +// --- Error types --- +#define cudaError_t hipError_t +#define cudaSuccess hipSuccess +#define cudaGetErrorString hipGetErrorString + +// --- Memory management --- +#define cudaMalloc hipMalloc +#define cudaFree hipFree +#define cudaMallocHost hipMallocHost +#define cudaFreeHost hipFreeHost +#define cudaMemset hipMemset + +// --- Stream --- +#define cudaStream_t hipStream_t +#define cudaStreamCreate hipStreamCreate +#define cudaStreamDestroy hipStreamDestroy +#define cudaStreamSynchronize hipStreamSynchronize + +// --- Event --- +#define cudaEvent_t hipEvent_t +#define cudaEventCreate hipEventCreate +#define cudaEventDestroy hipEventDestroy +#define cudaEventRecord hipEventRecord +#define cudaEventSynchronize hipEventSynchronize +#define cudaEventElapsedTime hipEventElapsedTime + +// --- Device management --- +#define cudaSetDevice hipSetDevice +#define cudaGetDevice hipGetDevice +#define cudaGetDeviceCount hipGetDeviceCount +#define cudaGetDeviceProperties hipGetDeviceProperties +#define cudaDeviceSynchronize hipDeviceSynchronize +#define cudaGetLastError hipGetLastError + +// --- Device property struct --- +#define cudaDeviceProp hipDeviceProp_t + +// --- Memory copy --- +#define cudaMemcpy hipMemcpy +#define cudaMemcpyAsync hipMemcpyAsync +#define cudaMemcpyHostToDevice hipMemcpyHostToDevice +#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost +#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice + +// --- Version --- +#define cudaRuntimeGetVersion hipRuntimeGetVersion +#define cudaDriverGetVersion hipDriverGetVersion + +// Provide CUDART_VERSION equivalent for HIP +#ifndef CUDART_VERSION +#define CUDART_VERSION (HIP_VERSION_MAJOR * 1000 + HIP_VERSION_MINOR * 10) +#endif + +#else +// ---- CUDA backend (NVIDIA, MetaX cu-bridge, CoreX) ---- +#include +#endif diff --git a/infinimetrics/hardware/hardware_adapter.py b/infinimetrics/hardware/hardware_adapter.py index e1392df9..1a86a457 100644 --- a/infinimetrics/hardware/hardware_adapter.py +++ b/infinimetrics/hardware/hardware_adapter.py @@ -1,77 +1,104 @@ #!/usr/bin/env python3 -"""Hardware Test Adapter for CUDA Unified Benchmark Suite""" +"""Hardware test adapter for CUDA-compatible accelerators.""" import logging -import subprocess import re import shutil +import subprocess from pathlib import Path -from typing import Any, Dict, Optional, List +from typing import Any, Dict, List, Optional from infinimetrics.adapter import BaseAdapter -from infinimetrics.common.csv_utils import save_csv, create_timeseries_metric from infinimetrics.common.command_builder import build_command_from_config from infinimetrics.common.constants import ( - TEST_TYPE_MAP, - MEMORY_DIRECTIONS, - STREAM_OPERATIONS, - MEMORY_CSV_FIELDS, + CACHE_TEST_TIMEOUT, + DEFAULT_TEST_TIMEOUT, L1_CACHE_CSV_FIELDS, - L2_CACHE_CSV_FIELDS, L1_CACHE_PATTERN, + L2_CACHE_CSV_FIELDS, L2_CACHE_PATTERN, - CACHE_TEST_TIMEOUT, - DEFAULT_TEST_TIMEOUT, + MEMORY_CSV_FIELDS, + MEMORY_DIRECTIONS, METRIC_PREFIX_MEM_SWEEP, + STREAM_OPERATIONS, + TEST_TYPE_MAP, InfiniMetricsJson, ) +from infinimetrics.common.csv_utils import create_timeseries_metric +from infinimetrics.hardware.constants import PLATFORM_ALIASES, PLATFORM_CONFIGS from infinimetrics.utils.time_utils import get_timestamp logger = logging.getLogger(__name__) def detect_platform() -> str: - """Detect GPU platform: 'metax' if MACA/cu-bridge is available, else 'cuda'.""" + """Detect the installed accelerator toolchain.""" + if shutil.which("mcc") or shutil.which("mthreads-gmi"): + return "moore" + maca_path = Path("/opt/maca") - cucc_path = maca_path / "tools" / "cu-bridge" / "bin" / "cucc" - if cucc_path.exists() or shutil.which("cucc") or shutil.which("mxcc"): + if ( + (maca_path / "tools" / "cu-bridge" / "bin" / "cucc").exists() + or shutil.which("cucc") + or shutil.which("mxcc") + ): return "metax" + + dtk_path = Path("/opt/dtk") + if ( + (dtk_path / "bin" / "hy-smi").exists() + or shutil.which("hy-smi") + or (dtk_path.exists() and shutil.which("hipcc")) + ): + return "hygon" + + corex_path = Path("/usr/local/corex") + if (corex_path / "bin" / "ixsmi").exists() or ( + corex_path / "bin" / "clang++" + ).exists(): + return "corex" return "cuda" class HardwareTestAdapter(BaseAdapter): - """Adapter for CUDA Unified hardware performance tests.""" + """Adapter for unified hardware performance tests. + + The existing CUDA constructor and private method signatures remain supported. + Additional platforms are selected through ``config.device`` or runtime + toolchain detection. + """ def __init__( self, cuda_perf_path: str = None, output_dir: str = "./output", + perf_binary_path: str = None, ): + self.hardware_dir = Path(__file__).parent self.cuda_perf_path = cuda_perf_path or str( - Path(__file__).parent - / "cuda-memory-benchmark" - / "build" - / "cuda_perf_suite" + self.hardware_dir / "cuda-memory-benchmark" / "build" / "cuda_perf_suite" ) + self.perf_binary_path = perf_binary_path self.output_dir = Path(output_dir) self.output_dir.mkdir(parents=True, exist_ok=True) - self.build_dir = Path(__file__).parent / "cuda-memory-benchmark" + + # Preserve attributes used by existing callers. + self.build_dir = self.hardware_dir / "cuda-memory-benchmark" self.build_script = self.build_dir / "build.sh" def setup(self, config: Dict[str, Any]) -> None: - """Initialize resources before running tests.""" - device = config.get("device", "cuda").lower() + """Build the selected benchmark binary if it is missing.""" + device = self._get_device_type(config) if ( device == "cpu" or config.get("skip_build", False) - or Path(self.cuda_perf_path).exists() + or Path(self._get_binary_path(device)).exists() ): return - self._build_cuda_project() + self._build_project(device) def process(self, test_input: Any) -> Dict[str, Any]: - """Process test input and return results.""" - # Normalize test input to dict format + """Process a hardware test input and return normalized metrics.""" test_input = self._normalize_test_input(test_input) if not test_input: raise ValueError(f"Invalid test_input type: {type(test_input)}") @@ -80,34 +107,33 @@ def process(self, test_input: Any) -> Dict[str, Any]: config = test_input.get(InfiniMetricsJson.CONFIG, {}) run_id = test_input.get(InfiniMetricsJson.RUN_ID, "unknown") - logger.info(f"HardwareTestAdapter: Processing {testcase}") - - # Put CSV files in hardware/ subdirectory to match JSON location + logger.info("HardwareTestAdapter: Processing %s", testcase) self.output_dir = Path(config.get("output_dir", "./output")) / "hardware" self.output_dir.mkdir(parents=True, exist_ok=True) - device = config.get("device", "cuda").lower() + device = self._get_device_type(config) test_type = config.get("test_type", "comprehensive") try: if device == "cpu": logger.info( - "CPU mode: Skipping hardware tests (not supported on CPU), returning empty results" + "CPU mode: Skipping hardware tests (not supported on CPU), " + "returning empty results" ) metrics = [] command = None else: - logger.info("GPU mode (device=%s): Executing CUDA tests", device) - cmd = self._build_command(config) - command = " ".join(cmd) # Store command as string - output = self._execute_test(cmd, test_type) - metrics = self._parse_output(output, test_type, run_id) + logger.info("Accelerator mode (platform=%s): Executing tests", device) + cmd = self._build_command(config, device) + command = " ".join(cmd) + output = self._execute_test(cmd, test_type, device) + metrics = self._parse_output(output, test_type, run_id, device) - # Add command to config for traceability result_config = config.copy() if command: result_config["command"] = command - + if device not in ("cpu", "cuda"): + result_config.setdefault("platform", device) return { InfiniMetricsJson.RESULT_CODE: 0, InfiniMetricsJson.TIME: get_timestamp(), @@ -116,66 +142,106 @@ def process(self, test_input: Any) -> Dict[str, Any]: InfiniMetricsJson.CONFIG: result_config, InfiniMetricsJson.METRICS: metrics, } - - except Exception as e: - # Log error with context, then re-raise for Executor to handle + except Exception as exc: logger.error( - f"HardwareTestAdapter: Test failed for {testcase}\n" - f" Device: {device}\n" - f" Test Type: {test_type}\n" - f" Error: {str(e)}", + "HardwareTestAdapter: Test failed for %s\n" + " Device: %s\n" + " Test Type: %s\n" + " Error: %s", + testcase, + device, + test_type, + str(exc), exc_info=True, ) raise - def _build_cuda_project(self) -> None: - """Build CUDA project if needed.""" - if not self.build_dir.exists(): - raise FileNotFoundError( - f"CUDA benchmark directory not found: {self.build_dir}" - ) - if not self.build_script.exists(): - raise FileNotFoundError(f"Build script not found: {self.build_script}") - platform = detect_platform() - logger.info( - "Building CUDA project in: %s (platform: %s)", self.build_dir, platform + def _get_device_type(self, config: Dict[str, Any]) -> str: + """Resolve an explicit device or the locally installed toolchain.""" + explicit = str(config.get("device", "")).lower().strip() + if explicit == "cpu": + return "cpu" + if explicit: + # Unknown explicit device values historically used the CUDA suite. + return PLATFORM_ALIASES.get(explicit, "cuda") + + return detect_platform() + + @staticmethod + def _get_device_config(device: str) -> Dict[str, Any]: + config = PLATFORM_CONFIGS.get(device) + if not config: + raise ValueError(f"Unknown device type: {device}") + return config + + def _get_binary_path(self, device: str) -> str: + if self.perf_binary_path: + return self.perf_binary_path + if device in ("cuda", "metax", "corex", "hygon", "moore"): + return self.cuda_perf_path + device_config = self._get_device_config(device) + return str( + self.hardware_dir + / device_config["benchmark_subdir"] + / "build" + / device_config["binary_name"] ) + + def _build_cuda_project(self) -> None: + """Build the detected CUDA-compatible project through the legacy entrypoint.""" + self._build_project(detect_platform()) + + def _build_project(self, device: str) -> None: + device_config = self._get_device_config(device) + build_dir = self.hardware_dir / device_config["benchmark_subdir"] + build_script = build_dir / "build.sh" + if not build_dir.exists(): + raise FileNotFoundError(f"Benchmark directory not found: {build_dir}") + if not build_script.exists(): + raise FileNotFoundError(f"Build script not found: {build_script}") + + command = ["bash", str(build_script)] + if device_config["build_platform"]: + command.extend(["--platform", device_config["build_platform"]]) + + logger.info("Building %s project in: %s", device, build_dir) try: result = subprocess.run( - ["bash", str(self.build_script), "--platform", platform], - cwd=str(self.build_dir), + command, + cwd=str(build_dir), capture_output=True, text=True, timeout=300, ) if result.returncode != 0: - raise RuntimeError(f"Failed to build CUDA project:\n{result.stderr}") - logger.info( - "CUDA project build completed successfully (platform: %s)", platform - ) + raise RuntimeError( + f"Failed to build {device} hardware project:\n{result.stderr}" + ) + logger.info("%s hardware project build completed successfully", device) except subprocess.TimeoutExpired: - raise RuntimeError("CUDA project build timed out after 5 minutes") + raise RuntimeError("Hardware project build timed out after 5 minutes") - def _build_command(self, config: Dict[str, Any]) -> List[str]: - """Build command for CUDA test suite.""" + def _build_command(self, config: Dict[str, Any], device: str = "cuda") -> List[str]: + """Build a benchmark command using the existing CUDA CLI contract.""" test_type = config.get("test_type", "all") - cuda_test_type = TEST_TYPE_MAP.get(test_type, test_type.lower()) - - base_command = [self.cuda_perf_path, f"--{cuda_test_type}"] - - # Use command builder for optional parameters + cli_test_type = TEST_TYPE_MAP.get(test_type, test_type.lower()) + base_command = [self._get_binary_path(device), f"--{cli_test_type}"] param_mappings = [ ("device_id", "--device"), ("iterations", "--iterations"), ("array_size", "--array-size"), ] - return build_command_from_config(base_command, config, param_mappings) - def _execute_test(self, cmd: List[str], test_type: str) -> str: - """Execute CUDA test and return output.""" - if not Path(self.cuda_perf_path).exists(): - raise RuntimeError(f"cuda_perf_suite not found: {self.cuda_perf_path}") + def _execute_test( + self, cmd: List[str], test_type: str, device: str = "cuda" + ) -> str: + """Execute a benchmark and return stdout.""" + binary = self._get_binary_path(device) + if not Path(binary).exists(): + if device == "cuda": + raise RuntimeError(f"cuda_perf_suite not found: {binary}") + raise RuntimeError(f"{device} benchmark binary not found: {binary}") logger.info("Executing: %s", " ".join(cmd)) timeout = ( @@ -186,42 +252,46 @@ def _execute_test(self, cmd: List[str], test_type: str) -> str: ) return result.stdout - def _parse_output(self, output: str, test_type: str, run_id: str) -> List[Dict]: - """Parse test output based on test type.""" + def _parse_output( + self, + output: str, + test_type: str, + run_id: str, + device: str = "cuda", + ) -> List[Dict]: + """Parse benchmark output while preserving existing CUDA metric names.""" + cache_parser = self._get_device_config(device)["cache_parser"] if test_type == "Comprehensive": return ( self._parse_memory_bandwidth(output, run_id, METRIC_PREFIX_MEM_SWEEP) + self._parse_stream_benchmark(output) - + self._parse_cache_bandwidth(output, run_id) + + self._parse_cache_for_platform(output, run_id, cache_parser) ) - # Single test type metric_map = { - "MemSweep": (METRIC_PREFIX_MEM_SWEEP, self._parse_memory_bandwidth), - "Stream": (None, self._parse_stream_benchmark), - "Cache": (None, self._parse_cache_bandwidth), + "MemSweep": lambda: self._parse_memory_bandwidth( + output, run_id, METRIC_PREFIX_MEM_SWEEP + ), + "Stream": lambda: self._parse_stream_benchmark(output), + "Cache": lambda: self._parse_cache_for_platform( + output, run_id, cache_parser + ), } - - if test_type in metric_map: - prefix, parser = metric_map[test_type] - return parser(output, run_id, prefix) if prefix else parser(output, run_id) - return [] + parser = metric_map.get(test_type) + return parser() if parser else [] def _parse_memory_bandwidth( self, output: str, run_id: str, metric_prefix: str ) -> List[Dict]: - """Parse memory bandwidth test output.""" + """Parse memory bandwidth sweep output.""" metrics = [] is_sweep = "sweep" in metric_prefix - for direction_label, key in MEMORY_DIRECTIONS: csv_data = self._parse_bandwidth_data(output, direction_label) - if not csv_data: continue metric_name = f"{metric_prefix}_{key}" - if is_sweep: metrics.append( self._create_timeseries_metric( @@ -241,36 +311,36 @@ def _parse_memory_bandwidth( "unit": "GB/s", } ) - return metrics def _parse_bandwidth_data(self, output: str, direction: str) -> List[Dict]: - """Parse bandwidth data for a specific direction.""" + """Parse bandwidth rows for one transfer direction.""" csv_data = [] - - # Sweep format - match from direction header to next section - # The pattern stops at: ==== (next section), Direction:, STREAM:, or end of string - sweep_pattern = rf"{direction}.*?Size \(MB\)\s+Time \(ms\)Bandwidth \(GB/s\)\s+CV \(%\)\s*-+\s*(.*?)\s*(?=\n=+|Direction:|STREAM:|\Z)" + sweep_pattern = ( + rf"{direction}.*?Size \(MB\)\s+Time \(ms\)\s*Bandwidth \(GB/s\)" + rf"\s+CV \(%\)\s*-+\s*(.*?)\s*" + rf"(?=\n=+|Direction:|STREAM:|\Z)" + ) sweep_match = re.search(sweep_pattern, output, re.DOTALL) - if sweep_match: - data_block = sweep_match.group(1) - for line in data_block.strip().split("\n"): + for line in sweep_match.group(1).strip().split("\n"): line = line.strip() - if line and not line.startswith("-"): + if line and not line.startswith("-") and not line.startswith("NOTE"): result = self._parse_sweep_line(line) if result: csv_data.append(result) - return csv_data @staticmethod def _parse_sweep_line(line: str) -> Optional[Dict]: - """Parse a line from sweep format output.""" + """Parse one memory sweep row.""" parts = line.split() if len(parts) >= 3: try: - return {"size_mb": float(parts[0]), "bandwidth_gbps": float(parts[2])} + return { + "size_mb": float(parts[0]), + "bandwidth_gbps": float(parts[2]), + } except (ValueError, IndexError): pass return None @@ -283,7 +353,7 @@ def _create_timeseries_metric( fields: List[str], unit: str = "GB/s", ) -> Dict: - """Create a timeseries metric with CSV file.""" + """Create a timeseries metric and its CSV file.""" return create_timeseries_metric( output_dir=self.output_dir, metric_name=name, @@ -296,12 +366,12 @@ def _create_timeseries_metric( def _parse_stream_benchmark(self, output: str, run_id: str = None) -> List[Dict]: """Parse STREAM benchmark output.""" metrics = [] - for op in STREAM_OPERATIONS: - match = re.search(rf"STREAM_{op.capitalize()}\s+(\d+\.\d+)", output) + for operation in STREAM_OPERATIONS: + match = re.search(rf"STREAM_{operation.capitalize()}\s+(\d+\.\d+)", output) if match: metrics.append( { - "name": f"hardware.stream_{op}", + "name": f"hardware.stream_{operation}", "value": float(match.group(1)), "type": "scalar", "unit": "GB/s", @@ -309,14 +379,19 @@ def _parse_stream_benchmark(self, output: str, run_id: str = None) -> List[Dict] ) return metrics + def _parse_cache_for_platform( + self, output: str, run_id: str, parser_name: str + ) -> List[Dict]: + if parser_name == "cuda": + return self._parse_cache_bandwidth(output, run_id) + raise ValueError(f"Unknown cache parser: {parser_name}") + def _parse_cache_bandwidth(self, output: str, run_id: str) -> List[Dict]: - """Parse cache bandwidth sweep test output.""" + """Parse the existing CUDA L1 and L2 cache output.""" metrics = [] - - # Parse L1 l1_match = re.search(L1_CACHE_PATTERN, output, re.DOTALL) if l1_match: - l1_data = self._parse_cache_lines(l1_match.group(1), cache_level="l1") + l1_data = self._parse_cache_lines(l1_match.group(1), "l1") if l1_data: metrics.append( self._create_timeseries_metric( @@ -327,10 +402,9 @@ def _parse_cache_bandwidth(self, output: str, run_id: str) -> List[Dict]: ) ) - # Parse L2 l2_match = re.search(L2_CACHE_PATTERN, output, re.DOTALL) if l2_match: - l2_data = self._parse_cache_lines(l2_match.group(1), cache_level="l2") + l2_data = self._parse_cache_lines(l2_match.group(1), "l2") if l2_data: metrics.append( self._create_timeseries_metric( @@ -340,41 +414,21 @@ def _parse_cache_bandwidth(self, output: str, run_id: str) -> List[Dict]: L2_CACHE_CSV_FIELDS, ) ) - return metrics def _parse_cache_lines(self, text: str, cache_level: str) -> List[Dict]: - """ - Parse cache metrics from text lines. - - Args: - text: Text containing cache data lines - cache_level: Either 'l1' or 'l2' - - Returns: - List of parsed cache metric dictionaries - """ - csv_data = [] - + """Parse cache metric rows.""" + rows = [] for line in text.strip().split("\n"): - if parsed := self._parse_cache_line(line, cache_level): - csv_data.append(parsed) + parsed = self._parse_cache_line(line, cache_level) + if parsed: + rows.append(parsed) + return rows - return csv_data - - def _parse_cache_line(self, line: str, cache_level: str) -> Optional[Dict]: - """ - Parse a single cache line. - - Args: - line: Line of text containing cache metrics - cache_level: Either 'l1' or 'l2' - - Returns: - Dictionary with parsed metrics or None if parsing fails - """ + @staticmethod + def _parse_cache_line(line: str, cache_level: str) -> Optional[Dict]: + """Parse one CUDA-style cache metric row.""" parts = line.split() - if cache_level == "l1" and len(parts) >= 5: try: return { @@ -386,7 +440,6 @@ def _parse_cache_line(self, line: str, cache_level: str) -> Optional[Dict]: } except (ValueError, IndexError): pass - elif cache_level == "l2" and len(parts) >= 7: try: return { @@ -399,5 +452,4 @@ def _parse_cache_line(self, line: str, cache_level: str) -> Optional[Dict]: } except (ValueError, IndexError): pass - return None diff --git a/infinimetrics/utils/hardware_detector.py b/infinimetrics/utils/hardware_detector.py index 2af64bcc..ad9bb40a 100644 --- a/infinimetrics/utils/hardware_detector.py +++ b/infinimetrics/utils/hardware_detector.py @@ -26,6 +26,7 @@ class HardwareDetector: "--format=csv,noheader", ] AMD_SMI_CANDIDATES = ["amd-smi", "rocm-smi"] + MTHREADS_SMI_QUERY = ["mthreads-gmi", "-q"] @classmethod def detect(cls, accel_type_hint: str = "") -> Dict[str, Any]: @@ -53,6 +54,12 @@ def detect(cls, accel_type_hint: str = "") -> Dict[str, Any]: hw["accelerator_type"] = "nvidia" hw["cuda_version"] = cls._get_cuda_version() or hw["cuda_version"] return hw + if probe == "moore" and cls._probe_mthreads(hw): + hw["accelerator_type"] = "moore" + musa_ver = cls._get_musa_version() + if musa_ver: + hw["cuda_version"] = f"MUSA {musa_ver}" + return hw if probe == "amd" and cls._probe_amd(hw): hw["accelerator_type"] = "amd" return hw @@ -107,12 +114,9 @@ def _detect_memory(cls, hw: Dict[str, Any]) -> None: @classmethod def _get_probe_order(cls, hint: str) -> List[str]: hint = hint.lower().strip() - probes = ( - [hint] - if hint in ("nvidia", "amd", "ascend", "cambricon", "generic") - else [] - ) - for p in ["nvidia", "amd", "ascend", "cambricon", "generic"]: + valid_types = ("nvidia", "moore", "amd", "ascend", "cambricon", "generic") + probes = [hint] if hint in valid_types else [] + for p in ["nvidia", "moore", "amd", "ascend", "cambricon", "generic"]: if p not in probes: probes.append(p) return probes @@ -140,6 +144,34 @@ def _probe_nvidia(cls, hw: Dict[str, Any]) -> bool: except Exception: return False + @classmethod + def _probe_mthreads(cls, hw: Dict[str, Any]) -> bool: + try: + if not _which("mthreads-gmi"): + return False + r = subprocess.run( + cls.MTHREADS_SMI_QUERY, capture_output=True, text=True, timeout=5 + ) + if r.returncode != 0 or not r.stdout.strip(): + return False + + gpu_count = 0 + gpu_name = None + for line in r.stdout.splitlines(): + stripped = line.strip() + if re.match(r"^GPU\d+\s", stripped): + gpu_count += 1 + if stripped.startswith("Product Name") and ":" in stripped: + gpu_name = stripped.split(":", 1)[1].strip() + + if gpu_count > 0: + hw["gpu_count"] = max(hw["gpu_count"], gpu_count) + if hw["gpu_model"] == "Unknown": + hw["gpu_model"] = gpu_name or "Moore Threads GPU" + return True + except Exception: + return False + @classmethod def _probe_amd(cls, hw: Dict[str, Any]) -> bool: try: @@ -238,3 +270,18 @@ def _get_cuda_version(cls) -> Optional[str]: except Exception: pass return None + + @classmethod + def _get_musa_version(cls) -> Optional[str]: + try: + r = subprocess.run( + ["mcc", "--version"], capture_output=True, text=True, timeout=2 + ) + if r.returncode == 0: + for line in r.stdout.splitlines(): + m = re.search(r"(\d+\.\d+\.\d+)", line) + if m: + return m.group(1) + except Exception: + pass + return None diff --git a/tests/test_hardware_adapter.py b/tests/test_hardware_adapter.py new file mode 100644 index 00000000..4262fab2 --- /dev/null +++ b/tests/test_hardware_adapter.py @@ -0,0 +1,216 @@ +from pathlib import Path + +import pytest + +from infinimetrics.dispatcher import Dispatcher +from infinimetrics.hardware import hardware_adapter +from infinimetrics.hardware.hardware_adapter import HardwareTestAdapter + + +CUDA_OUTPUT = """ +Direction: Host to Device +Size (MB) Time (ms)Bandwidth (GB/s) CV (%) +------------------------------------------------------- +64.00 1.000 12.50 1.0 + +=================================================== +STREAM Benchmark Suite +STREAM_Copy 100.00 +STREAM_Scale 90.00 +STREAM_Add 80.00 +STREAM_Triad 70.00 + +L1 Cache Bandwidth Sweep Test +Eff. bw +------------------------------------------------------- +4 kB 1.0ms 0.1% 200.0GB/s +L2 Cache Bandwidth Sweep Test +Eff. bw +------------------------------------------------------- +256 kB 64 kB 1.0ms 0.1% 300.0GB/s +""" + + +def test_constructor_preserves_cuda_perf_path_and_default_layout(tmp_path): + custom_binary = tmp_path / "custom_cuda_perf_suite" + adapter = HardwareTestAdapter(str(custom_binary), output_dir=str(tmp_path)) + + assert adapter.cuda_perf_path == str(custom_binary) + assert adapter.output_dir == tmp_path + assert adapter.build_dir.name == "cuda-memory-benchmark" + assert adapter.build_script == adapter.build_dir / "build.sh" + + +def test_build_command_preserves_cuda_cli_shape(tmp_path): + binary = tmp_path / "cuda_perf_suite" + adapter = HardwareTestAdapter(str(binary), output_dir=str(tmp_path)) + + command = adapter._build_command( + { + "test_type": "Comprehensive", + "device_id": 2, + "iterations": 7, + "array_size": 1024, + } + ) + + assert command == [ + str(binary), + "--all", + "--device", + "2", + "--iterations", + "7", + "--array-size", + "1024", + ] + + +def test_parse_stream_preserves_metric_names(tmp_path): + adapter = HardwareTestAdapter(output_dir=str(tmp_path)) + + metrics = adapter._parse_stream_benchmark(CUDA_OUTPUT) + + assert metrics == [ + { + "name": "hardware.stream_copy", + "value": 100.0, + "type": "scalar", + "unit": "GB/s", + }, + { + "name": "hardware.stream_scale", + "value": 90.0, + "type": "scalar", + "unit": "GB/s", + }, + { + "name": "hardware.stream_add", + "value": 80.0, + "type": "scalar", + "unit": "GB/s", + }, + { + "name": "hardware.stream_triad", + "value": 70.0, + "type": "scalar", + "unit": "GB/s", + }, + ] + + +def test_parse_comprehensive_preserves_cuda_metric_names(tmp_path): + adapter = HardwareTestAdapter(output_dir=str(tmp_path)) + + metrics = adapter._parse_output(CUDA_OUTPUT, "Comprehensive", "run-1") + names = [metric["name"] for metric in metrics] + + assert names == [ + "hardware.mem_sweep_h2d", + "hardware.stream_copy", + "hardware.stream_scale", + "hardware.stream_add", + "hardware.stream_triad", + "hardware.gpu_cache_l1", + "hardware.gpu_cache_l2", + ] + assert Path(tmp_path, "mem_sweep_h2d_run-1_").parent == tmp_path + assert any(tmp_path.glob("mem_sweep_h2d_run-1_*.csv")) + assert any(tmp_path.glob("cache_l1_bandwidth_run-1_*.csv")) + assert any(tmp_path.glob("cache_l2_bandwidth_run-1_*.csv")) + + +@pytest.mark.parametrize("header_spacing", ["", " "]) +def test_parse_memory_bandwidth_accepts_platform_header_spacing( + tmp_path, header_spacing +): + output = f""" +Direction: Host to Device +Size (MB) Time (ms){header_spacing}Bandwidth (GB/s) CV (%) +------------------------------------------------------- +64.00 1.000 12.50 1.0 +""" + adapter = HardwareTestAdapter(output_dir=str(tmp_path)) + + metrics = adapter._parse_output(output, "MemSweep", "run-spacing") + + assert [metric["name"] for metric in metrics] == ["hardware.mem_sweep_h2d"] + + +def test_parse_unknown_test_type_returns_no_metrics(tmp_path): + adapter = HardwareTestAdapter(output_dir=str(tmp_path)) + + assert adapter._parse_output(CUDA_OUTPUT, "Unknown", "run-1") == [] + + +@pytest.mark.parametrize( + ("device", "expected"), + [ + ("cuda", "cuda"), + ("nvidia", "cuda"), + ("metax", "metax"), + ("iluvatar", "corex"), + ("hygon", "hygon"), + ("moore", "moore"), + ("musa", "moore"), + ("legacy-unknown-device", "cuda"), + ], +) +def test_explicit_device_aliases_preserve_unknown_cuda_fallback( + tmp_path, device, expected +): + adapter = HardwareTestAdapter(output_dir=str(tmp_path)) + + assert adapter._get_device_type({"device": device}) == expected + + +def test_runtime_detection_ignores_testcase_framework(tmp_path, monkeypatch): + adapter = HardwareTestAdapter(output_dir=str(tmp_path)) + monkeypatch.setattr(hardware_adapter, "detect_platform", lambda: "moore") + + assert ( + adapter._get_device_type({"_testcase": "hardware.cudaUnified.Stream"}) + == "moore" + ) + assert adapter._get_device_type({}) == "moore" + + +@pytest.mark.parametrize("device", ["cuda", "metax", "corex", "hygon", "moore"]) +def test_cuda_compatible_platforms_share_binary(tmp_path, device): + cuda_binary = tmp_path / "cuda_perf_suite" + adapter = HardwareTestAdapter(str(cuda_binary), output_dir=str(tmp_path)) + + assert adapter._get_binary_path(device) == str(cuda_binary) + + +def test_build_cuda_project_preserves_runtime_platform_detection(tmp_path, monkeypatch): + adapter = HardwareTestAdapter(output_dir=str(tmp_path)) + built_platforms = [] + monkeypatch.setattr(hardware_adapter, "detect_platform", lambda: "metax") + monkeypatch.setattr(adapter, "_build_project", built_platforms.append) + + adapter._build_cuda_project() + + assert built_platforms == ["metax"] + + +def test_dispatcher_registers_cudaunified_hardware_framework(): + adapter = Dispatcher()._create_adapter("hardware", "cudaunified") + + assert isinstance(adapter, HardwareTestAdapter) + + +@pytest.mark.parametrize( + "device", + [ + "cuda", + "metax", + "corex", + "iluvatar", + "hygon", + "moore", + ], +) +def test_dispatcher_does_not_register_devices_as_frameworks(device): + with pytest.raises(ValueError, match="Adapter not registered"): + Dispatcher()._create_adapter("hardware", device) diff --git a/tests/test_hardware_detection.py b/tests/test_hardware_detection.py new file mode 100644 index 00000000..723cbf6d --- /dev/null +++ b/tests/test_hardware_detection.py @@ -0,0 +1,59 @@ +from types import SimpleNamespace + +from infinimetrics.common import hardware_info +from infinimetrics.common.hardware_info import HardwareCollector +from infinimetrics.utils import hardware_detector +from infinimetrics.utils.hardware_detector import HardwareDetector + + +MTHREADS_OUTPUT = """ +Attached GPUs : 2 + +GPU0 00000000:03:00.0 + Product Name : MTT S5000 + Product Brand : MTT + GPU UUID : first-uuid + GPU Link Info + +GPU1 00000000:05:00.0 + Product Name : MTT S5000 + Product Brand : MTT + GPU UUID : second-uuid + GPU Link Info +""" + + +def _empty_hardware(): + return { + "gpu_count": 0, + "gpu_model": "Unknown", + "cuda_version": "Unknown", + } + + +def _successful_probe(*args, **kwargs): + return SimpleNamespace(returncode=0, stdout=MTHREADS_OUTPUT) + + +def test_hardware_collector_counts_only_mthreads_device_headers(monkeypatch): + monkeypatch.setattr(hardware_info, "_which", lambda command: command) + monkeypatch.setattr(hardware_info.subprocess, "run", _successful_probe) + monkeypatch.setattr(HardwareCollector, "_collect_musa_version", lambda self: None) + hardware = _empty_hardware() + + result = HardwareCollector()._probe_mthreads("moore", hardware) + + assert result.success + assert result.count == 2 + assert hardware["gpu_count"] == 2 + assert hardware["gpu_model"] == "MTT S5000" + + +def test_hardware_detector_counts_only_mthreads_device_headers(monkeypatch): + monkeypatch.setattr(hardware_detector, "_which", lambda command: command) + monkeypatch.setattr(hardware_detector.subprocess, "run", _successful_probe) + hardware = _empty_hardware() + + assert HardwareDetector._probe_mthreads(hardware) + assert hardware["gpu_count"] == 2 + assert hardware["gpu_model"] == "MTT S5000"