From 8b575c46dcb69e7cab59c92a633a4dc26cfeede5 Mon Sep 17 00:00:00 2001 From: Konstantinos Korres <96335554+2jztricks@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:08:30 +0300 Subject: [PATCH 1/8] Add experimental gfx1100 HIP backend and native mmap support --- CMakeLists.txt | 271 ++++++++++++------- README.md | 3 + cmake/hip_backend.cmake | 51 ++++ docs/AMD_HIP.md | 133 +++++++++ include/strata/core/expert_source.hpp | 10 +- include/strata/hip_compat/cublas_v2.h | 19 ++ include/strata/hip_compat/cuda_fp16.h | 2 + include/strata/hip_compat/cuda_runtime.h | 98 +++++++ include/strata/hip_compat/cuda_runtime_api.h | 2 + include/strata/hip_compat/intrinsics.hpp | 125 +++++++++ include/strata/kernels/bf16_bits.hpp | 10 +- include/strata/kernels/f16_bits.hpp | 10 +- include/strata/kernels/mrope.hpp | 2 +- include/strata/kernels/rope.hpp | 2 +- src/core/device.cu | 10 + src/core/device_main.cpp | 4 + src/core/expert_source.cpp | 130 +++++++-- src/kernels/cuda/fused_gr.cu | 4 + src/kernels/cuda/native_qsa_score.cu | 42 +++ src/kernels/cuda/ple.cu | 5 +- src/kernels/gr_parity.cpp | 139 ++++++++++ tests/core/file_expert_source_test.cpp | 200 ++++++++++++++ tests/hip/handoff.cpp | 122 +++++++++ tests/hip/intrinsics.cpp | 167 ++++++++++++ tests/hip/native_qsa_score.cpp | 95 +++++++ 25 files changed, 1523 insertions(+), 133 deletions(-) create mode 100644 cmake/hip_backend.cmake create mode 100644 docs/AMD_HIP.md create mode 100644 include/strata/hip_compat/cublas_v2.h create mode 100644 include/strata/hip_compat/cuda_fp16.h create mode 100644 include/strata/hip_compat/cuda_runtime.h create mode 100644 include/strata/hip_compat/cuda_runtime_api.h create mode 100644 include/strata/hip_compat/intrinsics.hpp create mode 100644 tests/core/file_expert_source_test.cpp create mode 100644 tests/hip/handoff.cpp create mode 100644 tests/hip/intrinsics.cpp create mode 100644 tests/hip/native_qsa_score.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e876dd0aa0..774aa4b3b94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,13 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) endif() option(STRATA_ENABLE_CUDA "Build the CUDA targets (Phase 2+)" OFF) +option(STRATA_ENABLE_HIP "Build the HIP targets for gfx1100 wave32 hardware" OFF) +if(STRATA_ENABLE_CUDA AND STRATA_ENABLE_HIP) + message(FATAL_ERROR "STRATA_ENABLE_CUDA and STRATA_ENABLE_HIP are separate backends; enable only one") +endif() +if(STRATA_ENABLE_HIP) + include(cmake/hip_backend.cmake) +endif() # The test suite and the micro benchmarks live in tests/ and bench/micro/, which the published source leaves out. set(_strata_tests_default OFF) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/bench/micro) @@ -63,6 +70,13 @@ if(STRATA_ENABLE_CUDA) endif() endforeach() message(STATUS "Strata: CUDA enabled, arch ${CMAKE_CUDA_ARCHITECTURES}") + set(_strata_gpu_runtime_target CUDA::cudart) + set(_strata_gpu_blas_target CUDA::cublas) + set(_strata_gpu_include_directories ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) +elseif(STRATA_ENABLE_HIP) + set(_strata_gpu_runtime_target strata_hip_runtime) + set(_strata_gpu_blas_target roc::hipblas) + set(_strata_gpu_include_directories ${STRATA_HIP_COMPAT_INCLUDE_DIR}) endif() # Warnings live on an INTERFACE target the project's own code links, NOT in add_compile_options: applied @@ -78,11 +92,14 @@ target_compile_options(strata_warnings INTERFACE $<$,$>:/utf-8> $<$,$>>:-Wall> $<$,$>>:-Wextra> - $<$,$>:-Xcompiler=/W4>) + $<$,$>:-Xcompiler=/W4> + $<$:-Wall> + $<$:-Wextra>) if(STRATA_WERROR) target_compile_options(strata_warnings INTERFACE $<$,$>:/WX> - $<$,$>>:-Werror>) + $<$,$>>:-Werror> + $<$:-Werror>) endif() # ---- strata_artifact ------------------------------------------------------- @@ -111,17 +128,16 @@ target_link_libraries(strata_plan INTERFACE strata_warnings) add_executable(strata-plan src/plan/plan_main.cpp) target_link_libraries(strata-plan PRIVATE strata_plan) -# ---- strata_core (CUDA, Phase 2) ------------------------------------------ -# Only built when CUDA is on, so the Phase 1 targets keep working on a machine with no toolkit. The arch -# guard above has already refused anything but sm_120 by the time this runs. -if(STRATA_ENABLE_CUDA) +# ---- strata_core (GPU runtime) --------------------------------------------- +# Only built when a GPU backend is on, so the Phase 1 targets keep working without a toolkit. +if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) add_library(strata_core STATIC src/core/device.cu src/core/pinned.cu src/platform/memory.cpp src/core/graph.cpp src/core/weights.cpp src/core/layout.cpp) target_include_directories(strata_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) - # CUDA::cudart is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and + # The selected runtime is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and # cudaEventQuery: the .cu sources get cudart through the CUDA language rules, a .cpp does not, and the # linker error ("unresolved external symbol cudaGraphInstantiate") names the function rather than the cause. - target_link_libraries(strata_core PUBLIC strata_plan strata_warnings CUDA::cudart) + target_link_libraries(strata_core PUBLIC strata_plan strata_warnings ${_strata_gpu_runtime_target}) add_executable(strata-device src/core/device_main.cpp) target_link_libraries(strata-device PRIVATE strata_core) @@ -130,13 +146,13 @@ if(STRATA_ENABLE_CUDA) target_link_libraries(strata-load PRIVATE strata_core strata_kernels) add_executable(strata-concurrent src/core/concurrent_main.cpp) - target_include_directories(strata-concurrent PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels CUDA::cudart) + target_include_directories(strata-concurrent PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target}) add_executable(strata-overlap src/core/overlap_main.cpp) # a .cpp that calls cudaMemcpyAsync itself needs the toolkit headers and cudart explicitly - target_include_directories(strata-overlap PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels CUDA::cudart) + target_include_directories(strata-overlap PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target}) # The load test needs the 34 GB artifact, so it is registered ONLY where that artifact exists. A test that # silently skips is a test that stops running; an absent registration is visible in the configure output. @@ -201,30 +217,38 @@ if(STRATA_ENABLE_CUDA) # plan v0.3 P6: llama.cpp's block layouts and codebook grids for the i-quant formats (MIT, third_party/ggml) target_include_directories(strata_kernels PRIVATE third_party/ggml) # Match the pinned llama.cpp CUDA arithmetic contract for this adapter only. - set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu - src/kernels/cuda/native_gr_norm.cu - src/kernels/cuda/native_moe.cu - src/kernels/cuda/native_gdn.cu - src/kernels/cuda/native_gr_postops.cu - src/kernels/cuda/native_router.cu - src/kernels/cuda/native_gdn_preprocess.cu - src/kernels/cuda/native_qsa.cu - src/kernels/cuda/native_rope.cu - src/kernels/cuda/native_ple_postops.cu - src/kernels/cuda/native_flash_attn.cu - src/kernels/cuda/native_qsa_indexer.cu - src/kernels/cuda/native_qsa_score.cu - PROPERTIES COMPILE_OPTIONS "--use_fast_math") + if(STRATA_ENABLE_CUDA) + set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu + src/kernels/cuda/native_gr_norm.cu + src/kernels/cuda/native_moe.cu + src/kernels/cuda/native_gdn.cu + src/kernels/cuda/native_gr_postops.cu + src/kernels/cuda/native_router.cu + src/kernels/cuda/native_gdn_preprocess.cu + src/kernels/cuda/native_qsa.cu + src/kernels/cuda/native_rope.cu + src/kernels/cuda/native_ple_postops.cu + src/kernels/cuda/native_flash_attn.cu + src/kernels/cuda/native_qsa_indexer.cu + src/kernels/cuda/native_qsa_score.cu + PROPERTIES COMPILE_OPTIONS "--use_fast_math") + elseif(STRATA_ENABLE_HIP) + # Keep Q8_K's multiply and magic-bias rounding as separate FP32 operations on HIP; contraction to an + # FMA changes tie results. The embedding gather also relies on separately rounded operations. + set_source_files_properties(src/kernels/cuda/elementwise.cu src/kernels/cuda/quantize_act.cu + PROPERTIES COMPILE_OPTIONS "-ffp-contract=off") + endif() set(STRATA_ORACLE_SOURCE_DIR "" CACHE PATH "Pinned llama.cpp source for independent CUDA parity tests") set(STRATA_ORACLE_BUILD_DIR "" CACHE PATH "Matching existing shared llama.cpp build") - if(STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR) + # This oracle is a CUDA ggml build; do not pull CUDA libraries into a HIP build. + if(STRATA_ENABLE_CUDA AND STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR) foreach(parity IN ITEMS native_mmvq native_bf16 native_shared native_gr_norm native_moe native_gdn native_gr_postops native_router native_gdn_preprocess native_gdn_pipeline native_qsa native_rope native_ple_postops native_flash_attn native_qsa_indexer native_qsa_score) add_executable(${parity}_parity bench/micro/${parity}_parity.cpp) target_include_directories(${parity}_parity PRIVATE - "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) + "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${_strata_gpu_include_directories}) target_compile_definitions(${parity}_parity PRIVATE GGML_SHARED GGML_BACKEND_SHARED) - target_link_libraries(${parity}_parity PRIVATE strata_kernels CUDA::cudart) + target_link_libraries(${parity}_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) foreach(component IN ITEMS ggml ggml-base ggml-cpu ggml-cuda) string(REPLACE "-" "_" variable "STRATA_ORACLE_${component}") find_library(${variable} NAMES ${component} PATHS @@ -248,99 +272,132 @@ if(STRATA_ENABLE_CUDA) add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp) target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) - target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu CUDA::cudart) + target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target}) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/file_expert_source_test.cpp") + enable_testing() + add_executable(file_expert_source_test tests/core/file_expert_source_test.cpp) + target_link_libraries(file_expert_source_test PRIVATE strata_engine) + add_test(NAME file_expert_source_test COMMAND file_expert_source_test) + set_tests_properties(file_expert_source_test PROPERTIES TIMEOUT 60) + endif() # `strata generate` (P2.S6). The FIRST user-facing program: everything under it is a component, and this is # what composes them into a token. No tokenizer - the prompt arrives as ids - because Gate C1 compares logits # against llama.cpp on the SAME ids and a tokenizer on one side of that comparison is a second variable. # plan v0.3 P5: batched prompt processing (tensor-core GEMMs through cuBLAS). add_library(strata_prefill STATIC src/prefill/gemm.cu src/prefill/kernels.cu src/prefill/prefill.cpp) - target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(strata_prefill PUBLIC strata_engine CUDA::cublas CUDA::cudart) + target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}) + target_link_libraries(strata_prefill PUBLIC strata_engine ${_strata_gpu_blas_target} ${_strata_gpu_runtime_target}) add_executable(strata src/program/generate.cpp) - target_include_directories(strata PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec CUDA::cudart) + target_include_directories(strata PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec ${_strata_gpu_runtime_target}) + + if(STRATA_ENABLE_HIP) + enable_testing() + set(_strata_hip_tests) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/intrinsics.cpp") + add_executable(hip_intrinsics tests/hip/intrinsics.cpp) + target_link_libraries(hip_intrinsics PRIVATE strata_hip_runtime) + add_test(NAME hip_intrinsics COMMAND hip_intrinsics) + list(APPEND _strata_hip_tests hip_intrinsics) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/handoff.cpp") + add_executable(hip_handoff tests/hip/handoff.cpp) + target_link_libraries(hip_handoff PRIVATE strata_kernels strata_hip_runtime) + add_test(NAME hip_handoff COMMAND hip_handoff) + list(APPEND _strata_hip_tests hip_handoff) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/native_qsa_score.cpp") + add_executable(hip_native_qsa_score tests/hip/native_qsa_score.cpp) + target_link_libraries(hip_native_qsa_score PRIVATE strata_kernels strata_hip_runtime) + add_test(NAME hip_native_qsa_score COMMAND hip_native_qsa_score) + list(APPEND _strata_hip_tests hip_native_qsa_score) + endif() + if(_strata_hip_tests) + set_tests_properties(${_strata_hip_tests} PROPERTIES TIMEOUT 60) + endif() + endif() add_executable(dequant_s2_parity src/kernels/dequant_s2_parity.cpp) # The tests are HOST C++ files that call cudaMalloc/cudaMemcpy themselves, so they need the toolkit headers # and cudart explicitly: enabling the CUDA language gives those to .cu sources automatically, but a .cpp # target is just a C++ target and gets neither. - target_include_directories(dequant_s2_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(dequant_s2_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(dequant_s2_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(dequant_s2_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME dequant_s2_parity COMMAND dequant_s2_parity --selftest) add_executable(s2_gemv_parity src/kernels/s2_gemv_parity.cpp) - target_include_directories(s2_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(s2_gemv_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(s2_gemv_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(s2_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME s2_gemv_parity COMMAND s2_gemv_parity --selftest) add_executable(iq_parity src/kernels/iq_parity.cpp) - target_include_directories(iq_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(iq_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(iq_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(iq_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp) - target_include_directories(shared_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(shared_expert_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME shared_expert_parity COMMAND shared_expert_parity --selftest) add_executable(gr_parity src/kernels/gr_parity.cpp) - target_include_directories(gr_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(gr_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(gr_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(gr_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME gr_parity COMMAND gr_parity --selftest) add_executable(gdn_parity src/kernels/gdn_parity.cpp) - target_include_directories(gdn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(gdn_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(gdn_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(gdn_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME gdn_parity COMMAND gdn_parity --selftest) add_executable(s2_gemv_q8_parity src/kernels/s2_gemv_q8_parity.cpp) - target_include_directories(s2_gemv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(s2_gemv_q8_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME s2_gemv_q8_parity COMMAND s2_gemv_q8_parity --selftest) add_executable(sampler_parity src/kernels/sampler_parity.cpp) - target_include_directories(sampler_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(sampler_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(sampler_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(sampler_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME sampler_parity COMMAND sampler_parity --selftest) add_executable(rope_parity src/kernels/rope_parity.cpp) - target_include_directories(rope_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(rope_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(rope_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(rope_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME rope_parity COMMAND rope_parity --selftest) add_executable(quantize_act_parity src/kernels/quantize_act_parity.cpp) - target_include_directories(quantize_act_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(quantize_act_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(quantize_act_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(quantize_act_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME quantize_act_parity COMMAND quantize_act_parity --selftest) add_executable(router_top10_parity src/kernels/router_top10_parity.cpp) - target_include_directories(router_top10_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(router_top10_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(router_top10_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(router_top10_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME router_top10_parity COMMAND router_top10_parity --selftest) add_executable(s_gemv_parity src/kernels/s_gemv_parity.cpp) - target_include_directories(s_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(s_gemv_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(s_gemv_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(s_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME s_gemv_parity COMMAND s_gemv_parity --selftest) add_executable(elementwise_parity src/kernels/elementwise_parity.cpp) - target_include_directories(elementwise_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(elementwise_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(elementwise_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(elementwise_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME elementwise_parity COMMAND elementwise_parity --selftest) add_executable(bf16_gemv_parity src/kernels/bf16_gemv_parity.cpp) - target_include_directories(bf16_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(bf16_gemv_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME bf16_gemv_parity COMMAND bf16_gemv_parity --selftest) add_executable(s_gemv_q8k_parity src/kernels/s_gemv_q8k_parity.cpp) - target_include_directories(s_gemv_q8k_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(s_gemv_q8k_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME s_gemv_q8k_parity COMMAND s_gemv_q8k_parity --selftest) add_executable(qsa_parity src/kernels/qsa_parity.cpp) - target_include_directories(qsa_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(qsa_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(qsa_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(qsa_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME qsa_parity COMMAND qsa_parity --selftest) # ---- P2.S4: the PLE n-gram hash, the IQ4_NL table read, and the PLE block ---- @@ -352,8 +409,8 @@ if(STRATA_ENABLE_CUDA) # INSIDE the test. A test that passes by finding nothing to compare is the failure mode this project keeps # re-learning. add_executable(ple_parity src/kernels/ple_parity.cpp) - target_include_directories(ple_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(ple_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(ple_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(ple_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) set(STRATA_PLE_FIXTURE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro" CACHE PATH "Directory containing required ple_in.bin and ple_out.bin parity fixtures") add_test(NAME ple_parity COMMAND ple_parity --selftest @@ -362,73 +419,81 @@ if(STRATA_ENABLE_CUDA) # Plan v0.3 P7: INT8 KV storage vs a host reference and vs the FP16 path (GPU, synthetic, no model). add_executable(kv_q8_parity src/kernels/kv_q8_parity.cpp) - target_include_directories(kv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(kv_q8_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(kv_q8_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(kv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME kv_q8_parity COMMAND kv_q8_parity) # KV streaming (docs/kv-streaming-design.md): streamed vs resident attention, bitwise (GPU, synthetic, no model). add_executable(kv_stream_parity src/kernels/kv_stream_parity.cpp) - target_include_directories(kv_stream_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(kv_stream_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(kv_stream_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(kv_stream_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME kv_stream_parity COMMAND kv_stream_parity) # PR #21: Q4_0 KV with Walsh-Hadamard rotation vs a host reference (GPU, synthetic, no model). add_executable(kv_q4_parity src/kernels/kv_q4_parity.cpp) - target_include_directories(kv_q4_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(kv_q4_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(kv_q4_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(kv_q4_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME kv_q4_parity COMMAND kv_q4_parity) add_executable(cvec_parity src/kernels/cvec_parity.cpp) - target_include_directories(cvec_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(cvec_parity PRIVATE strata_kernels CUDA::cudart) + target_include_directories(cvec_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(cvec_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME cvec_parity COMMAND cvec_parity) # Plan v0.3 P3: native MMVQ for 1..8 tokens (bitwise selftest + --bench timing); synthetic data, no model. if(STRATA_BUILD_TESTS) add_executable(native_mmvq_multi bench/micro/native_mmvq_multi.cpp) - target_include_directories(native_mmvq_multi PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(native_mmvq_multi PRIVATE strata_kernels CUDA::cudart) + target_include_directories(native_mmvq_multi PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(native_mmvq_multi PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME native_mmvq_multi COMMAND native_mmvq_multi --selftest) endif() # Plan v0.3 P0.1: working-set lock of a host region (CPU only). add_executable(platform_memory_test src/platform/memory_test.cpp) - target_link_libraries(platform_memory_test PRIVATE strata_core CUDA::cudart) + target_link_libraries(platform_memory_test PRIVATE strata_core ${_strata_gpu_runtime_target}) add_test(NAME platform_memory_test COMMAND platform_memory_test) # Plan v0.3 P2: the n-gram table read straight from the SSD. The selftest needs only the CPU and a temp file. add_executable(ple_reader_test src/ngram/ple_reader_test.cpp) - target_include_directories(ple_reader_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(ple_reader_test PRIVATE strata_kernels CUDA::cudart) + target_include_directories(ple_reader_test PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(ple_reader_test PRIVATE strata_kernels ${_strata_gpu_runtime_target}) add_test(NAME ple_reader_selftest COMMAND ple_reader_test --selftest) - # The GPU test is registered only when CUDA is on, and it runs ON THE TARGET PC. `--selftest` allocates + # The GPU test runs ON THE TARGET PC. `--selftest` allocates # and writes real VRAM, so a machine without a device fails this test rather than skipping it silently - # which is correct: an engine that targets one GPU should not report a pass for having no GPU. - add_test(NAME cuda_device_selftest COMMAND strata-device --selftest) + if(STRATA_ENABLE_CUDA) + add_test(NAME cuda_device_selftest COMMAND strata-device --selftest) + else() + add_test(NAME hip_device_selftest COMMAND strata-device --selftest) + endif() # ---- P2.S5: the GraphRegistry -------------------------------------------------- # A .cu because the test needs real kernels to capture; it links `strata_core` and not `strata_kernels`, # because what it exercises is the capture machinery and not any kernel in it. - if(STRATA_BUILD_TESTS) - add_executable(graph_registry_test bench/micro/graph_registry_test.cu) - target_include_directories(graph_registry_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(graph_registry_test PRIVATE strata_core CUDA::cudart) - add_test(NAME graph_registry_test COMMAND graph_registry_test) + if(STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/graph_registry_test.cu") + add_executable(graph_registry_test bench/micro/graph_registry_test.cu) + target_include_directories(graph_registry_test PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(graph_registry_test PRIVATE strata_core ${_strata_gpu_runtime_target}) + add_test(NAME graph_registry_test COMMAND graph_registry_test) + endif() # ---- P2.S5's GATE: is a device<->pinned-memory handoff graph-capturable? -------------------- # The doorbell protocol puts the D2H of `x_f` and the miss list INSIDE G[l] - that is the whole overlap. The # phase doc says to verify this "before building on it", and this project has already lost two rounds to # building first (a memory-only spin that DEADLOCKS on Windows, and a `cudaEventRecord` inside a capture that # is SILENTLY DROPPED). It links only cudart: it is about the capture machinery, not about any kernel. - add_executable(pinned_capture bench/micro/pinned_capture.cu) - target_include_directories(pinned_capture PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(pinned_capture PRIVATE CUDA::cudart) - add_test(NAME pinned_capture COMMAND pinned_capture) + if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/pinned_capture.cu") + add_executable(pinned_capture bench/micro/pinned_capture.cu) + target_include_directories(pinned_capture PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(pinned_capture PRIVATE ${_strata_gpu_runtime_target}) + add_test(NAME pinned_capture COMMAND pinned_capture) endif() # Tells tests/CMakeLists.txt to add the CUDA-dependent Catch2 target. It cannot be declared there # unconditionally: tests/ is added at the end of this file, but the flag has to be known before it runs. - set(STRATA_CUDA_TESTS ON) + if(STRATA_ENABLE_CUDA) + set(STRATA_CUDA_TESTS ON) + endif() endif() # ---- strata_kernels_cpu (P2.S3) ---- @@ -551,28 +616,30 @@ if(STRATA_NATIVE_EXPERTS) if(MSVC) target_compile_options(strata_mmq PRIVATE $<$:-Xcompiler=/bigobj>) endif() - target_link_libraries(strata_mmq PUBLIC ggml-base CUDA::cudart) + target_link_libraries(strata_mmq PUBLIC ggml-base ${_strata_gpu_runtime_target}) target_link_libraries(strata_prefill PUBLIC strata_mmq) target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1) endif() - if(STRATA_ENABLE_CUDA) + if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) add_executable(native_expert_parity src/kernels/native_expert_parity.cpp) - target_include_directories(native_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact CUDA::cudart) + target_include_directories(native_expert_parity PRIVATE ${_strata_gpu_include_directories}) + target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact ${_strata_gpu_runtime_target}) endif() endif() if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS) add_executable(hit_cpu_order_parity bench/micro/hit_cpu_order_parity.cu) - target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu CUDA::cudart) + target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target}) endif() if(STRATA_BUILD_TESTS) # Plan v0.3 P6: one expert applied to several tokens, bitwise vs the single-token kernel (synthetic, CPU only). # plan v0.3 P5: the device dequantizers vs the artifact's CPU dequantizers on the real shards (not a ctest: needs # the 38 GB model; run it by hand or from tools/gpu_session.py) - add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp) - target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact CUDA::cudart) + if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) + add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp) + target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact ${_strata_gpu_runtime_target}) + endif() add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp) target_link_libraries(expert_multi_test PRIVATE strata_kernels_cpu) @@ -596,8 +663,8 @@ endif() if(STRATA_BUILD_TESTS) enable_testing() - # the unit tests above build from src/; the tests/ tree (model-level checks) is not part of the published repo - if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt) + # Model-level checks may be absent from source distributions. + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt") add_subdirectory(tests) endif() endif() diff --git a/README.md b/README.md index f8234adbcbd..c3d6fbbeddf 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,9 @@ one later with `SETUP.bat` (the same as `START-HERE.bat --setup`; on Linux `./se For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](docs/ORCA.md). It needs an explicit packing conversion and is not an installer menu option. +For an **experimental RX 7900 XTX (gfx1100) Linux source build**, see the +[AMD HIP instructions](docs/AMD_HIP.md). The automatic installer below remains NVIDIA-only. + ## Install **You need:** an NVIDIA RTX 30, 40 or 50 card with 12 GB of VRAM or more, enough RAM for the size you pick (above), diff --git a/cmake/hip_backend.cmake b/cmake/hip_backend.cmake new file mode 100644 index 00000000000..033c0aa835c --- /dev/null +++ b/cmake/hip_backend.cmake @@ -0,0 +1,51 @@ +# Opt-in HIP configuration. Strata's CUDA-shaped kernels currently target RDNA3 gfx1100 wave32. +# CMake/compiler discovery stays machine-independent; pass CMAKE_HIP_COMPILER when it is not on PATH. +if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "") + set(CMAKE_HIP_ARCHITECTURES gfx1100 CACHE STRING "Strata HIP target architecture") +endif() +if(NOT CMAKE_HIP_ARCHITECTURES STREQUAL "gfx1100") + message(FATAL_ERROR + "Strata HIP currently supports only gfx1100 wave32; CMAKE_HIP_ARCHITECTURES is '${CMAKE_HIP_ARCHITECTURES}'") +endif() + +enable_language(HIP) +find_package(hip CONFIG REQUIRED) +find_package(hipblas CONFIG REQUIRED) + +if(NOT TARGET hip::host) + message(FATAL_ERROR "The ROCm hip CMake package did not provide hip::host") +endif() +if(NOT TARGET roc::hipblas) + message(FATAL_ERROR "The ROCm hipblas CMake package did not provide roc::hipblas") +endif() + +# HIP's link step produces a PIE; make Strata and ggml objects PIC for the ROCm linker. +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +set(STRATA_HIP_COMPAT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/strata/hip_compat") +add_library(strata_hip_runtime INTERFACE) +target_include_directories(strata_hip_runtime BEFORE INTERFACE + "${STRATA_HIP_COMPAT_INCLUDE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/include") +target_compile_definitions(strata_hip_runtime INTERFACE STRATA_USE_HIP=1) +target_link_libraries(strata_hip_runtime INTERFACE hip::host) +foreach(_language IN ITEMS CXX HIP) + target_compile_options(strata_hip_runtime INTERFACE + "$<$:-include>" + "$<$:${STRATA_HIP_COMPAT_INCLUDE_DIR}/cuda_runtime.h>") +endforeach() + +# CMake does not infer HIP from Strata's existing CUDA-shaped .cu suffixes. +file(GLOB_RECURSE _strata_hip_sources CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cu" + "${CMAKE_CURRENT_SOURCE_DIR}/bench/*.cu" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cu") +if(_strata_hip_sources) + set_source_files_properties(${_strata_hip_sources} PROPERTIES LANGUAGE HIP) +endif() +foreach(_source IN ITEMS tests/hip/intrinsics.cpp tests/hip/native_qsa_score.cpp) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${_source}") + set_source_files_properties("${_source}" PROPERTIES LANGUAGE HIP) + endif() +endforeach() + +message(STATUS "Strata: HIP enabled, arch ${CMAKE_HIP_ARCHITECTURES}") diff --git a/docs/AMD_HIP.md b/docs/AMD_HIP.md new file mode 100644 index 00000000000..7f1343883ea --- /dev/null +++ b/docs/AMD_HIP.md @@ -0,0 +1,133 @@ +# Experimental AMD HIP backend (gfx1100) + +This is a manual Linux source build for the RX 7900 XTX. It is opt-in; the +NVIDIA installer and CUDA build remain the default. Other AMD architectures, +wave64, Windows HIP, and mixed AMD/NVIDIA execution are outside this contribution. + +The backend maps the CUDA-shaped runtime and BLAS calls to HIP/hipBLAS, uses +RDNA3's signed integer dot instruction for quantized kernels, and supplies +wave32 shuffle/packed-byte operations. CUDA-only QSA matrix instructions have +an ordered FP32 fallback. Prefill uses dequantization plus hipBLAS GEMM; the +CUDA ggml MMQ path is not compiled for HIP. This does not claim CUDA tensor-core +performance or bit-identical answers across the two backends. + +## Build + +Requirements: a working ROCm driver/runtime, HIP development headers and +compiler, hipBLAS development files, CMake 3.24+, a C++20 host compiler, and Git. +The model still needs sufficient system RAM and fast SSD storage for its PLE +table. VRAM occupancy alone is not a throughput measurement. + +```sh +cmake -S . -B build-hip \ + -DCMAKE_BUILD_TYPE=Release \ + -DSTRATA_ENABLE_HIP=ON -DSTRATA_ENABLE_CUDA=OFF \ + -DCMAKE_HIP_ARCHITECTURES=gfx1100 +cmake --build build-hip --target strata -j2 +``` + +If CMake cannot find the HIP compiler, add +`-DCMAKE_HIP_COMPILER=/path/to/rocm/llvm/bin/clang++`. On the Fedora-family test +host this was `/usr/lib64/rocm/llvm/bin/clang++`. Use the compiler and libraries +from the same ROCm installation. Do not enable both GPU backends in one build. + +Native IQ experts are enabled by default. CMake fetches the llama.cpp revision +pinned by this repository. For an offline build, point `STRATA_GGML_DIR` at a +checkout of that exact revision; using an arbitrary newer checkout changes the +dependency being tested. + +## Model and serving configuration + +Prepare a supported model pack and its MTP runtime using the existing tools. +For OrcaRouter IQ3_XXS, follow [the explicit compatibility conversion](ORCA.md); +this backend does not change quantization, model licenses, or tokenizers. +Add `--experts-bin` to the `tools/iq_pack.py` command: mmap requires the pack's +`experts.bin`, which the default native packing command does not emit. This +consumes additional disk space. Original GSQ-RCO IQ3_XXS uses shard 2 for PLE; +Orca uses shard 1. Keep each model's own pack and tokenizer together. + +Use `build-hip/strata` as the executable in the server JSON. Select the AMD +device with `ROCR_VISIBLE_DEVICES`/`HIP_VISIBLE_DEVICES` if necessary. Start with +a modest context and prefill chunk size before measuring larger workloads. +Keep the PLE file on an SSD. Do not assume a configured context length proves +successful full-window inference. + +Large pinned host allocations can fail on ROCm even when ordinary RAM is +available. The existing `--mmap-experts` path avoids allocating the full pinned +expert arena; it still depends on OS file-cache residency and may stall on +storage reads. It does not make SSD access equivalent to RAM. + +Starting args for the original GSQ-RCO IQ3_XXS model (replace the paths): + +```sh +build-hip/strata --serve \ + --pack packs/iq3xxs \ + --native /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf \ + --ple-gguf /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf \ + --mmap-experts --expert-profile data/expert-profile.bin --expert-cache auto \ + --prefill 512 --spec 4 --spec-min-p 0.5 --mtp mtp/rt \ + --max-context 4096 --kv int8 --pool-workers 15 \ + --adapt-every 0 --pcie-frac 0 --vram-reserve-mib 1024 +``` + +`--serve` is the engine's internal token protocol. For a browser or OpenAI API, +put these args in the `args` array of the [server JSON example](ORCA.md#local-server), +set `exe` to `build-hip/strata`, and use the matching pack's `tokenizer` directory. +Launch with `.venv/bin/python -m serve.server --engine strata --config strata-hip.json --port 8080`. +The worker count above was used on a 16-core CPU; measure it for your CPU. +The 4K context is a smoke-test starting point, not a model limit. The expert cache +sizes itself automatically and leaves 1 GiB of VRAM headroom. + +The automatic installer, +vision helper, local gateway integrations, and multi-GPU experiments are not +included in this backend patch. + +## Validation + +Tested against upstream `c1e903310f211e6630780c3bd2038778c071c68d` (0.1.20), +with pinned llama.cpp `3cf03257f219afbe7334045ff7c6a06ac68c627d`. +The host has an RX 7900 XTX (24 GiB), Ryzen 9 7950X3D, 64 GiB system RAM, +and Fedora-family Linux with ROCm 7.1.52802 / Clang 20. The original model used +for the smoke check was on mechanical RAID0; cold page faults were slow. Keep +latency-sensitive data on SSD and measure warm residency separately. + +- Complete HIP Release build, including the executable and parity targets. +- All 25 tests selected by the command below passed on 2026-09-29. +- Intrinsic parity includes signed packed-byte dot products and overflow. +- CPU/GPU mapped-memory handoff covers delayed publication, changing payloads, + and repeated graph replay in both directions. +- GR coverage includes the maximum eight-token fused batch and graph replay + after inputs change. Native QSA checks the scalar HIP fallback. +- Q8_K quantization is byte-exact against the existing reference. HIP's + `__fmul_rn` can become ordinary multiplication; disabling contraction in + `quantize_act.cu` preserves the separately rounded product before the magic + rounding bias is added. CUDA's compile flags remain unchanged. +- The mmap regression covers canonical and differing native layer sizes, + truncated/oversized files, lookup bounds, and reopening after errors. +- CUDA 13 / GCC 15 / sm_89 executable build passed; GR, sampler, quantization, + and mmap tests passed on the same tree. This is a regression check, not a + claim of complete CUDA inference validation. +- CPU-only `strata-plan` build passed without either GPU backend. +- Real GSQ-RCO IQ3_XXS server smoke with mmap, the shipped expert profile, + automatic GPU cache, and MTP passed arithmetic, executable Python addition, + system-marker recall, and 1,170-token batched-prefill recall. All four requests + ended normally with correct answers; one engine process served them all. + The cache held 11,142 experts (18.08 GiB), with 874 MiB of VRAM free after + startup. These short checks do not establish a throughput or quality benchmark. + +Build all targets before running the registered focused checks: + +```sh +cmake --build build-hip -j2 +ctest --test-dir build-hip --output-on-failure --timeout 60 \ + -E '^(ple_parity|platform_memory_test)$' +``` + +`ple_parity` requires an external model fixture. `platform_memory_test` requests +256 MiB of locked memory; the test shell's 8 MiB memlock limit prevented that +test from passing. These are explicit exclusions, not skipped tests counted as +passes. Additional unpublished upstream fixture suites are not covered. + +Earlier throughput measurements from a larger private working branch are not +results of this patch. Vision, long-context stress, broad answer-quality +equivalence, other AMD cards, and mixed-vendor inference are not validated here. diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp index adec516e647..549674c1a5e 100644 --- a/include/strata/core/expert_source.hpp +++ b/include/strata/core/expert_source.hpp @@ -42,7 +42,7 @@ class ExpertSource { public: virtual ~ExpertSource() = default; - /// The 1,382,400-byte blob for `(layer, expert)`, or nullptr if it cannot be produced. + /// The expert-layout blob for `(layer, expert)`, or nullptr if it cannot be produced. /// /// The pointer only has to stay valid until the next `blob()` call: with `h = 0` every expert is computed /// immediately and nothing is retained. A CACHING source must return pointers into the cache, not into a @@ -270,7 +270,7 @@ class FileExpertSource : public ExpertSource { FileExpertSource(const FileExpertSource&) = delete; FileExpertSource& operator=(const FileExpertSource&) = delete; - /// Maps `/experts.bin` and checks its size against `n_layers * n_expert * BLOB`. + /// Maps `/experts.bin` and checks its size against the loaded expert layout. /// /// The size check is not a formality: a short file would fault at the END of a long sequence, and an /// over-long one means the pack is not the one the geometry came from. Refuses with the two numbers. @@ -287,10 +287,16 @@ class FileExpertSource : public ExpertSource { int64_t reads() const override { return reads_; } private: + const uint8_t* mapped_blob(int64_t layer, int64_t expert) const; + const uint8_t* base_ = nullptr; int64_t blobs_ = 0; + int64_t n_layers_ = 0; int64_t n_expert_ = 0; int64_t reads_ = 0; + uint64_t mapped_bytes_ = 0; + std::vector layer_offsets_; + std::vector layer_blob_bytes_; #if defined(_WIN32) void* file_ = nullptr; void* mapping_ = nullptr; diff --git a/include/strata/hip_compat/cublas_v2.h b/include/strata/hip_compat/cublas_v2.h new file mode 100644 index 00000000000..f162474bb41 --- /dev/null +++ b/include/strata/hip_compat/cublas_v2.h @@ -0,0 +1,19 @@ +#pragma once +#include +#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F +#define CUBLAS_DEFAULT_MATH HIPBLAS_DEFAULT_MATH +#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT +#define CUBLAS_OP_N HIPBLAS_OP_N +#define CUBLAS_OP_T HIPBLAS_OP_T +#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS +#define CUDA_R_16BF HIP_R_16BF +#define CUDA_R_16F HIP_R_16F +#define CUDA_R_32F HIP_R_32F +#define cublasCreate hipblasCreate +#define cublasDestroy hipblasDestroy +#define cublasGemmEx hipblasGemmEx +#define cublasHandle_t hipblasHandle_t +#define cublasSetMathMode hipblasSetMathMode +#define cublasSetStream hipblasSetStream +#define cublasSetWorkspace hipblasSetWorkspace +#define cublasStatus_t hipblasStatus_t diff --git a/include/strata/hip_compat/cuda_fp16.h b/include/strata/hip_compat/cuda_fp16.h new file mode 100644 index 00000000000..969acef62cd --- /dev/null +++ b/include/strata/hip_compat/cuda_fp16.h @@ -0,0 +1,2 @@ +#pragma once +#include diff --git a/include/strata/hip_compat/cuda_runtime.h b/include/strata/hip_compat/cuda_runtime.h new file mode 100644 index 00000000000..77ee7f72968 --- /dev/null +++ b/include/strata/hip_compat/cuda_runtime.h @@ -0,0 +1,98 @@ +#pragma once +// Included only by STRATA_ENABLE_HIP builds. CUDA builds use NVIDIA headers. +#include +// Do not let HIP's legacy macro corrupt libstdc++ attribute names. +#ifdef __noinline__ +#undef __noinline__ +#endif +#define cudaDevAttrMaxSharedMemoryPerBlockOptin hipDeviceAttributeMaxSharedMemoryPerBlock +#define cudaDeviceGetAttribute hipDeviceGetAttribute +#define cudaDeviceProp hipDeviceProp_t +#define cudaDeviceSynchronize hipDeviceSynchronize +#define cudaDriverGetVersion hipDriverGetVersion +#define cudaErrorNotReady hipErrorNotReady +#define cudaErrorStreamCaptureUnsupported hipErrorStreamCaptureUnsupported +#define cudaError_t hipError_t +#define cudaEventCreate hipEventCreate +#define cudaEventCreateWithFlags hipEventCreateWithFlags +#define cudaEventDestroy hipEventDestroy +#define cudaEventDisableTiming hipEventDisableTiming +#define cudaEventElapsedTime hipEventElapsedTime +#define cudaEventQuery hipEventQuery +#define cudaEventRecord hipEventRecord +#define cudaEventSynchronize hipEventSynchronize +#define cudaEvent_t hipEvent_t +#define cudaFree hipFree +#define cudaFreeHost hipFreeHost +#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize +#define cudaGetDevice hipGetDevice +#define cudaGetDeviceCount hipGetDeviceCount +#define cudaGetDeviceProperties hipGetDeviceProperties +#define cudaGetErrorString hipGetErrorString +#define cudaGetLastError hipGetLastError +#define cudaGraphDestroy hipGraphDestroy +#define cudaGraphExecDestroy hipGraphExecDestroy +#define cudaGraphExec_t hipGraphExec_t +#define cudaGraphGetNodes hipGraphGetNodes +#define cudaGraphLaunch hipGraphLaunch +#define cudaGraphUpload hipGraphUpload +#define cudaGraph_t hipGraph_t +#define cudaHostAlloc hipHostMalloc +#define cudaHostAllocDefault hipHostAllocDefault +#define cudaHostAllocMapped hipHostAllocMapped +#define cudaHostAllocPortable hipHostAllocPortable +#define cudaHostGetDevicePointer hipHostGetDevicePointer +#define cudaHostRegister hipHostRegister +#define cudaHostRegisterMapped hipHostRegisterMapped +#define cudaHostRegisterPortable hipHostRegisterPortable +#define cudaHostUnregister hipHostUnregister +#define cudaLaunchHostFunc hipLaunchHostFunc +#define cudaMalloc hipMalloc +#define cudaMallocHost(...) (::strata::hip_compat::malloc_host(__VA_ARGS__)) +#define cudaMemGetInfo hipMemGetInfo +#define cudaMemcpy hipMemcpy +#define cudaMemcpy2DAsync hipMemcpy2DAsync +#define cudaMemcpyAsync hipMemcpyAsync +#define cudaMemcpyDefault hipMemcpyDefault +#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice +#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost +#define cudaMemcpyHostToDevice hipMemcpyHostToDevice +#define cudaMemset hipMemset +#define cudaMemsetAsync hipMemsetAsync +#define cudaPeekAtLastError hipPeekAtLastError +#define cudaRuntimeGetVersion hipRuntimeGetVersion +#define cudaSetDevice hipSetDevice +#define cudaStreamBeginCapture hipStreamBeginCapture +#define cudaStreamCaptureModeThreadLocal hipStreamCaptureModeThreadLocal +#define cudaStreamCreate hipStreamCreate +#define cudaStreamCreateWithFlags hipStreamCreateWithFlags +#define cudaStreamDestroy hipStreamDestroy +#define cudaStreamEndCapture hipStreamEndCapture +#define cudaStreamNonBlocking hipStreamNonBlocking +#define cudaStreamQuery hipStreamQuery +#define cudaStreamSynchronize hipStreamSynchronize +#define cudaStreamWaitEvent hipStreamWaitEvent +#define cudaStream_t hipStream_t +#define cudaSuccess hipSuccess + +namespace strata::hip_compat { +template +inline hipError_t malloc_host(T** pointer, size_t bytes) { + return hipHostMalloc(reinterpret_cast(pointer), bytes, hipHostMallocDefault); +} +} // namespace strata::hip_compat + +template +inline hipError_t cudaFuncSetAttribute(Kernel kernel, hipFuncAttribute attribute, int value) { + return hipFuncSetAttribute(reinterpret_cast(kernel), attribute, value); +} +inline hipError_t cudaGraphInstantiate(hipGraphExec_t* exec, hipGraph_t graph, unsigned long long flags) { + return hipGraphInstantiateWithFlags(exec, graph, flags); +} +inline hipError_t cudaGraphInstantiate(hipGraphExec_t* exec, hipGraph_t graph, + hipGraphNode_t* error, char* log, size_t size) { + return hipGraphInstantiate(exec, graph, error, log, size); +} +#define cudaMemcpyToSymbol(symbol, ...) hipMemcpyToSymbol(HIP_SYMBOL(symbol), __VA_ARGS__) + +#include "intrinsics.hpp" diff --git a/include/strata/hip_compat/cuda_runtime_api.h b/include/strata/hip_compat/cuda_runtime_api.h new file mode 100644 index 00000000000..cf546fc3b35 --- /dev/null +++ b/include/strata/hip_compat/cuda_runtime_api.h @@ -0,0 +1,2 @@ +#pragma once +#include "cuda_runtime.h" diff --git a/include/strata/hip_compat/intrinsics.hpp b/include/strata/hip_compat/intrinsics.hpp new file mode 100644 index 00000000000..1ab2462f5bf --- /dev/null +++ b/include/strata/hip_compat/intrinsics.hpp @@ -0,0 +1,125 @@ +#pragma once + +// CUDA device intrinsics used by Strata kernels that HIP does not provide on gfx1100. +// This header is included only from the HIP cuda_runtime compatibility shim. +#if defined(__HIPCC__) + +#include + +namespace strata::hip_compat { + +__device__ __forceinline__ int signed_byte(uint32_t word, int lane) { + const unsigned value = (word >> (lane * 8)) & 0xffu; + return value < 0x80u ? static_cast(value) : static_cast(value) - 0x100; +} + +// CUDA's signed __dp4a: four signed byte products accumulated modulo 2^32. +__device__ __forceinline__ int dp4a(int a, int b, int c) { +#if defined(__gfx1100__) && __has_builtin(__builtin_amdgcn_sudot4) + // RDNA3 exposes the signed/unsigned dot4 form. Mark both packed operands + // signed to preserve CUDA __dp4a semantics; keep the portable path for + // other HIP compilers/targets. + return __builtin_amdgcn_sudot4(true, a, true, b, c, false); +#else + const uint32_t ua = static_cast(a); + const uint32_t ub = static_cast(b); + uint32_t sum = static_cast(c); +#pragma unroll + for (int lane = 0; lane < 4; ++lane) + sum += static_cast(signed_byte(ua, lane) * signed_byte(ub, lane)); + return static_cast(sum); +#endif +} + +// CUDA's packed byte subtract wraps independently in each unsigned byte lane. +__device__ __forceinline__ int vsub4(int a, int b) { + const uint32_t ua = static_cast(a); + const uint32_t ub = static_cast(b); + uint32_t out = 0; +#pragma unroll + for (int lane = 0; lane < 4; ++lane) { + const uint32_t x = (ua >> (lane * 8)) & 0xffu; + const uint32_t y = (ub >> (lane * 8)) & 0xffu; + out |= ((x - y) & 0xffu) << (lane * 8); + } + return static_cast(out); +} + +// CUDA's packed signed-byte saturating subtract. +__device__ __forceinline__ int vsubss4(int a, int b) { + const uint32_t ua = static_cast(a); + const uint32_t ub = static_cast(b); + uint32_t out = 0; +#pragma unroll + for (int lane = 0; lane < 4; ++lane) { + int value = signed_byte(ua, lane) - signed_byte(ub, lane); + value = value < -128 ? -128 : (value > 127 ? 127 : value); + out |= (static_cast(value) & 0xffu) << (lane * 8); + } + return static_cast(out); +} + +// CUDA's four-lane byte compare, returning 0xff for each unequal lane and 0 otherwise. +__device__ __forceinline__ int vcmpne4(int a, int b) { + const uint32_t ua = static_cast(a); + const uint32_t ub = static_cast(b); + uint32_t out = 0; +#pragma unroll + for (int lane = 0; lane < 4; ++lane) { + if (((ua >> (lane * 8)) & 0xffu) != ((ub >> (lane * 8)) & 0xffu)) + out |= 0xffu << (lane * 8); + } + return static_cast(out); +} + +// CUDA's mask argument describes participating lanes. The current kernel set uses full +// wave32 masks; reject future partial-mask use instead of silently dropping its semantics. +__device__ __forceinline__ void require_full_wave_mask(uint32_t mask) { + if (mask != 0xffffffffu) __builtin_trap(); +} + +template +__device__ __forceinline__ T shfl_xor_sync(uint32_t mask, T value, int lane_mask, int width = 32) { + require_full_wave_mask(mask); + return __shfl_xor(value, lane_mask, width); +} + +template +__device__ __forceinline__ T shfl_down_sync(uint32_t mask, T value, unsigned delta, int width = 32) { + require_full_wave_mask(mask); + return __shfl_down(value, delta, width); +} + +template +__device__ __forceinline__ T shfl_up_sync(uint32_t mask, T value, unsigned delta, int width = 32) { + require_full_wave_mask(mask); + return __shfl_up(value, delta, width); +} + +template +__device__ __forceinline__ T shfl_sync(uint32_t mask, T value, int source_lane, int width = 32) { + require_full_wave_mask(mask); + return __shfl(value, source_lane, width); +} + +__device__ __forceinline__ unsigned ballot_sync(uint32_t mask, int predicate) { + require_full_wave_mask(mask); + return static_cast(__ballot(predicate)); +} + +} // namespace strata::hip_compat + +#define __dp4a(a, b, c) (::strata::hip_compat::dp4a((a), (b), (c))) +#define __vsub4(a, b) (::strata::hip_compat::vsub4((a), (b))) +#define __vsubss4(a, b) (::strata::hip_compat::vsubss4((a), (b))) +#define __vcmpne4(a, b) (::strata::hip_compat::vcmpne4((a), (b))) +#define __shfl_xor_sync(...) (::strata::hip_compat::shfl_xor_sync(__VA_ARGS__)) +#define __shfl_down_sync(...) (::strata::hip_compat::shfl_down_sync(__VA_ARGS__)) +#define __shfl_up_sync(...) (::strata::hip_compat::shfl_up_sync(__VA_ARGS__)) +#define __shfl_sync(...) (::strata::hip_compat::shfl_sync(__VA_ARGS__)) +#define __ballot_sync(mask, predicate) (::strata::hip_compat::ballot_sync((mask), (predicate))) +// AMD's sleep instruction accepts only 0..15; the synchronization loops use it as a +// backoff hint, so use its smallest portable delay independently of CUDA cycle counts. +#define __nanosleep(cycles) __builtin_amdgcn_s_sleep(1) + +#endif // defined(__HIPCC__) diff --git a/include/strata/kernels/bf16_bits.hpp b/include/strata/kernels/bf16_bits.hpp index 8c72feb8052..5c37701fb92 100644 --- a/include/strata/kernels/bf16_bits.hpp +++ b/include/strata/kernels/bf16_bits.hpp @@ -19,7 +19,7 @@ #include #include -#if defined(__CUDACC__) +#if defined(__CUDACC__) || defined(__HIPCC__) #define STRATA_BF16_HD __host__ __device__ #else #define STRATA_BF16_HD @@ -30,7 +30,11 @@ namespace strata::kernels { /// Round-to-nearest-even f32 -> bf16, returned as raw bits in the LOW half of the uint16. STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) { uint32_t i; +#if defined(__HIP_DEVICE_COMPILE__) + __builtin_memcpy(&i, &f, 4); +#else std::memcpy(&i, &f, 4); +#endif i = (i + ((i >> 16) & 1u) + 0x7FFFu) & 0xFFFF0000u; return (uint16_t) (i >> 16); } @@ -39,7 +43,11 @@ STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) { STRATA_BF16_HD inline float f32_from_bf16(uint16_t h) { const uint32_t i = (uint32_t) h << 16; float f; +#if defined(__HIP_DEVICE_COMPILE__) + __builtin_memcpy(&f, &i, 4); +#else std::memcpy(&f, &i, 4); +#endif return f; } diff --git a/include/strata/kernels/f16_bits.hpp b/include/strata/kernels/f16_bits.hpp index 4f3e8fa659c..396d690a392 100644 --- a/include/strata/kernels/f16_bits.hpp +++ b/include/strata/kernels/f16_bits.hpp @@ -25,7 +25,7 @@ #include #include -#if defined(__CUDACC__) +#if defined(__CUDACC__) || defined(__HIPCC__) #define STRATA_HD __host__ __device__ #else #define STRATA_HD @@ -36,7 +36,11 @@ namespace strata::kernels { /// Round-to-nearest-even f32 -> fp16, returned as raw bits. STRATA_HD inline uint16_t f16_from_f32(float f) { uint32_t x; +#if defined(__HIP_DEVICE_COMPILE__) + __builtin_memcpy(&x, &f, 4); +#else std::memcpy(&x, &f, 4); +#endif const uint32_t sign = (x >> 16) & 0x8000u; const uint32_t rawexp = (x >> 23) & 0xFFu; int exp = (int) rawexp - 127 + 15; @@ -89,7 +93,11 @@ STRATA_HD inline float f32_from_f16(uint16_t h) { out = sign | ((ex - 15 + 127) << 23) | (man << 13); } float f; +#if defined(__HIP_DEVICE_COMPILE__) + __builtin_memcpy(&f, &out, 4); +#else std::memcpy(&f, &out, 4); +#endif return f; } diff --git a/include/strata/kernels/mrope.hpp b/include/strata/kernels/mrope.hpp index 165f3edb2f2..f733116f36e 100644 --- a/include/strata/kernels/mrope.hpp +++ b/include/strata/kernels/mrope.hpp @@ -21,7 +21,7 @@ namespace strata::kernels { void mrope_table_set(const int32_t* device_table); const int32_t* mrope_table(); -#if defined(__CUDACC__) +#if defined(__CUDACC__) || defined(__HIPCC__) /// ggml rope_multi, is_imrope, sections {11, 11, 10, 0}: sector = pair % 32; sector % 3 == 1 -> h (sector < 33), /// == 2 -> w (sector < 30), == 0 -> t (sector < 33). For pairs 0..31 all three bounds hold, so it is pair % 3. __device__ __forceinline__ int mrope_pos(const int32_t* tab, int pos, int pair) { diff --git a/include/strata/kernels/rope.hpp b/include/strata/kernels/rope.hpp index bf93ced0ee3..59b66c88557 100644 --- a/include/strata/kernels/rope.hpp +++ b/include/strata/kernels/rope.hpp @@ -10,7 +10,7 @@ #include -#if defined(__CUDACC__) +#if defined(__CUDACC__) || defined(__HIPCC__) #define STRATA_ROPE_HD __host__ __device__ #else #define STRATA_ROPE_HD diff --git a/src/core/device.cu b/src/core/device.cu index 9b8a53731c9..d0373d8b45a 100644 --- a/src/core/device.cu +++ b/src/core/device.cu @@ -30,7 +30,11 @@ DeviceInfo device_info(int ordinal) { int count = 0; check(cudaGetDeviceCount(&count), "cudaGetDeviceCount"); if (count == 0) { +#if defined(STRATA_USE_HIP) + throw CudaError("no HIP device is present; this backend targets gfx1100 wave32", -1); +#else throw CudaError("no CUDA device is present; Strata targets sm_120 (RTX 5000 series)", -1); +#endif } if (ordinal < 0 || ordinal >= count) { throw CudaError("device ordinal " + std::to_string(ordinal) + " is out of range (have " + @@ -59,12 +63,18 @@ DeviceInfo device_info(int ordinal) { // The engine is written against sm_120. Compiling for it is enforced by CMake; RUNNING on something else // is caught here, because a binary can be carried to a machine with an older card and would otherwise // silently take whatever path the driver chose. +#if defined(STRATA_USE_HIP) + if (std::strncmp(p.gcnArchName, "gfx1100", 7) != 0 || p.warpSize != 32) { + throw CudaError("HIP backend requires validated gfx1100 wave32 hardware", -1); + } +#else if (d.cc_major != 12) { throw CudaError("device " + d.name + " reports compute capability " + std::to_string(d.cc_major) + "." + std::to_string(d.cc_minor) + "; Strata targets sm_120 (RTX 5000 series / Blackwell) only", -1); } +#endif return d; } diff --git a/src/core/device_main.cpp b/src/core/device_main.cpp index e14657e57b7..e97cefca789 100644 --- a/src/core/device_main.cpp +++ b/src/core/device_main.cpp @@ -33,7 +33,11 @@ int main(int argc, char** argv) { try { const strata::core::DeviceInfo d = strata::core::device_info(0); std::printf("device %d: %s\n", d.ordinal, d.name.c_str()); +#if defined(STRATA_USE_HIP) + std::printf(" HIP target gfx1100 wave32\n"); +#else std::printf(" compute capability %d.%d (sm_%d%d)\n", d.cc_major, d.cc_minor, d.cc_major, d.cc_minor); +#endif std::printf(" multiprocessors %d\n", d.multi_processor_count); std::printf(" VRAM total / free %s / %s\n", human(d.total_bytes).c_str(), human(d.free_bytes).c_str()); std::printf(" driver / runtime %d / %d\n", d.driver_version, d.runtime_version); diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp index 6533894b882..7568a255099 100644 --- a/src/core/expert_source.cpp +++ b/src/core/expert_source.cpp @@ -17,7 +17,9 @@ #include #include #include +#include #include +#include #include #if defined(_WIN32) @@ -39,9 +41,74 @@ FileExpertSource::~FileExpertSource() { close(); } bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err) { close(); if (n_layers <= 0 || n_expert <= 0) { err = "FileExpertSource: the geometry is empty"; return false; } - n_expert_ = n_expert; - blobs_ = n_layers * n_expert; - const uint64_t want = (uint64_t) blobs_ * (uint64_t) strata::kernels::cpu::BLOB; + const auto& layout = strata::kernels::cpu::expert_layout(); + if (layout.n_layers != n_layers || layout.n_expert != n_expert) { + err = "FileExpertSource: the requested geometry does not match the loaded expert layout"; + return false; + } + if ((uint64_t) n_layers > (uint64_t) std::numeric_limits::max() / (uint64_t) n_expert) { + err = "FileExpertSource: the expert count overflows"; + return false; + } + const uint64_t blob_count = (uint64_t) n_layers * (uint64_t) n_expert; + if (blob_count > (uint64_t) std::numeric_limits::max() || + (uint64_t) n_layers > (uint64_t) std::numeric_limits::max()) { + err = "FileExpertSource: the expert count overflows"; + return false; + } + + std::vector layer_offsets((size_t) n_layers), layer_blob_bytes((size_t) n_layers); + const uint64_t want = layout.total; + if (want == 0 || want > (uint64_t) std::numeric_limits::max()) { + err = "FileExpertSource: the loaded expert layout has an invalid size"; + return false; + } + if (!layout.native) { + if (blob_count > std::numeric_limits::max() / (uint64_t) strata::kernels::cpu::BLOB) { + err = "FileExpertSource: the canonical expert size overflows"; + return false; + } + const uint64_t canonical_size = blob_count * (uint64_t) strata::kernels::cpu::BLOB; + if (want != canonical_size) { + err = "FileExpertSource: the canonical expert layout has an inconsistent size"; + return false; + } + const uint64_t bytes = (uint64_t) strata::kernels::cpu::BLOB; + const uint64_t layer_bytes = (uint64_t) n_expert * bytes; + for (int64_t layer = 0; layer < n_layers; ++layer) { + layer_offsets[(size_t) layer] = (uint64_t) layer * layer_bytes; + layer_blob_bytes[(size_t) layer] = bytes; + } + } else { + if (layout.offset.size() != (size_t) n_layers || layout.bytes.size() != (size_t) n_layers || + layout.fmt.size() != (size_t) n_layers) { + err = "FileExpertSource: the native expert layout is incomplete"; + return false; + } + uint64_t at = 0; + for (int64_t layer = 0; layer < n_layers; ++layer) { + const size_t i = (size_t) layer; + const uint64_t bytes = (uint64_t) layout.fmt[i].bytes; + if (layout.offset[i] != at || bytes == 0 || layout.bytes[i] != bytes || + bytes > std::numeric_limits::max() / (uint64_t) n_expert) { + err = "FileExpertSource: the native expert layout is invalid at layer " + std::to_string(layer); + return false; + } + const uint64_t layer_bytes = bytes * (uint64_t) n_expert; + if (at > want || layer_bytes > want - at) { + err = "FileExpertSource: the native expert layout exceeds its declared size at layer " + + std::to_string(layer); + return false; + } + layer_offsets[i] = layout.offset[i]; + layer_blob_bytes[i] = bytes; + at += layer_bytes; + } + if (at != want) { + err = "FileExpertSource: the native expert layout has an inconsistent size"; + return false; + } + } const std::string path = pack_dir + "/experts.bin"; #if defined(_WIN32) @@ -76,10 +143,9 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64 if ((uint64_t) sz.QuadPart != want) { char buf[400]; std::snprintf(buf, sizeof buf, - "FileExpertSource: %s is %llu B but %lld layers x %lld experts x %d B is %llu B - this " - "is not the pack this geometry came from", - path.c_str(), (unsigned long long) sz.QuadPart, (long long) n_layers, - (long long) n_expert, (int) strata::kernels::cpu::BLOB, (unsigned long long) want); + "FileExpertSource: %s is %llu B but the loaded expert layout requires %llu B - this is not " + "the pack this geometry came from", + path.c_str(), (unsigned long long) sz.QuadPart, (unsigned long long) want); CloseHandle(f); err = buf; return false; @@ -105,13 +171,13 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64 if (fd < 0) { err = "FileExpertSource: cannot open " + path; return false; } struct stat st{}; if (fstat(fd, &st) != 0) { ::close(fd); err = "FileExpertSource: cannot stat " + path; return false; } - if ((uint64_t) st.st_size != want) { + if (st.st_size < 0 || (uint64_t) st.st_size != want) { char buf[400]; std::snprintf(buf, sizeof buf, - "FileExpertSource: %s is %llu B but %lld layers x %lld experts x %d B is %llu B - this " - "is not the pack this geometry came from", - path.c_str(), (unsigned long long) st.st_size, (long long) n_layers, (long long) n_expert, - (int) strata::kernels::cpu::BLOB, (unsigned long long) want); + "FileExpertSource: %s is %llu B but the loaded expert layout requires %llu B - this is not " + "the pack this geometry came from", + path.c_str(), (unsigned long long) (st.st_size < 0 ? 0 : st.st_size), + (unsigned long long) want); ::close(fd); err = buf; return false; @@ -121,6 +187,12 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64 fd_ = fd; base_ = (const uint8_t*) view; #endif + blobs_ = (int64_t) blob_count; + n_layers_ = n_layers; + n_expert_ = n_expert; + mapped_bytes_ = want; + layer_offsets_ = std::move(layer_offsets); + layer_blob_bytes_ = std::move(layer_blob_bytes); return true; } @@ -132,27 +204,39 @@ void FileExpertSource::close() { mapping_ = nullptr; file_ = nullptr; #else - if (base_ != nullptr) munmap((void*) base_, (size_t) blobs_ * (size_t) strata::kernels::cpu::BLOB); + if (base_ != nullptr) munmap((void*) base_, (size_t) mapped_bytes_); if (fd_ >= 0) ::close(fd_); fd_ = -1; #endif base_ = nullptr; blobs_ = 0; + n_layers_ = 0; + n_expert_ = 0; + mapped_bytes_ = 0; + layer_offsets_.clear(); + layer_blob_bytes_.clear(); reads_ = 0; } +const uint8_t* FileExpertSource::mapped_blob(int64_t layer, int64_t expert) const { + if (base_ == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return nullptr; + const size_t i = (size_t) layer; + if (i >= layer_offsets_.size() || i >= layer_blob_bytes_.size()) return nullptr; + const uint64_t blob_bytes = layer_blob_bytes_[i]; + if (blob_bytes == 0 || (uint64_t) expert > std::numeric_limits::max() / blob_bytes) return nullptr; + const uint64_t expert_offset = (uint64_t) expert * blob_bytes; + const uint64_t layer_offset = layer_offsets_[i]; + if (layer_offset > mapped_bytes_ || expert_offset > mapped_bytes_ - layer_offset) return nullptr; + const uint64_t offset = layer_offset + expert_offset; + if (blob_bytes > mapped_bytes_ - offset) return nullptr; + return base_ + (size_t) offset; +} + const uint8_t* FileExpertSource::blob(int64_t layer, int64_t expert) { - if (base_ == nullptr) return nullptr; - if (layer < 0 || expert < 0) return nullptr; - // **`expert >= n_expert_` IS CHECKED SEPARATELY, AND THE FLAT INDEX ALONE DOES NOT CATCH IT.** The blob - // index is `layer * n_expert + expert`, so `blob(0, 512)` has flat index 512 - which is in range, and is - // `blob(1, 0)`. A router id one past the end of a layer would then read the NEXT LAYER's first expert: - // finite, correctly sized, and wrong. Layer and expert are separate axes and are validated as such. - if (expert >= n_expert_) return nullptr; - const int64_t i = layer * n_expert_ + expert; - if (i >= blobs_) return nullptr; + const uint8_t* result = mapped_blob(layer, expert); + if (result == nullptr) return nullptr; ++reads_; - return base_ + (size_t) i * strata::kernels::cpu::BLOB; + return result; } // ================================ THE ADAPTER ================================ diff --git a/src/kernels/cuda/fused_gr.cu b/src/kernels/cuda/fused_gr.cu index 4b461cc7ca1..203cbb195ba 100644 --- a/src/kernels/cuda/fused_gr.cu +++ b/src/kernels/cuda/fused_gr.cu @@ -178,7 +178,11 @@ __global__ void __launch_bounds__(THREADS) gr_norm_multi_kernel(GrMulti m) { for (int i = t; i < D; i += THREADS) xn[i] *= s_rs[i / N]; } +#if defined(__HIPCC__) +constexpr int TILE = 1280; // eight-token tile fits gfx1100's 64 KiB LDS limit +#else constexpr int TILE = 2560; // xn floats per token staged at a time: 320 chunks of 8, 10 per lane +#endif constexpr int TQ = TILE / 8 / 32; // uint4 weight chunks per lane per tile // Step 2 of `gr_down_kernel` for T tokens. One warp per row (so each lane accumulates the same chunks in the diff --git a/src/kernels/cuda/native_qsa_score.cu b/src/kernels/cuda/native_qsa_score.cu index fe5d447a3ee..cdcfeadd330 100644 --- a/src/kernels/cuda/native_qsa_score.cu +++ b/src/kernels/cuda/native_qsa_score.cu @@ -32,6 +32,7 @@ constexpr int D=128, HEADS=4, R=4, ROWS=32, WARPS=2, STRIDE=36, COMBINE=68; struct TileA { uint32_t x[4]; }; struct TileB { uint32_t x[2]; }; struct TileC { float x[4]={0.0f,0.0f,0.0f,0.0f}; }; +#if !defined(__HIPCC__) #if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800 __device__ __forceinline__ void load_a(TileA& a,const float* p) { const float* src=p+(threadIdx.x%16)*STRIDE+(threadIdx.x/16)*4; @@ -139,6 +140,42 @@ __global__ __launch_bounds__(64,1) void score_kernel( } #endif } +#else +// gfx1100 has no CUDA ldmatrix/mma instruction sequence. Keep the same entry point and +// score contract with an ordered scalar F32 dot for each indexer head. The four heads +// run independently; their ReLU'd scores are then added in the documented head order. +// This path favors a well-defined fallback over pretending the CUDA PTX is portable. +__global__ void scalar_score_kernel( + const float* __restrict__ pooled,const float* __restrict__ query, + const float* __restrict__ bias,const int32_t* __restrict__ step, + int max_cells,float* __restrict__ cells) { + const int n=step[kStepNKv],full=step[kStepNBid]; + if(n<1||n>max_cells||step[kStepPos]!=n-1||full!=n/R|| + step[kStepWidth]!=(n<2051?n:2051))return; + const int row=blockIdx.x; + if(row>full)return; + __shared__ float head_score[HEADS]; + const int head=threadIdx.x; + if(head0.0f?dot:0.0f; + } + __syncthreads(); + if(head==0){ + float sum=__fadd_rn(0.0f,head_score[0]); + sum=__fadd_rn(sum,head_score[1]); + sum=__fadd_rn(sum,head_score[2]); + sum=__fadd_rn(sum,head_score[3]); + if(bias)sum=__fadd_rn(sum,bias[row]); + sum=__fadd_rn(sum,row==full&&n%R?1e9f:0.0f); + sum=__fadd_rn(sum,0.0f); + for(int i=row*R;i(s.p); @@ -163,8 +200,13 @@ void native_qsa_score(const float* pooled,const float* query,const float* bias, for(int i=0;i(stream)>>>( + pooled,query,bias,step,int(max_cells),cells); +#else score_kernel<<(stream)>>>( pooled,query,bias,step,int(max_cells),cells); +#endif const auto error=cudaGetLastError(); if(error!=cudaSuccess)throw std::runtime_error(cudaGetErrorString(error)); } diff --git a/src/kernels/cuda/ple.cu b/src/kernels/cuda/ple.cu index 5a9174e0246..66bb4220618 100644 --- a/src/kernels/cuda/ple.cu +++ b/src/kernels/cuda/ple.cu @@ -228,8 +228,9 @@ void ple_block(const float* emb, const float* hidden, const float* hist_rows, co PleOut& out, void* scratch, void* stream) { const bool native_key = w.key_native_data != nullptr && w.key_bf16 == nullptr; if (native_key && (!emb || !hidden || !hist_rows || !out.result || !scratch || !stream || - !w.key_native_q8_1 || w.key_native_type != 42)) - throw std::invalid_argument("ple_block: native key requires Q2_0 weights, input/output, private scratch and explicit stream"); + !w.key_native_q8_1 || + (w.key_native_type != 42 && w.key_native_type != 18 && w.key_native_type != 23))) + throw std::invalid_argument("ple_block: native key requires Q2_0, IQ3_XXS or IQ4_XS weights, input/output, private scratch and explicit stream"); if (emb == nullptr || hidden == nullptr || hist_rows == nullptr || out.result == nullptr) return; const int n_embd = NG_N_EMBD, hc = NG_HC, hc_dim = NG_HC_DIM; static_assert(NG_N_EMBD == 2560 && NG_HC_DIM == 10240, "native PLE key geometry changed"); diff --git a/src/kernels/gr_parity.cpp b/src/kernels/gr_parity.cpp index b690af1bd38..d1f79fa38b3 100644 --- a/src/kernels/gr_parity.cpp +++ b/src/kernels/gr_parity.cpp @@ -20,6 +20,7 @@ // 6. `gr_write`'s `2*sigmoid`, which centres the gate on 1 so a ZERO injection is a plain residual add. // Asserted as a property, not as a value, because that is what the source comment claims. #include "strata/kernels/gr.hpp" +#include "strata/kernels/fused_gr.hpp" #include @@ -268,6 +269,143 @@ int scalar_activation_contract() { return bad; } +int fused_multi_lds_parity(const float* d_norm, const uint16_t* d_down, const uint16_t* d_up, + const uint16_t* d_inject, float eps) { + using namespace strata::kernels; + constexpr int N = 2560, HC = 4, LR = 320, D = N * HC, T = kFusedGrMaxT; + std::mt19937 rng(0x6f8a); + std::normal_distribution normal(0.0f, 0.3f); + std::vector r((size_t) T * D), bo((size_t) T * N), inj((size_t) T * HC); + for (auto& x : r) x = normal(rng); + for (auto& x : bo) x = normal(rng); + for (auto& x : inj) x = normal(rng); + + float *d_r = nullptr, *d_r_out = nullptr, *d_bo = nullptr, *d_inj = nullptr; + float *d_lo = nullptr, *d_rs = nullptr, *d_inj_out = nullptr, *d_mixed = nullptr, *d_xn = nullptr; + check(cudaMalloc(&d_r, r.size() * sizeof(float)), "multi R"); + check(cudaMalloc(&d_r_out, r.size() * sizeof(float)), "multi R_out"); + check(cudaMalloc(&d_bo, bo.size() * sizeof(float)), "multi bo"); + check(cudaMalloc(&d_inj, inj.size() * sizeof(float)), "multi inj"); + check(cudaMalloc(&d_lo, (size_t) T * LR * sizeof(float)), "multi lo"); + check(cudaMalloc(&d_rs, (size_t) T * HC * sizeof(float)), "multi rs"); + check(cudaMalloc(&d_inj_out, (size_t) T * HC * sizeof(float)), "multi injection"); + check(cudaMalloc(&d_mixed, (size_t) T * N * sizeof(float)), "multi mixed"); + check(cudaMalloc(&d_xn, (size_t) T * D * sizeof(float)), "multi xn"); + check(cudaMemcpy(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy R"); + check(cudaMemcpy(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy bo"); + check(cudaMemcpy(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy inj"); + + std::vector args(T); + for (int t = 0; t < T; ++t) { + auto& a = args[t]; + a.R = d_r + (size_t) t * D; + a.R_out = d_r_out + (size_t) t * D; + a.apply = true; + a.bo_prev = d_bo + (size_t) t * N; + a.inj_prev = d_inj + (size_t) t * HC; + a.w_norm = d_norm; + a.w_down = d_down; + a.w_up = d_up; + a.w_inject = d_inject; + a.eps = eps; + a.lo = d_lo + (size_t) t * LR; + a.rs = d_rs + (size_t) t * HC; + a.inject_out = d_inj_out + (size_t) t * HC; + a.mixed = d_mixed + (size_t) t * N; + } + + struct Snapshot { + std::vector r_out, lo, rs, inject, mixed; + }; + auto snapshot = [&]() { + Snapshot s; + s.r_out.resize((size_t) T * D); + s.lo.resize((size_t) T * LR); + s.rs.resize((size_t) T * HC); + s.inject.resize((size_t) T * HC); + s.mixed.resize((size_t) T * N); + check(cudaMemcpy(s.r_out.data(), d_r_out, s.r_out.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read R_out"); + check(cudaMemcpy(s.lo.data(), d_lo, s.lo.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read lo"); + check(cudaMemcpy(s.rs.data(), d_rs, s.rs.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read rs"); + check(cudaMemcpy(s.inject.data(), d_inj_out, s.inject.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read inject"); + check(cudaMemcpy(s.mixed.data(), d_mixed, s.mixed.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read mixed"); + return s; + }; + auto same = [](const Snapshot& a, const Snapshot& b) { + return std::memcmp(a.r_out.data(), b.r_out.data(), a.r_out.size() * sizeof(float)) == 0 && + std::memcmp(a.lo.data(), b.lo.data(), a.lo.size() * sizeof(float)) == 0 && + std::memcmp(a.rs.data(), b.rs.data(), a.rs.size() * sizeof(float)) == 0 && + std::memcmp(a.inject.data(), b.inject.data(), a.inject.size() * sizeof(float)) == 0 && + std::memcmp(a.mixed.data(), b.mixed.data(), a.mixed.size() * sizeof(float)) == 0; + }; + + cudaStream_t stream = nullptr; + check(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), "multi stream"); + // Max T forces the HIP kernel's full dynamic-LDS request: 8 * 1280 * sizeof(float) = 40 KiB. + fused_gr_read_multi(args.data(), T, d_xn, stream); + check(cudaStreamSynchronize(stream), "multi max-T sync"); + const Snapshot multi = snapshot(); + for (int t = 0; t < T; ++t) fused_gr_read(args[t], stream); + check(cudaStreamSynchronize(stream), "single reference sync"); + const Snapshot single = snapshot(); + int bad = 0; + if (!same(multi, single)) { + std::printf(" fused GR multi max-T differs from single-token calls\n"); + ++bad; + } + + cudaGraph_t graph = nullptr; + cudaGraphExec_t graph_exec = nullptr; + check(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal), "multi graph begin"); + fused_gr_read_multi(args.data(), T, d_xn, stream); + check(cudaStreamEndCapture(stream, &graph), "multi graph end"); + check(cudaGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0), "multi graph instantiate"); + check(cudaGraphLaunch(graph_exec, stream), "multi graph initial replay"); + check(cudaStreamSynchronize(stream), "multi graph initial sync"); + const Snapshot captured = snapshot(); + if (!same(multi, captured)) { + std::printf(" fused GR multi captured graph differs from direct max-T call\n"); + ++bad; + } + + // Reuse the same captured pointers with new payloads; then independently run the single-token path again. + for (size_t i = 0; i < r.size(); ++i) r[i] = -0.7f * r[i] + 0.001f * (float) (i % 17); + for (size_t i = 0; i < bo.size(); ++i) bo[i] = -0.4f * bo[i] + 0.02f; + for (size_t i = 0; i < inj.size(); ++i) inj[i] += 0.3f; + check(cudaMemcpyAsync(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay R"); + check(cudaMemcpyAsync(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay bo"); + check(cudaMemcpyAsync(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay inj"); + check(cudaGraphLaunch(graph_exec, stream), "multi graph changed replay"); + check(cudaStreamSynchronize(stream), "multi graph changed sync"); + const Snapshot replay = snapshot(); + if (std::memcmp(multi.mixed.data(), replay.mixed.data(), multi.mixed.size() * sizeof(float)) == 0) { + std::printf(" fused GR graph replay ignored changed inputs\n"); + ++bad; + } + for (int t = 0; t < T; ++t) fused_gr_read(args[t], stream); + check(cudaStreamSynchronize(stream), "changed single reference sync"); + if (!same(replay, snapshot())) { + std::printf(" fused GR changed graph replay differs from single-token calls\n"); + ++bad; + } + + std::printf(" fused GR multi max-T=8 LDS launch and changing graph replay %s\n", + bad == 0 ? "pass" : "FAIL"); + check(cudaGraphExecDestroy(graph_exec), "multi graph exec destroy"); + check(cudaGraphDestroy(graph), "multi graph destroy"); + check(cudaStreamDestroy(stream), "multi stream destroy"); + check(cudaFree(d_xn), "free multi xn"); + check(cudaFree(d_mixed), "free multi mixed"); + check(cudaFree(d_inj_out), "free multi injection"); + check(cudaFree(d_rs), "free multi rs"); + check(cudaFree(d_lo), "free multi lo"); + check(cudaFree(d_inj), "free multi inj"); + check(cudaFree(d_bo), "free multi bo"); + check(cudaFree(d_r_out), "free multi R_out"); + check(cudaFree(d_r), "free multi R"); + return bad; +} + } // namespace int main(int argc, char** argv) { @@ -602,6 +740,7 @@ int main(int argc, char** argv) { activation_mode_name(mode), ok ? "pass" : "*** FAIL ***", rm, ri); if (!ok) ++bad; } + bad += fused_multi_lds_parity(dN, dD, dU, dJ, eps); select_activation_mode(0); cudaFree(rws_raw); cudaFree(dR); cudaFree(dN); cudaFree(dD); cudaFree(dU); cudaFree(dJ); cudaFree(dM); cudaFree(dI); diff --git a/tests/core/file_expert_source_test.cpp b/tests/core/file_expert_source_test.cpp new file mode 100644 index 00000000000..3a25248ff99 --- /dev/null +++ b/tests/core/file_expert_source_test.cpp @@ -0,0 +1,200 @@ +#include "strata/core/expert_source.hpp" +#include "strata/kernels/cpu/expert.hpp" +#include "strata/kernels/cpu/expert_layout.hpp" + +#if defined(STRATA_NATIVE_EXPERTS) +#include "strata/kernels/cpu/native_expert.hpp" +#include "ggml.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +void require(bool ok, const std::string& message) { + if (!ok) throw std::runtime_error(message); +} + +struct TempDirectory { + fs::path path; + + TempDirectory() { + const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count(); + path = fs::temp_directory_path() / ("strata-file-source-test-" + std::to_string(stamp)); + fs::create_directories(path); + } + + ~TempDirectory() { + std::error_code ignored; + fs::remove_all(path, ignored); + } +}; + +void create_pack(const fs::path& dir, uint64_t bytes, const std::vector>& markers = {}) { + const fs::path file = dir / "experts.bin"; + { + std::ofstream out(file, std::ios::binary | std::ios::trunc); + require((bool) out, "could not create synthetic experts.bin"); + if (bytes > 0) { + out.seekp((std::streamoff) (bytes - 1)); + out.put('\0'); + } + require((bool) out, "could not size synthetic experts.bin"); + } + for (const auto& [offset, marker] : markers) { + require(offset < bytes, "synthetic marker lies beyond experts.bin"); + std::fstream file_out(file, std::ios::binary | std::ios::in | std::ios::out); + require((bool) file_out, "could not open synthetic experts.bin for markers"); + file_out.seekp((std::streamoff) offset); + file_out.put(marker); + require((bool) file_out, "could not write synthetic expert marker"); + } +} + +void check_file_size_rejection(strata::core::FileExpertSource& source, const fs::path& dir, int64_t layers, + int64_t experts, uint64_t expected_bytes) { + std::string err; + source.close(); + fs::resize_file(dir / "experts.bin", expected_bytes - 1); + require(!source.open(dir.string(), layers, experts, err), "a truncated expert file was accepted"); + require(!source.mapped() && !err.empty(), "truncated-file rejection left the source mapped or unreported"); + + fs::resize_file(dir / "experts.bin", expected_bytes + 1); + err.clear(); + require(!source.open(dir.string(), layers, experts, err), "an oversized expert file was accepted"); + require(!source.mapped() && !err.empty(), "oversized-file rejection left the source mapped or unreported"); +} + +void test_canonical_layout() { + using namespace strata::core; + using namespace strata::kernels::cpu; + constexpr int64_t layers = 2; + constexpr int64_t experts = 3; + const uint64_t layer_bytes = (uint64_t) experts * BLOB; + const uint64_t total = (uint64_t) layers * layer_bytes; + TempDirectory dir; + + std::string err; + const bool layout_ok = expert_layout_load(dir.path.string(), layers, experts, err); + require(layout_ok, "could not load canonical layout: " + err); + create_pack(dir.path, total, {{0, 'a'}, {(uint64_t) BLOB, 'b'}, {layer_bytes, 'c'}, + {layer_bytes + (uint64_t) BLOB, 'd'}}); + + FileExpertSource source; + bool opened = source.open(dir.path.string(), layers, experts, err); + require(opened, "could not map canonical pack: " + err); + require(source.blobs() == layers * experts, "canonical blob count is wrong"); + const uint8_t* first = source.blob(0, 0); + const uint8_t* second = source.blob(0, 1); + const uint8_t* next_layer = source.blob(1, 0); + require(first && second && next_layer, "valid canonical blob lookup failed"); + require(second - first == (ptrdiff_t) BLOB && next_layer - first == (ptrdiff_t) layer_bytes, + "canonical expert or layer stride is wrong"); + require(first[0] == 'a' && second[0] == 'b' && next_layer[0] == 'c', + "canonical blob lookup returned bytes from the wrong expert"); + require(source.blob(-1, 0) == nullptr && source.blob(0, -1) == nullptr && + source.blob(layers, 0) == nullptr && source.blob(0, experts) == nullptr, + "canonical bounds check accepted an invalid layer or expert"); + require(source.reads() == 3, "invalid canonical lookups changed the read count"); + + check_file_size_rejection(source, dir.path, layers, experts, total); + create_pack(dir.path, total, {{layer_bytes, 'c'}}); + err.clear(); + opened = source.open(dir.path.string(), layers, experts, err); + require(opened, "source failed to reopen after size errors: " + err); + const uint8_t* reopened = source.blob(1, 0); + require(reopened && source.reads() == 1 && reopened[0] == 'c', "reopened canonical source kept stale state"); + source.close(); +} + +#if defined(STRATA_NATIVE_EXPERTS) +void test_native_variable_layout() { + using namespace strata::core; + using namespace strata::kernels::cpu; + constexpr int64_t layers = 2; + constexpr int64_t experts = 3; + constexpr int64_t embedding = 2560; + constexpr int64_t feed_forward = 640; + TempDirectory dir; + + NativeFmt first_fmt, second_fmt; + std::string err; + const bool first_ok = native_fmt(GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ4_NL, embedding, feed_forward, first_fmt, err); + require(first_ok, "IQ3_XXS/IQ4_NL synthetic format is unavailable: " + err); + err.clear(); + const bool second_ok = native_fmt(GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_NL, embedding, feed_forward, second_fmt, err); + require(second_ok, "IQ2_XS/IQ4_NL synthetic format is unavailable: " + err); + require(first_fmt.bytes != second_fmt.bytes, "chosen native formats do not exercise variable layer sizes"); + + const uint64_t layer0_bytes = (uint64_t) first_fmt.bytes * experts; + const uint64_t total = layer0_bytes + (uint64_t) second_fmt.bytes * experts; + { + std::ofstream metadata(dir.path / "native_experts.txt"); + require((bool) metadata, "could not create synthetic native_experts.txt"); + metadata << "0 " << GGML_TYPE_IQ3_XXS << ' ' << GGML_TYPE_IQ4_NL << " 0 " << first_fmt.bytes << '\n'; + metadata << "1 " << GGML_TYPE_IQ2_XS << ' ' << GGML_TYPE_IQ4_NL << ' ' << layer0_bytes << ' ' + << second_fmt.bytes << '\n'; + require((bool) metadata, "could not write synthetic native_experts.txt"); + } + create_pack(dir.path, total, {{0, 'a'}, {(uint64_t) first_fmt.bytes, 'b'}, + {layer0_bytes, 'c'}, {layer0_bytes + (uint64_t) second_fmt.bytes, 'd'}}); + + const bool layout_ok = expert_layout_load(dir.path.string(), layers, experts, err); + require(layout_ok, "could not load synthetic native layout: " + err); + FileExpertSource source; + bool opened = source.open(dir.path.string(), layers, experts, err); + require(opened, "could not map native pack: " + err); + require(source.blobs() == layers * experts, "native blob count is wrong"); + const uint8_t* first = source.blob(0, 0); + const uint8_t* second = source.blob(0, 1); + const uint8_t* next_layer = source.blob(1, 0); + const uint8_t* next_layer_second = source.blob(1, 1); + require(first && second && next_layer && next_layer_second, "valid native blob lookup failed"); + require(second - first == (ptrdiff_t) first_fmt.bytes && next_layer - first == (ptrdiff_t) layer0_bytes && + next_layer_second - next_layer == (ptrdiff_t) second_fmt.bytes, + "native per-layer expert stride is wrong"); + require(first[0] == 'a' && second[0] == 'b' && next_layer[0] == 'c' && next_layer_second[0] == 'd', + "native blob lookup returned bytes from the wrong expert"); + require(source.blob(-1, 0) == nullptr && source.blob(0, -1) == nullptr && + source.blob(layers, 0) == nullptr && source.blob(0, experts) == nullptr, + "native bounds check accepted an invalid layer or expert"); + require(source.reads() == 4, "invalid native lookups changed the read count"); + + check_file_size_rejection(source, dir.path, layers, experts, total); + create_pack(dir.path, total, {{layer0_bytes, 'c'}}); + err.clear(); + opened = source.open(dir.path.string(), layers, experts, err); + require(opened, "source failed to reopen after size errors: " + err); + const uint8_t* reopened = source.blob(1, 0); + require(reopened && source.reads() == 1 && reopened[0] == 'c', "reopened native source kept stale state"); + source.close(); +} +#endif + +} // namespace + +int main() { + try { + test_canonical_layout(); +#if defined(STRATA_NATIVE_EXPERTS) + test_native_variable_layout(); +#endif + std::cout << "file_expert_source_test: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "file_expert_source_test: " << error.what() << '\n'; + return 1; + } +} diff --git a/tests/hip/handoff.cpp b/tests/hip/handoff.cpp new file mode 100644 index 00000000000..5d358bf6db0 --- /dev/null +++ b/tests/hip/handoff.cpp @@ -0,0 +1,122 @@ +// Real asynchronous CPU/GPU handoff and graph replay, including changing payloads. +#include +#include "strata/kernels/elementwise.hpp" +#include +#include +#include +#include +#include +#define CHECK(x) do {auto e=(x);if(e!=hipSuccess){std::fprintf(stderr,"%s: %s\n",#x,hipGetErrorString(e));return 2;}} while(0) +int main(){ + constexpr int N=4096, rounds=100; + float *host=nullptr,*mapped=nullptr,*device=nullptr; + uint32_t *flag=nullptr,*seq=nullptr,*dflag=nullptr,*dseq=nullptr; + CHECK(hipHostMalloc((void**)&host,N*4,hipHostMallocMapped)); + CHECK(hipHostMalloc((void**)&flag,4,hipHostMallocMapped)); + CHECK(hipHostMalloc((void**)&seq,4,hipHostMallocMapped)); + CHECK(hipHostGetDevicePointer((void**)&mapped,host,0)); + CHECK(hipHostGetDevicePointer((void**)&dflag,flag,0)); + CHECK(hipHostGetDevicePointer((void**)&dseq,seq,0)); + CHECK(hipMalloc((void**)&device,N*4)); + hipStream_t stream;CHECK(hipStreamCreateWithFlags(&stream,hipStreamNonBlocking)); + hipGraph_t graph;hipGraphExec_t exec; + *flag=0;*seq=1; + CHECK(hipStreamBeginCapture(stream,hipStreamCaptureModeThreadLocal)); + strata::kernels::doorbell_wait(dflag,dseq,(void*)stream); + strata::kernels::copy_from_mapped(device,mapped,N,(void*)stream); + strata::kernels::scale_inplace(device,N,2.0f,(void*)stream); + CHECK(hipStreamEndCapture(stream,&graph));CHECK(hipGraphInstantiateWithFlags(&exec,graph,0)); + std::vector got(N); + for(int r=1;r<=rounds;r++){ + *seq=r; + CHECK(hipGraphLaunch(exec,stream)); + // Producer deliberately arrives after the GPU consumer; a missing wait must fail. + std::this_thread::sleep_for(std::chrono::microseconds(50+(r%17)*10)); + for(int i=0;i copy_route_ids(10);std::vector copy_route_weights(10); + CHECK(hipStreamBeginCapture(stream,hipStreamCaptureModeThreadLocal)); + CHECK(hipMemcpyAsync(mapped_copy_out,device,N*4,hipMemcpyDeviceToDevice,stream)); + CHECK(hipMemcpyAsync(mapped_copy_ids,copy_ids_device,10*4,hipMemcpyDeviceToDevice,stream)); + CHECK(hipMemcpyAsync(mapped_copy_weights,copy_weights_device,10*4,hipMemcpyDeviceToDevice,stream)); + strata::kernels::doorbell_ring(device_copy_seq,(void*)stream); + CHECK(hipStreamEndCapture(stream,&graph));CHECK(hipGraphInstantiateWithFlags(&exec,graph,0)); + for(int r=1;r<=rounds;r++){ + for(int i=0;ideadline){std::fprintf(stderr,"separate copy/ring timeout\n");return 10;} + std::this_thread::yield(); + } + for(int i=0;i ids_src(10);std::vector ws(10); + *flag=0; + CHECK(hipStreamBeginCapture(stream,hipStreamCaptureModeThreadLocal)); + strata::kernels::doorbell_publish(device,gi,gw,N,10,dpublished,dids,dweights,dflag,(void*)stream); + CHECK(hipStreamEndCapture(stream,&graph));CHECK(hipGraphInstantiateWithFlags(&exec,graph,0)); + for(int r=1;r<=rounds;r++){ + for(int i=0;ideadline){std::fprintf(stderr,"publish timeout\n");return 7;} + std::this_thread::yield(); + } + for(int i=0;i +#include "strata/hip_compat/intrinsics.hpp" + +#include +#include +#include + +#define CHECK(call) \ + do { \ + const hipError_t error = (call); \ + if (error != hipSuccess) { \ + std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error)); \ + return 2; \ + } \ + } while (0) + +namespace { + +constexpr int kLanes = 32; +constexpr int kIntegerResults = 7; + +__global__ void intrinsic_probe(int* integer_results, float* float_shuffle, double* double_shuffle, + unsigned* ballot, const uint32_t* dot_inputs) { + const int lane = static_cast(threadIdx.x); + constexpr unsigned mask = 0xffffffffu; + const uint32_t a = 0x80ff017fu; + const uint32_t b = 0x0203fe81u; + + integer_results[lane * kIntegerResults + 0] = __dp4a(static_cast(a), static_cast(b), 7); + integer_results[lane * kIntegerResults + 1] = __dp4a(0x7f7f7f7f, 0x7f7f7f7f, 0x7fffffff); + integer_results[lane * kIntegerResults + 2] = __vsub4(0x00ff0102, 0x01010103); + integer_results[lane * kIntegerResults + 3] = __vsub4(0x00000000, 0x01020304); + integer_results[lane * kIntegerResults + 4] = __vsubss4(0x807f0080, 0x017f0180); + integer_results[lane * kIntegerResults + 5] = __vcmpne4(0x00ff0102, 0x01010103); + integer_results[lane * kIntegerResults + 6] = + __dp4a(static_cast(dot_inputs[lane * 2]), static_cast(dot_inputs[lane * 2 + 1]), + 0x7fffffff - lane); + + float_shuffle[lane * 4 + 0] = __shfl_xor_sync(mask, static_cast(lane), 1); + float_shuffle[lane * 4 + 1] = static_cast(__shfl_down_sync(mask, lane, 4, 8)); + float_shuffle[lane * 4 + 2] = static_cast(__shfl_up_sync(mask, lane, 1)); + float_shuffle[lane * 4 + 3] = static_cast(__shfl_sync(mask, lane, 0)); + double_shuffle[lane] = __shfl_xor_sync(mask, static_cast(lane) + 0.25, 1); + + const unsigned hit = __ballot_sync(mask, (lane & 1) == 0); + if (lane == 0) { + *ballot = hit; + __nanosleep(100); + } +} + +int signed_byte(uint32_t word, int lane) { + const unsigned value = (word >> (lane * 8)) & 0xffu; + return value < 0x80u ? static_cast(value) : static_cast(value) - 0x100; +} + +int dp4a_reference(uint32_t a, uint32_t b, int c) { + int64_t sum = c; + for (int lane = 0; lane < 4; ++lane) sum += signed_byte(a, lane) * signed_byte(b, lane); + return static_cast(static_cast(sum)); +} + +uint32_t byte_sub_reference(uint32_t a, uint32_t b) { + uint32_t out = 0; + for (int lane = 0; lane < 4; ++lane) + out |= ((((a >> (lane * 8)) & 0xffu) - ((b >> (lane * 8)) & 0xffu)) & 0xffu) << (lane * 8); + return out; +} + +uint32_t signed_saturating_sub_reference(uint32_t a, uint32_t b) { + uint32_t out = 0; + for (int lane = 0; lane < 4; ++lane) { + int value = signed_byte(a, lane) - signed_byte(b, lane); + if (value < -128) value = -128; + if (value > 127) value = 127; + out |= (static_cast(value) & 0xffu) << (lane * 8); + } + return out; +} + +uint32_t not_equal_reference(uint32_t a, uint32_t b) { + uint32_t out = 0; + for (int lane = 0; lane < 4; ++lane) + if (((a >> (lane * 8)) & 0xffu) != ((b >> (lane * 8)) & 0xffu)) out |= 0xffu << (lane * 8); + return out; +} + +} // namespace + +int main() { + int device = 0; + CHECK(hipGetDevice(&device)); + hipDeviceProp_t properties{}; + CHECK(hipGetDeviceProperties(&properties, device)); + if (properties.warpSize != kLanes) { + std::fprintf(stderr, "expected wave32, got wave%d\n", properties.warpSize); + return 1; + } + + int* device_integers = nullptr; + float* device_float_shuffle = nullptr; + double* device_double_shuffle = nullptr; + unsigned* device_ballot = nullptr; + uint32_t* device_dot_inputs = nullptr; + CHECK(hipMalloc(reinterpret_cast(&device_integers), kLanes * kIntegerResults * sizeof(int))); + CHECK(hipMalloc(reinterpret_cast(&device_float_shuffle), kLanes * 4 * sizeof(float))); + CHECK(hipMalloc(reinterpret_cast(&device_double_shuffle), kLanes * sizeof(double))); + CHECK(hipMalloc(reinterpret_cast(&device_ballot), sizeof(unsigned))); + CHECK(hipMalloc(reinterpret_cast(&device_dot_inputs), kLanes * 2 * sizeof(uint32_t))); + + uint32_t dot_inputs[kLanes * 2]{}; + for (int lane = 0; lane < kLanes; ++lane) { + dot_inputs[lane * 2] = 0x80ff017fu ^ (0x01010101u * static_cast(lane)); + dot_inputs[lane * 2 + 1] = 0x0203fe81u + (0x11111111u * static_cast(lane)); + } + CHECK(hipMemcpy(device_dot_inputs, dot_inputs, sizeof(dot_inputs), hipMemcpyHostToDevice)); + + hipLaunchKernelGGL(intrinsic_probe, dim3(1), dim3(kLanes), 0, 0, + device_integers, device_float_shuffle, device_double_shuffle, device_ballot, device_dot_inputs); + CHECK(hipDeviceSynchronize()); + + int integers[kLanes * kIntegerResults]{}; + float float_shuffle[kLanes * 4]{}; + double double_shuffle[kLanes]{}; + unsigned ballot = 0; + CHECK(hipMemcpy(integers, device_integers, sizeof(integers), hipMemcpyDeviceToHost)); + CHECK(hipMemcpy(float_shuffle, device_float_shuffle, sizeof(float_shuffle), hipMemcpyDeviceToHost)); + CHECK(hipMemcpy(double_shuffle, device_double_shuffle, sizeof(double_shuffle), hipMemcpyDeviceToHost)); + CHECK(hipMemcpy(&ballot, device_ballot, sizeof(ballot), hipMemcpyDeviceToHost)); + + constexpr uint32_t a = 0x80ff017fu, b = 0x0203fe81u; + const uint32_t expected_integer[kIntegerResults - 1] = { + static_cast(dp4a_reference(a, b, 7)), + static_cast(dp4a_reference(0x7f7f7f7fu, 0x7f7f7f7fu, std::numeric_limits::max())), + byte_sub_reference(0x00ff0102u, 0x01010103u), + byte_sub_reference(0x00000000u, 0x01020304u), + signed_saturating_sub_reference(0x807f0080u, 0x017f0180u), + not_equal_reference(0x00ff0102u, 0x01010103u), + }; + bool ok = ballot == 0x55555555u; + for (int lane = 0; lane < kLanes; ++lane) { + for (int i = 0; i < kIntegerResults - 1; ++i) + ok = ok && static_cast(integers[lane * kIntegerResults + i]) == expected_integer[i]; + ok = ok && static_cast(integers[lane * kIntegerResults + kIntegerResults - 1]) == + static_cast(dp4a_reference(dot_inputs[lane * 2], dot_inputs[lane * 2 + 1], + 0x7fffffff - lane)); + ok = ok && float_shuffle[lane * 4 + 0] == static_cast(lane ^ 1); + const int down = lane % 8 < 4 ? lane + 4 : lane; + ok = ok && float_shuffle[lane * 4 + 1] == static_cast(down); + ok = ok && float_shuffle[lane * 4 + 2] == static_cast(lane == 0 ? 0 : lane - 1); + ok = ok && float_shuffle[lane * 4 + 3] == 0.0f; + ok = ok && double_shuffle[lane] == static_cast(lane ^ 1) + 0.25; + } + + CHECK(hipFree(device_ballot)); + CHECK(hipFree(device_dot_inputs)); + CHECK(hipFree(device_double_shuffle)); + CHECK(hipFree(device_float_shuffle)); + CHECK(hipFree(device_integers)); + if (!ok) { + std::fprintf(stderr, "HIP intrinsic parity failed (ballot 0x%08x)\n", ballot); + return 1; + } + std::puts("HIP intrinsics parity OK: dynamic signed dot4/overflow, packed integer boundaries, wave32 shuffles, ballot and sleep"); + return 0; +} diff --git a/tests/hip/native_qsa_score.cpp b/tests/hip/native_qsa_score.cpp new file mode 100644 index 00000000000..6d60eb06ba1 --- /dev/null +++ b/tests/hip/native_qsa_score.cpp @@ -0,0 +1,95 @@ +// Device parity for the scalar HIP implementation of the optional native QSA scorer. +#include +#include "strata/kernels/native_qsa_score.hpp" + +#include +#include +#include + +#define CHECK(call) \ + do { \ + const hipError_t error = (call); \ + if (error != hipSuccess) { \ + std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error)); \ + return 2; \ + } \ + } while (0) + +int main() { + using namespace strata::kernels; + constexpr int kDim = 128, kHeads = 4, kBlock = 4; + constexpr int kMaxCells = 8, kMaxBlocks = 3, kCells = 7; + const QsaShapes shapes = qsa_real_shapes(); + + std::vector pooled(kMaxBlocks * kDim), query(kHeads * kDim), bias(kMaxBlocks), + scores(kMaxCells, -123.0f), expected(kMaxCells, -123.0f); + const float head_sign[kHeads] = {1.0f, -1.0f, 1.0f, -1.0f}; + for (int row = 0; row < kMaxBlocks; ++row) + for (int d = 0; d < kDim; ++d) + pooled[row * kDim + d] = static_cast(row + 1) * (1.0f + static_cast(d % 7) / 8.0f) / 128.0f; + for (int head = 0; head < kHeads; ++head) + for (int d = 0; d < kDim; ++d) + query[head * kDim + d] = head_sign[head] * (1.0f + static_cast((head + d) % 5) / 16.0f) / 16.0f; + for (int row = 0; row < kMaxBlocks; ++row) bias[row] = (static_cast(row) - 1.0f) / 8.0f; + + int32_t step[kStepCount] = {kCells - 1, kCells, kCells / kBlock, static_cast(kCells)}; + bool saw_positive_head = false, saw_negative_head = false; + for (int row = 0; row <= step[kStepNBid]; ++row) { + float sum = 0.0f; + for (int head = 0; head < kHeads; ++head) { + float dot = 0.0f; + for (int d = 0; d < kDim; ++d) + dot = std::fma(pooled[row * kDim + d], query[head * kDim + d], dot); + saw_positive_head = saw_positive_head || dot > 0.0f; + saw_negative_head = saw_negative_head || dot < 0.0f; + sum = sum + (dot > 0.0f ? dot : 0.0f); + } + sum = sum + bias[row]; + if (row == step[kStepNBid] && kCells % kBlock != 0) sum = sum + 1.0e9f; + const int lo = row * kBlock; + const int hi = (row + 1) * kBlock < kCells ? (row + 1) * kBlock : kCells; + for (int cell = lo; cell < hi; ++cell) expected[cell] = sum; + } + if (!saw_positive_head || !saw_negative_head) { + std::fprintf(stderr, "fixture did not exercise both positive and negative head scores\n"); + return 1; + } + + float *device_pooled = nullptr, *device_query = nullptr, *device_bias = nullptr, *device_scores = nullptr; + int32_t* device_step = nullptr; + CHECK(hipMalloc(reinterpret_cast(&device_pooled), pooled.size() * sizeof(float))); + CHECK(hipMalloc(reinterpret_cast(&device_query), query.size() * sizeof(float))); + CHECK(hipMalloc(reinterpret_cast(&device_bias), bias.size() * sizeof(float))); + CHECK(hipMalloc(reinterpret_cast(&device_scores), scores.size() * sizeof(float))); + CHECK(hipMalloc(reinterpret_cast(&device_step), sizeof(step))); + CHECK(hipMemcpy(device_pooled, pooled.data(), pooled.size() * sizeof(float), hipMemcpyHostToDevice)); + CHECK(hipMemcpy(device_query, query.data(), query.size() * sizeof(float), hipMemcpyHostToDevice)); + CHECK(hipMemcpy(device_bias, bias.data(), bias.size() * sizeof(float), hipMemcpyHostToDevice)); + CHECK(hipMemcpy(device_scores, scores.data(), scores.size() * sizeof(float), hipMemcpyHostToDevice)); + CHECK(hipMemcpy(device_step, step, sizeof(step), hipMemcpyHostToDevice)); + + hipStream_t stream; + CHECK(hipStreamCreate(&stream)); + native_qsa_score(device_pooled, device_query, device_bias, shapes, device_step, kMaxBlocks, kMaxCells, + device_scores, static_cast(stream)); + CHECK(hipStreamSynchronize(stream)); + CHECK(hipMemcpy(scores.data(), device_scores, scores.size() * sizeof(float), hipMemcpyDeviceToHost)); + + bool ok = true; + for (int cell = 0; cell < kMaxCells; ++cell) { + if (scores[cell] != expected[cell]) { + std::fprintf(stderr, "score[%d] got %.9g, expected %.9g\n", cell, scores[cell], expected[cell]); + ok = false; + } + } + + CHECK(hipStreamDestroy(stream)); + CHECK(hipFree(device_step)); + CHECK(hipFree(device_scores)); + CHECK(hipFree(device_bias)); + CHECK(hipFree(device_query)); + CHECK(hipFree(device_pooled)); + if (!ok) return 1; + std::puts("native QSA HIP scalar parity OK: FMA order, per-head ReLU, bias, partial tail and padding"); + return 0; +} From ea2a87b91728f6dfc512b697dda5c0307752ec8e Mon Sep 17 00:00:00 2001 From: Konstantinos Korres <96335554+2jztricks@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:04:42 +0300 Subject: [PATCH 2/8] Add tuned gfx1100 prefill and resident expert serving support --- CMakeLists.txt | 77 ++++ cmake/hip_backend.cmake | 7 + docs/AMD_HIP.md | 19 +- docs/AMD_HIP_PERFORMANCE.md | 106 +++++ include/strata/core/expert_source.hpp | 51 ++- include/strata/hip_compat/cuda_runtime.h | 2 + include/strata/kernels/native_qsa_indexer.hpp | 6 + include/strata/platform/direct_file.hpp | 2 +- include/strata/prefill/gemm.hpp | 2 + src/core/expert_source.cpp | 356 ++++++++++++++- src/core/remote_experts.cpp | 3 + src/kernels/cuda/native_qsa_indexer.cu | 102 +++++ src/ngram/ple_reader.cpp | 82 +++- src/platform/direct_file.cpp | 237 +++++++++- src/prefill/gemm.cu | 279 ++++++++++++ src/prefill/ggml_cuda_host.cu | 57 ++- src/prefill/hipblaslt_tuning.hpp | 123 +++++ src/prefill/prefill.cpp | 85 +++- src/program/generate.cpp | 55 ++- tests/core/file_expert_source_test.cpp | 23 + tests/hip/prefill_hipblaslt_gemm.cpp | 188 ++++++++ tests/hip/prefill_mmq_parity.cpp | 281 ++++++++++++ tests/hip/prefill_native_batch.cpp | 289 ++++++++++++ tests/platform/direct_file_async.cpp | 62 +++ tools/hip/bench_prefill.py | 75 ++++ tools/hip/gfx1100-hipblaslt-100100.txt | 32 ++ tools/hip/tune_hipblaslt.cpp | 419 ++++++++++++++++++ 27 files changed, 2956 insertions(+), 64 deletions(-) create mode 100644 docs/AMD_HIP_PERFORMANCE.md create mode 100644 src/prefill/hipblaslt_tuning.hpp create mode 100644 tests/hip/prefill_hipblaslt_gemm.cpp create mode 100644 tests/hip/prefill_mmq_parity.cpp create mode 100644 tests/hip/prefill_native_batch.cpp create mode 100644 tests/platform/direct_file_async.cpp create mode 100644 tools/hip/bench_prefill.py create mode 100644 tools/hip/gfx1100-hipblaslt-100100.txt create mode 100644 tools/hip/tune_hipblaslt.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 774aa4b3b94..52ef525b29f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,10 @@ endif() option(STRATA_ENABLE_CUDA "Build the CUDA targets (Phase 2+)" OFF) option(STRATA_ENABLE_HIP "Build the HIP targets for gfx1100 wave32 hardware" OFF) +option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF) +if(STRATA_PREFILL_MMQ AND NOT STRATA_ENABLE_HIP) + message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_ENABLE_HIP=ON") +endif() if(STRATA_ENABLE_CUDA AND STRATA_ENABLE_HIP) message(FATAL_ERROR "STRATA_ENABLE_CUDA and STRATA_ENABLE_HIP are separate backends; enable only one") endif() @@ -288,6 +292,13 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) add_library(strata_prefill STATIC src/prefill/gemm.cu src/prefill/kernels.cu src/prefill/prefill.cpp) target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}) target_link_libraries(strata_prefill PUBLIC strata_engine ${_strata_gpu_blas_target} ${_strata_gpu_runtime_target}) + if(STRATA_ENABLE_HIP AND STRATA_HIPBLASLT_AVAILABLE) + target_compile_definitions(strata_prefill PRIVATE STRATA_HIPBLASLT_AVAILABLE=1) + target_link_libraries(strata_prefill PRIVATE roc::hipblaslt) + + add_executable(tune_hipblaslt tools/hip/tune_hipblaslt.cpp) + target_link_libraries(tune_hipblaslt PRIVATE hip::host roc::hipblas roc::hipblaslt) + endif() add_executable(strata src/program/generate.cpp) target_include_directories(strata PRIVATE ${_strata_gpu_include_directories}) @@ -314,9 +325,29 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) add_test(NAME hip_native_qsa_score COMMAND hip_native_qsa_score) list(APPEND _strata_hip_tests hip_native_qsa_score) endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_native_batch.cpp") + add_executable(hip_prefill_native_batch tests/hip/prefill_native_batch.cpp) + target_link_libraries(hip_prefill_native_batch PRIVATE strata_kernels strata_hip_runtime) + add_test(NAME hip_prefill_native_batch COMMAND hip_prefill_native_batch) + list(APPEND _strata_hip_tests hip_prefill_native_batch) + endif() + if(STRATA_HIPBLASLT_AVAILABLE AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_hipblaslt_gemm.cpp") + add_executable(hip_prefill_hipblaslt_gemm tests/hip/prefill_hipblaslt_gemm.cpp) + target_include_directories(hip_prefill_hipblaslt_gemm PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/prefill") + target_link_libraries(hip_prefill_hipblaslt_gemm PRIVATE strata_prefill strata_hip_runtime roc::hipblaslt roc::hipblas) + add_test(NAME hip_prefill_hipblaslt_gemm COMMAND hip_prefill_hipblaslt_gemm) + set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77) + list(APPEND _strata_hip_tests hip_prefill_hipblaslt_gemm) + endif() if(_strata_hip_tests) set_tests_properties(${_strata_hip_tests} PROPERTIES TIMEOUT 60) endif() + if(TARGET hip_prefill_native_batch) + set_tests_properties(hip_prefill_native_batch PROPERTIES TIMEOUT 120) + endif() + if(TARGET hip_prefill_hipblaslt_gemm) + set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77) + endif() endif() add_executable(dequant_s2_parity src/kernels/dequant_s2_parity.cpp) @@ -550,6 +581,9 @@ target_link_libraries(strata_kernels_cpu PUBLIC Threads::Threads) # with -DSTRATA_GGML_DIR=..., otherwise the pinned upstream commit (third_party/ggml/VERSION.txt). option(STRATA_NATIVE_EXPERTS "Native (IQ) expert support through ggml-cpu" ON) set(STRATA_GGML_DIR "" CACHE PATH "llama.cpp checkout to take ggml from (empty: fetch the pinned commit)") +if(STRATA_PREFILL_MMQ AND NOT STRATA_NATIVE_EXPERTS) + message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_NATIVE_EXPERTS=ON for GGML sources") +endif() if(STRATA_NATIVE_EXPERTS) enable_language(C) set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) @@ -619,6 +653,42 @@ if(STRATA_NATIVE_EXPERTS) target_link_libraries(strata_mmq PUBLIC ggml-base ${_strata_gpu_runtime_target}) target_link_libraries(strata_prefill PUBLIC strata_mmq) target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1) + elseif(STRATA_PREFILL_MMQ) + # Compile only the pinned GGML MMQ and activation-quantizer sources used by prefill. This uses + # common.cuh's native HIP path without enabling a second GGML HIP backend. + if(STRATA_GGML_DIR) + set(_strata_gg "${STRATA_GGML_DIR}/ggml") + else() + set(_strata_gg "${strata_llamacpp_SOURCE_DIR}/ggml") + endif() + set(_strata_mmq "${_strata_gg}/src/ggml-cuda") + if(NOT EXISTS "${_strata_mmq}/common.cuh" OR NOT EXISTS "${_strata_mmq}/quantize.cu") + message(FATAL_ERROR "The selected GGML source is missing the ggml-cuda MMQ sources: ${_strata_mmq}") + endif() + set(_strata_mmq_sources + src/prefill/moe_mmq.cu + src/prefill/ggml_cuda_host.cu + "${_strata_mmq}/quantize.cu") + foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs) + list(APPEND _strata_mmq_sources "${_strata_mmq}/template-instances/mmq-instance-${_t}.cu") + endforeach() + set_source_files_properties(${_strata_mmq_sources} PROPERTIES LANGUAGE HIP) + add_library(strata_mmq STATIC ${_strata_mmq_sources}) + target_include_directories(strata_mmq + PRIVATE "${_strata_gg}/include" "${_strata_gg}/src" "${_strata_mmq}" + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) + target_compile_definitions(strata_mmq PRIVATE GGML_USE_HIP=1) + target_link_libraries(strata_mmq PUBLIC ggml-base hip::host roc::hipblas) + target_link_libraries(strata_prefill PUBLIC strata_mmq) + target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1) + + # Numerical diagnostic: synthetic layouts by default, optional first-expert IQ3_XXS/IQ4_NL check with a pack. + add_executable(hip_prefill_mmq_parity tests/hip/prefill_mmq_parity.cpp) + target_include_directories(hip_prefill_mmq_parity PRIVATE "${_strata_gg}/include") + target_link_libraries(hip_prefill_mmq_parity PRIVATE strata_prefill) + add_test(NAME hip_prefill_mmq_parity COMMAND hip_prefill_mmq_parity) + set_tests_properties(hip_prefill_mmq_parity PROPERTIES TIMEOUT 120) + list(APPEND _strata_hip_tests hip_prefill_mmq_parity) endif() if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) add_executable(native_expert_parity src/kernels/native_expert_parity.cpp) @@ -659,6 +729,13 @@ if(STRATA_BUILD_TESTS) add_executable(pool_stress src/kernels/cpu/pool_stress.cpp) target_link_libraries(pool_stress PRIVATE strata_kernels_cpu) add_test(NAME pool_stress COMMAND pool_stress 20) + + if(UNIX AND NOT APPLE) + add_executable(direct_file_async_test tests/platform/direct_file_async.cpp src/platform/direct_file.cpp) + target_include_directories(direct_file_async_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) + target_link_libraries(direct_file_async_test PRIVATE Threads::Threads) + add_test(NAME direct_file_async_test COMMAND direct_file_async_test) + endif() endif() if(STRATA_BUILD_TESTS) diff --git a/cmake/hip_backend.cmake b/cmake/hip_backend.cmake index 033c0aa835c..42f50e11daa 100644 --- a/cmake/hip_backend.cmake +++ b/cmake/hip_backend.cmake @@ -11,6 +11,7 @@ endif() enable_language(HIP) find_package(hip CONFIG REQUIRED) find_package(hipblas CONFIG REQUIRED) +find_package(hipblaslt CONFIG QUIET) if(NOT TARGET hip::host) message(FATAL_ERROR "The ROCm hip CMake package did not provide hip::host") @@ -18,6 +19,12 @@ endif() if(NOT TARGET roc::hipblas) message(FATAL_ERROR "The ROCm hipblas CMake package did not provide roc::hipblas") endif() +if(TARGET roc::hipblaslt) + set(STRATA_HIPBLASLT_AVAILABLE ON) +else() + set(STRATA_HIPBLASLT_AVAILABLE OFF) + message(STATUS "Strata: hipBLASLt not found; solution-table dispatch is unavailable") +endif() # HIP's link step produces a PIE; make Strata and ggml objects PIC for the ROCm linker. set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/docs/AMD_HIP.md b/docs/AMD_HIP.md index 7f1343883ea..abb360a0b3f 100644 --- a/docs/AMD_HIP.md +++ b/docs/AMD_HIP.md @@ -7,14 +7,15 @@ wave64, Windows HIP, and mixed AMD/NVIDIA execution are outside this contributio The backend maps the CUDA-shaped runtime and BLAS calls to HIP/hipBLAS, uses RDNA3's signed integer dot instruction for quantized kernels, and supplies wave32 shuffle/packed-byte operations. CUDA-only QSA matrix instructions have -an ordered FP32 fallback. Prefill uses dequantization plus hipBLAS GEMM; the -CUDA ggml MMQ path is not compiled for HIP. This does not claim CUDA tensor-core -performance or bit-identical answers across the two backends. +an ordered FP32 fallback. Prefill supports both dequantization plus hipBLAS GEMM and opt-in HIP ggml MMQ. +An optional, calibrated hipBLASLt path accelerates dense projections on gfx1100. +This does not claim bit-identical model answers across backends. See +[performance settings and evidence](AMD_HIP_PERFORMANCE.md). ## Build Requirements: a working ROCm driver/runtime, HIP development headers and -compiler, hipBLAS development files, CMake 3.24+, a C++20 host compiler, and Git. +compiler, hipBLAS and (for tuned dense prefill) hipBLASLt development files, CMake 3.24+, a C++20 host compiler, and Git. The model still needs sufficient system RAM and fast SSD storage for its PLE table. VRAM occupancy alone is not a throughput measurement. @@ -82,7 +83,11 @@ The automatic installer, vision helper, local gateway integrations, and multi-GPU experiments are not included in this backend patch. -## Validation +## Original backend validation (PR #94) + +The following is historical validation of the original backend, not a fresh +test count for this replacement. Current build/test evidence and performance +limits are recorded in [AMD_HIP_PERFORMANCE.md](AMD_HIP_PERFORMANCE.md). Tested against upstream `c1e903310f211e6630780c3bd2038778c071c68d` (0.1.20), with pinned llama.cpp `3cf03257f219afbe7334045ff7c6a06ac68c627d`. @@ -128,6 +133,6 @@ ctest --test-dir build-hip --output-on-failure --timeout 60 \ test from passing. These are explicit exclusions, not skipped tests counted as passes. Additional unpublished upstream fixture suites are not covered. -Earlier throughput measurements from a larger private working branch are not -results of this patch. Vision, long-context stress, broad answer-quality +The published performance table explicitly identifies the measured development +snapshot; it must not be read as a benchmark of every subsequent rebase. Vision, long-context stress, broad answer-quality equivalence, other AMD cards, and mixed-vendor inference are not validated here. diff --git a/docs/AMD_HIP_PERFORMANCE.md b/docs/AMD_HIP_PERFORMANCE.md new file mode 100644 index 00000000000..a192a4d6463 --- /dev/null +++ b/docs/AMD_HIP_PERFORMANCE.md @@ -0,0 +1,106 @@ +# RX 7900 XTX support and performance evidence + +This opt-in Linux `gfx1100` backend supersedes the initial support in +[PR #94](https://github.com/Niko1221/Strata/pull/94). It retains HIP runtime and +wave32 integer-dot compatibility, native mmap layout validation, and MTP, then +adds HIP MMQ, optional calibrated dense hipBLASLt GEMM, native prefill batching, +and the host-memory / SSD paths used by the measured configuration. + +## Reproduce the configuration + +Build instructions are in [AMD_HIP.md](AMD_HIP.md). Use the repository-pinned +llama.cpp dependency; do not silently substitute another revision. Build with +`STRATA_PREFILL_MMQ=ON` and use these runtime variables for the measured arm: + +```sh +export STRATA_PREFILL_MMQ=1 +export STRATA_HIPBLASLT_TUNING="$PWD/tools/hip/gfx1100-hipblaslt-100100.txt" +export STRATA_PF_IDX_BATCH=1 +export STRATA_PF_EMBED_BATCH=1 +export STRATA_PREFILL_RING=96 +export STRATA_PLE_IO_THREADS=32 +``` + +The supplied table is calibrated for gfx1100 and hipBLASLt version 100100. +It is not a universal ROCm tuning table. The runtime guards architecture, +library version and actual shape/stride/workspace requirements, falling back +when a table entry is unavailable or incompatible. Batched embedding/indexer +paths and HIP MMQ are opt-in at runtime. Default CUDA selection is preserved. + +Measured engine configuration: Orca Flash Next IQ3_XXS, native pack plus matching +GGUF/tokenizer/template, `--mmap-experts --resident-cpu-experts`, fixed ranked +`--expert-profile`, `--expert-cache auto`, `--prefill 8192`, `--spec 4`, +`--spec-min-p 0.5`, matching MTP runtime, `--max-context 262144`, `--kv int8`, +`--kv-resident 32768`, `--pool-workers 15`, `--adapt-every 0`, `--pcie-frac 0`, +and `--vram-reserve-mib 1024`. Keep PLE on SSD. + +`--resident-cpu-experts` copies the complement of the static GPU cache to ordinary +RAM. It requires mmap, a fixed expert profile, and disabled adaptation. It does +not lock all data into RAM: temporary prefill borrowing and cache refill may +still use the mapped fallback. Allocation needs sufficient available RAM; +on Linux this option requires readable standard cgroup-v2 mounts. The guard +checks available RAM and ancestor limits with 8 GiB headroom, +but cannot reserve memory against concurrent allocations by other applications. +The POSIX PLE path issues direct reads through a configurable worker pool. + +Use a dedicated idle server, restart it between arms, and capture its engine log: + +```sh +python3 tools/hip/bench_prefill.py \ + --model MODEL_NAME --url http://127.0.0.1:8080 \ + --engine-log /path/to/engine.log --label candidate --output candidate.json +``` + +The script uses the same synthetic source prompts and request order as the +reported measurements: a small warmup, then 140/280/140/280 functions, with one +follow-up after each fresh prompt. It enforces zero reused tokens for the fresh +prompts and records prefill, decode, wall time and stop reason. It requires the +unbuffered log to contain exactly one completed timing record per request. The +128-token output cap is intentional for throughput measurement, not task success. +Never use cancelled-request timing lines as throughput evidence. + +## Final revision benchmark + +The final revision is being built and measured against the existing runtime. +The published table below will contain only this fresh matched run. +Both arms use an RX 7900 XTX 24 GiB / gfx1100, Ryzen 9 7950X3D, +64 GiB installed RAM, Fedora-family Linux, ROCm 7.1 and a 272 W GPU cap. +Other model services are stopped for measurement and restored afterward. +Sampling: temperature 0, top-k 1, top-p 1, min-p 0, seed 42, reasoning disabled. + +Zero KV reuse is not equivalent to a cold filesystem cache. First-use speed +and warmed speed are reported separately. Individual observations do not +establish confidence intervals or a general rate at every context length. + +## Correctness and quality limits + +Development checks passed eight MMQ numerical comparisons, four actual Lt GEMM +comparisons, and native QSA/indexer/embedding parity including tail states, +chunk continuation and image overrides. Real quantized MMQ relative L2 error +was approximately 0.0026 against raw-FP32/dequantized-weight reference, reflecting +Q8 activation arithmetic; this is not bitwise numerical equivalence. + +The actual Claude Code concurrency repair ran once per build. Original: +91.098 seconds, normal CLI completion, no edits and no repair. Optimized snapshot: +480.531 seconds, timeout, no edits and no repair. Six of seven hidden groups +already passed in the unchanged fixture; that is not a model-authored quality +score. These trials do not establish quality equivalence or prove the cause of +the timeout. The optimized snapshot was not promoted to the daily gateway. + +The changes do not claim better model reasoning, verified full-context behavior, +end-to-end vision validation, Windows HIP, other AMD architectures, or mixed +AMD/NVIDIA execution. Existing CUDA multi-GPU code remains present but is not +validation of HIP multi-GPU support. + +## Attribution and rejected experiments + +Native embedding gather and QSA batching are selectively adapted from +[PR #108](https://github.com/Niko1221/Strata/pull/108), commit +`acd487233c0bbe2217a6881c5bb43f8a283b0de5`. Its optional GDN parallel/split path is +not included. Existing upstream MMQ orchestration is retained and enabled for +HIP with AMD architecture identification and the correct backend compilation. + +Static 16K chunks, expanded FP16 expert tuning, and a 192-slot ring did not offer +a consistent end-to-end win in the evaluated workload. The table includes only +the selected 30 dense-shape rows. Microkernel speedups alone were not sufficient +to select a configuration. No increase in GPU power cap was used. diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp index 549674c1a5e..959221a993d 100644 --- a/include/strata/core/expert_source.hpp +++ b/include/strata/core/expert_source.hpp @@ -26,6 +26,8 @@ #include "strata/kernels/cpu/pool.hpp" #include +#include +#include #include #include @@ -33,6 +35,28 @@ namespace strata::core { class RemoteExperts; +namespace detail { + +/// Sentinel used by the pure complement planner for a blob that remains in the mmap fallback. +inline constexpr uint64_t kNoCacheComplement = ~uint64_t{0}; + +/// Build compact offsets for experts absent from both the primary GPU cache and an optional second GPU tier. +/// Kept CPU-only so selection and byte accounting can be tested without initializing a GPU. +bool make_cache_complement_plan( + int64_t n_layers, int64_t n_expert, const std::vector& layer_blob_bytes, + const std::vector>& primary_gpu_pairs, + const std::vector>& additional_gpu_pairs, + std::vector& offsets, uint64_t& bytes, std::string& err); + +/// Resolve one blob through the compact copy when present, otherwise preserve its exact mapped-file fallback. +const uint8_t* cache_complement_blob_or_fallback( + size_t index, const std::vector& offsets, const uint8_t* complement_host, + const uint8_t* mapped_fallback); + +} // namespace detail + + + /// Where one routed expert's bytes come from. /// /// Phase 2 has NO cache (`phase-2-correct-engine.md`: hit rate `h = 0`), so the only implementation is a @@ -270,17 +294,28 @@ class FileExpertSource : public ExpertSource { FileExpertSource(const FileExpertSource&) = delete; FileExpertSource& operator=(const FileExpertSource&) = delete; - /// Maps `/experts.bin` and checks its size against the loaded expert layout. + /// Maps `/experts.bin` and checks its size against the loaded expert layout. Canonical packs use + /// `n_layers * n_expert * BLOB`; native packs use their variable per-layer blob sizes and offsets. /// /// The size check is not a formality: a short file would fault at the END of a long sequence, and an /// over-long one means the pack is not the one the geometry came from. Refuses with the two numbers. bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err); + /// Pin a compact host mirror of experts absent from a fully filled static GPU cache. The mmap remains open + /// as a fallback for later cache reloads. This is opt-in because the complement may still be a large allocation. + bool pin_cache_complement( + const ExpertCache& cache, std::string& err, bool pin = true, + const std::vector>& additional_gpu_pairs = {}); void close(); bool mapped() const { return base_ != nullptr; } int64_t blobs() const { return blobs_; } + uint64_t pinned_bytes() const { return complement_pinned_ ? complement_bytes_ : 0; } + uint64_t resident_bytes() const { return complement_bytes_; } + bool complement_pinned() const { return complement_pinned_; } const uint8_t* blob(int64_t layer, int64_t expert) override; + bool pinned(int64_t layer, int64_t expert) const override; + const uint8_t* device_alias(int64_t layer, int64_t expert) const override; /// Blobs touched, for the driver to report. With `h = 0` this is `48 * k` per token and the number is only /// interesting once Phase 3 makes it not so. @@ -288,15 +323,21 @@ class FileExpertSource : public ExpertSource { private: const uint8_t* mapped_blob(int64_t layer, int64_t expert) const; - + static constexpr uint64_t kNoComplement = detail::kNoCacheComplement; const uint8_t* base_ = nullptr; int64_t blobs_ = 0; int64_t n_layers_ = 0; int64_t n_expert_ = 0; - int64_t reads_ = 0; uint64_t mapped_bytes_ = 0; - std::vector layer_offsets_; - std::vector layer_blob_bytes_; + std::vector layer_offsets_, layer_blob_bytes_; + void* complement_arena_ = nullptr; + const uint8_t* complement_host_ = nullptr; + const uint8_t* complement_device_ = nullptr; + uint64_t complement_bytes_ = 0; + std::vector complement_offsets_; + bool complement_pinned_ = false; + bool complement_ready_ = false; + int64_t reads_ = 0; #if defined(_WIN32) void* file_ = nullptr; void* mapping_ = nullptr; diff --git a/include/strata/hip_compat/cuda_runtime.h b/include/strata/hip_compat/cuda_runtime.h index 77ee7f72968..243d2509716 100644 --- a/include/strata/hip_compat/cuda_runtime.h +++ b/include/strata/hip_compat/cuda_runtime.h @@ -6,6 +6,8 @@ #undef __noinline__ #endif #define cudaDevAttrMaxSharedMemoryPerBlockOptin hipDeviceAttributeMaxSharedMemoryPerBlock +#define cudaDevAttrMultiProcessorCount hipDeviceAttributeMultiprocessorCount +#define cudaDevAttrClockRate hipDeviceAttributeClockRate #define cudaDeviceGetAttribute hipDeviceGetAttribute #define cudaDeviceProp hipDeviceProp_t #define cudaDeviceSynchronize hipDeviceSynchronize diff --git a/include/strata/kernels/native_qsa_indexer.hpp b/include/strata/kernels/native_qsa_indexer.hpp index a79fd47cef7..39054f46a54 100644 --- a/include/strata/kernels/native_qsa_indexer.hpp +++ b/include/strata/kernels/native_qsa_indexer.hpp @@ -32,4 +32,10 @@ void native_qsa_indexer_append(const float* raw, const int32_t* relative_pos_dev const QsaIndexerBuffers& buffers, const QsaShapes& shapes, int64_t max_cells, float freq_base, void* stream); +// Prompt path: append T contiguous cells in two launches instead of one launch per cell. +// The resulting tail, dead key, pooled keys, and block position match T single appends. +void native_qsa_indexer_append_chunk(const float* raw, int64_t T, int64_t pos0, int32_t pos_base, + const float* gamma, float epsilon, const QsaIndexerBuffers& buffers, + const QsaShapes& shapes, int64_t max_cells, float freq_base, void* stream); + } // namespace strata::kernels diff --git a/include/strata/platform/direct_file.hpp b/include/strata/platform/direct_file.hpp index 1d196d860ab..f8b4c644b8b 100644 --- a/include/strata/platform/direct_file.hpp +++ b/include/strata/platform/direct_file.hpp @@ -2,7 +2,7 @@ // // The n-gram table (26.8 GiB) stays on the SSD and must never occupy RAM, including the OS file cache. A memory // map cannot promise that; an unbuffered read can. On Windows this is FILE_FLAG_NO_BUFFERING | OVERLAPPED with -// an I/O completion port; on Linux, O_DIRECT (synchronous pread for now; io_uring is phase L). +// an I/O completion port; on Linux, O_DIRECT with a configurable worker pool (STRATA_PLE_IO_THREADS). // // Contract of every read: offset, length and buffer address are multiples of `alignment()` (4096 here). // Reads past end of file return the bytes that exist; `Completion::bytes` says how many. diff --git a/include/strata/prefill/gemm.hpp b/include/strata/prefill/gemm.hpp index cddce1a6e23..c9ddc6f4dc3 100644 --- a/include/strata/prefill/gemm.hpp +++ b/include/strata/prefill/gemm.hpp @@ -6,6 +6,7 @@ // rounded to BF16, which is also what llama.cpp's batched CUDA path does. Tensor-core GEMM through cuBLAS. #pragma once +#include #include #include @@ -50,6 +51,7 @@ class Gemm { int64_t scratch_elems_ = 0; void* workspace_ = nullptr; bool external_ = false; + void* hipblaslt_state_ = nullptr; }; diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp index 7568a255099..fcb2aae03b3 100644 --- a/src/core/expert_source.cpp +++ b/src/core/expert_source.cpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -34,6 +36,157 @@ namespace strata::core { +namespace detail { + +bool make_cache_complement_plan( + int64_t n_layers, int64_t n_expert, const std::vector& layer_blob_bytes, + const std::vector>& primary_gpu_pairs, + const std::vector>& additional_gpu_pairs, + std::vector& offsets, uint64_t& bytes, std::string& err) { + offsets.clear(); + bytes = 0; + err.clear(); + if (n_layers <= 0 || n_expert <= 0 || layer_blob_bytes.size() != (size_t) n_layers) { + err = "FileExpertSource: invalid geometry for the cache complement plan"; + return false; + } + if ((uint64_t) n_layers > (uint64_t) std::numeric_limits::max() / (uint64_t) n_expert) { + err = "FileExpertSource: cache complement index table is too large"; + return false; + } + const size_t count = (size_t) n_layers * (size_t) n_expert; + std::vector omitted(count, 0); + auto mark_pairs = [&](const std::vector>& pairs, uint8_t bit, + const char* label) -> bool { + for (const auto& pair : pairs) { + if (pair.first < 0 || pair.second < 0 || pair.first >= n_layers || pair.second >= n_expert) { + err = std::string("FileExpertSource: ") + label + " pair is outside the expert geometry"; + return false; + } + const size_t index = (size_t) pair.first * (size_t) n_expert + (size_t) pair.second; + if ((omitted[index] & bit) != 0) { + err = std::string("FileExpertSource: duplicate ") + label + " pair in the cache complement plan"; + return false; + } + if (bit == 2 && (omitted[index] & 1) != 0) { + err = "FileExpertSource: the primary and additional GPU expert tiers overlap"; + return false; + } + omitted[index] |= bit; + } + return true; + }; + if (!mark_pairs(primary_gpu_pairs, 1, "primary GPU") || + !mark_pairs(additional_gpu_pairs, 2, "additional GPU")) return false; + for (uint64_t blob_bytes : layer_blob_bytes) { + if (blob_bytes == 0) { + err = "FileExpertSource: cache complement layer has zero-sized expert blobs"; + return false; + } + } + + offsets.assign(count, kNoCacheComplement); + for (int64_t layer = 0; layer < n_layers; ++layer) { + const uint64_t blob_bytes = layer_blob_bytes[(size_t) layer]; + for (int64_t expert = 0; expert < n_expert; ++expert) { + const size_t index = (size_t) layer * (size_t) n_expert + (size_t) expert; + if (omitted[index] != 0) continue; + if (bytes > std::numeric_limits::max() - blob_bytes) { + offsets.clear(); + bytes = 0; + err = "FileExpertSource: cache complement size overflows"; + return false; + } + offsets[index] = bytes; + bytes += blob_bytes; + } + } + if (bytes > (uint64_t) std::numeric_limits::max()) { + offsets.clear(); + bytes = 0; + err = "FileExpertSource: cache complement exceeds the host address space"; + return false; + } + return true; +} + +const uint8_t* cache_complement_blob_or_fallback( + size_t index, const std::vector& offsets, const uint8_t* complement_host, + const uint8_t* mapped_fallback) { + if (complement_host != nullptr && index < offsets.size() && offsets[index] != kNoCacheComplement) + return complement_host + (size_t) offsets[index]; + return mapped_fallback; +} + +} // namespace detail + +namespace { +constexpr uint64_t kPinnedMemoryHeadroom = 8ull << 30; + +bool available_memory_bytes(uint64_t& bytes) { +#if defined(_WIN32) + MEMORYSTATUSEX status{}; + status.dwLength = sizeof(status); + if (!GlobalMemoryStatusEx(&status)) return false; + bytes = (uint64_t) status.ullAvailPhys; + return bytes > 0; +#elif defined(__linux__) + // MemAvailable includes reclaimable page cache, unlike _SC_AVPHYS_PAGES. + std::ifstream info("/proc/meminfo"); + std::string line; + bytes = 0; + while (std::getline(info, line)) { + std::istringstream fields(line); + std::string key, unit; + uint64_t value = 0; + if (fields >> key >> value >> unit && key == "MemAvailable:" && unit == "kB" && + value <= std::numeric_limits::max() / 1024) bytes = value * 1024; + } + if (bytes == 0) return false; + // Account for the tightest cgroup-v2 ancestor limit when its normal mount is visible. + // This is a point-in-time guard, not a reservation against concurrent allocations. + std::ifstream groups("/proc/self/cgroup"); + if (!groups) return false; + bool resolved_v2 = false; + while (std::getline(groups, line)) { + if (line.rfind("0::/", 0) != 0) continue; + const std::filesystem::path root("/sys/fs/cgroup"); + auto path = (root / line.substr(4)).lexically_normal(); + if (path.string().rfind(root.string(), 0) != 0 || !std::filesystem::is_directory(path)) return false; + resolved_v2 = true; + while (path.string().rfind(root.string(), 0) == 0) { + std::ifstream limit_file(path / "memory.max"), current_file(path / "memory.current"); + std::string limit; + uint64_t current = 0; + const bool readable = bool(limit_file >> limit) && bool(current_file >> current); + // The host's root cgroup has no memory.max; ordinary child groups must expose their limits. + if (!readable && !(path == root && !std::filesystem::exists(path / "memory.max") && + std::filesystem::exists(path / "cgroup.controllers"))) return false; + if (readable && limit != "max") { + try { + size_t consumed = 0; + const uint64_t cap = std::stoull(limit, &consumed); + if (consumed != limit.size()) return false; + bytes = std::min(bytes, current < cap ? cap - current : uint64_t{0}); + } catch (...) { return false; } + } + if (path == root) break; + path = path.parent_path(); + } + } + return resolved_v2; +#else + const long pages = sysconf(_SC_AVPHYS_PAGES); + const long page_bytes = sysconf(_SC_PAGESIZE); + if (pages <= 0 || page_bytes <= 0 || + (uint64_t) pages > std::numeric_limits::max() / (uint64_t) page_bytes) return false; + bytes = (uint64_t) pages * (uint64_t) page_bytes; + return bytes > 0; +#endif +} + +} // namespace + // ================================ THE FILE-BACKED SOURCE ================================ FileExpertSource::~FileExpertSource() { close(); } @@ -197,6 +350,17 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64 } void FileExpertSource::close() { + if (complement_arena_ != nullptr) { + if (complement_pinned_) (void) cudaFreeHost(complement_arena_); + else std::free(complement_arena_); + } + complement_arena_ = nullptr; + complement_host_ = nullptr; + complement_device_ = nullptr; + complement_bytes_ = 0; + complement_offsets_.clear(); + complement_pinned_ = false; + complement_ready_ = false; #if defined(_WIN32) if (base_ != nullptr) UnmapViewOfFile((LPCVOID) base_); if (mapping_ != nullptr) CloseHandle((HANDLE) mapping_); @@ -232,13 +396,199 @@ const uint8_t* FileExpertSource::mapped_blob(int64_t layer, int64_t expert) cons return base_ + (size_t) offset; } +bool FileExpertSource::pin_cache_complement( + const ExpertCache& cache, std::string& err, bool pin, + const std::vector>& additional_gpu_pairs) { + err.clear(); + if (base_ == nullptr) { err = "FileExpertSource: open the mapped experts before pinning a complement"; return false; } + if (complement_ready_) { err = "FileExpertSource: the cache complement is already pinned"; return false; } + if (!cache.valid()) { err = "FileExpertSource: the GPU expert cache is not open"; return false; } + if (cache.fills() != cache.resident()) { + err = "FileExpertSource: the GPU expert cache is not fully filled"; + return false; + } + const cudaError_t sync = cudaDeviceSynchronize(); + if (sync != cudaSuccess) { + err = std::string("FileExpertSource: GPU expert cache is not ready: ") + cudaGetErrorString(sync); + (void) cudaGetLastError(); + return false; + } + + std::vector> primary_gpu_pairs; + primary_gpu_pairs.reserve((size_t) cache.resident()); + for (int64_t layer = 0; layer < n_layers_; ++layer) { + for (int64_t expert = 0; expert < n_expert_; ++expert) { + if (cache.slot_of(layer, expert) != kNotResident) + primary_gpu_pairs.emplace_back((int32_t) layer, (int32_t) expert); + } + } + std::vector offsets; + uint64_t bytes = 0; + if (!detail::make_cache_complement_plan(n_layers_, n_expert_, layer_blob_bytes_, primary_gpu_pairs, + additional_gpu_pairs, offsets, bytes, err)) return false; + + if (bytes > 0) { + uint64_t physical = 0; + if (!available_memory_bytes(physical)) { + err = "FileExpertSource: cannot determine available RAM for the resident-memory safety check"; + return false; + } + if (physical <= kPinnedMemoryHeadroom || bytes > physical - kPinnedMemoryHeadroom) { + char message[256]; + std::snprintf(message, sizeof message, + "FileExpertSource: resident complement %.2f GiB exceeds available RAM minus the 8 GiB safety headroom", + (double) bytes / 1073741824.0); + err = message; + return false; + } + } + + void* arena = nullptr; + const uint8_t* host = nullptr; + const uint8_t* device = nullptr; + if (bytes > 0) { + std::fprintf(stderr, "FileExpertSource: allocating %.2f GiB %s cache complement\n", + (double) bytes / 1073741824.0, pin ? "mapped pinned" : "pageable resident"); + std::fflush(stderr); + if (pin) { + const cudaError_t allocated = cudaHostAlloc(&arena, (size_t) bytes, + cudaHostAllocMapped | cudaHostAllocPortable); + if (allocated != cudaSuccess) { + err = std::string("FileExpertSource: mapped pinned complement allocation failed: ") + + cudaGetErrorString(allocated); + std::fprintf(stderr, "%s\n", err.c_str()); + std::fflush(stderr); + (void) cudaGetLastError(); + return false; + } + host = (const uint8_t*) arena; + void* alias = nullptr; + const cudaError_t aliased = cudaHostGetDevicePointer(&alias, arena, 0); + if (aliased != cudaSuccess || alias == nullptr) { + err = std::string("FileExpertSource: mapped pinned complement device alias failed: ") + + cudaGetErrorString(aliased); + std::fprintf(stderr, "%s\n", err.c_str()); + std::fflush(stderr); + (void) cudaGetLastError(); + (void) cudaFreeHost(arena); + return false; + } + device = (const uint8_t*) alias; + } else { + arena = std::malloc((size_t) bytes); + if (arena == nullptr) { + err = "FileExpertSource: pageable resident complement allocation failed"; + return false; + } + host = (const uint8_t*) arena; + } + } + + uint64_t copied = 0; + for (int64_t layer = 0; layer < n_layers_; ++layer) { + const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer]; + for (int64_t expert = 0; expert < n_expert_; ++expert) { + const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert; + const uint64_t offset = offsets[index]; + if (offset == kNoComplement) continue; + const uint8_t* source = mapped_blob(layer, expert); + if (source == nullptr || offset > bytes || blob_bytes > bytes - offset) { + err = "FileExpertSource: invalid blob bounds while building the cache complement"; + if (arena != nullptr) { + if (pin) (void) cudaFreeHost(arena); + else std::free(arena); + } + return false; + } + std::memcpy((uint8_t*) host + (size_t) offset, source, (size_t) blob_bytes); + copied += blob_bytes; + } +#if !defined(_WIN32) + const uint64_t layer_offset = layer_offsets_[(size_t) layer]; + const uint64_t layer_bytes = blob_bytes * (uint64_t) n_expert_; + const uint64_t layer_end = layer_offset + layer_bytes; + const long page_bytes = sysconf(_SC_PAGESIZE); + if (page_bytes <= 0) { + err = "FileExpertSource: cannot determine page size for mapped-page release"; + if (arena != nullptr) { + if (pin) (void) cudaFreeHost(arena); + else std::free(arena); + } + return false; + } + const uint64_t page = (uint64_t) page_bytes; + const uint64_t advice_start = layer_offset - layer_offset % page; + const uint64_t end_remainder = layer_end % page; + const uint64_t extra = end_remainder == 0 ? 0 : page - end_remainder; + const uint64_t advice_end = extra > mapped_bytes_ - layer_end ? mapped_bytes_ : layer_end + extra; + if (advice_end > advice_start && + madvise((void*) (base_ + (size_t) advice_start), (size_t) (advice_end - advice_start), MADV_DONTNEED) != 0) { + err = "FileExpertSource: madvise could not release mapped expert layer " + std::to_string(layer); + if (arena != nullptr) { + if (pin) (void) cudaFreeHost(arena); + else std::free(arena); + } + return false; + } + const int advice = posix_fadvise(fd_, (off_t) layer_offset, (off_t) layer_bytes, POSIX_FADV_DONTNEED); + if (advice != 0) { + err = "FileExpertSource: posix_fadvise could not release expert layer " + std::to_string(layer); + if (arena != nullptr) { + if (pin) (void) cudaFreeHost(arena); + else std::free(arena); + } + return false; + } +#endif + if ((layer + 1) % 8 == 0 || layer + 1 == n_layers_) { + std::fprintf(stderr, "FileExpertSource: copied cache complement through layer %lld/%lld (%.2f GiB)\n", + (long long) (layer + 1), (long long) n_layers_, (double) copied / 1073741824.0); + std::fflush(stderr); + } + } + + complement_arena_ = arena; + complement_host_ = host; + complement_device_ = device; + complement_bytes_ = bytes; + complement_offsets_ = std::move(offsets); + complement_pinned_ = pin && bytes > 0; + complement_ready_ = true; + std::fprintf(stderr, "FileExpertSource: %s cache complement ready: resident %.2f GiB, pinned %.2f GiB\n", + complement_pinned_ ? "mapped pinned" : "pageable resident", + (double) resident_bytes() / 1073741824.0, (double) pinned_bytes() / 1073741824.0); + if (!additional_gpu_pairs.empty()) { + std::fprintf(stderr, "FileExpertSource: %zu verified additional-GPU experts remain on the mmap fallback\n", + additional_gpu_pairs.size()); + } + std::fflush(stderr); + return true; +} + const uint8_t* FileExpertSource::blob(int64_t layer, int64_t expert) { - const uint8_t* result = mapped_blob(layer, expert); - if (result == nullptr) return nullptr; - ++reads_; + if (base_ == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return nullptr; + const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert; + const uint8_t* mapped_fallback = mapped_blob(layer, expert); + const uint8_t* result = complement_ready_ + ? detail::cache_complement_blob_or_fallback(index, complement_offsets_, complement_host_, mapped_fallback) + : mapped_fallback; + if (result != nullptr) ++reads_; return result; } +bool FileExpertSource::pinned(int64_t layer, int64_t expert) const { + if (!complement_ready_ || !complement_pinned_ || complement_host_ == nullptr || layer < 0 || expert < 0 || + layer >= n_layers_ || expert >= n_expert_) return false; + const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert; + return index < complement_offsets_.size() && complement_offsets_[index] != kNoComplement; +} + +const uint8_t* FileExpertSource::device_alias(int64_t layer, int64_t expert) const { + if (!pinned(layer, expert) || complement_device_ == nullptr) return nullptr; + const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert; + return complement_device_ + (size_t) complement_offsets_[index]; +} + // ================================ THE ADAPTER ================================ void expert_pool_dispatch(void* user, const float* x_f, const int32_t* ids, const float* weights, int64_t n_embd, diff --git a/src/core/remote_experts.cpp b/src/core/remote_experts.cpp index d1be42313ed..53d8d7643c4 100644 --- a/src/core/remote_experts.cpp +++ b/src/core/remote_experts.cpp @@ -73,9 +73,12 @@ bool RemoteExperts::preflight(int device, double& free_gib, std::string& err) { // The layer waits for this GPU on the CPU pool's critical path: spin instead of sleeping, whose wake-up // costs more than a small expert batch takes (measured: ~0.3 ms per round trip on Windows). Only possible // before the device's context exists, so first thing; STRATA_REMOTE_SPIN=0 keeps the driver's default. +#if !defined(STRATA_USE_HIP) const char* spin = std::getenv("STRATA_REMOTE_SPIN"); if (!(spin && spin[0] == '0')) cudaInitDevice(device, cudaDeviceScheduleSpin | cudaDeviceMapHost, 0); cudaGetLastError(); +#endif + // HIP has no cudaInitDevice equivalent; retain its default scheduling policy. DeviceScope scope(device); if (!scope.ok) { err = scope.error(device); return false; } size_t free_bytes = 0, total_bytes = 0; diff --git a/src/kernels/cuda/native_qsa_indexer.cu b/src/kernels/cuda/native_qsa_indexer.cu index b80009b52b6..206b8080e1d 100644 --- a/src/kernels/cuda/native_qsa_indexer.cu +++ b/src/kernels/cuda/native_qsa_indexer.cu @@ -93,6 +93,84 @@ __global__ void append(const float* __restrict__ raw, const int32_t* __restrict_ else pooled[std::size_t(b + 1) * D + d] = dead[d]; if (d == 0 && pos != 0) *block_pos = rope_pos; } +// Batch the appends at block completion points. Intermediate per-token state is not visible to the QSA queries: +// their step positions mask future blocks, while the final tail/dead/spare state is restored by the finish kernel. +__global__ void append_batch(const float* __restrict__ raw, int64_t T, int pos0, int pos_base, + const float* __restrict__ gamma, float epsilon, const float* __restrict__ tail, + float* __restrict__ dead, float* __restrict__ pooled, int max_cells, float theta_scale, + const int32_t* __restrict__ mtab) { + const int t = blockIdx.x, d = threadIdx.x; + const int pos = pos0 + t; + if (pos < 0 || pos >= max_cells) return; + const int slot = pos % R; + if (pos != 0 && slot != R - 1) return; + // Members before pos0 come from the previous chunk's tail; members inside this chunk come from raw rows. + auto member = [&](int j) -> float { + const int tt = t - (R - 1) + j; + return tt >= 0 ? __half2float(__float2half_rn(raw[(int64_t) tt * D + d])) : tail[j * D + d]; + }; + __shared__ float values[D]; + __shared__ float partials[32]; + float mean = 0.0f; + if (d < D) { + const float incoming = __half2float(__float2half_rn(raw[(int64_t) t * D + d])); + float sum = pos == 0 ? incoming : member(0); +#pragma unroll + for (int j = 1; j < R; ++j) + sum = __fadd_rn(sum, pos == 0 || j == R - 1 ? incoming : member(j)); + mean = __fmaf_rn(0.25f, sum, 0.0f); + } + float square_sum = 0.0f; + if (d < D) square_sum += mean * mean; + square_sum = warp_sum(square_sum); + const int lane = d % 32; + if (lane == 0) partials[d / 32] = square_sum; + __syncthreads(); + square_sum = lane < THREADS / 32 ? partials[lane] : 0.0f; + square_sum = warp_sum(square_sum); + const float scale = rsqrtf(square_sum / D + epsilon); + if (d < D) values[d] = scale * mean * gamma[d]; + __syncthreads(); + if (d >= D) return; + const int b = pos / R; + const int rope_pos = pos == 0 ? 0 : pos_base + R * b; + float y = values[d]; + if (d < ROT) { + const int pair = d % (ROT / 2); + const float theta = (pos == 0 ? 0 : mrope_pos(mtab, rope_pos, pair)) * powf(theta_scale, float(pair)); + const float c = cosf(theta), s = sinf(theta); + const float a = values[pair], z = values[pair + ROT / 2]; + y = d < ROT / 2 ? a * c - z * s : a * s + z * c; + } + if (pos == 0) { + dead[d] = y; + // If block zero completes in this chunk, its completion kernel writes pooled[0]. + if (pos0 + T - 1 < R - 1 || R - 1 >= max_cells) pooled[d] = y; + } else { + pooled[std::size_t(b) * D + d] = y; + } +} +// Restore the state left after the final position of the chunk: last spare and block position, plus the raw tail. +__global__ void append_batch_finish(const float* __restrict__ raw, int64_t T, int pos0, int pos_base, + float* __restrict__ tail, const float* __restrict__ dead, + float* __restrict__ pooled, int32_t* __restrict__ block_pos, int max_cells) { + const int d = threadIdx.x; + const int64_t last64 = (int64_t) pos0 + T - 1 < (int64_t) max_cells - 1 + ? (int64_t) pos0 + T - 1 + : (int64_t) max_cells - 1; + const int last = (int) last64; + if (last < pos0) return; + const int lc = (last + 1) / R * R - 1; + if (lc >= pos0 && lc > 0) { + const int b = lc / R; + pooled[std::size_t(b + 1) * D + d] = dead[d]; + if (d == 0) *block_pos = pos_base + R * b; + } + for (int j = 0; j < R - 1; ++j) { + const int p = last - ((last % R) - j + R) % R; + if (p >= pos0) tail[j * D + d] = __half2float(__float2half_rn(raw[(int64_t) (p - pos0) * D + d])); + } +} struct Span { const void* p; std::size_t n; }; void validate(Span s) { const auto p = reinterpret_cast(s.p); @@ -126,4 +204,28 @@ void native_qsa_indexer_append(const float* raw, const int32_t* relative_pos_dev const auto error = cudaGetLastError(); if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error)); } +void native_qsa_indexer_append_chunk(const float* raw, int64_t T, int64_t pos0, int32_t pos_base, const float* gamma, + float epsilon, const QsaIndexerBuffers& b, const QsaShapes& s, + int64_t max_cells, float freq_base, void* stream) { + if (T <= 0) return; + if (!stream || s.idx_dim != D || s.idx_block != R || s.n_rot != ROT || pos0 < 0 || pos0 > INT32_MAX || + T > (int64_t) INT32_MAX - pos0 || + max_cells < 1 || max_cells > INT32_MAX || pos_base < 0 || pos_base % R || + int64_t(pos_base) + max_cells > INT32_MAX || !std::isfinite(epsilon) || epsilon <= 0.0f || + !std::isfinite(freq_base) || freq_base <= 1.0f) + throw std::invalid_argument("native QSA indexer (chunk) requires fixed geometry and valid positions"); + const Span spans[] = {{raw,std::size_t(T)*D*4},{gamma,D*4},{b.tail,(R-1)*D*4}, + {b.dead,D*4},{b.pooled,std::size_t(max_cells/R+1)*D*4},{b.block_pos,4}}; + for (const auto& span : spans) validate(span); + for (int i = 0; i < 6; ++i) for (int j = i + 1; j < 6; ++j) + if (overlaps(spans[i], spans[j])) throw std::invalid_argument("native QSA indexer buffers overlap"); + const float theta_scale = powf(freq_base, -2.0f / ROT); + const auto st = static_cast(stream); + append_batch<<<(unsigned) T, THREADS, 0, st>>>(raw, T, (int) pos0, pos_base, gamma, epsilon, b.tail, b.dead, + b.pooled, int(max_cells), theta_scale, mrope_table()); + append_batch_finish<<<1, D, 0, st>>>(raw, T, (int) pos0, pos_base, b.tail, b.dead, b.pooled, b.block_pos, + int(max_cells)); + const auto error = cudaGetLastError(); + if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error)); +} } // namespace strata::kernels diff --git a/src/ngram/ple_reader.cpp b/src/ngram/ple_reader.cpp index 44acd6d4855..ff6d5139cd6 100644 --- a/src/ngram/ple_reader.cpp +++ b/src/ngram/ple_reader.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -118,6 +119,14 @@ struct PleReader::Impl { uint8_t* slot_buf(uint32_t s) { return slab + (size_t) s * 2 * PAGE; } bool busy() const { return free_slots.size() < max_inflight || !delayed.empty(); } + void cancel_queued() { + for (const Job& job : queue) { + auto it = tickets.find(job.ticket); + if (it != tickets.end() && it->second.pending > 0) --it->second.pending; + } + queue.clear(); + } + void record_latency(double us) { stats.read_us_sum += us; if (stats.read_us.size() < LATENCY_RING) stats.read_us.push_back((float) us); @@ -132,7 +141,14 @@ struct PleReader::Impl { queue.pop_front(); Job& j = inflight[s]; j.issued_us = now_us(); - if (!file.submit(j.offset, slot_buf(s), j.length, s, error)) return false; + if (!file.submit(j.offset, slot_buf(s), j.length, s, error)) { + j.uses.clear(); + auto it = tickets.find(j.ticket); + if (it != tickets.end() && it->second.pending > 0) --it->second.pending; + free_slots.push_back(s); + cancel_queued(); + return false; + } stats.submit_us += now_us() - j.issued_us; ++stats.reads; } @@ -141,9 +157,19 @@ struct PleReader::Impl { bool finish(const Completion& c) { const uint32_t s = (uint32_t) c.tag; + if (s >= inflight.size()) { + error = "PleReader: invalid table-read completion"; + cancel_queued(); + return false; + } Job& j = inflight[s]; if (!c.ok) { error = "PleReader: a table read failed"; + cancel_queued(); + j.uses.clear(); + auto it = tickets.find(j.ticket); + if (it != tickets.end() && it->second.pending > 0) --it->second.pending; + free_slots.push_back(s); return false; } record_latency(now_us() - j.issued_us); @@ -152,8 +178,15 @@ struct PleReader::Impl { for (const Use& u : j.uses) { if (u.in_page + ROW_BYTES > c.bytes) { error = "PleReader: short read inside the table"; + cancel_queued(); + j.uses.clear(); + auto it = tickets.find(j.ticket); + if (it != tickets.end() && it->second.pending > 0) --it->second.pending; + free_slots.push_back(s); return false; } + } + for (const Use& u : j.uses) { std::memcpy(u.dst, buf + u.in_page, ROW_BYTES); cache.insert(u.row, buf + u.in_page); } @@ -161,21 +194,28 @@ struct PleReader::Impl { if (it != tickets.end() && it->second.pending > 0) --it->second.pending; j.uses.clear(); free_slots.push_back(s); - return pump(); + return error.empty() ? pump() : true; } /// Completions (or wake packets) just returned by `file.wait`, applying fault injection. bool process(const Completion* got, int n) { + bool ok = true; for (int i = 0; i < n; ++i) { if (got[i].tag == DirectFile::WAKE_TAG) continue; + if (got[i].tag >= inflight.size()) { + error = "PleReader: invalid table-read completion"; + cancel_queued(); + ok = false; + continue; + } if (delay_us > 0 && now_us() - inflight[(uint32_t) got[i].tag].issued_us < delay_us) { delayed.push_back(got[i]); ++stats.late_injected; continue; } - if (!finish(got[i])) return false; + if (!finish(got[i])) ok = false; // drain the rest of this batch so no completed slot is stranded } - return true; + return ok; } bool release_delayed() { @@ -256,7 +296,19 @@ bool PleReader::open(const std::string& path, uint64_t table_offset, uint64_t n_ reset_stats(); impl_->stop = false; impl_->threaded = io_thread; - if (io_thread) impl_->worker = std::thread([this] { impl_->worker_loop(); }); + if (io_thread) { + try { + impl_->worker = std::thread([this] { impl_->worker_loop(); }); + } catch (const std::exception& e) { + err = std::string("PleReader: cannot create I/O worker: ") + e.what(); + close(); + return false; + } catch (...) { + err = "PleReader: cannot create I/O worker"; + close(); + return false; + } + } return true; } @@ -341,10 +393,11 @@ PleReader::Ticket PleReader::issue(const uint32_t* rows, size_t n, uint8_t* out_ // Sorted by offset: prefill chunks then read the SSD in near-sequential order. std::sort(jobs.begin(), jobs.end(), [](const Job& a, const Job& b) { return a.offset < b.offset; }); ts.pending = (uint32_t) jobs.size(); + const bool has_jobs = ts.pending > 0; for (Job& j : jobs) m.queue.push_back(std::move(j)); if (m.threaded) { lk.unlock(); - if (ts.pending > 0) { + if (has_jobs) { m.cv_work.notify_one(); m.file.wake(); // in case the worker is blocked in the port } @@ -361,21 +414,26 @@ bool PleReader::collect(Ticket t, std::string& err) { std::unique_lock lk(m.mu); auto it = m.tickets.find(t.id); if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; } - m.cv_done.wait(lk, [&] { return !m.error.empty() || m.tickets[t.id].pending == 0; }); + m.cv_done.wait(lk, [&] { + const auto current = m.tickets.find(t.id); + return current == m.tickets.end() || current->second.pending == 0; + }); + it = m.tickets.find(t.id); + if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; } if (!m.error.empty()) { err = m.error; return false; } m.stats.wait_us += now_us() - start; - m.tickets.erase(t.id); + m.tickets.erase(it); return true; } auto it = m.tickets.find(t.id); if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; } while (it->second.pending > 0) { - if (!m.error.empty() || !m.drain(-1)) { - err = m.error.empty() ? "PleReader: read failed" : m.error; - return false; - } + const bool drained = m.drain(-1); + if (!drained && m.error.empty()) m.error = "PleReader: read failed"; it = m.tickets.find(t.id); + if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; } } + if (!m.error.empty()) { err = m.error; return false; } m.stats.wait_us += now_us() - start; m.tickets.erase(it); return true; diff --git a/src/platform/direct_file.cpp b/src/platform/direct_file.cpp index 0e46017440e..3ff76ddfa58 100644 --- a/src/platform/direct_file.cpp +++ b/src/platform/direct_file.cpp @@ -1,10 +1,16 @@ // src/platform/direct_file.cpp - see include/strata/platform/direct_file.hpp. #include "strata/platform/direct_file.hpp" +#include #include +#include #include #include #include +#include +#include +#include +#include #include #if defined(_WIN32) @@ -185,12 +191,79 @@ void DirectFile::wake() { #else // ------------------------------------------------------------------------------------------------ POSIX -// Phase L replaces this with io_uring. Until then a read completes inside `submit`, which is correct and -// keeps the tree compiling on Linux (plan v0.3 section 5.1 rule 4). +// On Linux, submit queues direct reads to a bounded worker pool. wait() and wake() retain +// their completion contract while the workers provide the queue depth unavailable to sync pread. +namespace { +constexpr unsigned kDefaultIoThreads = 32; +constexpr unsigned kMaxIoThreads = 128; + +bool configured_io_threads(unsigned& count, std::string& err) { + const char* text = std::getenv("STRATA_PLE_IO_THREADS"); + if (text == nullptr) { + count = kDefaultIoThreads; + return true; + } + errno = 0; + char* end = nullptr; + const long parsed = std::strtol(text, &end, 10); + if (errno == ERANGE || end == text || *end != '\0' || parsed < 1) { + err = "DirectFile: STRATA_PLE_IO_THREADS must be a positive integer"; + return false; + } + count = parsed > (long) kMaxIoThreads ? kMaxIoThreads : (unsigned) parsed; + return true; +} +} // namespace + struct DirectFile::Impl { + struct Req { uint64_t offset; void* buffer; uint32_t length; uint64_t tag; }; int fd = -1; uint64_t size = 0; + std::mutex mu; + std::condition_variable cv_req, cv_done; + std::deque reqs; std::deque done; + std::vector workers; + size_t active = 0; + size_t wakes = 0; + bool stop = false; + + void work() { + for (;;) { + Req r{}; + int request_fd = -1; + { + std::unique_lock lk(mu); + cv_req.wait(lk, [&] { return stop || !reqs.empty(); }); + if (stop && reqs.empty()) return; + r = reqs.front(); + reqs.pop_front(); + request_fd = fd; + ++active; + } + uint32_t got = 0; + bool ok = true; + while (got < r.length) { + const uint64_t at = r.offset + got; // submit validated the full offset range. + const ssize_t n = ::pread(request_fd, (uint8_t*) r.buffer + got, r.length - got, (off_t) at); + if (n < 0) { + if (errno == EINTR) continue; + ok = false; + break; + } + if (n == 0) break; // end of file: a legal short read at the table's last page + const uint32_t remaining = r.length - got; + got += (uint32_t) n; + if ((uint32_t) n < remaining) break; // Short EOF may not preserve O_DIRECT alignment. + } + { + std::lock_guard lk(mu); + done.push_back(Completion{r.tag, got, ok}); + --active; + } + cv_done.notify_all(); + } + } }; DirectFile::DirectFile() : impl_(new Impl) {} @@ -198,38 +271,168 @@ DirectFile::~DirectFile() { close(); delete impl_; } bool DirectFile::open(const std::string& path, std::string& err) { close(); - impl_->fd = ::open(path.c_str(), O_RDONLY | O_DIRECT); - if (impl_->fd < 0) { err = "DirectFile: cannot open " + path; return false; } + unsigned thread_count = 0; + if (!configured_io_threads(thread_count, err)) return false; + + const int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT); + if (fd < 0) { err = "DirectFile: cannot open " + path; return false; } struct stat st; - if (fstat(impl_->fd, &st) != 0) { err = "DirectFile: cannot size " + path; close(); return false; } - impl_->size = (uint64_t) st.st_size; + if (fstat(fd, &st) != 0) { + err = "DirectFile: cannot size " + path; + ::close(fd); + return false; + } + + { + std::lock_guard lk(impl_->mu); + impl_->fd = fd; + impl_->size = (uint64_t) st.st_size; + impl_->stop = false; + impl_->active = 0; + impl_->wakes = 0; + impl_->reqs.clear(); + impl_->done.clear(); + } + try { + impl_->workers.reserve(thread_count); + for (unsigned i = 0; i < thread_count; ++i) + impl_->workers.emplace_back([this] { impl_->work(); }); + } catch (const std::exception& e) { + { + std::lock_guard lk(impl_->mu); + impl_->stop = true; + } + impl_->cv_req.notify_all(); + for (std::thread& worker : impl_->workers) + if (worker.joinable()) worker.join(); + impl_->workers.clear(); + { + std::lock_guard lk(impl_->mu); + impl_->fd = -1; + impl_->size = 0; + impl_->stop = false; + impl_->active = 0; + impl_->wakes = 0; + impl_->reqs.clear(); + impl_->done.clear(); + } + ::close(fd); + err = std::string("DirectFile: cannot create I/O workers: ") + e.what(); + return false; + } catch (...) { + { + std::lock_guard lk(impl_->mu); + impl_->stop = true; + } + impl_->cv_req.notify_all(); + for (std::thread& worker : impl_->workers) + if (worker.joinable()) worker.join(); + impl_->workers.clear(); + { + std::lock_guard lk(impl_->mu); + impl_->fd = -1; + impl_->size = 0; + impl_->stop = false; + impl_->active = 0; + impl_->wakes = 0; + impl_->reqs.clear(); + impl_->done.clear(); + } + ::close(fd); + err = "DirectFile: cannot create I/O workers"; + return false; + } return true; } void DirectFile::close() { - if (impl_->fd >= 0) ::close(impl_->fd); - impl_->fd = -1; - impl_->size = 0; - impl_->done.clear(); + { + std::lock_guard lk(impl_->mu); + impl_->stop = true; + } + impl_->cv_req.notify_all(); + impl_->cv_done.notify_all(); + for (std::thread& worker : impl_->workers) + if (worker.joinable()) worker.join(); + impl_->workers.clear(); + + int fd = -1; + { + std::lock_guard lk(impl_->mu); + fd = impl_->fd; + impl_->fd = -1; + impl_->size = 0; + impl_->reqs.clear(); + impl_->done.clear(); + impl_->active = 0; + impl_->wakes = 0; + impl_->stop = false; + } + if (fd >= 0) ::close(fd); } -bool DirectFile::is_open() const { return impl_->fd >= 0; } -uint64_t DirectFile::size() const { return impl_->size; } +bool DirectFile::is_open() const { + std::lock_guard lk(impl_->mu); + return impl_->fd >= 0; +} +uint64_t DirectFile::size() const { + std::lock_guard lk(impl_->mu); + return impl_->size; +} bool DirectFile::submit(uint64_t offset, void* buffer, uint32_t length, uint64_t tag, std::string& err) { - if (offset % alignment() || length % alignment() || ((uintptr_t) buffer) % alignment() || length == 0) { + if (buffer == nullptr || offset % alignment() || length % alignment() || + ((uintptr_t) buffer) % alignment() || length == 0) { err = "DirectFile: unaligned request"; return false; } - const ssize_t got = pread(impl_->fd, buffer, length, (off_t) offset); - impl_->done.push_back(Completion{tag, got < 0 ? 0u : (uint32_t) got, got >= 0}); + const uint64_t max_offset = (uint64_t) std::numeric_limits::max(); + if (offset > max_offset || (uint64_t) length - 1 > max_offset - offset) { + err = "DirectFile: request offset exceeds POSIX file range"; + return false; + } + { + std::lock_guard lk(impl_->mu); + if (impl_->fd < 0 || impl_->stop) { err = "DirectFile: not open"; return false; } + try { + impl_->reqs.push_back(Impl::Req{offset, buffer, length, tag}); + } catch (const std::exception& e) { + err = std::string("DirectFile: cannot queue read: ") + e.what(); + return false; + } catch (...) { + err = "DirectFile: cannot queue read"; + return false; + } + } + impl_->cv_req.notify_one(); return true; } -void DirectFile::wake() {} // reads complete inside submit; nothing ever blocks in wait +void DirectFile::wake() { + { + std::lock_guard lk(impl_->mu); + if (impl_->fd < 0) return; + ++impl_->wakes; + } + impl_->cv_done.notify_all(); +} -int DirectFile::wait(Completion* out, int max, int) { +int DirectFile::wait(Completion* out, int max, int timeout_ms) { + if (out == nullptr || max <= 0) return 0; + std::unique_lock lk(impl_->mu); + const auto ready = [&] { + return impl_->fd < 0 || !impl_->done.empty() || impl_->wakes > 0 || + (impl_->stop && impl_->reqs.empty() && impl_->active == 0); + }; + if (!ready()) { + if (timeout_ms < 0) impl_->cv_done.wait(lk, ready); + else if (timeout_ms > 0) impl_->cv_done.wait_for(lk, std::chrono::milliseconds(timeout_ms), ready); + } int n = 0; + if (impl_->wakes > 0 && n < max) { + --impl_->wakes; + out[n++] = Completion{WAKE_TAG, 0, true}; + } while (n < max && !impl_->done.empty()) { out[n++] = impl_->done.front(); impl_->done.pop_front(); diff --git a/src/prefill/gemm.cu b/src/prefill/gemm.cu index 60720ee65a3..4742d423b3e 100644 --- a/src/prefill/gemm.cu +++ b/src/prefill/gemm.cu @@ -5,8 +5,32 @@ #include #include +#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE) +// The HIP compatibility shim maps CUDA shuffle spellings to Strata helpers. +// hipBLASLt's public headers declare native HIP shuffle functions, so keep +// those declarations from being macro-expanded in this translation unit. +#undef __shfl_xor_sync +#undef __shfl_down_sync +#undef __shfl_up_sync +#undef __shfl_sync +#undef __ballot_sync +#endif + +#include #include #include +#include +#include + +#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE) +#include "hipblaslt_tuning.hpp" +#include +#include +#include +#include +#include +#include +#endif namespace strata::prefill { namespace { @@ -18,9 +42,239 @@ void ck(cublasStatus_t s, const char* what) { } } +#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE) +struct HipLtCallKey { + strata::prefill::hipblaslt::InputType type; + int t; + int n; + int k; + int ldy; + uint32_t beta_bits; + + bool operator<(const HipLtCallKey& other) const { + return std::tie(type, n, k, ldy, t, beta_bits) < + std::tie(other.type, other.n, other.k, other.ldy, other.t, other.beta_bits); + } +}; + +struct HipLtCachedAlgo { + bool supported = false; + hipblasLtMatmulAlgo_t algo{}; + size_t workspace_bytes = 0; +}; + +struct HipLtState { + hipblasLtHandle_t handle = nullptr; + void* workspace = nullptr; + size_t workspace_bytes = 0; + strata::prefill::hipblaslt::TuningTable table; + std::map cache; + uint64_t lt_launches = 0; + uint64_t fallbacks = 0; + std::set> fallback_shapes; + + ~HipLtState() { + if (std::getenv("STRATA_HIPBLASLT_VERBOSE")) { + std::fprintf(stderr, "prefill gemm: hipBLASLt summary launches=%llu fallbacks=%llu unique_fallback_shapes=%zu\n", + (unsigned long long) lt_launches, (unsigned long long) fallbacks, fallback_shapes.size()); + for (const auto& shape : fallback_shapes) { + const auto type = std::get<0>(shape); + std::fprintf(stderr, "prefill gemm: fallback shape dtype=%s T=%d N=%d K=%d ldy=%d\n", + type == strata::prefill::hipblaslt::InputType::bf16 ? "bf16" : "f16", + std::get<1>(shape), std::get<2>(shape), std::get<3>(shape), std::get<4>(shape)); + } + } + if (handle) hipblasLtDestroy(handle); + } +}; + +struct HipLtDescriptors { + hipblasLtMatmulDesc_t op = nullptr; + hipblasLtMatrixLayout_t a = nullptr; + hipblasLtMatrixLayout_t b = nullptr; + hipblasLtMatrixLayout_t c = nullptr; + + ~HipLtDescriptors() { + if (op) hipblasLtMatmulDescDestroy(op); + if (a) hipblasLtMatrixLayoutDestroy(a); + if (b) hipblasLtMatrixLayoutDestroy(b); + if (c) hipblasLtMatrixLayoutDestroy(c); + } + + bool init(hipDataType type, int t, int n, int k, int ldy) { + const hipblasOperation_t trans_a = HIPBLAS_OP_T; + const hipblasOperation_t trans_b = HIPBLAS_OP_N; + if (hipblasLtMatmulDescCreate(&op, HIPBLAS_COMPUTE_32F, HIP_R_32F) != HIPBLAS_STATUS_SUCCESS || + hipblasLtMatmulDescSetAttribute(op, HIPBLASLT_MATMUL_DESC_TRANSA, &trans_a, sizeof(trans_a)) != + HIPBLAS_STATUS_SUCCESS || + hipblasLtMatmulDescSetAttribute(op, HIPBLASLT_MATMUL_DESC_TRANSB, &trans_b, sizeof(trans_b)) != + HIPBLAS_STATUS_SUCCESS || + hipblasLtMatrixLayoutCreate(&a, type, k, n, k) != HIPBLAS_STATUS_SUCCESS || + hipblasLtMatrixLayoutCreate(&b, type, k, t, k) != HIPBLAS_STATUS_SUCCESS || + hipblasLtMatrixLayoutCreate(&c, HIP_R_32F, n, t, ldy) != HIPBLAS_STATUS_SUCCESS) { + return false; + } + return true; + } +}; + +std::unique_ptr create_hipblaslt_state(void* workspace, size_t workspace_bytes) { + const char* path = std::getenv("STRATA_HIPBLASLT_TUNING"); + if (!path || !*path) return nullptr; + + auto state = std::make_unique(); + state->workspace = workspace; + state->workspace_bytes = workspace_bytes; + if (hipblasLtCreate(&state->handle) != HIPBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "prefill gemm: hipBLASLt handle creation failed; using hipBLASEx\n"); + return nullptr; + } + + int version = 0; + if (hipblasLtGetVersion(state->handle, &version) != HIPBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "prefill gemm: hipBLASLt version query failed; using hipBLASEx\n"); + return nullptr; + } + int device = 0; + hipDeviceProp_t properties{}; + if (hipGetDevice(&device) != hipSuccess || hipGetDeviceProperties(&properties, device) != hipSuccess) { + std::fprintf(stderr, "prefill gemm: HIP device query failed; using hipBLASEx\n"); + return nullptr; + } + std::string arch(properties.gcnArchName); + const auto suffix = arch.find(':'); + if (suffix != std::string::npos) arch.resize(suffix); + + std::string error; + if (!state->table.load(path, arch, version, error)) { + std::fprintf(stderr, "prefill gemm: %s; using hipBLASEx\n", error.c_str()); + return nullptr; + } + std::fprintf(stderr, "prefill gemm: hipBLASLt tuning enabled (%zu rows, %s, version %d)\n", + state->table.rows().size(), arch.c_str(), version); + return state; +} + +HipLtCachedAlgo resolve_hipblaslt_algo(HipLtState& state, strata::prefill::hipblaslt::InputType type, int t, + int n, int k, int ldy, float beta) { + uint32_t beta_bits = 0; + static_assert(sizeof(beta_bits) == sizeof(beta)); + std::memcpy(&beta_bits, &beta, sizeof(beta)); + const HipLtCallKey key{type, t, n, k, ldy, beta_bits}; + const auto cached = state.cache.find(key); + if (cached != state.cache.end()) return cached->second; + + HipLtCachedAlgo resolved; + const bool verbose = std::getenv("STRATA_HIPBLASLT_VERBOSE") != nullptr; + const auto* row = state.table.closest(type, n, k, ldy, t); + if (!row) { + if (verbose) { + std::fprintf(stderr, "prefill gemm: Lt fallback; no calibration for dtype=%s T=%d N=%d K=%d ldy=%d\n", + type == strata::prefill::hipblaslt::InputType::bf16 ? "bf16" : "f16", t, n, k, ldy); + } + return state.cache.emplace(key, resolved).first->second; + } + + HipLtDescriptors desc; + const hipDataType input_type = type == strata::prefill::hipblaslt::InputType::bf16 ? HIP_R_16BF : HIP_R_16F; + if (!desc.init(input_type, t, n, k, ldy)) { + return state.cache.emplace(key, resolved).first->second; + } + + std::vector solution_ids{row->solution_id}; + std::vector candidates; + if (hipblaslt_ext::getAlgosFromIndex(state.handle, solution_ids, candidates) != HIPBLAS_STATUS_SUCCESS || + candidates.empty() || candidates.front().state != HIPBLAS_STATUS_SUCCESS || + hipblaslt_ext::getIndexFromAlgo(candidates.front().algo) != row->solution_id) { + if (verbose) { + std::fprintf(stderr, "prefill gemm: Lt fallback; solution %d unavailable for T=%d N=%d K=%d ldy=%d\n", + row->solution_id, t, n, k, ldy); + } + return state.cache.emplace(key, resolved).first->second; + } + + const float alpha = 1.0f; + size_t required_workspace = 0; + auto algo = candidates.front().algo; + if (hipblaslt_ext::matmulIsAlgoSupported(state.handle, desc.op, &alpha, desc.a, desc.b, &beta, desc.c, desc.c, + algo, required_workspace) != HIPBLAS_STATUS_SUCCESS) { + if (verbose) { + std::fprintf(stderr, "prefill gemm: Lt fallback; solution %d rejects actual T=%d N=%d K=%d ldy=%d beta=%.9g\n", + row->solution_id, t, n, k, ldy, beta); + } + return state.cache.emplace(key, resolved).first->second; + } + + resolved.supported = true; + resolved.algo = algo; + resolved.workspace_bytes = required_workspace; + if (verbose) { + std::fprintf(stderr, + "prefill gemm: Lt solution=%d dtype=%s T=%d N=%d K=%d ldy=%d beta=%.9g workspace=%zu\n", + row->solution_id, type == strata::prefill::hipblaslt::InputType::bf16 ? "bf16" : "f16", t, n, + k, ldy, beta, required_workspace); + } + return state.cache.emplace(key, resolved).first->second; +} + +bool try_hipblaslt(void* opaque_state, strata::prefill::hipblaslt::InputType type, const uint16_t* x, + const uint16_t* w, float* y, int64_t t, int64_t n, int64_t k, int64_t ldy, float beta, + void* stream) { + auto* state = static_cast(opaque_state); + if (!state || t <= 0 || n <= 0 || k <= 0 || t > INT_MAX || n > INT_MAX || k > INT_MAX || ldy > INT_MAX || + ldy < n) { + return false; + } + const auto resolved = resolve_hipblaslt_algo(*state, type, (int) t, (int) n, (int) k, (int) ldy, beta); + if (!resolved.supported) { + ++state->fallbacks; + state->fallback_shapes.emplace(type, (int) t, (int) n, (int) k, (int) ldy); + return false; + } + if (resolved.workspace_bytes > state->workspace_bytes) { + ++state->fallbacks; + state->fallback_shapes.emplace(type, (int) t, (int) n, (int) k, (int) ldy); + if (std::getenv("STRATA_HIPBLASLT_VERBOSE")) { + std::fprintf(stderr, "prefill gemm: Lt fallback; solution needs %zu workspace bytes, have %zu\n", + resolved.workspace_bytes, state->workspace_bytes); + } + return false; + } + + HipLtDescriptors desc; + const hipDataType input_type = type == strata::prefill::hipblaslt::InputType::bf16 ? HIP_R_16BF : HIP_R_16F; + if (!desc.init(input_type, (int) t, (int) n, (int) k, (int) ldy)) return false; + const float alpha = 1.0f; + const hipblasStatus_t status = hipblasLtMatmul(state->handle, desc.op, &alpha, w, desc.a, x, desc.b, &beta, y, + desc.c, y, desc.c, &resolved.algo, state->workspace, + state->workspace_bytes, (hipStream_t) stream); + if (status == HIPBLAS_STATUS_SUCCESS) { + ++state->lt_launches; + return true; + } + + std::fprintf(stderr, "prefill gemm: hipBLASLt launch failed with status %d\n", (int) status); + if (beta != 0.0f) { + std::fprintf(stderr, "prefill gemm: refusing a fallback after hipBLASLt failed with nonzero beta\n"); + std::exit(1); + } + auto* mutable_state = static_cast(opaque_state); + uint32_t beta_bits = 0; + std::memcpy(&beta_bits, &beta, sizeof(beta_bits)); + auto cached = mutable_state->cache.find(HipLtCallKey{type, (int) t, (int) n, (int) k, (int) ldy, beta_bits}); + if (cached != mutable_state->cache.end()) cached->second.supported = false; + ++mutable_state->fallbacks; + mutable_state->fallback_shapes.emplace(type, (int) t, (int) n, (int) k, (int) ldy); + return false; +} +#endif + } // namespace Gemm::~Gemm() { +#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE) + delete static_cast(hipblaslt_state_); +#endif if (handle_) cublasDestroy((cublasHandle_t) handle_); if (!external_) { if (scratch_) cudaFree(scratch_); @@ -41,6 +295,9 @@ bool Gemm::init_external(void* stream, uint16_t* scratch, int64_t scratch_elems, cublasSetMathMode(h, CUBLAS_DEFAULT_MATH); scratch_ = scratch; scratch_elems_ = scratch_elems; +#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE) + hipblaslt_state_ = create_hipblaslt_state(workspace_, ws_bytes).release(); +#endif return true; } @@ -49,6 +306,13 @@ void Gemm::rebind(uint16_t* scratch, int64_t scratch_elems, void* workspace, siz scratch_elems_ = scratch_elems; workspace_ = workspace; cublasSetWorkspace((cublasHandle_t) handle_, workspace_, ws_bytes); +#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE) + if (hipblaslt_state_) { + auto* state = static_cast(hipblaslt_state_); + state->workspace = workspace_; + state->workspace_bytes = ws_bytes; + } +#endif } bool Gemm::init(void* stream, int64_t scratch_elems, std::string& err) { @@ -62,6 +326,9 @@ bool Gemm::init(void* stream, int64_t scratch_elems, std::string& err) { if (cudaMalloc(&workspace_, ws) != cudaSuccess) { err = "prefill gemm: workspace"; return false; } cublasSetWorkspace(h, workspace_, ws); cublasSetMathMode(h, CUBLAS_DEFAULT_MATH); +#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE) + hipblaslt_state_ = create_hipblaslt_state(workspace_, ws).release(); +#endif if (scratch_elems > 0 && cudaMalloc((void**) &scratch_, (size_t) scratch_elems * 2) != cudaSuccess) { err = "prefill gemm: dequant scratch of " + std::to_string(scratch_elems * 2 >> 20) + " MiB"; return false; @@ -75,6 +342,12 @@ void Gemm::bf16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64 if (T <= 0 || N <= 0) return; if (ldy <= 0) ldy = N; const float alpha = 1.0f; +#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE) + if (try_hipblaslt(hipblaslt_state_, strata::prefill::hipblaslt::InputType::bf16, X, W, Y, T, N, K, ldy, + beta, stream_)) { + return; + } +#endif // Column-major view: Y^T[N, T] = W[N, K] (stored K x N col-major, transposed) . X^T[K, T]. ck(cublasGemmEx((cublasHandle_t) handle_, CUBLAS_OP_T, CUBLAS_OP_N, (int) N, (int) T, (int) K, &alpha, W, CUDA_R_16BF, (int) K, X, CUDA_R_16BF, (int) K, &beta, Y, CUDA_R_32F, (int) ldy, @@ -87,6 +360,12 @@ void Gemm::f16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_ if (T <= 0 || N <= 0) return; if (ldy <= 0) ldy = N; const float alpha = 1.0f; +#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE) + if (try_hipblaslt(hipblaslt_state_, strata::prefill::hipblaslt::InputType::f16, X, W, Y, T, N, K, ldy, + beta, stream_)) { + return; + } +#endif ck(cublasGemmEx((cublasHandle_t) handle_, CUBLAS_OP_T, CUBLAS_OP_N, (int) N, (int) T, (int) K, &alpha, W, CUDA_R_16F, (int) K, X, CUDA_R_16F, (int) K, &beta, Y, CUDA_R_32F, (int) ldy, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT), diff --git a/src/prefill/ggml_cuda_host.cu b/src/prefill/ggml_cuda_host.cu index 88effa42e25..12f4c8affda 100644 --- a/src/prefill/ggml_cuda_host.cu +++ b/src/prefill/ggml_cuda_host.cu @@ -5,6 +5,7 @@ #include #include #include +#include #include [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) { @@ -18,6 +19,45 @@ int ggml_cuda_get_device() { return id; } +#if defined(GGML_USE_HIP) +namespace { + +unsigned parse_hex(const std::string& value) { + if (value.empty()) return 0; + char* end = nullptr; + const unsigned long parsed = std::strtoul(value.c_str(), &end, 16); + return end == value.c_str() || *end != '\0' ? 0 : static_cast(parsed); +} + +// Match GGML's AMD cc encoding: OFFSET_AMD + gfx major * 0x100 + minor/stepping. +// For example, gfx1100 maps to GGML_CUDA_CC_RDNA3 (0x01001100). +int ggml_cuda_parse_amd_id(const char* device_name) { + if (device_name == nullptr) return GGML_CUDA_CC_OFFSET_AMD; + + std::string arch(device_name); + if (arch.compare(0, 3, "gfx") == 0) arch.erase(0, 3); + const size_t suffix = arch.find(':'); + if (suffix != std::string::npos) arch.resize(suffix); + + const size_t generic = arch.rfind("-generic"); + if (generic != std::string::npos && generic + 8 == arch.size()) arch.resize(generic); + + unsigned major = 0; + unsigned minor = 0; + const size_t separator = arch.find('-'); + if (separator != std::string::npos) { + major = parse_hex(arch.substr(0, separator)); + minor = parse_hex(arch.substr(separator + 1)) * 0x10; + } else if (arch.size() >= 3) { + minor = parse_hex(arch.substr(arch.size() - 2)); + major = parse_hex(arch.substr(0, arch.size() - 2)); + } + return GGML_CUDA_CC_OFFSET_AMD + static_cast(major * 0x100 + minor); +} + +} // namespace +#endif + const ggml_cuda_device_info & ggml_cuda_info() { static ggml_cuda_device_info info = [] { ggml_cuda_device_info in = {}; @@ -30,15 +70,26 @@ const ggml_cuda_device_info & ggml_cuda_info() { cudaDeviceProp prop; CUDA_CHECK(cudaGetDeviceProperties(&prop, id)); auto & d = in.devices[id]; +#if defined(GGML_USE_HIP) + d.cc = ggml_cuda_parse_amd_id(prop.gcnArchName); + if ((d.cc & 0xff00) == 0) { + // Fall back to HIP's device version fields if its architecture name is unavailable. + d.cc = GGML_CUDA_CC_OFFSET_AMD + prop.major * 0x100 + prop.minor * 0x10; + } + d.smpbo = prop.sharedMemPerBlock; + d.integrated = false; + d.supports_cooperative_launch = false; +#else d.cc = 100 * prop.major + 10 * prop.minor; - d.nsm = prop.multiProcessorCount; - d.smpb = prop.sharedMemPerBlock; d.smpbo = prop.sharedMemPerBlockOptin; d.integrated = prop.integrated != 0; + d.supports_cooperative_launch = prop.cooperativeLaunch != 0; +#endif + d.nsm = prop.multiProcessorCount; + d.smpb = prop.sharedMemPerBlock; d.vmm = false; d.total_vram = prop.totalGlobalMem; d.warp_size = prop.warpSize; - d.supports_cooperative_launch = prop.cooperativeLaunch != 0; d.physical_device = id; d.physical_share_count = 1; d.virtual_index = 0; diff --git a/src/prefill/hipblaslt_tuning.hpp b/src/prefill/hipblaslt_tuning.hpp new file mode 100644 index 00000000000..b9d7f54be1b --- /dev/null +++ b/src/prefill/hipblaslt_tuning.hpp @@ -0,0 +1,123 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace strata::prefill::hipblaslt { + +enum class InputType : uint8_t { f16, bf16 }; + +struct TuningRow { + InputType type = InputType::bf16; + int n = 0; + int k = 0; + int ldy = 0; + int t_bucket = 0; + int solution_id = -1; +}; + +class TuningTable { +public: + bool load(const std::string& path, const std::string& expected_arch, int expected_version, std::string& err) { + std::ifstream input(path); + if (!input) { + err = "cannot open tuning file"; + return false; + } + + bool saw_header = false; + std::string line; + size_t line_number = 0; + while (std::getline(input, line)) { + ++line_number; + const auto first = line.find_first_not_of(" \t\r\n"); + if (first == std::string::npos || line[first] == '#') continue; + + std::istringstream row(line.substr(first)); + if (!saw_header) { + std::string magic; + std::string arch; + int version = 0; + std::string extra; + if (!(row >> magic >> arch >> version) || (row >> extra) || magic != "STRATA_HIPBLASLT_TUNING_V1") { + err = "invalid tuning header at line " + std::to_string(line_number); + return false; + } + if (arch != expected_arch) { + err = "tuning architecture mismatch: file=" + arch + " runtime=" + expected_arch; + return false; + } + if (version != expected_version) { + err = "hipBLASLt version mismatch: file=" + std::to_string(version) + + " runtime=" + std::to_string(expected_version); + return false; + } + arch_ = std::move(arch); + version_ = version; + saw_header = true; + continue; + } + + std::string type_name; + int n = 0, k = 0, ldy = 0, t_bucket = 0, solution_id = -1; + std::string extra; + if (!(row >> type_name >> n >> k >> ldy >> t_bucket >> solution_id) || (row >> extra) || + (type_name != "f16" && type_name != "bf16") || n <= 0 || k <= 0 || ldy < n || t_bucket <= 0 || + solution_id < 0) { + err = "invalid tuning row at line " + std::to_string(line_number); + return false; + } + TuningRow parsed; + parsed.type = type_name == "f16" ? InputType::f16 : InputType::bf16; + parsed.n = n; + parsed.k = k; + parsed.ldy = ldy; + parsed.t_bucket = t_bucket; + parsed.solution_id = solution_id; + for (const auto& existing : rows_) { + if (existing.type == parsed.type && existing.n == parsed.n && existing.k == parsed.k && + existing.ldy == parsed.ldy && existing.t_bucket == parsed.t_bucket) { + err = "duplicate tuning key at line " + std::to_string(line_number); + return false; + } + } + rows_.push_back(parsed); + } + if (!saw_header) { + err = "tuning file has no header"; + return false; + } + return true; + } + + const TuningRow* closest(InputType type, int n, int k, int ldy, int t) const { + const TuningRow* best = nullptr; + int64_t best_distance = std::numeric_limits::max(); + for (const auto& row : rows_) { + if (row.type != type || row.n != n || row.k != k || row.ldy != ldy) continue; + const int64_t distance = row.t_bucket >= t ? int64_t(row.t_bucket) - t : int64_t(t) - row.t_bucket; + if (distance < best_distance || + (distance == best_distance && (!best || row.t_bucket < best->t_bucket))) { + best = &row; + best_distance = distance; + } + } + return best; + } + + const std::vector& rows() const { return rows_; } + const std::string& arch() const { return arch_; } + int version() const { return version_; } + +private: + std::string arch_; + int version_ = 0; + std::vector rows_; +}; + +} // namespace strata::prefill::hipblaslt diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp index 73c43ee80ca..f6e60c51775 100644 --- a/src/prefill/prefill.cpp +++ b/src/prefill/prefill.cpp @@ -2,6 +2,7 @@ #include "strata/prefill/prefill.hpp" #include "strata/core/progress.hpp" #include "strata/core/on_device.hpp" +#include "strata/core/native_head.hpp" #include "strata/core/layout.hpp" #include "strata/kernels/cpu/expert.hpp" @@ -30,6 +31,8 @@ #include #include #include +#include +#include #include #include #include @@ -85,6 +88,11 @@ constexpr int DQ = 2; // dequantized-expert ring (FP16 gate/up + do double ms_since(Clock::time_point t) { return std::chrono::duration(Clock::now() - t).count(); } +bool prefill_env_enabled(const char* name) { + const char* value = std::getenv(name); + return value != nullptr && std::atoi(value) != 0; +} + // Either cudaMalloc (owned, freed with the object) or a bump allocation from a borrowed region; with no base and // no region it only counts, which is how `bytes_needed` sizes the region. struct Alloc { @@ -255,6 +263,8 @@ struct Prefill::Impl { uint16_t* attn_h = nullptr; int32_t* steps_dev = nullptr; std::vector steps_host; + int32_t* embed_tokens_dev = nullptr; + std::vector embed_tokens_host; int32_t* sel_ids = nullptr; float* sel_scores = nullptr; // [sel_batch, max_blocks] int64_t sel_batch = 256, max_blocks = 0; @@ -340,6 +350,7 @@ Prefill::Prefill() : impl_(new Impl) {} Prefill::~Prefill() { if (!impl_) return; if (impl_->cs) cudaStreamSynchronize(impl_->cs); + if (impl_->copy) cudaStreamSynchronize(impl_->copy); for (int i = 0; i < RING_MAX; ++i) { if (impl_->copied[i]) cudaEventDestroy(impl_->copied[i]); if (impl_->used[i]) cudaEventDestroy(impl_->used[i]); @@ -456,6 +467,17 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, co m.T_max = chunk; m.borrowed = borrow != nullptr; bool ok = true; + if (prefill_env_enabled("STRATA_PF_EMBED_BATCH")) { + m.embed_tokens_host.resize(T); + void* ids = nullptr; + if (cudaMalloc(&ids, T * sizeof(int32_t)) == cudaSuccess) { + m.embed_tokens_dev = (int32_t*) ids; + m.owned.push_back(ids); + } else { + cudaGetLastError(); + m.embed_tokens_host.clear(); + } + } // one-time: events, the stager, the host buffers (for the largest chunk), the identity page table for (int i = 0; i < RING_MAX; ++i) { if (cudaEventCreateWithFlags(&m.copied[i], cudaEventDisableTiming) != cudaSuccess) ok = false; @@ -780,6 +802,37 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e pt.mark(kPfStart, cs); // ---- embeddings, broadcast to the four streams - or, in a later stage of a layer split, the rows the // previous stage handed on + const core::NativeEmbed* native_embed = core::native_embed(); + const bool embed_batched = hand_in_ == nullptr && m.embed_tokens_dev != nullptr && native_embed != nullptr && + prefill_env_enabled("STRATA_PF_EMBED_BATCH"); + if (embed_batched) { + const size_t row_bytes = strata::kernels::iq_row_bytes(native_embed->type(), g.n_embd); + const uint64_t native_vocab = row_bytes != 0 && native_embed->bytes() % row_bytes == 0 + ? native_embed->bytes() / row_bytes : 0; + if (native_vocab == 0) { + err = "prefill: invalid native embedding row geometry"; + return false; + } + for (int64_t t = 0; t < T; ++t) { + // Image rows replace token rows; use a valid ID because the device gather has no bounds check. + if (embd_rows && embd_rows[p0 + t] != nullptr) { + m.embed_tokens_host[(size_t) t] = 0; + continue; + } + const int64_t token = tokens[c0 + t]; + if (token < 0 || (uint64_t) token >= native_vocab || token > INT32_MAX) { + err = "prefill: native batched embedding token is outside the table or int32 range"; + return false; + } + m.embed_tokens_host[(size_t) t] = (int32_t) token; + } + if (cudaMemcpyAsync(m.embed_tokens_dev, m.embed_tokens_host.data(), (size_t) T * sizeof(int32_t), + cudaMemcpyHostToDevice, m.cs) != cudaSuccess) { + err = "prefill: the batched embedding token upload failed"; + return false; + } + native_embed->gather_dev(m.embed_tokens_dev, T, m.emb, m.cs); + } if (hand_in_ != nullptr) { if (cudaMemcpyAsync(m.R, hand_in_ + (size_t) c0 * D, (size_t) T * D * 4, cudaMemcpyHostToDevice, m.cs) != cudaSuccess) { @@ -794,6 +847,8 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e err = "prefill: the image embedding upload failed"; return false; } + } else if (embed_batched) { + continue; } else if (!core::embed_row(*m.wt, g, tokens[c0 + t], m.emb + t * N, m.cs, err)) { return false; } @@ -1017,18 +1072,25 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e if (st.kv_q4) strata::kernels::fwht256_inplace_cuda(m.q, T * 24, m.cs); rms_rows(m.q_idx, (const float*) wiqn->data, T * 4, 128, 128, EPS, m.cs); rope(m.q_idx, T, 4, 128, 512, p0, (float) strata::kernels::qsa_freq_base(), m.cs); - // the indexer appends, token by token; then scores + selection for many queries at once: + // The indexer appends token by token by default; then scores + selection for many queries at once: // a query reads completed blocks (final once completed) and `dead` for its own tail block const strata::kernels::QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos}; pt.mark(kPfQsaIdx, cs); - for (int64_t t = 0; t < T; ++t) { - const int32_t* step_t = m.steps_dev + t * strata::kernels::kStepCount; - try { - strata::kernels::native_qsa_indexer_append(m.idx_raw + t * 128, step_t + strata::kernels::kStepPos, 0, - (const float*) wikn->data, EPS, ib, s, st.max_cells, - (float) strata::kernels::qsa_freq_base(), m.cs); - } catch (const std::exception& e) { err = std::string("prefill indexer: ") + e.what(); return false; } - } + try { + if (prefill_env_enabled("STRATA_PF_IDX_BATCH")) { + strata::kernels::native_qsa_indexer_append_chunk( + m.idx_raw, T, p0, 0, (const float*) wikn->data, EPS, ib, s, st.max_cells, + (float) strata::kernels::qsa_freq_base(), m.cs); + } else { + for (int64_t t = 0; t < T; ++t) { + const int32_t* step_t = m.steps_dev + t * strata::kernels::kStepCount; + strata::kernels::native_qsa_indexer_append( + m.idx_raw + t * 128, step_t + strata::kernels::kStepPos, 0, + (const float*) wikn->data, EPS, ib, s, st.max_cells, + (float) strata::kernels::qsa_freq_base(), m.cs); + } + } + } catch (const std::exception& e) { err = std::string("prefill indexer: ") + e.what(); return false; } pt.mark(kPfQsaSel, cs); for (int64_t t0 = 0; t0 < T; t0 += m.sel_batch) { const int64_t nb = std::min(m.sel_batch, T - t0); @@ -1435,6 +1497,11 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e err = std::string("prefill: ") + cudaGetErrorString(cudaGetLastError()); return false; } + const cudaError_t copy_status = cudaStreamSynchronize(m.copy); + if (copy_status != cudaSuccess) { + err = std::string("prefill: expert copy stream: ") + cudaGetErrorString(copy_status); + return false; + } stats_.ms_total += ms_since(t_start); if (pt.on) { pt.fold(); diff --git a/src/program/generate.cpp b/src/program/generate.cpp index 0364a73ed2a..f187c1a04f8 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -76,6 +76,7 @@ #include #include #include +#include #include #include #include @@ -174,6 +175,7 @@ struct Options { /// is reserved for a host thread that has nothing to do while the drain runs. bool no_host_worker = false; bool mmap_experts = false; ///< R2.1: opt OUT of the resident arena, back to MapViewOfFile + bool resident_cpu_experts = false; ///< mmap-backed static-cache misses copied into ordinary RAM /// R4: slots of VRAM-resident experts. **0 = off, and off is the default.** /// **THE COMMENT THAT USED TO BE HERE WAS FALSE AND ROUND 328 MEASURED IT.** It said "the cache has no /// consumer yet - `moe_hit_grouped_s2` does not exist - so switching it on costs the fill traffic and @@ -423,7 +425,9 @@ void usage() { " deviation from `cpu_s2` is attributed.\n" " --mmap-experts R2.1: opt OUT of the resident expert arena, back to MapViewOfFile.\n" " The A/B arm: the mmap's rate depends on the OS page cache holding\n" - " 34 GB, and measured 71.97 vs 34.78 ms/token cold vs warm.\n"); + " 34 GB, and measured 71.97 vs 34.78 ms/token cold vs warm.\n" + " --resident-cpu-experts with mmap and a static profile, keep CPU misses resident in ordinary RAM.\n" + " Borrowed GPU-cache entries may read from mmap during prompt prefill.\n"); } /// All shards of a split GGUF, from shard 1's path ("...-00001-of-00002.gguf"); just the path when it is not split. @@ -1043,6 +1047,7 @@ int main(int argc, char** argv) { else if (a == "--expert-profile") o.expert_profile = next("--expert-profile"); else if (a == "--gpu-stages") o.gpu_stages = true; else if (a == "--mmap-experts") o.mmap_experts = true; + else if (a == "--resident-cpu-experts") o.resident_cpu_experts = true; else if (a == "--stats") o.stats = true; else if (a == "--shared-late") o.shared_late = true; else if (a == "--keep-canonical") o.keep_canonical = true; @@ -1111,6 +1116,16 @@ int main(int argc, char** argv) { } } const bool multi_gpu = !split_devs.empty() && !split_same; + if (o.resident_cpu_experts && (!o.mmap_experts || o.expert_profile.empty() || o.adapt_every != 0)) { + std::fprintf(stderr, "strata generate: --resident-cpu-experts requires --mmap-experts, a static --expert-profile and --adapt-every 0\n"); + return 2; + } + if (o.resident_cpu_experts && + (!o.layer_split.empty() || o.expert_cache_remote[0] > 0 || o.expert_cache_remote[1] > 0 || + o.expert_cache_remote[2] > 0)) { + std::fprintf(stderr, "strata generate: --resident-cpu-experts does not support layer splits or remote expert caches\n"); + return 2; + } if (multi_gpu) { if (o.vision || !o.cvec_files.empty() || o.expert_cache_remote[0] > 0 || o.mmap_experts || o.expert_profile.empty()) { @@ -2067,6 +2082,14 @@ int main(int argc, char** argv) { (long long) prefilled, (long long) want); } + if (o.resident_cpu_experts) { + if (!src.pin_cache_complement(xcache, err, /*pin=*/false)) { + std::fprintf(stderr, "strata generate: CPU expert residency: %s\n", err.c_str()); + return 1; + } + std::fprintf(stderr, "strata generate: steady-state CPU cache misses are resident in ordinary RAM; borrowed cache entries may use mmap during prompt prefill\n"); + } + for (auto& stp : stages) { GpuStage& st = *stp; const auto& lay = strata::kernels::cpu::expert_layout(); @@ -2873,6 +2896,13 @@ int main(int argc, char** argv) { : (uint64_t) (xcache.slots() - first) * (uint64_t) strata::kernels::cpu::expert_layout().max_blob; }; + auto request_chunk = [](int64_t tokens, int64_t max_chunk) -> int64_t { + if (tokens <= 0 || max_chunk <= 0) return 0; + const int64_t rounded = tokens > std::numeric_limits::max() - 255 + ? tokens + : ((tokens + 255) / 256) * 256; + return std::min(max_chunk, rounded); + }; // The prompt path's chunk and the slots it borrows for its buffers: the requested chunk halved until it fits, // or with --prefill auto the largest of kAutoChunks whose buffers take at most kAutoLendPct % of the slots (a // lent slot's expert is streamed during the prompt and refilled after it; measured on a 12 GB card, 32K Q2_0 @@ -3688,12 +3718,21 @@ int main(int argc, char** argv) { // rounded up to 256), laid out in the last of the slots it may borrow auto lend = [&](int64_t tokens, std::string& e) -> bool { if (lend_first < 0) return true; // its own buffers: nothing to lend - const int64_t want = std::min(o.prefill_chunk, (tokens + 255) / 256 * 256); + const int64_t want = request_chunk(tokens, o.prefill_chunk); + if (want <= 0) { + e = "prefill: cannot lend buffers for an empty request segment"; + return false; + } if (!lent_now.empty()) { if (want <= lent_chunk) return true; if (!refill(e)) return false; } - const int32_t first = std::max(lend_first, (int32_t) (xcache.slots() - lend_slots(want))); + const int64_t slots = lend_slots(want); + if (slots <= 0 || slots > xcache.slots() - lend_first) { + e = "prefill: request-sized buffers exceed the configured lend region"; + return false; + } + const int32_t first = std::max(lend_first, (int32_t) (xcache.slots() - slots)); if (want != sp.chunk() || first != lend_first_now) { if (!sp.relayout(want, xcache.device_slot(first), lend_bytes(first), e)) return false; lend_first_now = first; @@ -4062,19 +4101,22 @@ int main(int argc, char** argv) { std::FILE* final_r = o.dump_final_r.empty() ? nullptr : std::fopen(o.dump_final_r.c_str(), "wb"); std::vector final_r_host(final_r ? (size_t) (g.hc * g.n_embd) : 0); std::vector> lent; // (residency index, slot) lent to the prompt path + const int64_t n_batched = (o.prefill_until > 0 && o.prefill_until < n_prompt - 1) ? o.prefill_until : n_prompt - 1; if (o.prefill_chunk > 0 && n_prompt > 1) { void* borrow = nullptr; uint64_t borrow_bytes = 0; if (!o.no_prefill_borrow && !host_res.empty() && d_res != nullptr) { int64_t chunk = o.prefill_chunk; int64_t k = plan_lend(chunk); // auto: the largest chunk that fits; fixed: halved to fit - if (k > 0 && chunk > (n_prompt - 1 + 255) / 256 * 256) { // no bigger than the prompt needs - chunk = std::max(256, (n_prompt - 1 + 255) / 256 * 256); + const int64_t request_sized = request_chunk(n_batched, chunk); + if (k > 0 && request_sized < chunk) { // no bigger than this prompt segment needs + chunk = request_sized; k = lend_slots(chunk); + if (k + 128 > xcache.slots()) k = 0; if (!o.prefill_auto) o.prefill_chunk = chunk; } if (o.prefill_auto) { - o.prefill_chunk = k > 0 ? chunk : 1024; + o.prefill_chunk = k > 0 ? chunk : request_chunk(n_batched, 1024); std::fprintf(stderr, "strata generate: prompt chunk auto: %lld tokens\n", (long long) o.prefill_chunk); } else if (chunk != o.prefill_chunk) { k = 0; // a fixed chunk that does not fit: its own buffers, as before @@ -4116,7 +4158,6 @@ int main(int argc, char** argv) { }; } const Clock::time_point tp0 = Clock::now(); - const int64_t n_batched = (o.prefill_until > 0 && o.prefill_until < n_prompt - 1) ? o.prefill_until : n_prompt - 1; if (!prefill.run(o.tokens.data(), n_batched, 0, err)) { std::fprintf(stderr, "strata generate: %s\n", err.c_str()); return 1; diff --git a/tests/core/file_expert_source_test.cpp b/tests/core/file_expert_source_test.cpp index 3a25248ff99..fdb86c24dca 100644 --- a/tests/core/file_expert_source_test.cpp +++ b/tests/core/file_expert_source_test.cpp @@ -183,10 +183,33 @@ void test_native_variable_layout() { } #endif +void test_complement_plan() { + using namespace strata::core::detail; + std::vector offsets; + uint64_t bytes = 0; + std::string error; + require(make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}, {1, 2}}, {}, offsets, bytes, error), error); + require(bytes == 16 && offsets == std::vector{0, kNoCacheComplement, 3, 6, 11, kNoCacheComplement}, + "wrong compact offsets for variable native layer sizes"); + const uint8_t resident[16] = {}, mapped[5] = {}; + require(cache_complement_blob_or_fallback(1, offsets, resident, mapped) == mapped, + "GPU-resident expert lost mmap fallback"); + require(cache_complement_blob_or_fallback(4, offsets, resident, mapped) == resident + 11, + "CPU miss did not use resident complement"); + require(!make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}, {0, 1}}, {}, offsets, bytes, error) + && offsets.empty() && bytes == 0, "duplicate pair accepted"); + require(!make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}}, {{0, 1}}, offsets, bytes, error), + "overlapping tiers accepted"); + require(!make_cache_complement_plan(2, 3, {3, 5}, {{2, 0}}, {}, offsets, bytes, error), + "out-of-range pair accepted"); + require(!make_cache_complement_plan(2, 3, {0, 5}, {}, {}, offsets, bytes, error), "zero-size layer accepted"); +} + } // namespace int main() { try { + test_complement_plan(); test_canonical_layout(); #if defined(STRATA_NATIVE_EXPERTS) test_native_variable_layout(); diff --git a/tests/hip/prefill_hipblaslt_gemm.cpp b/tests/hip/prefill_hipblaslt_gemm.cpp new file mode 100644 index 00000000000..e237af390c3 --- /dev/null +++ b/tests/hip/prefill_hipblaslt_gemm.cpp @@ -0,0 +1,188 @@ +// Opt-in HIP test for the calibrated hipBLASLt prefill route. Requires a matching tuning table. +#include +#include +#include +#include +#include + +#include "strata/prefill/gemm.hpp" +#include "hipblaslt_tuning.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define HIP_CHECK(call) do { \ + const hipError_t status_ = (call); \ + if (status_ != hipSuccess) { \ + std::fprintf(stderr, "HIP failure at %s:%d: %s: %s\n", __FILE__, __LINE__, #call, hipGetErrorString(status_)); \ + std::exit(2); \ + } \ +} while (0) + +#define HIPBLAS_CHECK(call) do { \ + const hipblasStatus_t status_ = (call); \ + if (status_ != HIPBLAS_STATUS_SUCCESS) { \ + std::fprintf(stderr, "hipBLAS failure at %s:%d: %s: %d\n", __FILE__, __LINE__, #call, (int) status_); \ + std::exit(2); \ + } \ +} while (0) + +struct DeviceBuffer { + void* p = nullptr; + explicit DeviceBuffer(size_t bytes) { if (bytes) HIP_CHECK(hipMalloc(&p, bytes)); } + ~DeviceBuffer() { if (p) (void) hipFree(p); } + DeviceBuffer(const DeviceBuffer&) = delete; + DeviceBuffer& operator=(const DeviceBuffer&) = delete; +}; + +uint16_t encode(float value, bool bf16) { + if (bf16) return hip_bfloat16(value).data; + const __half half = __float2half_rn(value); + uint16_t bits = 0; + std::memcpy(&bits, &half, sizeof(bits)); + return bits; +} + +bool run_case(strata::prefill::Gemm& gemm, hipblasHandle_t blas, hipStream_t stream, bool bf16, int t, int n, + int k, int ldy, int output_offset, float beta, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution dist(-0.25f, 0.25f); + const size_t x_count = (size_t) t * k; + const size_t w_count = (size_t) n * k; + std::vector x(x_count), w(w_count); + for (auto& value : x) value = encode(dist(rng), bf16); + for (auto& value : w) value = encode(dist(rng), bf16); + + const size_t output_count = (size_t) output_offset + (size_t) t * ldy + 8; + std::vector initial(output_count, -777.25f); + for (int row = 0; row < t; ++row) { + for (int col = 0; col < n; ++col) { + initial[(size_t) output_offset + (size_t) row * ldy + col] = + (float) ((row * 17 + col * 3) % 29 - 14) * 0.03125f; + } + } + + DeviceBuffer dx(x_count * sizeof(uint16_t)); + DeviceBuffer dw(w_count * sizeof(uint16_t)); + DeviceBuffer dy(initial.size() * sizeof(float)); + DeviceBuffer dref(initial.size() * sizeof(float)); + HIP_CHECK(hipMemcpyAsync(dx.p, x.data(), x_count * sizeof(uint16_t), hipMemcpyHostToDevice, stream)); + HIP_CHECK(hipMemcpyAsync(dw.p, w.data(), w_count * sizeof(uint16_t), hipMemcpyHostToDevice, stream)); + HIP_CHECK(hipMemcpyAsync(dy.p, initial.data(), initial.size() * sizeof(float), hipMemcpyHostToDevice, stream)); + HIP_CHECK(hipMemcpyAsync(dref.p, initial.data(), initial.size() * sizeof(float), hipMemcpyHostToDevice, stream)); + HIP_CHECK(hipStreamSynchronize(stream)); + + const hipDataType type = bf16 ? HIP_R_16BF : HIP_R_16F; + const float alpha = 1.0f; + HIPBLAS_CHECK(hipblasGemmEx(blas, HIPBLAS_OP_T, HIPBLAS_OP_N, n, t, k, &alpha, dw.p, type, k, dx.p, type, k, + &beta, (float*) dref.p + output_offset, HIP_R_32F, ldy, HIPBLAS_COMPUTE_32F, + HIPBLAS_GEMM_DEFAULT)); + if (bf16) { + gemm.bf16((const uint16_t*) dx.p, (const uint16_t*) dw.p, (float*) dy.p + output_offset, t, n, k, ldy, + beta); + } else { + gemm.f16((const uint16_t*) dx.p, (const uint16_t*) dw.p, (float*) dy.p + output_offset, t, n, k, ldy, + beta); + } + HIP_CHECK(hipStreamSynchronize(stream)); + + std::vector got(initial.size()), ref(initial.size()); + HIP_CHECK(hipMemcpy(got.data(), dy.p, got.size() * sizeof(float), hipMemcpyDeviceToHost)); + HIP_CHECK(hipMemcpy(ref.data(), dref.p, ref.size() * sizeof(float), hipMemcpyDeviceToHost)); + + std::vector active(initial.size(), 0); + double diff2 = 0.0, ref2 = 0.0, max_abs = 0.0; + for (int row = 0; row < t; ++row) { + for (int col = 0; col < n; ++col) { + const size_t index = (size_t) output_offset + (size_t) row * ldy + col; + active[index] = 1; + if (!std::isfinite(got[index]) || !std::isfinite(ref[index])) { + std::fprintf(stderr, "non-finite output dtype=%s T=%d N=%d K=%d beta=%.1f index=%zu\n", + bf16 ? "bf16" : "f16", t, n, k, beta, index); + return false; + } + const double delta = (double) got[index] - ref[index]; + diff2 += delta * delta; + ref2 += (double) ref[index] * ref[index]; + max_abs = std::max(max_abs, std::abs(delta)); + } + } + for (size_t i = 0; i < initial.size(); ++i) { + if (!active[i] && (got[i] != initial[i] || ref[i] != initial[i])) { + std::fprintf(stderr, "output touched padding/guard dtype=%s T=%d N=%d K=%d beta=%.1f index=%zu got=%.9g\n", + bf16 ? "bf16" : "f16", t, n, k, beta, i, got[i]); + return false; + } + } + const double relative_l2 = std::sqrt(diff2 / std::max(ref2, 1e-300)); + const bool passed = relative_l2 <= 1e-4 && max_abs <= 5e-3; + std::printf("case dtype=%s T=%d N=%d K=%d ldy=%d output_offset=%d beta=%.1f relative_l2=%.9g max_abs=%.9g %s\n", + bf16 ? "bf16" : "f16", t, n, k, ldy, output_offset, beta, relative_l2, max_abs, + passed ? "PASS" : "FAIL"); + return passed; +} + +int main() { + const char* tuning_path = std::getenv("STRATA_HIPBLASLT_TUNING"); + if (!tuning_path || !*tuning_path) { + std::fprintf(stderr, "SKIP: set STRATA_HIPBLASLT_TUNING to a calibrated matching table\n"); + return 77; + } + + int device = 0; + HIP_CHECK(hipGetDevice(&device)); + hipDeviceProp_t properties{}; + HIP_CHECK(hipGetDeviceProperties(&properties, device)); + std::string arch(properties.gcnArchName); + const auto suffix = arch.find(':'); + if (suffix != std::string::npos) arch.resize(suffix); + hipblasLtHandle_t lt = nullptr; + HIPBLAS_CHECK(hipblasLtCreate(<)); + int version = 0; + HIPBLAS_CHECK(hipblasLtGetVersion(lt, &version)); + HIPBLAS_CHECK(hipblasLtDestroy(lt)); + + strata::prefill::hipblaslt::TuningTable table; + std::string error; + if (!table.load(tuning_path, arch, version, error)) { + std::fprintf(stderr, "tuning table rejected: %s\n", error.c_str()); + return 1; + } + if (!table.closest(strata::prefill::hipblaslt::InputType::bf16, 48, 2560, 96, 4096) || + !table.closest(strata::prefill::hipblaslt::InputType::f16, 512, 2560, 512, 4096)) { + std::fprintf(stderr, "tuning table lacks the focused BF16/F16 test rows\n"); + return 1; + } + + hipStream_t stream = nullptr; + HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); + bool ok = true; + { + std::string init_error; + strata::prefill::Gemm gemm; + if (!gemm.init((void*) stream, 0, init_error)) { + std::fprintf(stderr, "Gemm init failed: %s\n", init_error.c_str()); + HIP_CHECK(hipStreamDestroy(stream)); + return 2; + } + hipblasHandle_t blas = nullptr; + HIPBLAS_CHECK(hipblasCreate(&blas)); + HIPBLAS_CHECK(hipblasSetStream(blas, stream)); + + // Exact calibrated chunks exercise the selected solution; T=37 reuses the closest bucket, + // validates the actual shape before launch, and tests a non-tile-multiple tail. + ok &= run_case(gemm, blas, stream, true, 4096, 48, 2560, 96, 0, 0.0f, 101); + ok &= run_case(gemm, blas, stream, true, 37, 48, 2560, 96, 48, 1.0f, 102); + ok &= run_case(gemm, blas, stream, false, 4096, 512, 2560, 512, 0, 0.0f, 103); + ok &= run_case(gemm, blas, stream, false, 37, 512, 2560, 512, 7, 1.0f, 104); + HIPBLAS_CHECK(hipblasDestroy(blas)); + } + HIP_CHECK(hipStreamDestroy(stream)); + return ok ? 0 : 1; +} diff --git a/tests/hip/prefill_mmq_parity.cpp b/tests/hip/prefill_mmq_parity.cpp new file mode 100644 index 00000000000..2fd7a0d7a62 --- /dev/null +++ b/tests/hip/prefill_mmq_parity.cpp @@ -0,0 +1,281 @@ +// Standalone diagnostic harness for strata-prefill's optional MMQ path. +// Keep this artifact outside the project source until it is reviewed and wired into a HIP test target. +// It compares GGML's MMQ result (with its intended q8_1 activation rounding) to a CPU FP32 product using +// GGML's own dequantizer for the exact same weight blocks. The tolerances screen layout/stride/bounds errors; +// they do not assert bitwise or FP16 parity. +#include "strata/prefill/moe_mmq.hpp" +#include "ggml.h" +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using strata::prefill::mmq::Context; +using strata::prefill::mmq::Product; +using strata::prefill::mmq::matrix_bytes; +using strata::prefill::mmq::q8_bytes; +using strata::prefill::mmq::quantize; +using strata::prefill::mmq::supported; + +void hip_check(hipError_t e, const char * what) { + if (e != hipSuccess) throw std::runtime_error(std::string(what) + ": " + hipGetErrorString(e)); +} + +struct DeviceBuffer { + void * p = nullptr; + explicit DeviceBuffer(size_t n) { hip_check(hipMalloc(&p, n), "hipMalloc"); } + ~DeviceBuffer() { if (p) hipFree(p); } + DeviceBuffer(const DeviceBuffer &) = delete; + DeviceBuffer & operator=(const DeviceBuffer &) = delete; + template T * as() const { return static_cast(p); } +}; + +size_t row_bytes(ggml_type type, int64_t cols) { + return ggml_row_size(type, cols); +} + +std::vector decode_matrix(ggml_type type, const std::vector & q, int64_t rows, int64_t cols) { + const auto * tr = ggml_get_type_traits(type); + if (!tr || !tr->to_float) throw std::runtime_error("GGML type has no host dequantizer"); + const size_t rb = row_bytes(type, cols); + if (q.size() != (size_t) rows * rb) throw std::runtime_error("matrix byte count mismatch"); + std::vector out((size_t) rows * (size_t) cols); + for (int64_t r = 0; r < rows; ++r) { + tr->to_float(q.data() + (size_t) r * rb, out.data() + (size_t) r * (size_t) cols, cols); + } + return out; +} + +std::vector make_q2_matrix(int64_t rows, int64_t cols, int expert, int trial) { + const ggml_type type = GGML_TYPE_Q2_0; + const auto * tr = ggml_get_type_traits(type); + if (!tr || !tr->from_float_ref) throw std::runtime_error("Q2_0 host quantizer unavailable"); + const size_t rb = row_bytes(type, cols); + std::vector out((size_t) rows * rb); + std::vector row((size_t) cols); + for (int64_t r = 0; r < rows; ++r) { + for (int64_t k = 0; k < cols; ++k) { + const int code = (int) ((k * 37 + r * 19 + expert * 23 + trial * 11) % 101) - 50; + row[(size_t) k] = 0.025f * std::sin(0.013f * (float) (k + 1) + 0.17f * (float) r + 0.31f * (float) expert) + + 0.0015f * (float) code; + } + tr->from_float_ref(row.data(), out.data() + (size_t) r * rb, cols); + } + return out; +} + +std::vector make_activations(int rows, int64_t cols, int trial) { + std::vector x((size_t) rows * (size_t) cols); + for (int r = 0; r < rows; ++r) { + for (int64_t k = 0; k < cols; ++k) { + x[(size_t) r * (size_t) cols + (size_t) k] = + 0.45f * std::sin(0.009f * (float) (k + 1) + 0.23f * (float) r + 0.07f * (float) trial) + + 0.17f * std::cos(0.021f * (float) (k + 3) - 0.19f * (float) r) + + 0.002f * (float) (((k * 7 + r * 13 + trial * 5) % 19) - 9); + } + } + // Exercise a complete all-zero row; its expected output is exactly zero even though Q8_1 rounding is enabled. + std::fill(x.end() - cols, x.end(), 0.0f); + return x; +} + +std::vector permutation(int rows, int shift, bool reverse) { + std::vector p((size_t) rows); + for (int i = 0; i < rows; ++i) { + p[(size_t) i] = reverse ? (int32_t) ((rows - 1 - i + shift) % rows) + : (int32_t) ((i + shift) % rows); + } + return p; +} + +std::vector make_bounds(const std::vector & counts) { + std::vector b(counts.size() + 1, 0); + for (size_t i = 0; i < counts.size(); ++i) b[i + 1] = b[i] + counts[i]; + return b; +} + +struct Metrics { double rel_l2 = 0, max_over_rms = 0, zero_row_max = 0, ref_rms = 0; }; + +Metrics run_product(Context & ctx, hipStream_t stream, const std::string & name, ggml_type type, + int64_t out_rows, int64_t cols, const std::vector> & expert_weights, + const std::vector & counts, const std::vector & x, + const std::vector & src_ids, const std::vector & dst_ids) { + const int rows = (int) src_ids.size(); + const int experts = (int) expert_weights.size(); + const std::vector bounds = make_bounds(counts); + if (!supported((int) type) || experts <= 0 || counts.size() != expert_weights.size() || + bounds.back() != rows || dst_ids.size() != (size_t) rows || x.size() != (size_t) rows * (size_t) cols) { + throw std::runtime_error(name + ": invalid test geometry"); + } + const size_t expert_bytes = matrix_bytes((int) type, out_rows, cols); + const size_t tail = 4096; // match the production group's zero guard after its last expert + std::vector w((size_t) experts * expert_bytes + tail, 0); + for (int e = 0; e < experts; ++e) { + if (expert_weights[(size_t) e].size() != expert_bytes) throw std::runtime_error(name + ": expert size mismatch"); + std::copy(expert_weights[(size_t) e].begin(), expert_weights[(size_t) e].end(), + w.begin() + (size_t) e * expert_bytes); + } + + DeviceBuffer dx(x.size() * sizeof(float)); + DeviceBuffer dsrc(src_ids.size() * sizeof(int32_t)); + DeviceBuffer ddst(dst_ids.size() * sizeof(int32_t)); + DeviceBuffer dbounds(bounds.size() * sizeof(int32_t)); + DeviceBuffer dw(w.size()); + DeviceBuffer dxq(q8_bytes(rows, cols)); + DeviceBuffer dy((size_t) rows * (size_t) out_rows * sizeof(float)); + hip_check(hipMemcpy(dx.p, x.data(), x.size() * sizeof(float), hipMemcpyHostToDevice), "copy x"); + hip_check(hipMemcpy(dsrc.p, src_ids.data(), src_ids.size() * sizeof(int32_t), hipMemcpyHostToDevice), "copy src ids"); + hip_check(hipMemcpy(ddst.p, dst_ids.data(), dst_ids.size() * sizeof(int32_t), hipMemcpyHostToDevice), "copy dst ids"); + hip_check(hipMemcpy(dbounds.p, bounds.data(), bounds.size() * sizeof(int32_t), hipMemcpyHostToDevice), "copy bounds"); + hip_check(hipMemcpy(dw.p, w.data(), w.size(), hipMemcpyHostToDevice), "copy weights and zero tail"); + hip_check(hipMemset(dy.p, 0xff, (size_t) rows * (size_t) out_rows * sizeof(float)), "initialize output sentinel"); + + quantize(dx.as(), dsrc.as(), dxq.p, (int) type, cols, cols, rows, (void *) stream); + const int max_rows = *std::max_element(counts.begin(), counts.end()); + Product p; + p.w = dw.p; p.type = (int) type; p.w_rows = out_rows; p.w_cols = cols; p.expert_bytes = expert_bytes; + p.n = experts; p.xq = dxq.p; p.bounds = dbounds.as(); p.ids = ddst.as(); + p.total_rows = rows; p.max_rows = max_rows; p.dst = dy.as(); p.ld_dst = out_rows; + ctx.run(p, (void *) stream); + hip_check(hipGetLastError(), "MMQ launch"); + hip_check(hipStreamSynchronize(stream), "MMQ synchronize"); + std::vector got((size_t) rows * (size_t) out_rows); + hip_check(hipMemcpy(got.data(), dy.p, got.size() * sizeof(float), hipMemcpyDeviceToHost), "copy output"); + + std::vector ref(got.size(), 0.0f); + double err2 = 0, ref2 = 0; + float max_abs = 0, zero_max = 0; + for (int e = 0; e < experts; ++e) { + const std::vector wd = decode_matrix(type, expert_weights[(size_t) e], out_rows, cols); + for (int r = bounds[(size_t) e]; r < bounds[(size_t) e + 1]; ++r) { + const int src = src_ids[(size_t) r]; + const int dst = dst_ids[(size_t) r]; + const float * xr = x.data() + (size_t) src * (size_t) cols; + for (int64_t o = 0; o < out_rows; ++o) { + double acc = 0; + const float * wr = wd.data() + (size_t) o * (size_t) cols; + for (int64_t k = 0; k < cols; ++k) acc += (double) wr[k] * xr[k]; + ref[(size_t) dst * (size_t) out_rows + (size_t) o] = (float) acc; + } + } + } + for (size_t i = 0; i < got.size(); ++i) { + if (!std::isfinite(got[i])) throw std::runtime_error(name + ": non-finite or unwritten MMQ output"); + const double d = (double) got[i] - ref[i]; + err2 += d * d; ref2 += (double) ref[i] * ref[i]; max_abs = std::max(max_abs, (float) std::fabs(d)); + } + const double rms = std::sqrt(ref2 / (double) ref.size()); + for (int r = 0; r < rows; ++r) { + if (std::all_of(x.begin() + (size_t) src_ids[(size_t) r] * (size_t) cols, + x.begin() + ((size_t) src_ids[(size_t) r] + 1) * (size_t) cols, + [](float v) { return v == 0.0f; })) { + for (int64_t o = 0; o < out_rows; ++o) + zero_max = std::max(zero_max, std::fabs(got[(size_t) dst_ids[(size_t) r] * (size_t) out_rows + (size_t) o])); + } + } + Metrics m; + m.rel_l2 = rms > 0 ? std::sqrt(err2 / (double) ref.size()) / rms : std::sqrt(err2); + m.max_over_rms = rms > 0 ? max_abs / rms : max_abs; + m.zero_row_max = zero_max; m.ref_rms = rms; + std::cout << name << ": type=" << (int) type << " rows=" << rows << " experts=" << experts + << " ref_rms=" << m.ref_rms << " rel_l2=" << m.rel_l2 + << " max_abs/ref_rms=" << m.max_over_rms << " zero_row_max=" << m.zero_row_max << '\n'; + // Q8_1 rounding is expected. These deliberately broad limits catch wrong block order, row strides, ids, + // expert bounds, or stale output while allowing ordinary activation-quantization error. + if (m.rel_l2 > 0.04 || m.max_over_rms > 0.35 || m.zero_row_max > std::max(1e-5, 1e-4 * rms)) + throw std::runtime_error(name + ": parity screen failed"); + return m; +} + +std::vector> synthetic_q2(int experts, int64_t out_rows, int64_t cols, int trial) { + std::vector> w; + for (int e = 0; e < experts; ++e) w.push_back(make_q2_matrix(out_rows, cols, e, trial)); + return w; +} + +bool run_real_iq3_first_expert(Context & ctx, hipStream_t stream, const std::string & pack) { + std::ifstream meta(pack + "/native_experts.txt"); + if (!meta) throw std::runtime_error("cannot open native_experts.txt: " + pack); + int layer = -1, gt = -1, dt = -1; + uint64_t offset = 0, blob_bytes = 0; + std::string line; + bool found = false; + while (std::getline(meta, line)) { + if (line.empty() || line[0] == '#') continue; + std::istringstream ss(line); + if (ss >> layer >> gt >> dt >> offset >> blob_bytes && layer == 0) { found = true; break; } + } + if (!found) throw std::runtime_error("native metadata has no layer 0"); + if (gt != GGML_TYPE_IQ3_XXS || !supported(gt) || !supported(dt)) { + std::cout << "real-IQ3 first expert skipped: layer 0 types are " << gt << "/" << dt << '\n'; + return false; + } + std::ifstream file(pack + "/experts.bin", std::ios::binary); + if (!file) throw std::runtime_error("cannot open experts.bin: " + pack); + std::vector blob((size_t) blob_bytes); + file.seekg((std::streamoff) offset); + file.read((char *) blob.data(), (std::streamsize) blob.size()); + if (!file) throw std::runtime_error("short read of first native expert"); + + constexpr int64_t N = 2560, FF = 640; + const size_t gu_row = row_bytes((ggml_type) gt, N), d_row = row_bytes((ggml_type) dt, FF); + const size_t one_gu = (size_t) FF * gu_row, down_off = 2 * one_gu, one_down = (size_t) N * d_row; + if (down_off + one_down != blob.size()) throw std::runtime_error("native blob shape/offset mismatch"); + std::vector gu(blob.begin(), blob.begin() + (ptrdiff_t) down_off); + std::vector down(blob.begin() + (ptrdiff_t) down_off, blob.end()); + const int rows = 5; + const std::vector counts{rows}; + const auto src = permutation(rows, 2, false), dst = permutation(rows, 1, true); + run_product(ctx, stream, "real-first-IQ3_XXS-GU", (ggml_type) gt, 1280, N, {gu}, counts, + make_activations(rows, N, 71), src, dst); + run_product(ctx, stream, "real-first-down", (ggml_type) dt, N, FF, {down}, counts, + make_activations(rows, FF, 73), src, dst); + std::cout << "real first expert tested: layer=0 expert=0 blob_bytes=" << blob_bytes << '\n'; + return true; +} +} // namespace + +int main(int argc, char ** argv) { + try { + if (argc > 2) throw std::runtime_error("usage: mmq-parity-test [optional-native-pack-dir]"); + hip_check(hipSetDevice(0), "hipSetDevice"); + hipStream_t stream = nullptr; + hip_check(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking), "create stream"); + int failures = 0; + { + Context ctx; // reuse one MMQ context/pool across every pass below + const std::vector> batches{{1, 3, 3}, {4, 1, 2}, {2, 3}}; + int trial = 0; + for (const auto & counts : batches) { + const int rows = std::accumulate(counts.begin(), counts.end(), 0); + const int n = (int) counts.size(); + const auto src = permutation(rows, trial + 1, false); + const auto dst = permutation(rows, trial + 2, true); + run_product(ctx, stream, "synthetic-Q2_0-GU-pass" + std::to_string(trial), GGML_TYPE_Q2_0, + 1280, 2560, synthetic_q2(n, 1280, 2560, trial), counts, + make_activations(rows, 2560, trial), src, dst); + run_product(ctx, stream, "synthetic-Q2_0-down-pass" + std::to_string(trial), GGML_TYPE_Q2_0, + 2560, 640, synthetic_q2(n, 2560, 640, trial + 9), counts, + make_activations(rows, 640, trial + 3), src, dst); + ++trial; + } + if (argc == 2) run_real_iq3_first_expert(ctx, stream, argv[1]); + } + hip_check(hipStreamSynchronize(stream), "final synchronize"); + hip_check(hipStreamDestroy(stream), "destroy stream"); + std::cout << "MMQ parity screen passed\n"; + return failures ? 1 : 0; + } catch (const std::exception & e) { + std::cerr << "MMQ parity screen failed: " << e.what() << '\n'; + return 1; + } +} diff --git a/tests/hip/prefill_native_batch.cpp b/tests/hip/prefill_native_batch.cpp new file mode 100644 index 00000000000..3e5b51a5aa5 --- /dev/null +++ b/tests/hip/prefill_native_batch.cpp @@ -0,0 +1,289 @@ +// Focused parity for the opt-in native QSA append and native embedding gather +// selectively ported from q8atnight/Strata PR #108 +// (acd487233c0bbe2217a6881c5bb43f8a283b0de5). +#include +#include "strata/kernels/iq_kernels.hpp" +#include "strata/kernels/native_qsa_indexer.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(call) do { \ + const cudaError_t e = (call); \ + if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", #call, cudaGetErrorString(e)); return false; } \ +} while (0) + +namespace { +constexpr int IDX = 128; +constexpr int BLOCK = 4; +constexpr int EMB = 2560; +constexpr int IQ4_NL = 20; +constexpr float EPS = 1e-6f; + +struct IndexState { + float *tail = nullptr, *dead = nullptr, *pooled = nullptr; + int32_t* block_pos = nullptr; +}; + +bool alloc_state(IndexState& s, int64_t max_cells) { + CHECK(cudaMalloc((void**) &s.tail, (BLOCK - 1) * IDX * sizeof(float))); + CHECK(cudaMalloc((void**) &s.dead, IDX * sizeof(float))); + CHECK(cudaMalloc((void**) &s.pooled, (size_t) (max_cells / BLOCK + 1) * IDX * sizeof(float))); + CHECK(cudaMalloc((void**) &s.block_pos, sizeof(int32_t))); + return true; +} +void free_state(IndexState& s) { + if (s.tail) cudaFree(s.tail); + if (s.dead) cudaFree(s.dead); + if (s.pooled) cudaFree(s.pooled); + if (s.block_pos) cudaFree(s.block_pos); + s = {}; +} +bool upload_state(const IndexState& s, int64_t max_cells) { + std::vector tail((BLOCK - 1) * IDX, 0.0f), dead(IDX, 0.0f); + std::vector pooled((size_t) (max_cells / BLOCK + 1) * IDX); + for (size_t i = 0; i < pooled.size(); ++i) pooled[i] = -0.125f * (float) (i + 1); + const int32_t block_pos = -1; + CHECK(cudaMemcpy(s.tail, tail.data(), tail.size() * sizeof(float), cudaMemcpyHostToDevice)); + CHECK(cudaMemcpy(s.dead, dead.data(), dead.size() * sizeof(float), cudaMemcpyHostToDevice)); + CHECK(cudaMemcpy(s.pooled, pooled.data(), pooled.size() * sizeof(float), cudaMemcpyHostToDevice)); + CHECK(cudaMemcpy(s.block_pos, &block_pos, sizeof(block_pos), cudaMemcpyHostToDevice)); + return true; +} + +bool qsa_case(cudaStream_t stream, int64_t max_cells, int64_t pos0, int64_t T, std::mt19937& rng) { + const size_t raw_n = (size_t) (pos0 + T) * IDX; + std::vector raw(raw_n), gamma(IDX); + std::uniform_real_distribution dist(-1.0f, 1.0f); + for (float& x : raw) x = dist(rng); + for (float& x : gamma) x = 0.5f + 0.5f * std::fabs(dist(rng)); + + float *raw_d = nullptr, *gamma_d = nullptr; + int32_t* pos_d = nullptr; + CHECK(cudaMalloc((void**) &raw_d, raw.size() * sizeof(float))); + CHECK(cudaMalloc((void**) &gamma_d, gamma.size() * sizeof(float))); + CHECK(cudaMalloc((void**) &pos_d, sizeof(int32_t))); + CHECK(cudaMemcpyAsync(raw_d, raw.data(), raw.size() * sizeof(float), cudaMemcpyHostToDevice, stream)); + CHECK(cudaMemcpyAsync(gamma_d, gamma.data(), gamma.size() * sizeof(float), cudaMemcpyHostToDevice, stream)); + + IndexState serial, batched; + if (!alloc_state(serial, max_cells) || !alloc_state(batched, max_cells)) return false; + if (!upload_state(serial, max_cells) || !upload_state(batched, max_cells)) return false; + + const auto shapes = strata::kernels::qsa_real_shapes(); + const strata::kernels::QsaIndexerBuffers sb{serial.tail, serial.dead, serial.pooled, serial.block_pos}; + const strata::kernels::QsaIndexerBuffers bb{batched.tail, batched.dead, batched.pooled, batched.block_pos}; + std::vector positions((size_t) (pos0 + T)); + for (size_t p = 0; p < positions.size(); ++p) positions[p] = (int32_t) p; + + // The reference is the original per-cell sequence. The candidate uses the + // same per-cell prefix, then replaces only this chunk with the two-launch API. + for (int64_t p = 0; p < pos0 + T; ++p) { + CHECK(cudaMemcpyAsync(pos_d, positions.data() + p, sizeof(int32_t), cudaMemcpyHostToDevice, stream)); + strata::kernels::native_qsa_indexer_append(raw_d + p * IDX, pos_d, 0, gamma_d, EPS, sb, + shapes, max_cells, 10000.0f, stream); + if (p < pos0) { + CHECK(cudaMemcpyAsync(pos_d, positions.data() + p, sizeof(int32_t), cudaMemcpyHostToDevice, stream)); + strata::kernels::native_qsa_indexer_append(raw_d + p * IDX, pos_d, 0, gamma_d, EPS, bb, + shapes, max_cells, 10000.0f, stream); + } + } + strata::kernels::native_qsa_indexer_append_chunk(raw_d + pos0 * IDX, T, pos0, 0, gamma_d, EPS, + bb, shapes, max_cells, 10000.0f, stream); + CHECK(cudaStreamSynchronize(stream)); + + const size_t tail_n = (BLOCK - 1) * IDX, dead_n = IDX; + const size_t pooled_n = (size_t) (max_cells / BLOCK + 1) * IDX; + std::vector st(tail_n), bt(tail_n), sd(dead_n), bd(dead_n), sp(pooled_n), bp(pooled_n); + int32_t spos = 0, bpos = 0; + CHECK(cudaMemcpy(st.data(), serial.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(bt.data(), batched.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(sd.data(), serial.dead, dead_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(bd.data(), batched.dead, dead_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(sp.data(), serial.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(bp.data(), batched.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(&spos, serial.block_pos, sizeof(spos), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(&bpos, batched.block_pos, sizeof(bpos), cudaMemcpyDeviceToHost)); + const bool same = std::memcmp(st.data(), bt.data(), tail_n * sizeof(float)) == 0 && + std::memcmp(sd.data(), bd.data(), dead_n * sizeof(float)) == 0 && + std::memcmp(sp.data(), bp.data(), pooled_n * sizeof(float)) == 0 && spos == bpos; + if (!same) std::fprintf(stderr, "QSA append parity mismatch max=%lld pos0=%lld T=%lld\n", + (long long) max_cells, (long long) pos0, (long long) T); + + free_state(serial); free_state(batched); + CHECK(cudaFree(raw_d)); CHECK(cudaFree(gamma_d)); CHECK(cudaFree(pos_d)); + return same; +} + +bool qsa_chunks_case(cudaStream_t stream, std::mt19937& rng, int64_t max_cells, + int64_t first, int64_t second) { + const int64_t total = first + second; + std::vector raw((size_t) total * IDX), gamma(IDX); + std::uniform_real_distribution dist(-1.0f, 1.0f); + for (float& x : raw) x = dist(rng); + for (float& x : gamma) x = 0.5f + 0.5f * std::fabs(dist(rng)); + float *raw_d = nullptr, *gamma_d = nullptr; + int32_t* pos_d = nullptr; + CHECK(cudaMalloc((void**) &raw_d, raw.size() * sizeof(float))); + CHECK(cudaMalloc((void**) &gamma_d, gamma.size() * sizeof(float))); + CHECK(cudaMalloc((void**) &pos_d, sizeof(int32_t))); + CHECK(cudaMemcpyAsync(raw_d, raw.data(), raw.size() * sizeof(float), cudaMemcpyHostToDevice, stream)); + CHECK(cudaMemcpyAsync(gamma_d, gamma.data(), gamma.size() * sizeof(float), cudaMemcpyHostToDevice, stream)); + IndexState serial, chunked; + if (!alloc_state(serial, max_cells) || !alloc_state(chunked, max_cells)) return false; + if (!upload_state(serial, max_cells) || !upload_state(chunked, max_cells)) return false; + const auto shapes = strata::kernels::qsa_real_shapes(); + const strata::kernels::QsaIndexerBuffers sb{serial.tail, serial.dead, serial.pooled, serial.block_pos}; + const strata::kernels::QsaIndexerBuffers cb{chunked.tail, chunked.dead, chunked.pooled, chunked.block_pos}; + std::vector positions((size_t) total); + for (int i = 0; i < total; ++i) positions[(size_t) i] = i; + for (int p = 0; p < total; ++p) { + CHECK(cudaMemcpyAsync(pos_d, positions.data() + p, sizeof(int32_t), cudaMemcpyHostToDevice, stream)); + strata::kernels::native_qsa_indexer_append(raw_d + (size_t) p * IDX, pos_d, 0, gamma_d, EPS, sb, + shapes, max_cells, 10000.0f, stream); + } + strata::kernels::native_qsa_indexer_append_chunk(raw_d, first, 0, 0, gamma_d, EPS, cb, shapes, + max_cells, 10000.0f, stream); + strata::kernels::native_qsa_indexer_append_chunk(raw_d + (size_t) first * IDX, second, first, 0, + gamma_d, EPS, cb, shapes, max_cells, 10000.0f, stream); + CHECK(cudaStreamSynchronize(stream)); + const size_t tail_n = (BLOCK - 1) * IDX, pooled_n = (size_t) (max_cells / BLOCK + 1) * IDX; + std::vector st(tail_n), ct(tail_n), sd(IDX), cd(IDX), sp(pooled_n), cp(pooled_n); + int32_t sbp = 0, cbp = 0; + CHECK(cudaMemcpy(st.data(), serial.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(ct.data(), chunked.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(sd.data(), serial.dead, IDX * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(cd.data(), chunked.dead, IDX * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(sp.data(), serial.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(cp.data(), chunked.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(&sbp, serial.block_pos, sizeof(sbp), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(&cbp, chunked.block_pos, sizeof(cbp), cudaMemcpyDeviceToHost)); + const bool same = std::memcmp(st.data(), ct.data(), tail_n * sizeof(float)) == 0 && + std::memcmp(sd.data(), cd.data(), IDX * sizeof(float)) == 0 && + std::memcmp(sp.data(), cp.data(), pooled_n * sizeof(float)) == 0 && sbp == cbp; + if (!same) std::fprintf(stderr, "QSA consecutive chunk parity mismatch %lld+%lld, capacity=%lld\n", + (long long) first, (long long) second, (long long) max_cells); + free_state(serial); free_state(chunked); + CHECK(cudaFree(raw_d)); CHECK(cudaFree(gamma_d)); CHECK(cudaFree(pos_d)); + return same; +} + +bool embed_case(cudaStream_t stream, int type, int64_t T) { + constexpr int vocab = 19; + const size_t row_bytes = strata::kernels::iq_row_bytes(type, EMB); + const size_t block_bytes = type == 20 ? 18 : 110; // IQ4_NL or IQ3_S GGML block layout + const size_t table_bytes = (size_t) vocab * row_bytes; + if (row_bytes == 0 || row_bytes % block_bytes != 0) { + std::fprintf(stderr, "bad test embedding row size for type=%d\n", type); + return false; + } + void* host_table = nullptr; + CHECK(cudaHostAlloc(&host_table, table_bytes, cudaHostAllocMapped | cudaHostAllocPortable)); + auto* table_bytes_u8 = (uint8_t*) host_table; + std::memset(table_bytes_u8, 0, table_bytes); + // Both test formats begin with a finite fp16 scale; the remaining packed code bits are all legal values. + for (size_t r = 0; r < (size_t) vocab; ++r) { + for (size_t b = 0; b < row_bytes; b += block_bytes) { + table_bytes_u8[r * row_bytes + b] = 0x00; + table_bytes_u8[r * row_bytes + b + 1] = 0x3c; + for (size_t q = b + 2; q < b + block_bytes; ++q) + table_bytes_u8[r * row_bytes + q] = (uint8_t) ((q * 37 + r * 19) & 0xff); + } + } + void* table_dev = nullptr; + CHECK(cudaHostGetDevicePointer(&table_dev, host_table, 0)); + + std::vector image_rows((size_t) T, 0); + std::vector token_values((size_t) T); + std::vector ids((size_t) T); + for (int64_t t = 0; t < T; ++t) { + if (t == 1 || t == 3) { + image_rows[(size_t) t] = 1; + token_values[(size_t) t] = t == 1 ? -1 : vocab + 11; // image placeholders must never be gathered + ids[(size_t) t] = 0; // safe row, overwritten below + } else { + token_values[(size_t) t] = (t % 3 == 0) ? 7 : (t % 5); // includes duplicate text IDs + if (token_values[(size_t) t] < 0 || token_values[(size_t) t] >= vocab || + token_values[(size_t) t] > INT32_MAX) { + std::fprintf(stderr, "test text token failed host-side validation\n"); + return false; + } + ids[(size_t) t] = (int32_t) token_values[(size_t) t]; + } + } + + float *batched = nullptr, *serial = nullptr; + int32_t* ids_d = nullptr; + CHECK(cudaMalloc((void**) &batched, (size_t) T * EMB * sizeof(float))); + CHECK(cudaMalloc((void**) &serial, (size_t) T * EMB * sizeof(float))); + CHECK(cudaMalloc((void**) &ids_d, (size_t) T * sizeof(int32_t))); + CHECK(cudaMemcpyAsync(ids_d, ids.data(), (size_t) T * sizeof(int32_t), cudaMemcpyHostToDevice, stream)); + strata::kernels::iq_embed_rows(type, table_dev, row_bytes, ids_d, T, EMB, batched, stream); + for (int64_t t = 0; t < T; ++t) if (!image_rows[(size_t) t]) + strata::kernels::iq_dequant_f32(type, + (const uint8_t*) table_dev + (size_t) ids[(size_t) t] * row_bytes, EMB, serial + t * EMB, stream); + + std::vector image(EMB); + for (int i = 0; i < EMB; ++i) image[(size_t) i] = 0.001f * (float) (i - EMB / 2); + for (int64_t t = 0; t < T; ++t) if (image_rows[(size_t) t]) { + CHECK(cudaMemcpyAsync(batched + t * EMB, image.data(), EMB * sizeof(float), cudaMemcpyHostToDevice, stream)); + CHECK(cudaMemcpyAsync(serial + t * EMB, image.data(), EMB * sizeof(float), cudaMemcpyHostToDevice, stream)); + } + CHECK(cudaStreamSynchronize(stream)); + + constexpr int64_t rows_per_check = 64; + std::vector a((size_t) rows_per_check * EMB), b(a.size()); + bool same = true; + for (int64_t t0 = 0; t0 < T; t0 += rows_per_check) { + const int64_t rows = std::min(rows_per_check, T - t0); + const size_t count = (size_t) rows * EMB; + CHECK(cudaMemcpy(a.data(), batched + t0 * EMB, count * sizeof(float), cudaMemcpyDeviceToHost)); + CHECK(cudaMemcpy(b.data(), serial + t0 * EMB, count * sizeof(float), cudaMemcpyDeviceToHost)); + if (std::memcmp(a.data(), b.data(), count * sizeof(float)) != 0) { + size_t first = 0; + while (first < count && std::memcmp(&a[first], &b[first], sizeof(float)) == 0) ++first; + std::fprintf(stderr, "NativeEmbed batch parity mismatch type=%d T=%lld at element %lld: %.9g vs %.9g\n", + type, (long long) T, (long long) (t0 * EMB + first), + first < count ? a[first] : 0.0f, first < count ? b[first] : 0.0f); + same = false; + break; + } + } + if (T > 3 && (!image_rows[1] || !image_rows[3] || ids[1] != 0 || ids[3] != 0)) same = false; + if (!same) std::fprintf(stderr, "NativeEmbed image placeholder sanitization mismatch type=%d T=%lld\n", + type, (long long) T); + CHECK(cudaFree(batched)); CHECK(cudaFree(serial)); CHECK(cudaFree(ids_d)); + CHECK(cudaFreeHost(host_table)); + return same; +} + +} // namespace + +int main() { + std::mt19937 rng(0x50524631u); + cudaStream_t stream = nullptr; + const cudaError_t create = cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking); + if (create != cudaSuccess) { std::fprintf(stderr, "cudaStreamCreateWithFlags: %s\n", cudaGetErrorString(create)); return 2; } + bool ok = true; + // Covers every slot residue, partial and multiple block chunks, and the fixed-capacity tail. + const std::array starts = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}; + const std::array lengths = {1, 2, 3, 7, 17}; + for (int64_t pos0 : starts) for (int64_t T : lengths) + ok = qsa_case(stream, 13, pos0, T, rng) && ok; + ok = qsa_chunks_case(stream, rng, 33, 7, 17) && ok; + // Production-scale, non-block-aligned first chunk with a short continuation. + ok = qsa_chunks_case(stream, rng, 8210, 8193, 17) && ok; + for (int type : {20, 21}) for (int64_t T : {1, 2, 7, 17}) + ok = embed_case(stream, type, T) && ok; + ok = embed_case(stream, 21, 4096) && ok; + const cudaError_t destroy = cudaStreamDestroy(stream); + if (destroy != cudaSuccess) { std::fprintf(stderr, "cudaStreamDestroy: %s\n", cudaGetErrorString(destroy)); return 2; } + std::printf("Native prefill batch parity: %s\n", ok ? "PASS" : "FAIL"); + return ok ? 0 : 1; +} diff --git a/tests/platform/direct_file_async.cpp b/tests/platform/direct_file_async.cpp new file mode 100644 index 00000000000..1e51aafa851 --- /dev/null +++ b/tests/platform/direct_file_async.cpp @@ -0,0 +1,62 @@ +#include "strata/platform/direct_file.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +using strata::platform::DirectFile; +using strata::platform::Completion; +static void check(bool ok, const char* why) { if (!ok) throw std::runtime_error(why); } +int main() { + char name[] = "/tmp/strata-direct-test-XXXXXX"; + int fd = mkstemp(name); + if (fd < 0) return 1; + close(fd); + void* memory = DirectFile::alloc_aligned(4096 * 18); + try { + check(memory != nullptr, "buffer allocation"); + { std::ofstream f(name, std::ios::binary); std::string bytes(4096 * 16 + 17, 'x'); f.write(bytes.data(), bytes.size()); } + DirectFile f; + std::string error; + setenv("STRATA_PLE_IO_THREADS", "0", 1); + check(!f.open(name, error), "invalid thread count accepted"); + setenv("STRATA_PLE_IO_THREADS", "4", 1); + check(f.open(name, error), error.c_str()); + check(!f.submit(1, memory, 4096, 0, error), "unaligned request accepted"); + for (unsigned i = 0; i < 17; ++i) + check(f.submit(i * 4096, static_cast(memory) + i * 4096, 4096, i, error), error.c_str()); + std::vector seen(17); + unsigned count = 0; + while (count < 17) { + Completion c[17]; int n = f.wait(c, 17, 5000); + check(n > 0, "completion timeout"); + for (int i = 0; i < n; ++i) { + check(c[i].tag < 17 && !seen[c[i].tag], "wrong or repeated completion"); + seen[c[i].tag] = true; ++count; + check(c[i].ok && c[i].bytes == (c[i].tag == 16 ? 17u : 4096u), "incorrect full/EOF completion"); + check(static_cast(memory)[c[i].tag * 4096] == 'x', "read data mismatch"); + } + } + std::jthread waker([&] { f.wake(); }); + Completion c; check(f.wait(&c, 1, 5000) == 1 && c.tag == DirectFile::WAKE_TAG, "wake lost"); + waker.join(); + check(f.submit(0, memory, 4096, 99, error), error.c_str()); + f.close(); // Must drain outstanding buffer writes before returning. + std::memset(memory, 0, 4096); + check(f.open(name, error), error.c_str()); + check(f.wait(&c, 1, 0) == 0, "stale completion on reopen"); + check(f.submit(0, memory, 4096, 100, error), error.c_str()); + check(f.wait(&c, 1, 5000) == 1 && c.ok && c.tag == 100, "reopened read failed"); + f.close(); + DirectFile::free_aligned(memory); unlink(name); + std::cout << "direct_file_async: passed queued reads, short EOF, wake, close drain, reopen\n"; + return 0; + } catch (const std::exception& e) { + DirectFile::free_aligned(memory); unlink(name); + std::cerr << e.what() << '\n'; return 1; + } +} diff --git a/tools/hip/bench_prefill.py b/tools/hip/bench_prefill.py new file mode 100644 index 00000000000..8edb11f6bf9 --- /dev/null +++ b/tools/hip/bench_prefill.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Matched fresh/follow-up prefill probe against an otherwise idle local Strata server. + +Does not start/stop services. Use a dedicated server and its engine log, restart +between build/configuration arms, and keep model/template/settings identical. +""" +import argparse +import json +import os +from pathlib import Path +import re +import time +import urllib.request + +PATTERN = re.compile(r'prompt (\d+) tokens = (\d+) reused \+ (\d+) read in (\d+) ms \(([0-9.]+) tok/s\), (\d+) generated in (\d+) ms \(([0-9.]+) tok/s\)') +FIELDS = ('prompt_tokens', 'reused', 'fresh', 'prefill_ms', 'prefill_tps', 'generated', 'decode_ms', 'decode_tps') + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--url', default='http://127.0.0.1:8080') + p.add_argument('--model', required=True) + p.add_argument('--engine-log', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--label', required=True) + args = p.parse_args() + results = [] + + def request(messages, kind, trial): + offset = args.engine_log.stat().st_size + body = dict(model=args.model, messages=messages, max_tokens=128, + temperature=0, top_k=1, top_p=1, min_p=0, seed=42, + reasoning_effort='none') + headers = {'Content-Type': 'application/json'} + if os.environ.get('STRATA_API_KEY'): + headers['Authorization'] = 'Bearer ' + os.environ['STRATA_API_KEY'] + req = urllib.request.Request(args.url.rstrip('/') + '/v1/chat/completions', + data=json.dumps(body).encode(), headers=headers) + start = time.monotonic() + with urllib.request.urlopen(req, timeout=360) as response: + data = json.load(response) + elapsed = time.monotonic() - start + # The API response may arrive just before stderr's completion line is flushed. + deadline = time.monotonic() + 2 + while True: + with args.engine_log.open('rb') as log: + log.seek(offset) + matches = PATTERN.findall(log.read().decode(errors='replace')) + if matches or time.monotonic() >= deadline: + break + time.sleep(0.02) + if len(matches) != 1: + raise RuntimeError('Expected exactly one completed engine timing line; check idle server and log path') + metrics = dict(zip(FIELDS, map(float, matches[0]))) + if kind == 'fresh' and metrics['reused'] != 0: + raise RuntimeError('Fresh prompt reused cached tokens; restart this arm before comparison') + choice = data['choices'][0] + results.append(dict(label=args.label, kind=kind, trial=trial, wall_s=elapsed, + metrics=metrics, usage=data.get('usage'), + finish_reason=choice.get('finish_reason'), message=choice['message'])) + args.output.write_text(json.dumps(results, indent=2) + '\n') + print(json.dumps(results[-1]), flush=True) + return choice['message'] + + request([{'role': 'user', 'content': 'Reply READY.'}], 'warmup', 0) + for trial, n in enumerate((140, 280, 140, 280), 1): + code = '\n'.join(f'export function rule{i}(x) {{ return x === {i} ? x + {i+1} : x - {i}; }}' for i in range(n)) + messages = [{'role': 'user', 'content': f'CASE {trial}: Review this source and describe its behavior precisely in a paragraph.\n' + code}] + reply = request(messages, 'fresh', trial) + messages += [reply, {'role': 'user', 'content': 'Explain the most relevant boundary case and the smallest useful regression test. ' + 'Focus on integer equality, zero, negative input, unexpected types, and caller assumptions. ' * 5}] + request(messages, 'followup', trial) + + +if __name__ == '__main__': + main() diff --git a/tools/hip/gfx1100-hipblaslt-100100.txt b/tools/hip/gfx1100-hipblaslt-100100.txt new file mode 100644 index 00000000000..2d02ffd1211 --- /dev/null +++ b/tools/hip/gfx1100-hipblaslt-100100.txt @@ -0,0 +1,32 @@ +# solution IDs are scoped to this hipBLASLt version and device architecture +STRATA_HIPBLASLT_TUNING_V1 gfx1100 100100 +bf16 1 2560 1 4096 541 +bf16 1 2560 1 8192 541 +bf16 4 10240 4 4096 541 +bf16 4 10240 4 8192 541 +bf16 48 2560 96 4096 541 +bf16 48 2560 96 8192 542 +bf16 128 2560 128 4096 542 +bf16 128 2560 128 8192 542 +bf16 320 10240 320 4096 542 +bf16 320 10240 320 8192 542 +bf16 512 2560 512 4096 542 +bf16 512 2560 512 8192 542 +bf16 2560 2560 2560 4096 542 +bf16 2560 2560 2560 8192 542 +bf16 10240 320 10240 4096 542 +bf16 10240 320 10240 8192 542 +f16 512 2560 512 4096 1147 +f16 512 2560 512 8192 1147 +f16 640 2560 640 4096 1147 +f16 640 2560 640 8192 1147 +f16 2560 640 2560 4096 1147 +f16 2560 640 2560 8192 1147 +f16 2560 6144 2560 4096 1147 +f16 2560 6144 2560 8192 1147 +f16 6144 2560 6144 4096 1147 +f16 6144 2560 6144 8192 1147 +f16 10240 2560 10240 4096 1147 +f16 10240 2560 10240 8192 1147 +f16 12288 2560 12288 4096 1147 +f16 12288 2560 12288 8192 1147 diff --git a/tools/hip/tune_hipblaslt.cpp b/tools/hip/tune_hipblaslt.cpp new file mode 100644 index 00000000000..5af18bfc38d --- /dev/null +++ b/tools/hip/tune_hipblaslt.cpp @@ -0,0 +1,419 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +#define HIP_CHECK(call) do { const hipError_t e = (call); if (e != hipSuccess) { \ + std::fprintf(stderr, "HIP %s:%d: %s: %s\n", __FILE__, __LINE__, #call, hipGetErrorString(e)); std::exit(2); } } while (0) +#define BLAS_CHECK(call) do { const hipblasStatus_t e = (call); if (e != HIPBLAS_STATUS_SUCCESS) { \ + std::fprintf(stderr, "hipBLAS %s:%d: %s: status=%d\n", __FILE__, __LINE__, #call, (int)e); std::exit(2); } } while (0) +#define LT_CHECK(call) do { const hipblasStatus_t e = (call); if (e != HIPBLAS_STATUS_SUCCESS) { \ + std::fprintf(stderr, "hipBLASLt %s:%d: %s: status=%d\n", __FILE__, __LINE__, #call, (int)e); std::exit(2); } } while (0) + +struct Buffer { + void *p = nullptr; + explicit Buffer(size_t bytes) { if (bytes) HIP_CHECK(hipMalloc(&p, bytes)); } + ~Buffer() { if (p) (void)hipFree(p); } + Buffer(const Buffer &) = delete; + Buffer &operator=(const Buffer &) = delete; +}; +struct Events { + hipEvent_t a = nullptr, b = nullptr; + Events() { HIP_CHECK(hipEventCreate(&a)); HIP_CHECK(hipEventCreate(&b)); } + ~Events() { if (a) (void)hipEventDestroy(a); if (b) (void)hipEventDestroy(b); } +}; + +struct Shape { int t, n, k, ldy; bool bf16; }; +struct Options { + size_t workspace = 32U * 1024U * 1024U; + std::vector shapes; + std::vector tokens{4096, 8192}; + std::string shapes_file, tuning_out; + bool explicit_shapes = false; +}; +struct Err { double rel_l2, max_abs; bool finite; size_t padding_writes; }; +struct Best { + Shape s; + int heuristic_index, solution_id; + size_t required_workspace, algo_workspace; + float mean_ms; + Err error; + std::string solution, kernel, config; +}; + +constexpr int WARMUPS = 2; +constexpr int REPS = 3; +constexpr int MAX_ALGOS = 16; +constexpr double REL_L2_TOL = 1e-4; +constexpr double MAX_ABS_TOL = 1e-2; +constexpr float PADDING_CANARY = 123456.25f; + +const char *dtype(bool bf16) { return bf16 ? "bf16" : "f16"; } +const char *status_name(hipblasStatus_t s) { + switch (s) { + case HIPBLAS_STATUS_SUCCESS: return "success"; + case HIPBLAS_STATUS_NOT_SUPPORTED: return "not_supported"; + case HIPBLAS_STATUS_INVALID_VALUE: return "invalid_value"; + case HIPBLAS_STATUS_ARCH_MISMATCH: return "arch_mismatch"; + default: return "other"; + } +} +std::string json(const std::string &s) { + std::ostringstream o; o << '"'; + for (unsigned char c : s) { + if (c == '"') o << "\\\""; + else if (c == '\\') o << "\\\\"; + else if (c == '\n') o << "\\n"; + else if (c == '\r') o << "\\r"; + else if (c == '\t') o << "\\t"; + else if (c < 0x20) o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (unsigned)c << std::dec; + else o << (char)c; + } + o << '"'; return o.str(); +} +std::string config_hex(const hipblasLtMatmulAlgo_t &algo) { + std::ostringstream o; o << std::hex << std::setfill('0'); + for (uint8_t b : algo.data) o << std::setw(2) << (unsigned)b; + return o.str(); +} +uint16_t input_bits(float x, bool bf16) { + if (bf16) return hip_bfloat16(x).data; + const __half half = __float2half_rn(x); + uint16_t bits = 0; + std::memcpy(&bits, &half, sizeof(bits)); + return bits; +} +void fill(std::vector &v, uint32_t seed, bool bf16) { + std::mt19937 rng(seed); + std::uniform_real_distribution dist(-1.0f, 1.0f); + for (auto &x : v) x = input_bits(dist(rng), bf16); +} +float time_call(hipStream_t stream, const std::function &f) { + Events ev; + HIP_CHECK(hipEventRecord(ev.a, stream)); + f(); + HIP_CHECK(hipEventRecord(ev.b, stream)); + HIP_CHECK(hipEventSynchronize(ev.b)); + HIP_CHECK(hipDeviceSynchronize()); + float ms = 0; + HIP_CHECK(hipEventElapsedTime(&ms, ev.a, ev.b)); + return ms; +} +Err compare(const std::vector &ref, const std::vector &got, int n, int t, int ld) { + long double d2 = 0, r2 = 0; double max_abs = 0; bool finite = true; + for (int col = 0; col < t; ++col) for (int row = 0; row < n; ++row) { + const size_t i = (size_t)col * ld + row; + const double r = ref[i], g = got[i]; + if (!std::isfinite(r) || !std::isfinite(g)) { finite = false; continue; } + const double d = g - r; + if (!std::isfinite(d)) { finite = false; continue; } + d2 += (long double)d * d; r2 += (long double)r * r; + max_abs = std::max(max_abs, std::abs(d)); + } + if (!finite) return {std::numeric_limits::infinity(), + std::numeric_limits::infinity(), false, 0}; + return {std::sqrt((double)(d2 / std::max(r2, 1e-300L))), max_abs, true, 0}; +} +size_t padding_writes(const std::vector &got, int n, int t, int ld) { + size_t writes = 0; + for (int col = 0; col < t; ++col) for (int row = n; row < ld; ++row) + if (got[(size_t)col * ld + row] != PADDING_CANARY) ++writes; + return writes; +} +bool accuracy_ok(const Err &e) { + return e.finite && e.padding_writes == 0 && + e.rel_l2 <= REL_L2_TOL && e.max_abs <= MAX_ABS_TOL; +} +std::vector csv(const std::string &s) { + std::vector out; size_t b = 0; + for (;;) { + const size_t p = s.find(',', b); + out.push_back(s.substr(b, p == std::string::npos ? p : p - b)); + if (p == std::string::npos) return out; + b = p + 1; + } +} +int positive(const std::string &s, const char *label) { + char *e = nullptr; const long v = std::strtol(s.c_str(), &e, 10); + if (s.empty() || !e || *e || v < 1 || v > std::numeric_limits::max()) { + std::fprintf(stderr, "invalid %s: %s\n", label, s.c_str()); std::exit(2); + } + return (int)v; +} +void add(Options &o, bool bf16, int t, int n, int k, int ldy) { + if (ldy < n) { std::fprintf(stderr, "ldy must be >= N\n"); std::exit(2); } + o.shapes.push_back({t,n,k,ldy,bf16}); +} +void add_shape(Options &o, const std::string &v) { + const auto f = csv(v); + if (f.size() != 3) { std::fprintf(stderr, "--shape expects T,N,K\n"); std::exit(2); } + int t=positive(f[0],"T"), n=positive(f[1],"N"), k=positive(f[2],"K"); + add(o,false,t,n,k,n); add(o,true,t,n,k,n); o.explicit_shapes=true; +} +void add_case(Options &o, const std::string &v) { + const auto f = csv(v); + if (f.size()!=5 || (f[0]!="f16" && f[0]!="bf16")) { + std::fprintf(stderr, "--case expects dtype,T,N,K,ldy\n"); std::exit(2); + } + add(o,f[0]=="bf16",positive(f[1],"T"),positive(f[2],"N"),positive(f[3],"K"),positive(f[4],"ldy")); + o.explicit_shapes=true; +} +std::vector parse_tokens(const std::string &s) { + std::vector out; + for (const auto &x : csv(s)) out.push_back(positive(x,"token bucket")); + return out; +} +void load_shapes(Options &o) { + std::ifstream f(o.shapes_file); + if (!f) { std::fprintf(stderr, "cannot open shapes file: %s\n",o.shapes_file.c_str()); std::exit(2); } + const std::string text((std::istreambuf_iterator(f)),{}); + const std::regex obj(R"(\{[^{}]*\})"), dtype_re(R"re("dtype"\s*:\s*"(f16|bf16)")re"), + n_re(R"("N"\s*:\s*([0-9]+))"), k_re(R"("K"\s*:\s*([0-9]+))"), + ld_re(R"("ldy"\s*:\s*([0-9]+))"); + int count=0; + for (auto i=std::sregex_iterator(text.begin(),text.end(),obj); i!=std::sregex_iterator(); ++i) { + const std::string item=i->str(); std::smatch d,n,k,ld; + if (!std::regex_search(item,d,dtype_re) || !std::regex_search(item,n,n_re) || + !std::regex_search(item,k,k_re) || !std::regex_search(item,ld,ld_re)) continue; + for (int t:o.tokens) add(o,d[1]=="bf16",t,positive(n[1],"N"),positive(k[1],"K"),positive(ld[1],"ldy")); + ++count; + } + if (!count) { std::fprintf(stderr,"no dtype,N,K,ldy entries in %s\n",o.shapes_file.c_str()); std::exit(2); } + o.explicit_shapes=true; +} +void usage(const char *p) { + std::printf("Usage: %s [--workspace-mib N] [--shape T,N,K]... [--case dtype,T,N,K,ldy]...\n",p); + std::printf(" [--shapes-file PATH [--tokens T1,T2,...]] [--tuning-out PATH]\n"); + std::printf("Defaults: original three shapes, f16+bf16, T=8192, workspace=32 MiB.\n"); +} +Options options(int argc,char **argv) { + Options o; + for (int i=1;i std::string { if(i+1>=argc){std::fprintf(stderr,"missing option value\n");std::exit(2);} return argv[++i]; }; + if(a=="-h"||a=="--help"){usage(argv[0]);std::exit(0);} + else if(a=="--workspace-mib"){ + std::string v=next(); char *e=nullptr; unsigned long long x=std::strtoull(v.c_str(),&e,10); + if(v.empty()||!e||*e||x>std::numeric_limits::max()/(1024ULL*1024ULL)){std::fprintf(stderr,"bad workspace MiB\n");std::exit(2);} + o.workspace=(size_t)x*1024U*1024U; + } else if(a=="--shape") add_shape(o,next()); + else if(a=="--case") add_case(o,next()); + else if(a=="--shapes-file"){o.shapes_file=next();o.explicit_shapes=true;} + else if(a=="--tokens") o.tokens=parse_tokens(next()); + else if(a=="--tuning-out") o.tuning_out=next(); + else {std::fprintf(stderr,"unknown option: %s\n",a.c_str());usage(argv[0]);std::exit(2);} + } + if(!o.shapes_file.empty()) load_shapes(o); + if(!o.explicit_shapes) { + struct DefaultShape { int n,k,ldy; bool bf16; }; + const DefaultShape defs[]={{10240,2560,10240,false},{320,10240,320,false},{2560,320,2560,false}, + {10240,2560,10240,true},{320,10240,320,true},{2560,320,2560,true}}; + for(const auto &s:defs) add(o,s.bf16,8192,s.n,s.k,s.ldy); + } + if(o.shapes.empty()){std::fprintf(stderr,"no shapes requested\n");std::exit(2);} + return o; +} +std::string arch_name(const char *s) { + std::string a=s?s:"unknown"; size_t p=a.find(':'); if(p!=std::string::npos)a.resize(p); return a; +} +void emit_candidate(const Shape &s,size_t workspace,int h,int id,size_t req,size_t algows,float ms,const Err &error, + const std::string &arch,int version,const std::string &sol,const std::string &kernel,const std::string &config) { + std::cout<<"candidate_json={\"device_arch\":"<&bests) { + const size_t na=(size_t)s.k*s.n, nb=(size_t)s.k*s.t, out_elems=(size_t)s.ldy*s.t; + const hipDataType it=s.bf16?HIP_R_16BF:HIP_R_16F; + const hipblasOperation_t ta=HIPBLAS_OP_T,tb=HIPBLAS_OP_N; + const int m=s.n,n=s.t,k=s.k,lda=s.k,ldb=s.k,ldc=s.ldy; + const float alpha=1.f,beta=0.f; + std::vector ha(na),hb(nb); + uint32_t seed=0x5A17C3U+(uint32_t)case_id*0x10001U+(s.bf16?0xB16U:0xF16U); + fill(ha,seed,s.bf16);fill(hb,seed^0x9E3779B9U,s.bf16); + Buffer da(na*sizeof(uint16_t)),db(nb*sizeof(uint16_t)),dc(out_elems*sizeof(float)),dy(out_elems*sizeof(float)),dws(ws); + const std::vector canary(out_elems,PADDING_CANARY); + HIP_CHECK(hipMemcpyAsync(dc.p,canary.data(),out_elems*sizeof(float),hipMemcpyHostToDevice,stream)); + HIP_CHECK(hipMemcpyAsync(da.p,ha.data(),na*sizeof(uint16_t),hipMemcpyHostToDevice,stream)); + HIP_CHECK(hipMemcpyAsync(db.p,hb.data(),nb*sizeof(uint16_t),hipMemcpyHostToDevice,stream)); + HIP_CHECK(hipStreamSynchronize(stream)); + + hipblasLtMatmulDesc_t op=nullptr; + hipblasLtMatrixLayout_t ad=nullptr,bd=nullptr,cd=nullptr; + hipblasLtMatmulPreference_t pref=nullptr; + LT_CHECK(hipblasLtMatmulDescCreate(&op,HIPBLAS_COMPUTE_32F,HIP_R_32F)); + LT_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSA,&ta,sizeof(ta))); + LT_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSB,&tb,sizeof(tb))); + LT_CHECK(hipblasLtMatrixLayoutCreate(&ad,it,s.k,s.n,lda)); + LT_CHECK(hipblasLtMatrixLayoutCreate(&bd,it,s.k,s.t,ldb)); + LT_CHECK(hipblasLtMatrixLayoutCreate(&cd,HIP_R_32F,s.n,s.t,ldc)); + LT_CHECK(hipblasLtMatmulPreferenceCreate(&pref)); + const uint64_t limit=ws; + LT_CHECK(hipblasLtMatmulPreferenceSetAttribute(pref,HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES,&limit,sizeof(limit))); + + BLAS_CHECK(hipblasSetStream(blas,stream)); + auto base=[&]{BLAS_CHECK(hipblasGemmEx(blas,ta,tb,m,n,k,&alpha,da.p,it,lda,db.p,it,ldb,&beta,dc.p,HIP_R_32F,ldc,HIPBLAS_COMPUTE_32F,HIPBLAS_GEMM_DEFAULT));}; + for(int i=0;i ref(out_elems); + HIP_CHECK(hipMemcpyAsync(ref.data(),dc.p,out_elems*sizeof(float),hipMemcpyDeviceToHost,stream)); + HIP_CHECK(hipStreamSynchronize(stream)); + const size_t baseline_padding=padding_writes(ref,s.n,s.t,s.ldy); + std::printf("baseline dtype=%s T=%d N=%d K=%d ldy=%d padding_writes=%zu\n",dtype(s.bf16),s.t,s.n,s.k,s.ldy,baseline_padding); + float base_mean=(base_ms[0]+base_ms[1]+base_ms[2])/REPS; + + hipblasLtMatmulHeuristicResult_t hs[MAX_ALGOS]{}; + int count=0; + const hipblasStatus_t query=hipblasLtMatmulAlgoGetHeuristic(lt,op,ad,bd,cd,cd,pref,MAX_ALGOS,hs,&count); + std::printf("case dtype=%s T=%d N=%d K=%d ldy=%d workspace_limit_bytes=%zu baseline=hipblasGemmEx rep1_ms=%.4f rep2_ms=%.4f rep3_ms=%.4f mean_ms=%.4f\n", + dtype(s.bf16),s.t,s.n,s.k,s.ldy,ws,base_ms[0],base_ms[1],base_ms[2],base_mean); + if(query!=HIPBLAS_STATUS_SUCCESS||count==0){ + std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d status=%s(%d) heuristic_count=%d\n", + dtype(s.bf16),s.t,s.n,s.k,s.ldy,status_name(query),(int)query,count); + } else { + int best_h=-1,best_id=-1; float best_ms=std::numeric_limits::infinity(); + size_t best_req=0,best_maxws=0; std::string best_sol,best_kernel,best_config; + Err best_error{std::numeric_limits::infinity(),std::numeric_limits::infinity(),false,0}; + for(int h=0;hws){ + std::printf("lt_skip dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d status=%s(%d) required_workspace_bytes=%zu\n", + dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,status_name(hs[h].state),(int)hs[h].state,hs[h].workspaceSize); + continue; + } + auto algo=hs[h].algo; + const int id=hipblaslt_ext::getIndexFromAlgo(algo); + const std::string sol=hipblaslt_ext::getSolutionNameFromAlgo(lt,algo); + const std::string kernel=hipblaslt_ext::getKernelNameFromAlgo(lt,algo); + const std::string config=config_hex(algo); + auto run=[&]{LT_CHECK(hipblasLtMatmul(lt,op,&alpha,da.p,ad,db.p,bd,&beta,dy.p,cd,dy.p,cd,&algo,dws.p,ws,stream));}; + + HIP_CHECK(hipMemcpyAsync(dy.p,canary.data(),out_elems*sizeof(float),hipMemcpyHostToDevice,stream)); + HIP_CHECK(hipStreamSynchronize(stream)); + const hipblasStatus_t check_status=hipblasLtMatmul(lt,op,&alpha,da.p,ad,db.p,bd,&beta,dy.p,cd,dy.p,cd,&algo,dws.p,ws,stream); + if(check_status!=HIPBLAS_STATUS_SUCCESS){ + std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d status=%s(%d) phase=accuracy_check\n", + dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,status_name(check_status),(int)check_status);continue; + } + HIP_CHECK(hipStreamSynchronize(stream)); + std::vector checked_y(out_elems); + HIP_CHECK(hipMemcpyAsync(checked_y.data(),dy.p,out_elems*sizeof(float),hipMemcpyDeviceToHost,stream)); + HIP_CHECK(hipStreamSynchronize(stream)); + Err error=compare(ref,checked_y,s.n,s.t,s.ldy); + error.padding_writes=padding_writes(checked_y,s.n,s.t,s.ldy); + if(!accuracy_ok(error)){ + std::printf("lt_reject dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d finite=%s relative_l2=%.12g relative_l2_tolerance=%.3g max_abs=%.12g max_abs_tolerance=%.3g padding_writes=%zu reason=accuracy_gate\n", + dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,error.finite?"true":"false", + error.rel_l2,REL_L2_TOL,error.max_abs,MAX_ABS_TOL,error.padding_writes);continue; + } + + bool failed=false; + for(int i=0;i&rows){ + if(path.empty())return; + std::ofstream f(path); + if(!f){std::fprintf(stderr,"cannot write tuning output: %s\n",path.c_str());std::exit(2);} + f<<"# solution IDs are scoped to this hipBLASLt version and device architecture\n"; + f<<"STRATA_HIPBLASLT_TUNING_V1 "<bests; + for(size_t i=0;i Date: Tue, 29 Sep 2026 11:09:50 +0300 Subject: [PATCH 3/8] Load supported native Orca PLE keys and verify real-matrix replay --- CMakeLists.txt | 5 + src/core/native_dense.cpp | 6 +- src/program/generate.cpp | 2 +- tests/hip/ple_iq4.cpp | 202 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 211 insertions(+), 4 deletions(-) create mode 100644 tests/hip/ple_iq4.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 52ef525b29f..a588b8023b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -690,6 +690,11 @@ if(STRATA_NATIVE_EXPERTS) set_tests_properties(hip_prefill_mmq_parity PROPERTIES TIMEOUT 120) list(APPEND _strata_hip_tests hip_prefill_mmq_parity) endif() + if(STRATA_ENABLE_HIP) + # Optional real-model diagnostic: pass the GGUF shard containing the native PLE key. + add_executable(hip_ple_iq4 tests/hip/ple_iq4.cpp) + target_link_libraries(hip_ple_iq4 PRIVATE strata_kernels strata_hip_runtime ggml-base) + endif() if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) add_executable(native_expert_parity src/kernels/native_expert_parity.cpp) target_include_directories(native_expert_parity PRIVATE ${_strata_gpu_include_directories}) diff --git a/src/core/native_dense.cpp b/src/core/native_dense.cpp index 51f166f0c86..ba3099897d5 100644 --- a/src/core/native_dense.cpp +++ b/src/core/native_dense.cpp @@ -16,9 +16,9 @@ namespace { bool eligible(const strata::TensorInfo& tensor, bool include_ple_key) { const auto& name = tensor.name; if (name.rfind("blk.", 0) != 0) return false; - // The native PLE kernel accepts Q2_0. Other quantized keys can be converted to BF16 by iq_pack; - // keep their packed bytes resident instead of overriding them with an unsupported native key. - if (name == "blk.1.ple_key.weight") return include_ple_key && tensor.type == 42; // 42 = Q2_0 + // Match the native PLE kernel: Q2_0, IQ3_XXS and IQ4_XS. Other keys retain the packed BF16 fallback. + if (name == "blk.1.ple_key.weight") + return include_ple_key && (tensor.type == 42 || tensor.type == 18 || tensor.type == 23); static const char* suffixes[] = {".attn_qkv.weight", ".attn_gate.weight", ".ssm_out.weight", ".attn_q.weight", ".attn_k.weight", ".attn_v.weight", ".attn_output.weight", ".ffn_gate_shexp.weight", ".ffn_up_shexp.weight", ".ffn_down_shexp.weight"}; diff --git a/src/program/generate.cpp b/src/program/generate.cpp index f187c1a04f8..aa6c0793ddd 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -1550,7 +1550,7 @@ int main(int argc, char** argv) { ss.ple.w.key_scales = (const float*) ((const uint8_t*) wk->data + wk->codes_bytes); } if (o.native_ple_key && wk->quantized()) { - if (!wk->native_data || wk->native_type != 42 || !wk->native_q8_1) { + if (!wk->native_data || (wk->native_type != 42 && wk->native_type != 18 && wk->native_type != 23) || !wk->native_q8_1) { std::fprintf(stderr, "strata generate: native PLE key is absent or incompatible\n"); return 1; } diff --git a/tests/hip/ple_iq4.cpp b/tests/hip/ple_iq4.cpp new file mode 100644 index 00000000000..7215e21b918 --- /dev/null +++ b/tests/hip/ple_iq4.cpp @@ -0,0 +1,202 @@ +// Real PLE IQ3_XXS/IQ4_XS matrix parity: ggml CPU dequantization versus HIP native MMVQ. +// Usage: hip_ple_iq4 [expected-type-id] +#include + +#include "ggml.h" +#include "strata/artifact/gguf_reader.hpp" +#include "strata/kernels/f16_bits.hpp" +#include "strata/kernels/native_mmvq.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr int kNIn = 2560; +constexpr int kNOut = 10240; +constexpr int kTypeIq3Xxs = 18; +constexpr int kTypeIq4Xs = 23; +constexpr int kReplayCount = 3; + +#define CHECK(call) \ + do { \ + const hipError_t error = (call); \ + if (error != hipSuccess) { \ + std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error)); \ + return 2; \ + } \ + } while (0) + +struct Q8Reference { + std::vector dequantized; +}; + +Q8Reference quantize_q8_1_reference(const std::vector& x) { + Q8Reference result; + result.dequantized.resize(x.size()); + for (size_t offset = 0; offset < x.size(); offset += 32) { + float amax = 0.0f; + for (int i = 0; i < 32; ++i) amax = std::fmax(amax, std::fabs(x[offset + i])); + if (amax == 0.0f) { + std::fill(result.dequantized.begin() + offset, result.dequantized.begin() + offset + 32, 0.0f); + continue; + } + const float scale_f32 = amax / 127.0f; + const float scale_f16 = strata::kernels::f32_from_f16(strata::kernels::f16_from_f32(scale_f32)); + for (int i = 0; i < 32; ++i) { + const int q = static_cast(std::round(x[offset + i] / scale_f32)); + result.dequantized[offset + i] = static_cast(static_cast(q)) * scale_f16; + } + } + return result; +} + +struct Parity { + double rel_l1_q8 = 0.0; + double worst_terms_q8 = 0.0; + double rel_l1_f32 = 0.0; + double max_abs_q8 = 0.0; + bool finite = true; +}; + +Parity compare(const uint8_t* weights, size_t row_bytes, const ggml_type_traits* traits, + const std::vector& x, const std::vector& x_q8, + const std::vector& actual) { + std::vector row(kNIn); + double q8_error = 0.0, q8_magnitude = 0.0; + double f32_error = 0.0, f32_magnitude = 0.0; + Parity result; + for (int r = 0; r < kNOut; ++r) { + traits->to_float(weights + static_cast(r) * row_bytes, row.data(), kNIn); + double expected_q8 = 0.0, terms_q8 = 0.0, expected_f32 = 0.0; + for (int i = 0; i < kNIn; ++i) { + const double term_q8 = static_cast(row[i]) * x_q8[i]; + const double term_f32 = static_cast(row[i]) * x[i]; + expected_q8 += term_q8; + terms_q8 += std::fabs(term_q8); + expected_f32 += term_f32; + } + const double got = actual[r]; + if (!std::isfinite(got) || !std::isfinite(expected_q8) || !std::isfinite(expected_f32)) result.finite = false; + q8_error += std::fabs(got - expected_q8); + q8_magnitude += std::fabs(expected_q8); + f32_error += std::fabs(got - expected_f32); + f32_magnitude += std::fabs(expected_f32); + result.max_abs_q8 = std::fmax(result.max_abs_q8, std::fabs(got - expected_q8)); + result.worst_terms_q8 = std::fmax(result.worst_terms_q8, + std::fabs(got - expected_q8) / std::fmax(terms_q8, 1e-30)); + } + result.rel_l1_q8 = q8_error / std::fmax(q8_magnitude, 1e-30); + result.rel_l1_f32 = f32_error / std::fmax(f32_magnitude, 1e-30); + return result; +} + +std::vector make_input(uint32_t seed) { + std::mt19937 rng(seed); + std::normal_distribution normal(0.0f, 0.6f); + std::vector x(kNIn); + for (int i = 0; i < kNIn; ++i) + x[i] = normal(rng) + 0.1f * std::sin(static_cast(i) * 0.013f); + return x; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 2 && argc != 3) { + std::fprintf(stderr, "usage: hip_ple_iq4 [expected-type-id]\n"); + return 2; + } + + try { + strata::GgufFile gguf(argv[1]); + const strata::TensorInfo* tensor = gguf.find("blk.1.ple_key.weight"); + if (!tensor || (tensor->type != kTypeIq3Xxs && tensor->type != kTypeIq4Xs) || tensor->shape.size() != 2 || + tensor->shape[0] != kNIn || tensor->shape[1] != kNOut) { + std::fprintf(stderr, "expected blk.1.ple_key.weight IQ3_XXS or IQ4_XS [2560,10240] in %s\n", argv[1]); + return 1; + } + if (argc == 3 && std::atoi(argv[2]) != static_cast(tensor->type)) { + std::fprintf(stderr, "requested GGML type %s but blk.1.ple_key.weight is type %u (%s)\n", + argv[2], tensor->type, tensor->type_name()); + return 1; + } + const auto* traits = ggml_get_type_traits(static_cast(tensor->type)); + if (!traits || !traits->to_float) { + std::fprintf(stderr, "ggml has no %s CPU dequantizer\n", tensor->type_name()); + return 1; + } + const uint8_t* host_weights = gguf.tensor_data(*tensor); + const size_t weight_bytes = strata::kernels::native_mmvq_weight_bytes(static_cast(tensor->type), kNIn, kNOut); + const size_t row_bytes = weight_bytes / kNOut; + const size_t expected_bytes_per_block = tensor->type == kTypeIq3Xxs ? 98 : 136; + if (weight_bytes != static_cast(tensor->elements()) / 256 * expected_bytes_per_block || + row_bytes != static_cast(kNIn / 256) * expected_bytes_per_block) { + std::fprintf(stderr, "unexpected %s storage geometry: %zu bytes, row %zu\n", + tensor->type_name(), weight_bytes, row_bytes); + return 1; + } + + uint8_t* device_weights = nullptr; + float* device_x = nullptr; + float* device_y = nullptr; + void* device_q8_1 = nullptr; + CHECK(hipMalloc(reinterpret_cast(&device_weights), weight_bytes)); + CHECK(hipMalloc(reinterpret_cast(&device_x), kNIn * sizeof(float))); + CHECK(hipMalloc(reinterpret_cast(&device_y), kNOut * sizeof(float))); + CHECK(hipMalloc(&device_q8_1, strata::kernels::native_q8_1_bytes(kNIn))); + CHECK(hipMemcpy(device_weights, host_weights, weight_bytes, hipMemcpyHostToDevice)); + + hipStream_t stream = nullptr; + CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); + CHECK(hipStreamBeginCapture(stream, hipStreamCaptureModeThreadLocal)); + strata::kernels::native_quantize_q8_1(device_x, device_q8_1, kNIn, 1, static_cast(stream)); + strata::kernels::native_mmvq(static_cast(tensor->type), device_weights, device_q8_1, device_y, + kNIn, kNOut, 1, static_cast(stream)); + hipGraph_t graph = nullptr; + CHECK(hipStreamEndCapture(stream, &graph)); + hipGraphExec_t graph_exec = nullptr; + CHECK(hipGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0)); + + std::vector actual(kNOut); + bool ok = true; + for (int replay = 0; replay < kReplayCount; ++replay) { + const std::vector x = make_input(0x1572u + static_cast(replay)); + const Q8Reference q8 = quantize_q8_1_reference(x); + CHECK(hipMemcpyAsync(device_x, x.data(), x.size() * sizeof(float), hipMemcpyHostToDevice, stream)); + CHECK(hipGraphLaunch(graph_exec, stream)); + CHECK(hipMemcpyAsync(actual.data(), device_y, actual.size() * sizeof(float), hipMemcpyDeviceToHost, stream)); + CHECK(hipStreamSynchronize(stream)); + + const Parity parity = compare(host_weights, row_bytes, traits, x, q8.dequantized, actual); + const bool pass = parity.finite && parity.rel_l1_q8 <= 5e-5 && parity.worst_terms_q8 <= 1e-4 && + parity.rel_l1_f32 <= 3e-2; + std::printf("%s PLE graph replay %d/%d: q8-ref rel-L1 %.3e, worst terms %.3e, " + "original-f32 rel-L1 %.3e, max q8 abs %.3e %s\n", + tensor->type_name(), replay + 1, kReplayCount, parity.rel_l1_q8, parity.worst_terms_q8, + parity.rel_l1_f32, parity.max_abs_q8, pass ? "PASS" : "FAIL"); + ok = ok && pass; + } + + CHECK(hipGraphExecDestroy(graph_exec)); + CHECK(hipGraphDestroy(graph)); + CHECK(hipStreamDestroy(stream)); + CHECK(hipFree(device_q8_1)); + CHECK(hipFree(device_y)); + CHECK(hipFree(device_x)); + CHECK(hipFree(device_weights)); + if (!ok) return 1; + std::printf("PLE %s native MMVQ parity OK: real GGUF matrix, CPU dequant reference, %d graph replays\n", + tensor->type_name(), kReplayCount); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "hip_ple_iq4: %s\n", error.what()); + return 1; + } +} From b90591e88ce806da6e6966d2e5d45b444d040190 Mon Sep 17 00:00:00 2001 From: Konstantinos Korres <96335554+2jztricks@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:26:39 +0300 Subject: [PATCH 4/8] Bound HIP expert uploads with reusable pinned staging --- CMakeLists.txt | 4 +++ include/strata/core/expert_cache.hpp | 6 ++++ src/core/expert_cache.cpp | 44 ++++++++++++++++++++++++++++ tests/hip/expert_cache_staging.cpp | 34 +++++++++++++++++++++ 4 files changed, 88 insertions(+) create mode 100644 tests/hip/expert_cache_staging.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a588b8023b4..227cbad30a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,6 +307,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) if(STRATA_ENABLE_HIP) enable_testing() set(_strata_hip_tests) + add_executable(hip_expert_cache_staging tests/hip/expert_cache_staging.cpp) + target_link_libraries(hip_expert_cache_staging PRIVATE strata_engine) + add_test(NAME hip_expert_cache_staging COMMAND hip_expert_cache_staging) + list(APPEND _strata_hip_tests hip_expert_cache_staging) if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/intrinsics.cpp") add_executable(hip_intrinsics tests/hip/intrinsics.cpp) target_link_libraries(hip_intrinsics PRIVATE strata_hip_runtime) diff --git a/include/strata/core/expert_cache.hpp b/include/strata/core/expert_cache.hpp index da856d75d79..62b0c2a1e91 100644 --- a/include/strata/core/expert_cache.hpp +++ b/include/strata/core/expert_cache.hpp @@ -26,6 +26,7 @@ // kernel. This is that first step, and the step it unblocks is the one that can be measured. #pragma once +#include #include #include #include @@ -136,6 +137,11 @@ class ExpertCache { int64_t fills() const { return fills_; } private: +#if defined(STRATA_USE_HIP) + bool ensure_blocking_staging(std::size_t bytes, std::string& err); + uint8_t* blocking_staging_ = nullptr; + std::size_t blocking_staging_bytes_ = 0; +#endif uint8_t* base_ = nullptr; std::vector residency_; ///< [n_layers * n_expert] -> slot or kNotResident int64_t slots_ = 0; diff --git a/src/core/expert_cache.cpp b/src/core/expert_cache.cpp index 9f5fc981fb3..7db6eeb1f2b 100644 --- a/src/core/expert_cache.cpp +++ b/src/core/expert_cache.cpp @@ -69,6 +69,22 @@ bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_ex ExpertCache::~ExpertCache() { close(); } +#if defined(STRATA_USE_HIP) +bool ExpertCache::ensure_blocking_staging(std::size_t bytes, std::string& err) { + if (bytes <= blocking_staging_bytes_) return true; + void* next = nullptr; + const cudaError_t status = cudaHostAlloc(&next, bytes, cudaHostAllocDefault); + if (status != cudaSuccess) { + err = std::string("ExpertCache: HIP blocking staging allocation: ") + cudaGetErrorString(status); + return false; + } + if (blocking_staging_) (void) cudaFreeHost(blocking_staging_); + blocking_staging_ = static_cast(next); + blocking_staging_bytes_ = bytes; + return true; +} +#endif + bool ExpertCache::open(int64_t n_slots, int64_t n_layers, int64_t n_expert, int64_t blob_bytes, std::string& err) { close(); @@ -125,6 +141,12 @@ bool ExpertCache::open(int64_t n_slots, int64_t n_layers, int64_t n_expert, int6 n_layers_ = n_layers; n_expert_ = n_expert; blob_ = blob_bytes; +#if defined(STRATA_USE_HIP) + if (!ensure_blocking_staging((std::size_t) blob_, err)) { + close(); + return false; + } +#endif next_free_ = 0; fills_ = 0; admitted_ = 0; @@ -154,11 +176,22 @@ bool ExpertCache::open_sized(const std::vector& slot_bytes, int64_t n_l slots_ = (int64_t) slot_bytes.size(); blob_ = mx; off_ = std::move(off); +#if defined(STRATA_USE_HIP) + if (!ensure_blocking_staging((std::size_t) blob_, err)) { + close(); + return false; + } +#endif layer_next_.assign((size_t) (n_layers > 0 ? n_layers : 0), 0); return true; } void ExpertCache::close() { +#if defined(STRATA_USE_HIP) + if (blocking_staging_) (void) cudaFreeHost(blocking_staging_); + blocking_staging_ = nullptr; + blocking_staging_bytes_ = 0; +#endif off_.clear(); if (base_ != nullptr) { cudaFree(base_); @@ -256,7 +289,18 @@ bool ExpertCache::fill_slot_blocking(int32_t slot, const uint8_t* host_blob, std err = "ExpertCache::fill_slot_blocking: the host blob is null"; return false; } +#if defined(STRATA_USE_HIP) + // Bound HIP's pageable-source staging to one expert instead of repeatedly + // registering regions of the mmap. The blocking copy completes before reuse. + if (!blocking_staging_ || n > blocking_staging_bytes_) { + err = "ExpertCache::fill_slot_blocking: HIP staging buffer is too small"; + return false; + } + std::memcpy(blocking_staging_, host_blob, n); + const cudaError_t e = cudaMemcpy(dst, blocking_staging_, n, cudaMemcpyHostToDevice); +#else const cudaError_t e = cudaMemcpy(dst, host_blob, n, cudaMemcpyHostToDevice); +#endif if (e != cudaSuccess) { err = std::string("ExpertCache::fill_slot_blocking: ") + cudaGetErrorString(e); return false; diff --git a/tests/hip/expert_cache_staging.cpp b/tests/hip/expert_cache_staging.cpp new file mode 100644 index 00000000000..a276da1fcb5 --- /dev/null +++ b/tests/hip/expert_cache_staging.cpp @@ -0,0 +1,34 @@ +#include "strata/core/expert_cache.hpp" +#include +#include + +int main() { + strata::core::ExpertCache cache; + std::string err; + auto check = [&](bool ok) { + if (!ok) std::fprintf(stderr, "%s\n", err.c_str()); + return ok; + }; + // open_sized first allocates byte slots, then grows to the largest expert. + // Exercise that growth and repeated reuse with distinct pageable payloads. + for (int reopen = 0; reopen < 2; ++reopen) { + if (!check(cache.open_sized({4096, 8192, 16384}, 1, 3, err))) return 1; + for (int round = 0; round < 8; ++round) { + for (int slot = 0; slot < 3; ++slot) { + std::vector payload((size_t) 4096 << slot); + for (size_t i = 0; i < payload.size(); ++i) + payload[i] = (uint8_t) (i * 17 + round * 11 + slot + reopen); + if (!check(cache.fill_slot_blocking(slot, payload.data(), err, payload.size())) || + !check(cache.verify_slot(slot, payload.data(), err, payload.size()))) return 1; + } + } + if (cache.fill_slot_blocking(0, nullptr, err)) return 1; + cache.close(); + } + if (!check(cache.open(2, 1, 2, 1024, err))) return 1; + std::vector payload(1024, 73); + if (!check(cache.fill_slot_blocking(1, payload.data(), err)) || + !check(cache.verify_slot(1, payload.data(), err))) return 1; + std::puts("HIP expert staging: variable slots, repeated payloads, close/reopen passed"); + return 0; +} From 014d9fca2da5d45d2829a74b593cb755e8e8fbc0 Mon Sep 17 00:00:00 2001 From: Konstantinos Korres <96335554+2jztricks@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:32:01 +0300 Subject: [PATCH 5/8] Record fresh gfx1100 throughput and validation results --- docs/AMD_HIP_PERFORMANCE.md | 55 ++++- docs/benchmarks/2026-09-29-gfx1100.json | 299 ++++++++++++++++++++++++ 2 files changed, 344 insertions(+), 10 deletions(-) create mode 100644 docs/benchmarks/2026-09-29-gfx1100.json diff --git a/docs/AMD_HIP_PERFORMANCE.md b/docs/AMD_HIP_PERFORMANCE.md index a192a4d6463..4685c6dbdc3 100644 --- a/docs/AMD_HIP_PERFORMANCE.md +++ b/docs/AMD_HIP_PERFORMANCE.md @@ -4,7 +4,9 @@ This opt-in Linux `gfx1100` backend supersedes the initial support in [PR #94](https://github.com/Niko1221/Strata/pull/94). It retains HIP runtime and wave32 integer-dot compatibility, native mmap layout validation, and MTP, then adds HIP MMQ, optional calibrated dense hipBLASLt GEMM, native prefill batching, -and the host-memory / SSD paths used by the measured configuration. +and the host-memory / SSD paths used by the measured configuration. HIP blocking +expert uploads use one reusable pinned staging buffer, including prompt-cache +refills, to avoid repeated pageable-source registrations. ## Reproduce the configuration @@ -40,7 +42,7 @@ not lock all data into RAM: temporary prefill borrowing and cache refill may still use the mapped fallback. Allocation needs sufficient available RAM; on Linux this option requires readable standard cgroup-v2 mounts. The guard checks available RAM and ancestor limits with 8 GiB headroom, -but cannot reserve memory against concurrent allocations by other applications. +but cannot reserve memory against concurrent system or process allocations. The POSIX PLE path issues direct reads through a configurable worker pool. Use a dedicated idle server, restart it between arms, and capture its engine log: @@ -61,31 +63,64 @@ Never use cancelled-request timing lines as throughput evidence. ## Final revision benchmark -The final revision is being built and measured against the existing runtime. -The published table below will contain only this fresh matched run. +Measured on 2026-09-29, source revision `b90591e88ce806da6e6966d2e5d45b444d040190`. +HIP executable SHA-256: `5ca416d9864b5fddad54fb2310707675c4a10e8cd19cb0ab7d8371f4d4b057f0`. +The control and candidate below were both measured anew in this session; no +historical benchmark values are mixed into the table. The control binary SHA-256 +is `4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7`; +its source was an existing local runtime snapshot rather than a clean upstream commit. Both arms use an RX 7900 XTX 24 GiB / gfx1100, Ryzen 9 7950X3D, 64 GiB installed RAM, Fedora-family Linux, ROCm 7.1 and a 272 W GPU cap. Other model services are stopped for measurement and restored afterward. Sampling: temperature 0, top-k 1, top-p 1, min-p 0, seed 42, reasoning disabled. +| Fresh request, execution order | Control prefill t/s | Candidate prefill t/s | Candidate output t/s | Candidate request wall time | +| --- | ---: | ---: | ---: | ---: | +| 4,210 tokens, first use | 240.0 | 437.6 | 54.0 | 12.000 s | +| 8,830 tokens, first use at this size | 484.8 | 863.4 | 53.1 | 12.654 s | +| 4,210 tokens, warmed | 457.2 | 878.8 | 55.1 | 7.122 s | +| 8,830 tokens, warmed | 478.9 | 968.2 | 57.5 | 11.363 s | + +Each request generated exactly 128 tokens and finished at the intentional length +cap. All four fresh prompts had zero reused KV tokens. Fresh-prompt prefill was +1.78–2.02x the freshly measured existing AMD runtime; fresh-request wall time was +39–46% lower. This is a comparison with our existing HIP runtime, not unmodified +upstream, which does not provide this backend. The candidate also completed four +cached follow-ups in 3.06–3.40 seconds. Those follow-ups are not apples-to-apples +prefill comparisons: cache reuse differs with generated text and checkpoint +selection. Full sanitized measurements are in +[the fresh-run JSON](benchmarks/2026-09-29-gfx1100.json). + +Before this revision, an intermediate package missing the HIP upload staging +buffer stalled on the 8,830-token request and was rejected. Restoring staging +allowed the complete sequence above to finish. That failed attempt is not +included in the throughput table. No watchdog limit was raised to obtain these +results. + Zero KV reuse is not equivalent to a cold filesystem cache. First-use speed and warmed speed are reported separately. Individual observations do not establish confidence intervals or a general rate at every context length. ## Correctness and quality limits +Both HIP and CUDA executables built from the measured source revision. CUDA was +compile-checked, not performance-tested by this run. The selected HIP CTest suite +passed **29/29** with the tuning table enabled, including expert-upload readback, +asynchronous handoff, QSA, MMQ, Lt GEMM, KV streaming and PLE reading. The excluded +`ple_parity` requires an external fixture; `platform_memory_test` requires a +larger locked-memory limit than the test account provides. Separately, three +real IQ3_XXS PLE matrix graph replays passed, and the POSIX direct-file test passed +queued reads, short EOF, wake, close/drain and reopen checks. + Development checks passed eight MMQ numerical comparisons, four actual Lt GEMM comparisons, and native QSA/indexer/embedding parity including tail states, chunk continuation and image overrides. Real quantized MMQ relative L2 error was approximately 0.0026 against raw-FP32/dequantized-weight reference, reflecting Q8 activation arithmetic; this is not bitwise numerical equivalence. -The actual Claude Code concurrency repair ran once per build. Original: -91.098 seconds, normal CLI completion, no edits and no repair. Optimized snapshot: -480.531 seconds, timeout, no edits and no repair. Six of seven hidden groups -already passed in the unchanged fixture; that is not a model-authored quality -score. These trials do not establish quality equivalence or prove the cause of -the timeout. The optimized snapshot was not promoted to the daily gateway. +End-to-end coding quality is not established by these numerical checks or capped +throughput requests. Earlier coding trials did not demonstrate a successful repair, +so this contribution makes no broad answer-quality or agentic-reliability claim. The changes do not claim better model reasoning, verified full-context behavior, end-to-end vision validation, Windows HIP, other AMD architectures, or mixed diff --git a/docs/benchmarks/2026-09-29-gfx1100.json b/docs/benchmarks/2026-09-29-gfx1100.json new file mode 100644 index 00000000000..041505e9c6b --- /dev/null +++ b/docs/benchmarks/2026-09-29-gfx1100.json @@ -0,0 +1,299 @@ +{ + "source_commit": "b90591e88ce806da6e6966d2e5d45b444d040190", + "date": "2026-09-29", + "candidate_binary_sha256": "5ca416d9864b5fddad54fb2310707675c4a10e8cd19cb0ab7d8371f4d4b057f0", + "note": "Fresh same-session control and candidate runs. Throughput cap 128; not a coding-quality benchmark.", + "control": [ + { + "kind": "warmup", + "trial": 0, + "wall_s": 0.45770244899904355, + "metrics": { + "prompt_tokens": 15.0, + "reused": 0.0, + "fresh": 15.0, + "prefill_ms": 335.0, + "prefill_tps": 44.8, + "generated": 2.0, + "decode_ms": 115.0, + "decode_tps": 17.5 + }, + "finish_reason": "stop" + }, + { + "kind": "cold", + "trial": 1, + "wall_s": 20.068275763012934, + "metrics": { + "prompt_tokens": 4210.0, + "reused": 0.0, + "fresh": 4210.0, + "prefill_ms": 17538.0, + "prefill_tps": 240.0, + "generated": 128.0, + "decode_ms": 2518.0, + "decode_tps": 50.8 + }, + "finish_reason": "length" + }, + { + "kind": "followup", + "trial": 1, + "wall_s": 4.412000925018219, + "metrics": { + "prompt_tokens": 4451.0, + "reused": 4337.0, + "fresh": 114.0, + "prefill_ms": 1943.0, + "prefill_tps": 58.7, + "generated": 128.0, + "decode_ms": 2451.0, + "decode_tps": 52.2 + }, + "finish_reason": "length" + }, + { + "kind": "cold", + "trial": 2, + "wall_s": 20.731399575975956, + "metrics": { + "prompt_tokens": 8830.0, + "reused": 0.0, + "fresh": 8830.0, + "prefill_ms": 18213.0, + "prefill_tps": 484.8, + "generated": 128.0, + "decode_ms": 2505.0, + "decode_tps": 51.1 + }, + "finish_reason": "length" + }, + { + "kind": "followup", + "trial": 2, + "wall_s": 5.0592734439997, + "metrics": { + "prompt_tokens": 9071.0, + "reused": 8823.0, + "fresh": 248.0, + "prefill_ms": 2601.0, + "prefill_tps": 95.4, + "generated": 128.0, + "decode_ms": 2441.0, + "decode_tps": 52.4 + }, + "finish_reason": "length" + }, + { + "kind": "cold", + "trial": 3, + "wall_s": 11.695648370019626, + "metrics": { + "prompt_tokens": 4210.0, + "reused": 0.0, + "fresh": 4210.0, + "prefill_ms": 9209.0, + "prefill_tps": 457.2, + "generated": 128.0, + "decode_ms": 2477.0, + "decode_tps": 51.7 + }, + "finish_reason": "length" + }, + { + "kind": "followup", + "trial": 3, + "wall_s": 4.093982424004935, + "metrics": { + "prompt_tokens": 4451.0, + "reused": 4337.0, + "fresh": 114.0, + "prefill_ms": 1896.0, + "prefill_tps": 60.1, + "generated": 128.0, + "decode_ms": 2188.0, + "decode_tps": 58.5 + }, + "finish_reason": "length" + }, + { + "kind": "cold", + "trial": 4, + "wall_s": 20.84539221401792, + "metrics": { + "prompt_tokens": 8830.0, + "reused": 0.0, + "fresh": 8830.0, + "prefill_ms": 18437.0, + "prefill_tps": 478.9, + "generated": 128.0, + "decode_ms": 2392.0, + "decode_tps": 53.5 + }, + "finish_reason": "length" + }, + { + "kind": "followup", + "trial": 4, + "wall_s": 5.357054464984685, + "metrics": { + "prompt_tokens": 9071.0, + "reused": 8823.0, + "fresh": 248.0, + "prefill_ms": 2658.0, + "prefill_tps": 93.3, + "generated": 128.0, + "decode_ms": 2682.0, + "decode_tps": 47.7 + }, + "finish_reason": "length" + } + ], + "optimized": [ + { + "kind": "warmup", + "trial": 0, + "wall_s": 0.5317635319952387, + "metrics": { + "prompt_tokens": 15.0, + "reused": 0.0, + "fresh": 15.0, + "prefill_ms": 361.0, + "prefill_tps": 41.6, + "generated": 2.0, + "decode_ms": 164.0, + "decode_tps": 12.2 + }, + "finish_reason": "stop" + }, + { + "kind": "cold", + "trial": 1, + "wall_s": 11.999838335003005, + "metrics": { + "prompt_tokens": 4210.0, + "reused": 0.0, + "fresh": 4210.0, + "prefill_ms": 9620.0, + "prefill_tps": 437.6, + "generated": 128.0, + "decode_ms": 2369.0, + "decode_tps": 54.0 + }, + "finish_reason": "length" + }, + { + "kind": "followup", + "trial": 1, + "wall_s": 3.059695895004552, + "metrics": { + "prompt_tokens": 4451.0, + "reused": 4337.0, + "fresh": 114.0, + "prefill_ms": 790.0, + "prefill_tps": 144.2, + "generated": 128.0, + "decode_ms": 2253.0, + "decode_tps": 56.8 + }, + "finish_reason": "length" + }, + { + "kind": "cold", + "trial": 2, + "wall_s": 12.654022857022937, + "metrics": { + "prompt_tokens": 8830.0, + "reused": 0.0, + "fresh": 8830.0, + "prefill_ms": 10227.0, + "prefill_tps": 863.4, + "generated": 128.0, + "decode_ms": 2411.0, + "decode_tps": 53.1 + }, + "finish_reason": "length" + }, + { + "kind": "followup", + "trial": 2, + "wall_s": 3.3966909740120173, + "metrics": { + "prompt_tokens": 9071.0, + "reused": 8823.0, + "fresh": 248.0, + "prefill_ms": 1142.0, + "prefill_tps": 217.1, + "generated": 128.0, + "decode_ms": 2237.0, + "decode_tps": 57.2 + }, + "finish_reason": "length" + }, + { + "kind": "cold", + "trial": 3, + "wall_s": 7.121944882004755, + "metrics": { + "prompt_tokens": 4210.0, + "reused": 0.0, + "fresh": 4210.0, + "prefill_ms": 4790.0, + "prefill_tps": 878.8, + "generated": 128.0, + "decode_ms": 2324.0, + "decode_tps": 55.1 + }, + "finish_reason": "length" + }, + { + "kind": "followup", + "trial": 3, + "wall_s": 3.2425516709918156, + "metrics": { + "prompt_tokens": 4451.0, + "reused": 4337.0, + "fresh": 114.0, + "prefill_ms": 828.0, + "prefill_tps": 137.7, + "generated": 128.0, + "decode_ms": 2405.0, + "decode_tps": 53.2 + }, + "finish_reason": "length" + }, + { + "kind": "cold", + "trial": 4, + "wall_s": 11.362885618000291, + "metrics": { + "prompt_tokens": 8830.0, + "reused": 0.0, + "fresh": 8830.0, + "prefill_ms": 9120.0, + "prefill_tps": 968.2, + "generated": 128.0, + "decode_ms": 2227.0, + "decode_tps": 57.5 + }, + "finish_reason": "length" + }, + { + "kind": "followup", + "trial": 4, + "wall_s": 3.2115756749990396, + "metrics": { + "prompt_tokens": 9071.0, + "reused": 8957.0, + "fresh": 114.0, + "prefill_ms": 792.0, + "prefill_tps": 143.9, + "generated": 128.0, + "decode_ms": 2402.0, + "decode_tps": 53.3 + }, + "finish_reason": "length" + } + ], + "control_binary_sha256": "4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7" +} From 9568e78da663e6b224ab96d57baf450717b36e73 Mon Sep 17 00:00:00 2001 From: Konstantinos Korres <96335554+2jztricks@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:55:35 +0300 Subject: [PATCH 6/8] Account for clean reclaimable cache in cgroup resident budgets --- docs/AMD_HIP_PERFORMANCE.md | 9 ++-- include/strata/core/expert_source.hpp | 13 ++++++ src/core/expert_source.cpp | 58 +++++++++++++++++++++++++- tests/core/file_expert_source_test.cpp | 37 ++++++++++++++++ 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/docs/AMD_HIP_PERFORMANCE.md b/docs/AMD_HIP_PERFORMANCE.md index 4685c6dbdc3..7588a62b03e 100644 --- a/docs/AMD_HIP_PERFORMANCE.md +++ b/docs/AMD_HIP_PERFORMANCE.md @@ -40,9 +40,12 @@ and `--vram-reserve-mib 1024`. Keep PLE on SSD. RAM. It requires mmap, a fixed expert profile, and disabled adaptation. It does not lock all data into RAM: temporary prefill borrowing and cache refill may still use the mapped fallback. Allocation needs sufficient available RAM; -on Linux this option requires readable standard cgroup-v2 mounts. The guard -checks available RAM and ancestor limits with 8 GiB headroom, -but cannot reserve memory against concurrent system or process allocations. +on Linux this option requires readable standard cgroup-v2 mounts. For each +finite cgroup ancestor, the guard credits only `inactive_file` after subtracting +`file_dirty` and `file_writeback`, capped by current usage; it remains bounded by +the ancestor limit and host `MemAvailable`, with 8 GiB headroom. This accounts +for reclaimable clean file cache but cannot reserve memory against concurrent +system or process allocations. The POSIX PLE path issues direct reads through a configurable worker pool. Use a dedicated idle server, restart it between arms, and capture its engine log: diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp index 959221a993d..11d93160cf2 100644 --- a/include/strata/core/expert_source.hpp +++ b/include/strata/core/expert_source.hpp @@ -40,6 +40,19 @@ namespace detail { /// Sentinel used by the pure complement planner for a blob that remains in the mmap fallback. inline constexpr uint64_t kNoCacheComplement = ~uint64_t{0}; +/// Required cgroup-v2 usage counters for the conservative cache-reclaim allowance. +struct CgroupMemoryStat { + uint64_t current = 0; + uint64_t inactive_file = 0; + uint64_t file_dirty = 0; + uint64_t file_writeback = 0; + bool valid = false; +}; + +/// Calculate additional bytes under a finite cgroup limit after reclaiming only clean inactive file cache. +/// Returns false when the required memory.stat counters were unavailable. +bool cgroup_available_bytes(uint64_t limit, const CgroupMemoryStat& stat, uint64_t& bytes); + /// Build compact offsets for experts absent from both the primary GPU cache and an optional second GPU tier. /// Kept CPU-only so selection and byte accounting can be tested without initializing a GPU. bool make_cache_complement_plan( diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp index fcb2aae03b3..c7a7882648a 100644 --- a/src/core/expert_source.cpp +++ b/src/core/expert_source.cpp @@ -38,6 +38,21 @@ namespace strata::core { namespace detail { +bool cgroup_available_bytes(uint64_t limit, const CgroupMemoryStat& stat, uint64_t& bytes) { + bytes = 0; + if (!stat.valid) return false; + + // memory.stat's inactive_file can race memory.current, so bound it to charged usage first. + uint64_t reclaimable = std::min(stat.inactive_file, stat.current); + reclaimable = stat.file_dirty >= reclaimable ? 0 : reclaimable - stat.file_dirty; + reclaimable = stat.file_writeback >= reclaimable ? 0 : reclaimable - stat.file_writeback; + + // Reclaiming clean file pages reduces usage; saturating subtraction also handles a transient over-limit read. + const uint64_t usage_after_reclaim = stat.current - reclaimable; + bytes = usage_after_reclaim < limit ? limit - usage_after_reclaim : 0; + return true; +} + bool make_cache_complement_plan( int64_t n_layers, int64_t n_expert, const std::vector& layer_blob_bytes, const std::vector>& primary_gpu_pairs, @@ -123,6 +138,43 @@ const uint8_t* cache_complement_blob_or_fallback( namespace { constexpr uint64_t kPinnedMemoryHeadroom = 8ull << 30; +#if defined(__linux__) +bool read_cgroup_memory_stat(const std::filesystem::path& path, uint64_t current, + detail::CgroupMemoryStat& stat) { + std::ifstream input(path / "memory.stat"); + if (!input) return false; + + bool inactive_file = false, file_dirty = false, file_writeback = false; + std::string line; + while (std::getline(input, line)) { + std::istringstream fields(line); + std::string key; + uint64_t value = 0; + if (!(fields >> key >> value)) return false; + fields >> std::ws; + if (!fields.eof()) return false; + + if (key == "inactive_file") { + if (inactive_file) return false; + inactive_file = true; + stat.inactive_file = value; + } else if (key == "file_dirty") { + if (file_dirty) return false; + file_dirty = true; + stat.file_dirty = value; + } else if (key == "file_writeback") { + if (file_writeback) return false; + file_writeback = true; + stat.file_writeback = value; + } + } + if (!input.eof() || !inactive_file || !file_dirty || !file_writeback) return false; + stat.current = current; + stat.valid = true; + return true; +} +#endif + bool available_memory_bytes(uint64_t& bytes) { #if defined(_WIN32) MEMORYSTATUSEX status{}; @@ -167,7 +219,11 @@ bool available_memory_bytes(uint64_t& bytes) { size_t consumed = 0; const uint64_t cap = std::stoull(limit, &consumed); if (consumed != limit.size()) return false; - bytes = std::min(bytes, current < cap ? cap - current : uint64_t{0}); + detail::CgroupMemoryStat stat; + if (!read_cgroup_memory_stat(path, current, stat)) return false; + uint64_t cgroup_available = 0; + if (!detail::cgroup_available_bytes(cap, stat, cgroup_available)) return false; + bytes = std::min(bytes, cgroup_available); } catch (...) { return false; } } if (path == root) break; diff --git a/tests/core/file_expert_source_test.cpp b/tests/core/file_expert_source_test.cpp index fdb86c24dca..be0f091207e 100644 --- a/tests/core/file_expert_source_test.cpp +++ b/tests/core/file_expert_source_test.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -205,11 +206,47 @@ void test_complement_plan() { require(!make_cache_complement_plan(2, 3, {0, 5}, {}, {}, offsets, bytes, error), "zero-size layer accepted"); } +void test_cgroup_memory_budget() { + using namespace strata::core::detail; + constexpr uint64_t GiB = 1ull << 30; + uint64_t bytes = 0; + + CgroupMemoryStat clean_cache{40 * GiB, 32 * GiB, 0, 0, true}; + require(cgroup_available_bytes(56 * GiB, clean_cache, bytes), "valid cgroup memory.stat was rejected"); + require(bytes == 48 * GiB, "clean inactive file cache was not credited against the cgroup cap"); + + CgroupMemoryStat dirty_cache{40 * GiB, 32 * GiB, 4 * GiB, 2 * GiB, true}; + require(cgroup_available_bytes(56 * GiB, dirty_cache, bytes), "valid dirty-cache counters were rejected"); + require(bytes == 42 * GiB, "dirty/writeback pages were incorrectly counted as reclaimable"); + + CgroupMemoryStat oversized_inactive{10 * GiB, 20 * GiB, 0, 0, true}; + require(cgroup_available_bytes(12 * GiB, oversized_inactive, bytes), + "valid oversized inactive-file counter was rejected"); + require(bytes == 12 * GiB, "inactive-file accounting exceeded charged current usage"); + + const uint64_t max = std::numeric_limits::max(); + CgroupMemoryStat large_counters{max, max, max - 1, max, true}; + require(cgroup_available_bytes(max, large_counters, bytes), + "large memory.stat counters were rejected"); + require(bytes == 0, "dirty/writeback subtraction overflowed or escaped the usage cap"); + + CgroupMemoryStat usage_over_limit{60 * GiB, 0, 0, 0, true}; + require(cgroup_available_bytes(56 * GiB, usage_over_limit, bytes), + "valid over-limit memory counters were rejected"); + require(bytes == 0, "over-limit cgroup usage produced a positive budget"); + + CgroupMemoryStat missing_stat{}; + bytes = 123; + require(!cgroup_available_bytes(56 * GiB, missing_stat, bytes) && bytes == 0, + "missing memory.stat counters did not fail closed"); +} + } // namespace int main() { try { test_complement_plan(); + test_cgroup_memory_budget(); test_canonical_layout(); #if defined(STRATA_NATIVE_EXPERTS) test_native_variable_layout(); From d797787b4e388aac89933d6e26286294da3adb78 Mon Sep 17 00:00:00 2001 From: Konstantinos Korres <96335554+2jztricks@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:10:37 +0300 Subject: [PATCH 7/8] docs: record final guarded HIP runtime measurements --- docs/AMD_HIP_PERFORMANCE.md | 18 ++--- docs/benchmarks/2026-09-29-gfx1100.json | 94 ++++++++++++------------- 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/docs/AMD_HIP_PERFORMANCE.md b/docs/AMD_HIP_PERFORMANCE.md index 7588a62b03e..37a422499dc 100644 --- a/docs/AMD_HIP_PERFORMANCE.md +++ b/docs/AMD_HIP_PERFORMANCE.md @@ -66,8 +66,8 @@ Never use cancelled-request timing lines as throughput evidence. ## Final revision benchmark -Measured on 2026-09-29, source revision `b90591e88ce806da6e6966d2e5d45b444d040190`. -HIP executable SHA-256: `5ca416d9864b5fddad54fb2310707675c4a10e8cd19cb0ab7d8371f4d4b057f0`. +Measured on 2026-09-29, source revision `9568e78da663e6b224ab96d57baf450717b36e73`. +HIP executable SHA-256: `705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b`. The control and candidate below were both measured anew in this session; no historical benchmark values are mixed into the table. The control binary SHA-256 is `4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7`; @@ -79,17 +79,17 @@ Sampling: temperature 0, top-k 1, top-p 1, min-p 0, seed 42, reasoning disabled. | Fresh request, execution order | Control prefill t/s | Candidate prefill t/s | Candidate output t/s | Candidate request wall time | | --- | ---: | ---: | ---: | ---: | -| 4,210 tokens, first use | 240.0 | 437.6 | 54.0 | 12.000 s | -| 8,830 tokens, first use at this size | 484.8 | 863.4 | 53.1 | 12.654 s | -| 4,210 tokens, warmed | 457.2 | 878.8 | 55.1 | 7.122 s | -| 8,830 tokens, warmed | 478.9 | 968.2 | 57.5 | 11.363 s | +| 4,210 tokens, first use | 240.0 | 447.5 | 55.5 | 11.729 s | +| 8,830 tokens, first use at this size | 484.8 | 873.7 | 55.7 | 12.420 s | +| 4,210 tokens, warmed | 457.2 | 901.1 | 56.7 | 6.937 s | +| 8,830 tokens, warmed | 478.9 | 966.5 | 59.2 | 11.314 s | Each request generated exactly 128 tokens and finished at the intentional length cap. All four fresh prompts had zero reused KV tokens. Fresh-prompt prefill was -1.78–2.02x the freshly measured existing AMD runtime; fresh-request wall time was -39–46% lower. This is a comparison with our existing HIP runtime, not unmodified +1.80–2.02x the freshly measured existing AMD runtime; fresh-request wall time was +40–46% lower. This is a comparison with our existing HIP runtime, not unmodified upstream, which does not provide this backend. The candidate also completed four -cached follow-ups in 3.06–3.40 seconds. Those follow-ups are not apples-to-apples +cached follow-ups in 2.95–3.26 seconds. Those follow-ups are not apples-to-apples prefill comparisons: cache reuse differs with generated text and checkpoint selection. Full sanitized measurements are in [the fresh-run JSON](benchmarks/2026-09-29-gfx1100.json). diff --git a/docs/benchmarks/2026-09-29-gfx1100.json b/docs/benchmarks/2026-09-29-gfx1100.json index 041505e9c6b..bf0e5ef7174 100644 --- a/docs/benchmarks/2026-09-29-gfx1100.json +++ b/docs/benchmarks/2026-09-29-gfx1100.json @@ -1,7 +1,7 @@ { - "source_commit": "b90591e88ce806da6e6966d2e5d45b444d040190", + "source_commit": "9568e78da663e6b224ab96d57baf450717b36e73", "date": "2026-09-29", - "candidate_binary_sha256": "5ca416d9864b5fddad54fb2310707675c4a10e8cd19cb0ab7d8371f4d4b057f0", + "candidate_binary_sha256": "705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b", "note": "Fresh same-session control and candidate runs. Throughput cap 128; not a coding-quality benchmark.", "control": [ { @@ -153,144 +153,144 @@ { "kind": "warmup", "trial": 0, - "wall_s": 0.5317635319952387, + "wall_s": 0.46265592801501043, "metrics": { "prompt_tokens": 15.0, "reused": 0.0, "fresh": 15.0, - "prefill_ms": 361.0, - "prefill_tps": 41.6, + "prefill_ms": 335.0, + "prefill_tps": 44.8, "generated": 2.0, - "decode_ms": 164.0, - "decode_tps": 12.2 + "decode_ms": 121.0, + "decode_tps": 16.5 }, "finish_reason": "stop" }, { "kind": "cold", "trial": 1, - "wall_s": 11.999838335003005, + "wall_s": 11.728894605010282, "metrics": { "prompt_tokens": 4210.0, "reused": 0.0, "fresh": 4210.0, - "prefill_ms": 9620.0, - "prefill_tps": 437.6, + "prefill_ms": 9408.0, + "prefill_tps": 447.5, "generated": 128.0, - "decode_ms": 2369.0, - "decode_tps": 54.0 + "decode_ms": 2308.0, + "decode_tps": 55.5 }, "finish_reason": "length" }, { "kind": "followup", "trial": 1, - "wall_s": 3.059695895004552, + "wall_s": 2.9531024169991724, "metrics": { "prompt_tokens": 4451.0, "reused": 4337.0, "fresh": 114.0, - "prefill_ms": 790.0, - "prefill_tps": 144.2, + "prefill_ms": 740.0, + "prefill_tps": 154.0, "generated": 128.0, - "decode_ms": 2253.0, - "decode_tps": 56.8 + "decode_ms": 2193.0, + "decode_tps": 58.4 }, "finish_reason": "length" }, { "kind": "cold", "trial": 2, - "wall_s": 12.654022857022937, + "wall_s": 12.42044531099964, "metrics": { "prompt_tokens": 8830.0, "reused": 0.0, "fresh": 8830.0, - "prefill_ms": 10227.0, - "prefill_tps": 863.4, + "prefill_ms": 10107.0, + "prefill_tps": 873.7, "generated": 128.0, - "decode_ms": 2411.0, - "decode_tps": 53.1 + "decode_ms": 2298.0, + "decode_tps": 55.7 }, "finish_reason": "length" }, { "kind": "followup", "trial": 2, - "wall_s": 3.3966909740120173, + "wall_s": 3.2599652509961743, "metrics": { "prompt_tokens": 9071.0, "reused": 8823.0, "fresh": 248.0, - "prefill_ms": 1142.0, - "prefill_tps": 217.1, + "prefill_ms": 1070.0, + "prefill_tps": 231.7, "generated": 128.0, - "decode_ms": 2237.0, - "decode_tps": 57.2 + "decode_ms": 2173.0, + "decode_tps": 58.9 }, "finish_reason": "length" }, { "kind": "cold", "trial": 3, - "wall_s": 7.121944882004755, + "wall_s": 6.937167472002329, "metrics": { "prompt_tokens": 4210.0, "reused": 0.0, "fresh": 4210.0, - "prefill_ms": 4790.0, - "prefill_tps": 878.8, + "prefill_ms": 4672.0, + "prefill_tps": 901.1, "generated": 128.0, - "decode_ms": 2324.0, - "decode_tps": 55.1 + "decode_ms": 2256.0, + "decode_tps": 56.7 }, "finish_reason": "length" }, { "kind": "followup", "trial": 3, - "wall_s": 3.2425516709918156, + "wall_s": 3.1877827199932653, "metrics": { "prompt_tokens": 4451.0, "reused": 4337.0, "fresh": 114.0, - "prefill_ms": 828.0, - "prefill_tps": 137.7, + "prefill_ms": 827.0, + "prefill_tps": 137.8, "generated": 128.0, - "decode_ms": 2405.0, - "decode_tps": 53.2 + "decode_ms": 2350.0, + "decode_tps": 54.5 }, "finish_reason": "length" }, { "kind": "cold", "trial": 4, - "wall_s": 11.362885618000291, + "wall_s": 11.314008287008619, "metrics": { "prompt_tokens": 8830.0, "reused": 0.0, "fresh": 8830.0, - "prefill_ms": 9120.0, - "prefill_tps": 968.2, + "prefill_ms": 9136.0, + "prefill_tps": 966.5, "generated": 128.0, - "decode_ms": 2227.0, - "decode_tps": 57.5 + "decode_ms": 2161.0, + "decode_tps": 59.2 }, "finish_reason": "length" }, { "kind": "followup", "trial": 4, - "wall_s": 3.2115756749990396, + "wall_s": 3.0476601750124246, "metrics": { "prompt_tokens": 9071.0, "reused": 8957.0, "fresh": 114.0, - "prefill_ms": 792.0, - "prefill_tps": 143.9, + "prefill_ms": 734.0, + "prefill_tps": 155.3, "generated": 128.0, - "decode_ms": 2402.0, - "decode_tps": 53.3 + "decode_ms": 2296.0, + "decode_tps": 55.7 }, "finish_reason": "length" } From 49d0a4bfaea4a6b6fe433c44b2b255e318216814 Mon Sep 17 00:00:00 2001 From: Konstantinos Korres <96335554+2jztricks@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:51:39 +0300 Subject: [PATCH 8/8] docs: keep throughput evidence distinct from task correctness --- docs/AMD_HIP_PERFORMANCE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/AMD_HIP_PERFORMANCE.md b/docs/AMD_HIP_PERFORMANCE.md index 37a422499dc..23a684c8346 100644 --- a/docs/AMD_HIP_PERFORMANCE.md +++ b/docs/AMD_HIP_PERFORMANCE.md @@ -122,8 +122,8 @@ was approximately 0.0026 against raw-FP32/dequantized-weight reference, reflecti Q8 activation arithmetic; this is not bitwise numerical equivalence. End-to-end coding quality is not established by these numerical checks or capped -throughput requests. Earlier coding trials did not demonstrate a successful repair, -so this contribution makes no broad answer-quality or agentic-reliability claim. +throughput requests. Validate completed tasks with independent runtime tests; this +contribution makes no broad answer-quality or agentic-reliability claim. The changes do not claim better model reasoning, verified full-context behavior, end-to-end vision validation, Windows HIP, other AMD architectures, or mixed