replay (review only): githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922 — filtered squash of 920 commit(s) from https://github.com/eelgaev/Strata-AC922

CMakeLists.txt

@@ -48,6 +48,13 @@ endif()
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
+# aarch64 (and other non-x86) GCC/Clang fuse a * b + c into one FMA by default; an x86 build without -mfma cannot.
+# The host code's roundings (activation quantization, ggml's nearest_int) then differ at exact ties, so a non-x86
+# build would disagree with x86 and with ggml's reference (quantize_act_parity's Q8_K case).  Off, as on x86;
+# explicit FMA intrinsics (NEON vfmaq) are unaffected.
+if(NOT MSVC AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|i[3-6]86|x86)$")
+  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:-ffp-contract=off>)
+endif()
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
   set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE)
 endif()
@@ -377,7 +384,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
                           src/kernels/cuda/kv_q8.cu
                           src/kernels/cuda/kv_stream.cu
                           src/kernels/cuda/kv_q4.cu
-                          src/kernels/cuda/fused_gr.cu
+                          src/kernels/cuda/fused_gr.cu src/kernels/cuda/fused_expert.cu
                           src/kernels/cuda/cvec.cu
                           src/kernels/cuda/qsa_decode_attn.cu
                           src/kernels/cuda/qsa_prompt_attn.cu
@@ -483,9 +490,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   # about arithmetic, and this one knows the ORDER the arithmetic happens in.
   add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
     src/core/remote_expert_opt.cu
-    src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp
-    src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/peer_experts.cpp
-    src/core/arch_defaults.cpp src/core/vmm.cpp src/core/conversation_file.cpp)
+    src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/load_converted.cpp src/core/verify.cpp src/core/mtp.cpp
+    src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/conversation_file.cpp src/core/peer_experts.cpp
+    src/core/arch_defaults.cpp src/core/vmm.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
   target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
   if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA AND NOT STRATA_HIP_GFX906)
@@ -1208,12 +1215,25 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)
   add_test(NAME batch_groups_test COMMAND batch_groups_test)
 endif()
 
-add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp
-            src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp)
+# The AVX kernels exist for x86 only; on aarch64 (DGX Spark / GB10, Grace) src/kernels/cpu/portable.cpp stands in
+# for them and the native experts run on ggml-cpu's NEON dot products.
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|i[3-6]86|x86)$")
+  set(STRATA_CPU_X86 ON)
+else()
+  set(STRATA_CPU_X86 OFF)
+endif()
+if(STRATA_CPU_X86)
+  add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp
+              src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp)
+else()
+  add_library(strata_kernels_cpu STATIC src/kernels/cpu/portable.cpp src/kernels/cpu/pool.cpp
+              src/kernels/cpu/expert_layout.cpp)
+endif()
 target_include_directories(strata_kernels_cpu PUBLIC include)
 # Plan v0.3 P6: only the kernel files get the AVX-512 / AVX2 flags; the pool and the probe run on any x86-64,
 # and the engine picks the kernels at run time (cpu_avx512_ok).
-if(MSVC)
+if(NOT STRATA_CPU_X86)
+elseif(MSVC)
   set_source_files_properties(src/kernels/cpu/expert.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX512")
   set_source_files_properties(src/kernels/cpu/q2_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
 else()
@@ -1278,6 +1298,10 @@ if(STRATA_NATIVE_EXPERTS)
     foreach(_f GGML_AVX GGML_AVX2 GGML_FMA GGML_F16C GGML_BMI2)
       set(${_f} ON CACHE BOOL "" FORCE)
     endforeach()
+  elseif(NOT STRATA_CPU_X86 AND GGML_CPU_ARM_ARCH)
+    # ARM with an explicit -march (gcc's -mcpu=native finds no features on big.LITTLE parts like GB10's
+    # Cortex-X925/A725, which leaves ggml-cpu without dotprod/i8mm): e.g. -DGGML_CPU_ARM_ARCH=armv8.6-a+dotprod+i8mm+fp16
+    set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
   elseif(STRATA_ISA_FLOOR)
     # the experimental older-CPU build (STRATA_ISA_FLOOR above): one ggml-cpu for the floor, ggml's "sandybridge"
     # feature set (SSE4.2 + AVX) or SSE4.2 alone - not the build host's, which may be newer than the PC it runs on.
@@ -1297,6 +1321,7 @@ if(STRATA_NATIVE_EXPERTS)
   else()
     set(GGML_NATIVE ON CACHE BOOL "" FORCE)
   endif()
+<<<<<<< this fork
   add_subdirectory(${STRATA_GGML_DIR}/ggml ${CMAKE_BINARY_DIR}/ggml EXCLUDE_FROM_ALL)
   # part of strata_kernels_cpu (the pool calls these and they call its CPU probe: one library, so GNU ld's
   # one-pass static linking has no cycle to resolve)
@@ -1308,6 +1333,44 @@ if(STRATA_NATIVE_EXPERTS)
   if(MSVC)
     set_source_files_properties(src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/nvfp4_avx512.cpp
                                 PROPERTIES COMPILE_OPTIONS "/arch:AVX512")
+=======
+  if(STRATA_GGML_DIR)
+    add_subdirectory(${STRATA_GGML_DIR}/ggml ${CMAKE_BINARY_DIR}/ggml EXCLUDE_FROM_ALL)
+  else()
+    include(FetchContent)
+    FetchContent_Declare(strata_llamacpp
+      GIT_REPOSITORY https://github.com/ggml-org/llama.cpp.git
+      GIT_TAG 3cf03257f219afbe7334045ff7c6a06ac68c627d
+      GIT_SHALLOW FALSE)
+    FetchContent_GetProperties(strata_llamacpp)
+    if(NOT strata_llamacpp_POPULATED)
+      FetchContent_Populate(strata_llamacpp)
+    endif()
+    add_subdirectory(${strata_llamacpp_SOURCE_DIR}/ggml ${CMAKE_BINARY_DIR}/ggml EXCLUDE_FROM_ALL)
+  endif()
+  # POWER: ggml-cpu's VSX code spells the AltiVec types `vector float`, a GNU keyword that the strict -std=c++17 this
+  # project's CMAKE_CXX_EXTENSIONS OFF gives the subdirectory turns off; upstream llama.cpp builds it as gnu++17
+  if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc|powerpc)")
+    set_target_properties(ggml-cpu PROPERTIES CXX_EXTENSIONS ON)
+  endif()
+  # part of strata_kernels_cpu (the pool calls these and they call its CPU probe: one library, so GNU ld's
+  # one-pass static linking has no cycle to resolve)
+  target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/native_expert.cpp)
+  if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64|powerpc64)")
+    # POWER9 VSX expert kernels (STRATA_VSX_EXPERTS=1); the half->float vector conversion needs ISA 3.0
+    target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/kq_vsx.cpp)
+    set_source_files_properties(src/kernels/cpu/kq_vsx.cpp PROPERTIES COMPILE_OPTIONS "-mcpu=power9")
+  endif()
+  if(STRATA_CPU_X86)
+    target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/iq_avx2.cpp
+                                              src/kernels/cpu/kq_avx2.cpp
+                                              src/kernels/cpu/kq_avx1.cpp)
+  endif()
+  target_include_directories(strata_kernels_cpu PRIVATE third_party/ggml)
+  if(NOT STRATA_CPU_X86)
+  elseif(MSVC)
+    set_source_files_properties(src/kernels/cpu/iq_avx512.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX512")
+>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922)
     set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
     set_source_files_properties(src/kernels/cpu/kq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
     set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
@@ -1362,6 +1425,7 @@ if(STRATA_NATIVE_EXPERTS)
   endif()
   # the AVX-2 expert kernels' variants (iq_avx2.cpp, q2_avx2.cpp) bit for bit against each other at 1..8 tokens and
   # against ggml-cpu's dot, on random blocks; --bench: ms per expert on one thread, the weights streamed from DRAM
+<<<<<<< this fork
   add_executable(iq_avx2_parity src/kernels/cpu/iq_avx2_parity.cpp)
   target_link_libraries(iq_avx2_parity PRIVATE strata_kernels_cpu)
   # MODEL-UPDATE: the pool's expert rows per format on T threads against the RAM bandwidth (a benchmark, no test)
@@ -1370,6 +1434,15 @@ if(STRATA_NATIVE_EXPERTS)
   if(STRATA_BUILD_TESTS)
     add_test(NAME iq_avx2_parity COMMAND iq_avx2_parity)
     add_test(NAME iq_avx2_parity_iq3s_mt1 COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ3S_MT1=1 $<TARGET_FILE:iq_avx2_parity>)
+=======
+  if(STRATA_CPU_X86)   # x86 only (cpuid.h, the AVX-2 kernels); non-x86 builds have portable.cpp instead
+    add_executable(iq_avx2_parity src/kernels/cpu/iq_avx2_parity.cpp)
+    target_link_libraries(iq_avx2_parity PRIVATE strata_kernels_cpu)
+    if(STRATA_BUILD_TESTS)
+      add_test(NAME iq_avx2_parity COMMAND iq_avx2_parity)
+      add_test(NAME iq_avx2_parity_iq3s_mt1 COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ3S_MT1=1 $<TARGET_FILE:iq_avx2_parity>)
+    endif()
+>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922)
   endif()
   # the older CPUs' AVX1 expert rows (iq_avx1.cpp) against ggml-cpu's dots and against their own one-token
   # rows at every width; --bench: ms per expert, the weights streamed from DRAM.  Runs on any AVX CPU:
@@ -1667,10 +1740,12 @@ if(STRATA_BUILD_TESTS)
     target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact ${_strata_gpu_runtime_target})
   endif()
 
-  add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp)
-  target_link_libraries(expert_multi_test PRIVATE strata_kernels_cpu)
-  add_test(NAME expert_multi_test COMMAND expert_multi_test)
-  set_tests_properties(expert_multi_test PROPERTIES SKIP_RETURN_CODE 77)   # no AVX-512: skipped
+  if(STRATA_CPU_X86)   # the Q2_0 expert kernel it checks is AVX-512 (portable.cpp refuses it elsewhere)
+    add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp)
+    target_link_libraries(expert_multi_test PRIVATE strata_kernels_cpu)
+    add_test(NAME expert_multi_test COMMAND expert_multi_test)
+    set_tests_properties(expert_multi_test PROPERTIES SKIP_RETURN_CODE 77)   # no AVX-512: skipped
+  endif()
 
   # PR #706: the opt-in AVX-2 Q2_0 bit-plane kernel (STRATA_Q2_BITPLANE=1) against the legacy one, bitwise alone/window
   if(STRATA_BUILD_TESTS AND CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64|x86_64|x64|X86")

include/strata/core/expert_cache.hpp

@@ -54,6 +54,11 @@ inline constexpr int32_t kNotResident = -1;
 bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_expert,
                          std::vector<std::pair<int32_t, int32_t>>& ranked, int64_t& slots, std::string& err);
 
+/// Free device memory for the expert cache.  On a unified memory system - one memory for the CPU and the GPU, such as
+/// the DGX Spark (GB10); CUDA reports it as cudaDevAttrIntegrated - cudaMemGetInfo counts only MemFree, not the page cache - which, with the experts read from the GGUF in place, is mostly those
+/// files' clean pages and gives way to a device allocation.  There the figure is MemAvailable less 6 GiB for the OS
+/// and the engine's host side (STRATA_UMA_HEADROOM_GIB).  Elsewhere it is cudaMemGetInfo's.
+size_t device_free_bytes();
 /// #477 (--expert-profile-save): what the adaptive tier learned, as a profile ranking EVERY (layer, expert) pair -
 /// the experts resident in VRAM now first (where the swaps left the cache), then the rest; within each, by `heat`
 /// (the routing the adaptive tier counted since the start, descending), then by `prior` (the profile the engine

include/strata/core/expert_source.hpp

@@ -582,6 +582,24 @@ class FileExpertSource : public ExpertSource {
         stage_lend_regions_ = std::move(regions);
     }
     void close();
+    /// A layer split across sockets (an IBM AC922: GPUs 0-1 on NUMA node 0, 2-3 on node 8): call before
+    /// `pin_cache_complement`.  `layer_node[l]` is the NUMA node of the GPU that runs layer l (-1: unknown).  The
+    /// arena is laid out node by node, one cudaHostAlloc per node made by a thread on that node's CPUs: the driver
+    /// puts page-locked memory on the allocating thread's node (not where a memory policy asks), and a GPU reads its
+    /// own socket's memory at 68 GB/s on the AC922, the other socket's at 40 - by DMA and in place alike, whichever
+    /// thread issues the copy.  `whole_model`: every expert is copied, the GPU caches' too, so an expert a swap
+    /// evicts from any stage's cache is in RAM already (no exchange: those read only CUDA0's cache).
+    void set_arena_placement(std::vector<int> layer_node, bool whole_model) {
+        arena_layer_node_ = std::move(layer_node);
+        arena_whole_ = whole_model;
+    }
+    /// set_arena_placement's allocation on NUMA node `node` (its host address and size), for the stages to probe the
+    /// link against the memory they read; false when there is none.
+    bool arena_segment(int node, const uint8_t*& host, uint64_t& bytes) const {
+        for (const Segment& s : complement_segs_)
+            if (s.node == node) { host = (const uint8_t*) s.arena; bytes = s.end - s.start; return true; }
+        return false;
+    }
 
     bool mapped() const { return base_ != nullptr; }
     int64_t blobs() const { return blobs_; }
@@ -871,6 +889,14 @@ class FileExpertSource : public ExpertSource {
     detail::ExchangeStorage exchange_storage_; // authoritative when active; original arenas still own memory
     bool complement_pinned_ = false;
     bool complement_partial_ = false;         ///< CS-T: only the first complement_pin_limit_ bytes are registered
+    /// set_arena_placement: the arena's offsets [start, end) in one cudaHostAlloc per NUMA node (empty: one arena)
+    struct Segment { void* arena; uint64_t start, end; const uint8_t* device; int node; };
+    std::vector<Segment> complement_segs_;
+    std::vector<int> arena_layer_node_;
+    bool arena_whole_ = false;
+    const uint8_t* complement_at(uint64_t offset) const;       ///< the host address of an arena offset
+    const uint8_t* complement_dev_at(uint64_t offset) const;   ///< its device address (mapped)
+    const uint8_t* complement_blob(size_t index) const;        ///< an expert's bytes in the arena, or null
     bool complement_registered_ = false;      ///< the arena is ordinary memory registered with cudaHostRegister (not cudaHostAlloc)
     uint64_t complement_pin_limit_ = 0;
     uint64_t complement_lock_off_ = 0;        ///< the working-set lock covers [lock_off, lock_off + locked)

include/strata/core/fp16_mode.hpp

@@ -0,0 +1,52 @@
+// include/strata/core/fp16_mode.hpp - STRATA_FP16: where the model's BF16-form weights run as FP16 (V100: no BF16
+// tensor cores; cuBLAS BF16 GEMMs are 3-18x slower than FP16 there).
+//
+//   unset / off  the BF16 forms as the pack holds them (the default: byte-identical outputs)
+//   prefill      the prompt path converts each BF16 weight to FP16 right before its GEMM; decode is untouched
+//   load         at load, before serving, the weights are taken from the GGUF instead of the pack's BF16 rows:
+//                Q8_0 / BF16 / F16 sources become FP16 and the gates (router, shared gate, hc inject, SSM
+//                alpha/beta: F32 in the GGUF) stay F32; every reader uses those forms.  STRATA_FP16_GATES=1
+//                makes the gates FP16 too.  The MTP drafter keeps its BF16 weights.
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+namespace strata {
+
+enum class Fp16Mode { Off, Prefill, Load };
+
+inline Fp16Mode fp16_mode() {
+    static const Fp16Mode m = [] {
+        const char* e = std::getenv("STRATA_FP16");
+        if (e == nullptr || *e == 0 || std::strcmp(e, "off") == 0 || std::strcmp(e, "0") == 0) return Fp16Mode::Off;
+        if (std::strcmp(e, "prefill") == 0) return Fp16Mode::Prefill;
+        if (std::strcmp(e, "load") == 0) return Fp16Mode::Load;
+        std::fprintf(stderr, "STRATA_FP16=%s: expected off, prefill or load\n", e);
+        std::exit(2);
+    }();
+    return m;
+}
+
+/// STRATA_FP16_GATES=1 (with STRATA_FP16=load): the F32 gates become FP16 at load too.
+inline bool fp16_gates() {
+    static const bool v = [] {
+        const char* e = std::getenv("STRATA_FP16_GATES");
+        return e != nullptr && e[0] == '1';
+    }();
+    return v && fp16_mode() == Fp16Mode::Load;
+}
+
+/// STRATA_FP16_MTP=1: the MTP drafter's prompt-path GEMMs (its attention hyper-connection down/up, run per prompt chunk)
+/// as FP16: those two BF16 weights converted to FP16 when the drafter loads, FP16 activations.  Its decode is unchanged
+/// (FP32 activations).  Independent of STRATA_FP16.
+inline bool fp16_mtp() {
+    static const bool v = [] {
+        const char* e = std::getenv("STRATA_FP16_MTP");
+        return e != nullptr && e[0] == '1';
+    }();
+    return v;
+}
+
+}  // namespace strata

include/strata/core/load_converted.hpp

@@ -0,0 +1,46 @@
+// include/strata/core/load_converted.hpp - STRATA_FP16=load: the BF16-form weights taken from the GGUF at load.
+//
+// The pack holds these tensors as BF16 (its `Bf16InF32` form), rounded from the GGUF's Q8_0 (hyper-connection
+// down/up, output_hc, ple_value), BF16 (indexer) and F32 (router, shared gate, hc inject, SSM alpha/beta: BF16
+// values in F32 containers).  Here they are read from the GGUF instead, once on the host: Q8_0 / BF16 / F16 to FP16
+// (`F16InF32` form), F32 kept as F32 (`F32` form; FP16 with STRATA_FP16_GATES=1), and each GPU's table gets its own
+// copy in place of the pack's row, which the arena skips.
+#pragma once
+
+#include <cstdint>
+#include <set>
+#include <string>
+#include <vector>
+
+namespace strata::core {
+class WeightTable;
+
+class LoadConverted {
+public:
+    /// Read and convert every served tensor of `shards`.  Refuses a value FP16 cannot hold (|x| > 65504).
+    bool build(const std::vector<std::string>& shards, bool gates_fp16, std::string& err);
+    /// The canonical names `build` serves (for the arena's skip set).
+    void served_names(std::set<std::string>& out) const;
+    /// Upload into `table` (one GPU's): each row's data, bytes and kind replaced.  `owner` keeps the allocations.
+    bool attach(WeightTable& table, std::vector<void*>& owner, std::string& err) const;
+    static void free_all(std::vector<void*>& owner);
+
+    size_t tensors_f16 = 0, tensors_f32 = 0;
+    uint64_t bytes = 0;
+    uint64_t subnormal = 0;     ///< nonzero values that became FP16 subnormals (|x| < 6.1e-5)
+    uint64_t flushed = 0;       ///< nonzero values that became FP16 zero
+
+private:
+    struct Host {
+        std::string name;
+        int64_t ne0 = 0, ne1 = 0;
+        bool f32 = false;
+        std::vector<uint8_t> data;
+    };
+    std::vector<Host> host_;
+};
+
+/// The served kinds: true for a gate (F32 in the GGUF, kept F32 unless STRATA_FP16_GATES=1).
+bool load_converted_kind(const std::string& name, bool& gate);
+
+}  // namespace strata::core