From 55297b8509e4a2e1674c7096b3b46a62ba06c71c Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:01 +0200 Subject: [PATCH 01/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): include/strata from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 26 file(s) under include/strata -- 25 clean, 1 with conflict markers. Conflict-marked files -- a pull request with conflicts: resolve them, do not discard the work: include/strata/kernels/verify_kernels.hpp Original authors: 4EverBuilder,Andy,Anon,BlueKingMuch Original commits touching these paths (152 subject(s)): - #253: the 8 GiB arena pin cap only under WDDM; STRATA_ARENA_PIN_GIB; the split's prompt chunk steps down - #284 follow-up: wait for the async commit on an event, at every point that touches the session afterwards - #315 follow-up: its variants and switch renamed (plain / split / staged, STRATA_HC_SPLIT) - no clash with STRATA_GR_V3 - #340 prototype: a split's streamed ring is 96 slots when most experts are resident (STRATA_SPLIT_RING); STRATA_SPLIT_SMALL_OWN=S keeps an S-token chunk's buffers per lending stage for the session; --no-prefill-borrow on a split books the ring (it no longer starves the draft head) - #465 engine: batch slots for one GPU - warn instead of refuse, windows over the active slots, slot conversation cache, prompts read beside decoding slots, BYIELD - #583 default on (owner): STRATA_RING_BYTES=0 opts out; DETAILS.md with the measured quality - #583: the byte-budget ring only where it gains; short prompts keep 0.1.39's ring - AMD HIP backend for gfx1100 (PR #121, squashed onto its 0.1.21 base) Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: include/strata Fork-Replay-Automated: true --- include/strata/core/expert_cache.hpp | 5 ++ include/strata/core/expert_source.hpp | 26 ++++++++ include/strata/core/fp16_mode.hpp | 52 +++++++++++++++ include/strata/core/load_converted.hpp | 46 +++++++++++++ include/strata/core/mtp.hpp | 8 +++ include/strata/core/pinned.hpp | 12 ++++ include/strata/core/verify.hpp | 16 ++++- include/strata/core/vmm.hpp | 1 + include/strata/core/weight_form.hpp | 24 +++++++ include/strata/core/weights.hpp | 1 + include/strata/hip_compat/cuda_runtime.h | 1 + include/strata/kernels/bf16_gemv.hpp | 8 +++ include/strata/kernels/cpu/expert_layout.hpp | 9 +++ include/strata/kernels/cpu/kq_vsx.hpp | 18 +++++ include/strata/kernels/fused_expert.hpp | 34 ++++++++++ include/strata/kernels/fused_gdn.hpp | 6 +- include/strata/kernels/fused_gr.hpp | 1 + include/strata/kernels/gr.hpp | 10 ++- include/strata/kernels/ple.hpp | 1 + include/strata/kernels/shared_expert.hpp | 14 ++-- include/strata/kernels/verify_kernels.hpp | 29 ++++++++ include/strata/platform/cpu_relax.hpp | 26 ++++++++ include/strata/platform/thread_affinity.hpp | 69 ++++++++++++++++++++ include/strata/prefill/gemm.hpp | 6 ++ include/strata/prefill/kernels.hpp | 31 +++++++++ include/strata/prefill/prefill.hpp | 4 ++ 26 files changed, 448 insertions(+), 10 deletions(-) create mode 100644 include/strata/core/fp16_mode.hpp create mode 100644 include/strata/core/load_converted.hpp create mode 100644 include/strata/core/weight_form.hpp create mode 100644 include/strata/kernels/cpu/kq_vsx.hpp create mode 100644 include/strata/kernels/fused_expert.hpp create mode 100644 include/strata/platform/cpu_relax.hpp create mode 100644 include/strata/platform/thread_affinity.hpp diff --git a/include/strata/core/expert_cache.hpp b/include/strata/core/expert_cache.hpp index 17e277debc9..bd51a86d06d 100644 --- a/include/strata/core/expert_cache.hpp +++ b/include/strata/core/expert_cache.hpp @@ -54,6 +54,11 @@ inline constexpr int32_t kNotResident = -1; bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_expert, std::vector>& ranked, int64_t& slots, std::string& err); +/// Free device memory for the expert cache. On a unified memory system - one memory for the CPU and the GPU, such as +/// the DGX Spark (GB10); CUDA reports it as cudaDevAttrIntegrated - cudaMemGetInfo counts only MemFree, not the page cache - which, with the experts read from the GGUF in place, is mostly those +/// files' clean pages and gives way to a device allocation. There the figure is MemAvailable less 6 GiB for the OS +/// and the engine's host side (STRATA_UMA_HEADROOM_GIB). Elsewhere it is cudaMemGetInfo's. +size_t device_free_bytes(); /// #477 (--expert-profile-save): what the adaptive tier learned, as a profile ranking EVERY (layer, expert) pair - /// the experts resident in VRAM now first (where the swaps left the cache), then the rest; within each, by `heat` /// (the routing the adaptive tier counted since the start, descending), then by `prior` (the profile the engine diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp index cf517690997..c587a58a302 100644 --- a/include/strata/core/expert_source.hpp +++ b/include/strata/core/expert_source.hpp @@ -582,6 +582,24 @@ class FileExpertSource : public ExpertSource { stage_lend_regions_ = std::move(regions); } void close(); + /// A layer split across sockets (an IBM AC922: GPUs 0-1 on NUMA node 0, 2-3 on node 8): call before + /// `pin_cache_complement`. `layer_node[l]` is the NUMA node of the GPU that runs layer l (-1: unknown). The + /// arena is laid out node by node, one cudaHostAlloc per node made by a thread on that node's CPUs: the driver + /// puts page-locked memory on the allocating thread's node (not where a memory policy asks), and a GPU reads its + /// own socket's memory at 68 GB/s on the AC922, the other socket's at 40 - by DMA and in place alike, whichever + /// thread issues the copy. `whole_model`: every expert is copied, the GPU caches' too, so an expert a swap + /// evicts from any stage's cache is in RAM already (no exchange: those read only CUDA0's cache). + void set_arena_placement(std::vector layer_node, bool whole_model) { + arena_layer_node_ = std::move(layer_node); + arena_whole_ = whole_model; + } + /// set_arena_placement's allocation on NUMA node `node` (its host address and size), for the stages to probe the + /// link against the memory they read; false when there is none. + bool arena_segment(int node, const uint8_t*& host, uint64_t& bytes) const { + for (const Segment& s : complement_segs_) + if (s.node == node) { host = (const uint8_t*) s.arena; bytes = s.end - s.start; return true; } + return false; + } bool mapped() const { return base_ != nullptr; } int64_t blobs() const { return blobs_; } @@ -871,6 +889,14 @@ class FileExpertSource : public ExpertSource { detail::ExchangeStorage exchange_storage_; // authoritative when active; original arenas still own memory bool complement_pinned_ = false; bool complement_partial_ = false; ///< CS-T: only the first complement_pin_limit_ bytes are registered + /// set_arena_placement: the arena's offsets [start, end) in one cudaHostAlloc per NUMA node (empty: one arena) + struct Segment { void* arena; uint64_t start, end; const uint8_t* device; int node; }; + std::vector complement_segs_; + std::vector arena_layer_node_; + bool arena_whole_ = false; + const uint8_t* complement_at(uint64_t offset) const; ///< the host address of an arena offset + const uint8_t* complement_dev_at(uint64_t offset) const; ///< its device address (mapped) + const uint8_t* complement_blob(size_t index) const; ///< an expert's bytes in the arena, or null bool complement_registered_ = false; ///< the arena is ordinary memory registered with cudaHostRegister (not cudaHostAlloc) uint64_t complement_pin_limit_ = 0; uint64_t complement_lock_off_ = 0; ///< the working-set lock covers [lock_off, lock_off + locked) diff --git a/include/strata/core/fp16_mode.hpp b/include/strata/core/fp16_mode.hpp new file mode 100644 index 00000000000..fe00f21d7f1 --- /dev/null +++ b/include/strata/core/fp16_mode.hpp @@ -0,0 +1,52 @@ +// include/strata/core/fp16_mode.hpp - STRATA_FP16: where the model's BF16-form weights run as FP16 (V100: no BF16 +// tensor cores; cuBLAS BF16 GEMMs are 3-18x slower than FP16 there). +// +// unset / off the BF16 forms as the pack holds them (the default: byte-identical outputs) +// prefill the prompt path converts each BF16 weight to FP16 right before its GEMM; decode is untouched +// load at load, before serving, the weights are taken from the GGUF instead of the pack's BF16 rows: +// Q8_0 / BF16 / F16 sources become FP16 and the gates (router, shared gate, hc inject, SSM +// alpha/beta: F32 in the GGUF) stay F32; every reader uses those forms. STRATA_FP16_GATES=1 +// makes the gates FP16 too. The MTP drafter keeps its BF16 weights. +#pragma once + +#include +#include +#include + +namespace strata { + +enum class Fp16Mode { Off, Prefill, Load }; + +inline Fp16Mode fp16_mode() { + static const Fp16Mode m = [] { + const char* e = std::getenv("STRATA_FP16"); + if (e == nullptr || *e == 0 || std::strcmp(e, "off") == 0 || std::strcmp(e, "0") == 0) return Fp16Mode::Off; + if (std::strcmp(e, "prefill") == 0) return Fp16Mode::Prefill; + if (std::strcmp(e, "load") == 0) return Fp16Mode::Load; + std::fprintf(stderr, "STRATA_FP16=%s: expected off, prefill or load\n", e); + std::exit(2); + }(); + return m; +} + +/// STRATA_FP16_GATES=1 (with STRATA_FP16=load): the F32 gates become FP16 at load too. +inline bool fp16_gates() { + static const bool v = [] { + const char* e = std::getenv("STRATA_FP16_GATES"); + return e != nullptr && e[0] == '1'; + }(); + return v && fp16_mode() == Fp16Mode::Load; +} + +/// STRATA_FP16_MTP=1: the MTP drafter's prompt-path GEMMs (its attention hyper-connection down/up, run per prompt chunk) +/// as FP16: those two BF16 weights converted to FP16 when the drafter loads, FP16 activations. Its decode is unchanged +/// (FP32 activations). Independent of STRATA_FP16. +inline bool fp16_mtp() { + static const bool v = [] { + const char* e = std::getenv("STRATA_FP16_MTP"); + return e != nullptr && e[0] == '1'; + }(); + return v; +} + +} // namespace strata diff --git a/include/strata/core/load_converted.hpp b/include/strata/core/load_converted.hpp new file mode 100644 index 00000000000..b1757fcf424 --- /dev/null +++ b/include/strata/core/load_converted.hpp @@ -0,0 +1,46 @@ +// include/strata/core/load_converted.hpp - STRATA_FP16=load: the BF16-form weights taken from the GGUF at load. +// +// The pack holds these tensors as BF16 (its `Bf16InF32` form), rounded from the GGUF's Q8_0 (hyper-connection +// down/up, output_hc, ple_value), BF16 (indexer) and F32 (router, shared gate, hc inject, SSM alpha/beta: BF16 +// values in F32 containers). Here they are read from the GGUF instead, once on the host: Q8_0 / BF16 / F16 to FP16 +// (`F16InF32` form), F32 kept as F32 (`F32` form; FP16 with STRATA_FP16_GATES=1), and each GPU's table gets its own +// copy in place of the pack's row, which the arena skips. +#pragma once + +#include +#include +#include +#include + +namespace strata::core { +class WeightTable; + +class LoadConverted { +public: + /// Read and convert every served tensor of `shards`. Refuses a value FP16 cannot hold (|x| > 65504). + bool build(const std::vector& shards, bool gates_fp16, std::string& err); + /// The canonical names `build` serves (for the arena's skip set). + void served_names(std::set& out) const; + /// Upload into `table` (one GPU's): each row's data, bytes and kind replaced. `owner` keeps the allocations. + bool attach(WeightTable& table, std::vector& owner, std::string& err) const; + static void free_all(std::vector& owner); + + size_t tensors_f16 = 0, tensors_f32 = 0; + uint64_t bytes = 0; + uint64_t subnormal = 0; ///< nonzero values that became FP16 subnormals (|x| < 6.1e-5) + uint64_t flushed = 0; ///< nonzero values that became FP16 zero + +private: + struct Host { + std::string name; + int64_t ne0 = 0, ne1 = 0; + bool f32 = false; + std::vector data; + }; + std::vector host_; +}; + +/// The served kinds: true for a gate (F32 in the GGUF, kept F32 unless STRATA_FP16_GATES=1). +bool load_converted_kind(const std::string& name, bool& gate); + +} // namespace strata::core diff --git a/include/strata/core/mtp.hpp b/include/strata/core/mtp.hpp index d3c05a98256..f273a05955f 100644 --- a/include/strata/core/mtp.hpp +++ b/include/strata/core/mtp.hpp @@ -154,6 +154,8 @@ class MtpDrafter { /// state, the first cell a round can still read, its device, and a wait for its own stream. const float* tensor_f32(const char* name) const { return f32(name); } const uint16_t* tensor_bf16(const char* name) const { return bf16(name); } + /// STRATA_FP16_MTP=1: the FP16 copy (made at load) of a BF16 tensor the prompt path reads; null otherwise. + const uint16_t* tensor_f16(const char* name) const; const void* tensor_q8(const char* name) const { return q8(name); } QsaState& kv_state_rw() { return st_; } int64_t first_needed() const { return (window_ > 0 && prompt_len_ > 0) ? prompt_len_ - window_ - 64 : 0; } @@ -241,7 +243,13 @@ class MtpDrafter { struct Tensor { std::string name, kind; int64_t rows = 0, cols = 0; uint64_t off = 0, bytes = 0; }; std::vector tensors_; uint8_t* dense_ = nullptr; + uint16_t* f16_ = nullptr; ///< STRATA_FP16_MTP=1: FP16 copies, see tensor_f16 + std::vector> f16_off_; ///< name -> element offset in f16_ uint8_t* experts_ = nullptr; + /// STRATA_MTP_EXPERTS=: the experts in the native GGUF layout (gate/up and down ggml types; -1 = the pack's + /// Q2_0 blobs) and the bytes per expert + int xgu_ = -1, xd_ = -1; + uint64_t xblob_ = 0; // Slot drafters borrow immutable weights and head; each still owns its state and scratch. bool owns_weights_ = true; bool owns_draft_head_ = true; diff --git a/include/strata/core/pinned.hpp b/include/strata/core/pinned.hpp index 4d5ef2be61e..93340756a00 100644 --- a/include/strata/core/pinned.hpp +++ b/include/strata/core/pinned.hpp @@ -14,6 +14,7 @@ // fallback is the common case and not an error path. #pragma once +#include #include #include #include @@ -29,10 +30,21 @@ enum class PageBacking { LargePages, NormalPages, PinnedByCuda }; /// -2 for "auto" (Windows: the sliced pin stays below the GPU's shared-memory budget). int arena_pin_cap_gib(); +/// The calling thread onto the CPUs of the NUMA node that holds `p`, when `p` is in an expert arena this process +/// allocated with cudaHostAlloc (PinnedArena below; Linux). The arena's loaders call it: on a POWER9 a GPU reads +/// memory the other socket's CPUs wrote at 45 GB/s, its own socket's at 68 - with the pages on its node either way. +void run_on_arena_node(const void* p); + +/// cudaHostAlloc made by a thread on the NUMA node of the current CUDA device (Linux; elsewhere a plain +/// cudaHostAlloc): the driver puts page-locked memory on the allocating thread's node, and a GPU reads the other +/// socket's memory at 40 GB/s on the AC922 against 68 from its own. For a layer split's per-GPU host buffers. +cudaError_t cuda_host_alloc_near_gpu(void** p, size_t bytes, unsigned int flags); + struct PinnedArena { void* base = nullptr; uint64_t capacity = 0; PageBacking backing = PageBacking::NormalPages; + int node = -1; ///< PinnedByCuda: the NUMA node of the GPU it was allocated for std::string note; // why the backing is what it is, for the startup print uint64_t locked_bytes = 0; // resident via the working-set lock when CUDA could not pin it /// Plan v0.3 P5: when the whole arena cannot be registered, it is registered in `slice`-byte pieces from the diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hpp index a9dc19dbd6c..77a6fab51be 100644 --- a/include/strata/core/verify.hpp +++ b/include/strata/core/verify.hpp @@ -227,7 +227,7 @@ class Verifier { } const int32_t* batch_out() const { return b_out_; } bool last_stage() const { return g_ != nullptr && le_ == g_->n_layers; } - /// commit() returns without waiting for its graph (a single-GPU session sets it): the next window follows it on + /// commit() returns without waiting for its graph (every stage of a layer split too): the next window follows it on /// the same stream and the drafter reads nothing it writes, so it overlaps the draft. Whoever reads or writes /// the session from another stream or the host afterwards (a new request, a checkpoint, a snapshot, the prompt /// path, the end of a run) calls wait_commit() first. STRATA_COMMIT_SYNC=1 keeps the wait. @@ -307,6 +307,10 @@ class Verifier { /// arena directly, 2 = a copy kernel stages it inside the graph (no API calls on the pool's thread; best when /// the CPU is RAM-bound, Q2_0). Set before the first `run`. void set_pcie_mode(int mode) { sink_.pcie_mode = mode; } + /// A layer split's idle NVLink peer fetches half of this stage's PCIe experts (partner_fetch, verify_kernels.hpp): + /// call after set_pcie_mode, before the first window. STRATA_FETCH_PARTNER=0: off. No-op unless the copy kernel + /// overlaps (pcie_mode 2, STRATA_FETCH_OVERLAP) and the peer reaches this GPU. + bool set_partner(int dev, std::string& err); /// the pool never plans a PCIe share (--pcie-frac 0): the window skips that path. Before the first run. double ms_wait = 0, ms_pool = 0, ms_host = 0, ms_commit = 0; @@ -483,6 +487,16 @@ class Verifier { cudaEvent_t commit_done_ = nullptr; // recorded after an async commit (set_commit_async); see wait_commit bool commit_pending_ = false; cudaStream_t copy_ = nullptr; // the copy engine's stream (DMA of missed experts) + // the PCIe share's copy kernel on its own stream beside the VRAM experts (STRATA_FETCH_OVERLAP, default on) + cudaStream_t fetch_s_ = nullptr; + cudaEvent_t ev_plan_ = nullptr, ev_fetched_ = nullptr; + bool fetch_overlap_ = false; + int partner_ = -1; // set_partner: the peer GPU, or -1 + cudaStream_t partner_s_ = nullptr; // on the peer + unsigned* partner_arrive_ = nullptr; // on the peer, one per step + long long* partner_bytes_ = nullptr; // on the peer, blob bytes per layer + uint32_t* h_pdone_ = nullptr; uint32_t* m_pdone_ = nullptr; // mapped, one per step: the peer's share landed + static constexpr int kMaxSteps = 2 * 64 * 2; struct FlagSet { uint32_t* flag; uint32_t value; }; FlagSet flag_sets_[2 * 64 * 2] = {}; // host-function arguments, one per (layer, group) static void fetch_dma(void* ctx, const uint8_t* const* src, int n, size_t bytes); diff --git a/include/strata/core/vmm.hpp b/include/strata/core/vmm.hpp index d8bcf0b57d6..7f344715177 100644 --- a/include/strata/core/vmm.hpp +++ b/include/strata/core/vmm.hpp @@ -4,6 +4,7 @@ // expert cache gives up, and neither moves: every captured graph and stored pointer stays valid. The driver calls // are reached through the runtime's entry-point query, so nothing links the driver library. Not under HIP. +#include #include #include diff --git a/include/strata/core/weight_form.hpp b/include/strata/core/weight_form.hpp new file mode 100644 index 00000000000..4b5faa9f3e9 --- /dev/null +++ b/include/strata/core/weight_form.hpp @@ -0,0 +1,24 @@ +// include/strata/core/weight_form.hpp - the kernels' WForm of a BF16-form weight as the table holds it: the pack's +// BF16, or under STRATA_FP16=load FP16 (`F16InF32`) or F32 (the gates). +#pragma once + +#include "strata/core/weights.hpp" +#include "strata/kernels/bf16_gemv.hpp" + +#include +#include + +namespace strata::core { + +inline strata::kernels::WForm wform(const WeightRef* w, const char* what) { + switch (w->kind) { + case WeightKind::Bf16InF32: return strata::kernels::WForm::Bf16; + case WeightKind::F16InF32: return strata::kernels::WForm::F16; + case WeightKind::F32: return strata::kernels::WForm::F32; + default: + throw std::runtime_error(std::string(what) + " is engine form " + std::to_string((int) w->kind) + + ", not BF16, FP16 or F32"); + } +} + +} // namespace strata::core diff --git a/include/strata/core/weights.hpp b/include/strata/core/weights.hpp index d352ba64189..7c6c5c13aa2 100644 --- a/include/strata/core/weights.hpp +++ b/include/strata/core/weights.hpp @@ -141,6 +141,7 @@ class WeightTable { private: friend class NativeDense; + friend class LoadConverted; std::map table_; LoadReport report_; std::string pack_dir_; diff --git a/include/strata/hip_compat/cuda_runtime.h b/include/strata/hip_compat/cuda_runtime.h index d669f0fe694..d692d5e7cd2 100644 --- a/include/strata/hip_compat/cuda_runtime.h +++ b/include/strata/hip_compat/cuda_runtime.h @@ -19,6 +19,7 @@ #define cudaDevAttrMultiProcessorCount hipDeviceAttributeMultiprocessorCount #define cudaDevAttrClockRate hipDeviceAttributeClockRate #define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor +#define cudaDevAttrIntegrated hipDeviceAttributeIntegrated #define cudaDevAttrComputeCapabilityMinor hipDeviceAttributeComputeCapabilityMinor #define cudaDevAttrIntegrated hipDeviceAttributeIntegrated #define cudaDeviceGetAttribute hipDeviceGetAttribute diff --git a/include/strata/kernels/bf16_gemv.hpp b/include/strata/kernels/bf16_gemv.hpp index f040b04cdbe..05500346627 100644 --- a/include/strata/kernels/bf16_gemv.hpp +++ b/include/strata/kernels/bf16_gemv.hpp @@ -59,4 +59,12 @@ bool bf16_gemv_fp32_mmvf_multi_aux(const float* x, int64_t ldx, const uint16_t* int64_t n_in, int64_t n_out, int n_tok, const uint16_t* w_aux, float* y_aux, int64_t ldy_aux, void* stream); +/// The stored form of a BF16-form weight: the pack's BF16, or under STRATA_FP16=load FP16 or F32 (the gates). +enum class WForm : int { Bf16 = 0, F16 = 1, F32 = 2 }; +/// bf16_gemv_fp32_mmvf / _multi for a weight in form `f` (F32: w 8-byte aligned). Bf16 is the same kernel and +/// the same bits as the BF16 entry points; F16 and F32 widen each weight to FP32 and keep the order of operations. +void gemv_fp32_mmvf(const float* x, const void* w, WForm f, float* y, int64_t n_in, int64_t n_out, void* stream); +void gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const void* w, WForm f, float* y, int64_t ldy, + int64_t n_in, int64_t n_out, int n_tok, void* stream); + } // namespace strata::kernels diff --git a/include/strata/kernels/cpu/expert_layout.hpp b/include/strata/kernels/cpu/expert_layout.hpp index 6b3fddb6646..fdd7cf85dbd 100644 --- a/include/strata/kernels/cpu/expert_layout.hpp +++ b/include/strata/kernels/cpu/expert_layout.hpp @@ -49,6 +49,15 @@ bool cpu_avx512bw_ok(); /// Whether this CPU (and its OS) runs the AVX2 kernels (AVX, AVX2, FMA, F16C): the floor of every expert kernel /// (q2_avx2.cpp, iq_avx2.cpp, and ggml-cpu in the portable build). STRATA_FORCE_AVX2 does not change it. bool cpu_avx2_ok(); +/// Whether Q2_0 rows of a native pack take this repository's own kernels (q2_rows_any: AVX-512 / AVX2, x86 only). +/// Elsewhere (aarch64) they go through ggml-cpu like every other native format (its NEON vec_dot_q2_0_q8_0). +constexpr bool q2_own_kernels() { +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) + return true; +#else + return false; +#endif +} /// Whether this CPU (and its OS) runs AVX (Sandy Bridge and newer): the AVX1 router dot (kq_avx1.cpp) and ggml-cpu in /// a STRATA_ISA_FLOOR=avx build. The other expert rows then run on ggml-cpu (experimental, slow). bool cpu_avx1_ok(); diff --git a/include/strata/kernels/cpu/kq_vsx.hpp b/include/strata/kernels/cpu/kq_vsx.hpp new file mode 100644 index 00000000000..6e80768251b --- /dev/null +++ b/include/strata/kernels/cpu/kq_vsx.hpp @@ -0,0 +1,18 @@ +#pragma once +// POWER9 VSX row kernels for the CPU expert share (STRATA_VSX_EXPERTS=1): UD-Q4_K_XL's Q4_K gate/up (Q8_K activations) +// and Q5_1 down (Q8_1 activations), each weight block decoded once for all the tokens routed to the expert. ggml's +// arithmetic per block (exact integer dots, the same scale products) summed in another order: ~2e-7 relative to +// ggml-cpu's vec_dot, not bitwise. +#include +#include + +namespace strata::kernels::cpu { +/// true on a POWER9 build with STRATA_VSX_EXPERTS set +bool kq_vsx_on() noexcept; +/// gate/up rows [r0, r1): ff[t][r] = silu(gate . x_t) * (up . x_t), act[t] = Q8_K rows of n_embd +void kq_vsx_gu_rows(const uint8_t* blob, size_t gu_row, size_t up_off, int n_embd, const void* const* act, int nt, + float* const* ff, int r0, int r1); +/// Q5_1 rows [r0, r1) of row_bytes each (n_ff values, n_ff % 128 == 0): out[t][r] = row . h_t, hq[t] = Q8_1 rows +void kq_vsx_q51_rows(const uint8_t* W, size_t row_bytes, int n_ff, const void* const* hq, int nt, float* const* out, + int r0, int r1); +} // namespace strata::kernels::cpu diff --git a/include/strata/kernels/fused_expert.hpp b/include/strata/kernels/fused_expert.hpp new file mode 100644 index 00000000000..83d4f5c5c09 --- /dev/null +++ b/include/strata/kernels/fused_expert.hpp @@ -0,0 +1,34 @@ +#pragma once +// Fused prefill expert products (STRATA_FUSED_EXPERTS=1): the GGUF blocks of a native pack's expert are dequantized +// tile by tile into shared memory and multiplied on the FP16 tensor cores (Volta wmma, FP32 accumulation), so the +// FP16 copy of the expert is never written to or read from VRAM, and up to kFusedExpertMax experts share one launch. +// The weights are the same FP16 values iq_dequant_*_f16 produces; only the summation order differs from cuBLAS. +#include +#include + +namespace strata::kernels { + +constexpr int kFusedExpertMax = 32; + +struct FusedExpertLayout { + int gu_type = 0, d_type = 0; + int64_t n_embd = 0, n_ff = 0; + size_t gu_row = 0, d_row = 0, up_off = 0, down_off = 0; +}; + +struct FusedExpertGroup { + const uint8_t* blob[kFusedExpertMax] = {}; + int64_t row0[kFusedExpertMax] = {}; // first row of the expert's tokens in X / H / D + int ne[kFusedExpertMax] = {}; // its token count + int n = 0; +}; + +// gate/up Q4_K or Q5_K with down Q5_1 or Q8_0, n_embd % 256 == 0, n_ff % 64 == 0 +bool fused_expert_supported(int gu_type, int d_type, int64_t n_embd, int64_t n_ff) noexcept; + +// H[rows][n_ff] (FP16) = SwiGLU of X[rows][n_embd] (FP16) times gate/up (interleaved as iq_dequant_gu_f16), +// then D[rows][n_embd] (FP32) = H times down, for every expert of `g` +void fused_expert_run(const FusedExpertLayout& L, const FusedExpertGroup& g, const uint16_t* X, uint16_t* H, float* D, + void* stream); + +} // namespace strata::kernels diff --git a/include/strata/kernels/fused_gdn.hpp b/include/strata/kernels/fused_gdn.hpp index d1f39cbd32c..a12b9bf3b82 100644 --- a/include/strata/kernels/fused_gdn.hpp +++ b/include/strata/kernels/fused_gdn.hpp @@ -12,6 +12,8 @@ // y = rmsnorm(o) * gamma * sigmoid(z) #pragma once +#include "strata/kernels/bf16_gemv.hpp" + #include namespace strata::kernels { @@ -25,8 +27,8 @@ void fused_gdn_conv_l2(float* history, const float* qkv, const float* conv_w, fl /// alpha and beta, both `(h_v, n_embd)` BF16 against the FP32 activation, with their epilogues: /// gate = softplus(alpha + dt) * ssm_a, beta = sigmoid(beta). One warp per row; replaces two MMVF launches and /// two elementwise kernels. -void fused_gdn_ab(const float* x, const uint16_t* w_alpha, const uint16_t* w_beta, const float* dt, const float* ssm_a, - float* gate, float* beta, int n_embd, int h_v, void* stream); +void fused_gdn_ab(const float* x, const void* w_alpha, const void* w_beta, const float* dt, const float* ssm_a, + float* gate, float* beta, int n_embd, int h_v, void* stream, WForm form = WForm::Bf16); void fused_gdn_step_norm(float* state, const float* q, const float* k, const float* v, const float* gate, const float* beta, const float* z, const float* gamma, float eps, float* y, int h_k, int h_v, diff --git a/include/strata/kernels/fused_gr.hpp b/include/strata/kernels/fused_gr.hpp index 0032424736a..f915822ea6f 100644 --- a/include/strata/kernels/fused_gr.hpp +++ b/include/strata/kernels/fused_gr.hpp @@ -36,6 +36,7 @@ struct FusedGrArgs { float* rs = nullptr; ///< workspace, hc floats float* inject_out = nullptr; ///< hc floats (when w_inject) float* mixed = nullptr; ///< n_embd + bool w_f16 = false; ///< STRATA_FP16=load: w_down / w_up / w_inject hold FP16 bits, not BF16 /// S23 experiment (STRATA_HC_Q8=1): the GGUF's Q8_0 projections (null: the BF16 ones above); fused_gr_read_multi /// only const uint8_t* q8_down = nullptr; ///< Q8_0 [hc_lr][hc*n_embd] diff --git a/include/strata/kernels/gr.hpp b/include/strata/kernels/gr.hpp index 2a7e85fcda2..b396571bcf2 100644 --- a/include/strata/kernels/gr.hpp +++ b/include/strata/kernels/gr.hpp @@ -19,6 +19,8 @@ // activation precision must be pinned with the oracle backend and batch geometry, not inferred from weights. #pragma once +#include "strata/kernels/bf16_gemv.hpp" + #include #include @@ -109,9 +111,11 @@ inline size_t gr_workspace_bytes(const GrShapes& s) { GrWorkspace w; return gr_w /// This version splits the work the way the shapes want: `hc` blocks for the per-stream norm, one warp per /// row for each of the two projections, and one block-tile for the mean. The transposes are gone because the /// warp-per-row mapping makes them unnecessary. -void gr_read(const float* R, const float* w_norm, const uint16_t* w_down, const uint16_t* w_up, - const uint16_t* w_inject, float eps, const GrShapes& s, const GrWorkspace& ws, float* mixed, - float* inject, void* stream); +/// `f_down_up` / `f_inject`: the weights' stored forms (STRATA_FP16=load: FP16, an F32 inject); not BF16 needs +/// the native MMVF read. +void gr_read(const float* R, const float* w_norm, const void* w_down, const void* w_up, + const void* w_inject, float eps, const GrShapes& s, const GrWorkspace& ws, float* mixed, + float* inject, void* stream, WForm f_down_up = WForm::Bf16, WForm f_inject = WForm::Bf16); void gr_read_multi(const float* R, const float* w_norm, const uint16_t* w_down, const uint16_t* w_up, const uint16_t* w_inject, float eps, const GrShapes& s, const GrWorkspace& ws, diff --git a/include/strata/kernels/ple.hpp b/include/strata/kernels/ple.hpp index 98bfc388def..c68905e9ec8 100644 --- a/include/strata/kernels/ple.hpp +++ b/include/strata/kernels/ple.hpp @@ -55,6 +55,7 @@ struct PleWeights { const uint8_t* key_codes = nullptr; // hc_dim * (n_embd/4) bytes const float* key_scales = nullptr; // hc_dim * (n_embd/64) floats const uint16_t* value_bf16 = nullptr; // n_embd * n_embd + bool value_f16 = false; // STRATA_FP16=load: value_bf16 holds FP16 bits const float* norm_key = nullptr; // hc_dim const float* norm_query = nullptr; // hc_dim const float* norm_conv = nullptr; // hc_dim diff --git a/include/strata/kernels/shared_expert.hpp b/include/strata/kernels/shared_expert.hpp index 5a898b5f84b..c5fc28a53f0 100644 --- a/include/strata/kernels/shared_expert.hpp +++ b/include/strata/kernels/shared_expert.hpp @@ -14,6 +14,8 @@ // contract. Optional native projection weights select the pinned CUDA Q8_1 MMVQ contract independently. #pragma once +#include "strata/kernels/bf16_gemv.hpp" + #include #include "strata/kernels/s_gemv.hpp" @@ -24,6 +26,8 @@ namespace strata::kernels { /// Configure before session capture; captured graphs retain their selected kernels. /// In this mode shared_expert requires its optional unrounded x_f32 input. void shared_expert_set_native_bf16(bool enabled); +/// Whether the native (FP32-activation) scalar gate is on: then the BF16 image of x is never read. +bool shared_expert_native_bf16(); bool shared_expert_native_bf16_enabled(); /// Optional native GGUF projections. Each supported type with nonnull data @@ -66,21 +70,23 @@ void shared_expert(const uint8_t* x_q8_0, const uint8_t* x_q8k, const uint16_t* const uint8_t* gate_codes, const float* gate_scales, const float* gate_off, const SForm& up_form, const uint8_t* up_codes, const float* up_scales, const float* up_off, const SForm& down_form, const uint8_t* down_codes, const float* down_scales, - const float* down_off, const uint16_t* gate_inp_bf16, float* scratch, float* out, + const float* down_off, const void* gate_inp_bf16, float* scratch, float* out, int64_t n_embd, int64_t n_ff, int tpr, void* stream, const float* x_f32 = nullptr, - const NativeSharedWeights* native = nullptr); + const NativeSharedWeights* native = nullptr, WForm gate_inp_form = WForm::Bf16); /// Plan v0.3 P6: the shared expert for `n_tok` <= 8 tokens (a verify window) with all three projections native: /// multi-column MMVQ, so the weights are read once; every token is bitwise `shared_expert` on that token. /// `x` (n_tok, n_embd) f32, `x_bf16` the same rounded (only read when the native BF16 gate is off), `gate`/`up` /// (n_tok, n_ff) scratch, `g` n_tok floats, `out` (n_tok, n_embd). `nw.q8_1` must hold n_tok columns of n_embd. +/// `gate_inp_form`: the scalar gate's stored form (STRATA_FP16=load: F32 or FP16; those need the native gate). /// x_q8_1_ready: the q8_1 image of x (n_tok rows), already quantized by the caller with the same quantizer (the /// same bytes native_quantize_q8_1 writes): its gate/up read it and x is not quantized again, and the SwiGLU and /// its q8_1 run as one launch (same values). Null: the separate launches. void shared_expert_multi(int n_tok, const float* x, const uint16_t* x_bf16, const NativeSharedWeights& nw, - const uint16_t* gate_inp_bf16, float* gate, float* up, float* g, float* out, int64_t n_embd, + const void* gate_inp_bf16, float* gate, float* up, float* g, float* out, int64_t n_embd, int64_t n_ff, void* stream, const void* x_q8_1_ready = nullptr, - int lfuse = 0); // S26 STRATA_LFUSE: bit 0 gate computed + applied by the caller, bit 1 gate/up pair + int lfuse = 0, // S26 STRATA_LFUSE: bit 0 gate computed + applied by the caller, bit 1 gate/up pair + WForm gate_inp_form = WForm::Bf16); /// The MoE block's final combination, `ref/moe.py::moe` L156: /// diff --git a/include/strata/kernels/verify_kernels.hpp b/include/strata/kernels/verify_kernels.hpp index 5b37cdf49b7..53a005372f4 100644 --- a/include/strata/kernels/verify_kernels.hpp +++ b/include/strata/kernels/verify_kernels.hpp @@ -14,6 +14,8 @@ // Everything that varies per window (token ids, `n_keep`) is read from DEVICE memory so the kernels can be captured. #pragma once +#include "strata/kernels/bf16_gemv.hpp" + #include namespace strata::kernels { @@ -28,6 +30,7 @@ void gdn_conv_l2_multi(const float* history, const float* qkv, const float* conv /// history <- the last 3 entries of [history | qkv_0 .. qkv_{n-1}], n = *n_keep (0 leaves it as it was). void gdn_conv_commit(float* history, const float* qkv, int channels, const int32_t* n_keep, void* stream); /// alpha/beta for T columns of x (T, n_embd): gate (T, h_v), beta (T, h_v). Bitwise `fused_gdn_ab` per column. +<<<<<<< this fork /// gdn_conv_l2_multi and gdn_ab_multi in one launch (143-amdk) void gdn_conv_ab_multi(const float* history, const float* qkv, const float* conv_w, float* h, int channels, int qk_heads, float eps, int n_tok, int t_begin, bool commit, const float* x, const uint16_t* w_alpha, @@ -35,6 +38,11 @@ void gdn_conv_ab_multi(const float* history, const float* qkv, const float* conv int h_v, void* stream); void gdn_ab_multi(const float* x, const uint16_t* w_alpha, const uint16_t* w_beta, const float* dt, const float* ssm_a, float* gate, float* beta, int n_embd, int h_v, int n_tok, void* stream); +======= +/// `form`: the weights' stored form (STRATA_FP16=load: F32 gates, or FP16); Bf16 is the original kernel's bits. +void gdn_ab_multi(const float* x, const void* w_alpha, const void* w_beta, const float* dt, const float* ssm_a, + float* gate, float* beta, int n_embd, int h_v, int n_tok, void* stream, WForm form = WForm::Bf16); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) /// The recurrence + output norm for T tokens (h = (T, conv_channels) as q|k|v, gate/beta (T, h_v), z/y /// (T, value_dim)). With `n_keep == nullptr` the state is read and NOT written (verify); otherwise the first /// *n_keep tokens are run and the state is written (commit; `y` may be scratch). Bitwise `fused_gdn_step_norm`. @@ -103,6 +111,27 @@ void copy_indexed(float* dst, const float* src, int64_t stride, const int32_t* i void fetch_blobs(const unsigned long long* src, const int32_t* n, uint8_t* dst, int64_t blob_bytes, int cap, void* stream); /// ptr[k] = base + k * blob_bytes for k < *n (the staged copies `fetch_blobs` made). void rebase_ptrs(unsigned long long* ptr, const int32_t* n, uint8_t* base, int64_t blob_bytes, void* stream); +/// fetch_blobs for the blobs q = q0, q0 + qstep, ... below *n only (the rest come from a partner GPU). +void fetch_blobs_strided(const unsigned long long* src, const int32_t* n, uint8_t* dst, int64_t blob_bytes, int q0, + int qstep, void* stream); +/// Spin until *flag (mapped host memory) is non-zero, then clear it: the partner GPU's share of a layer has landed. +void wait_and_clear(uint32_t* flag, void* stream); +/// The partner GPU's share of a verify window's PCIe experts (a layer split's idle NVLink peer). Step k (ring k+1): +/// wait for the plan (flagA >= k+1, mapped), copy the plan's odd PCIe blobs (q = 1, 3, ...) from host memory into the +/// active GPU's staging (peer writes over NVLink), then the last block sets done[k] (mapped). flagA = UINT32_MAX +/// (release_gpu_waits) ends it. +struct PartnerFetchArgs { + const uint32_t* flagA; // mapped, portable + const int32_t* plan; // the mapped plan, group g at plan + g * plan_i32 + long long plan_i32, ptr_off, capx; + int G, steps, lb; + const long long* blob_bytes; // per layer, on the partner GPU + uint8_t* staging; // the active GPU's staging (peer) + long long per, max_blob; // staging blobs per group, slot size + uint32_t* done; // mapped, one per step + unsigned* arrive; // on the partner GPU, one per step, zeroed per window +}; +void partner_fetch(const PartnerFetchArgs& a, int blocks, void* stream); // ---- the MTP draft layer (src/core/mtp.cpp) /// R[t][c][:] = h[t][c][:] + e[t][:] (the embedding branch added to every stream). diff --git a/include/strata/platform/cpu_relax.hpp b/include/strata/platform/cpu_relax.hpp new file mode 100644 index 00000000000..6882227298d --- /dev/null +++ b/include/strata/platform/cpu_relax.hpp @@ -0,0 +1,26 @@ +// include/strata/platform/cpu_relax.hpp - the spin-wait hint and the store fence, per architecture. +// +// x86: `_mm_pause` and `_mm_sfence` (the fence orders the write-combining stores into pinned host memory before +// the doorbell the GPU polls). aarch64: `yield` and `dmb oshst` - the outer-shareable store barrier is what +// makes host stores visible to a coherent device (GB10 / Grace NVLink-C2C) before a later store. POWER: `or 27,27,27` +// (glibc's __ppc_yield) lets the other SMT threads of the core issue while this one spins; the fence stays the +// full `sync` of the generic branch, which orders the stores for a coherent NVLink 2.0 device too (AC922). +#pragma once + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) +#include +#define STRATA_X86 1 +inline void strata_cpu_pause() { _mm_pause(); } +inline void strata_store_fence() { _mm_sfence(); } +#elif defined(__aarch64__) +inline void strata_cpu_pause() { __asm__ __volatile__("yield" ::: "memory"); } +inline void strata_store_fence() { __asm__ __volatile__("dmb oshst" ::: "memory"); } +#elif defined(__powerpc64__) +#include +inline void strata_cpu_pause() { __asm__ __volatile__("or 27,27,27" ::: "memory"); } +inline void strata_store_fence() { std::atomic_thread_fence(std::memory_order_seq_cst); } +#else +#include +inline void strata_cpu_pause() {} +inline void strata_store_fence() { std::atomic_thread_fence(std::memory_order_seq_cst); } +#endif diff --git a/include/strata/platform/thread_affinity.hpp b/include/strata/platform/thread_affinity.hpp new file mode 100644 index 00000000000..a5d407bad66 --- /dev/null +++ b/include/strata/platform/thread_affinity.hpp @@ -0,0 +1,69 @@ +// include/strata/platform/thread_affinity.hpp - helper threads leave the host thread's core. +// +// **A LINUX THREAD STARTS WITH ITS CREATOR'S AFFINITY** (Windows gives it the process's). The session pins the host +// thread to one core (pin_current_thread, pool.hpp), and every thread the host creates after that - the prompt +// path's stager and issuer, the router lookahead, the PLE and direct-file readers - would share that core with it +// and with each other. On an IBM AC922 (160 logical processors) the prompt path's issuer and the host then handed +// every expert to each other through `yield` on CPU 0, and a 10K prompt left the GPU idle 91% of the time. +// +// pin_current_thread records the affinity from before the first pin and the host's core here; a helper thread calls +// release_inherited_pin() first, and when it carries exactly the host's pin it gets that earlier affinity back. +// Anywhere else it is a no-op. Header-only, so the platform and reader libraries need not link the CPU pool. +#pragma once + +#if defined(_WIN32) +namespace strata::platform { +inline void release_inherited_pin() {} +} // namespace strata::platform +#else +#include +#include + +#include +#include + +namespace strata::platform { + +struct HostPin { + std::mutex mu; + bool valid = false; + cpu_set_t before; // the whole set: a 64-bit mask loses CPUs 64 and up (a POWER9, a big EPYC) + std::atomic core{-1}; +}; +inline HostPin& host_pin() { + static HostPin p; + return p; +} + +/// The host thread was pinned to `core`; `before` is its affinity from before (the first pin's is kept). +inline void note_host_pin(const cpu_set_t& before, int core) { + HostPin& p = host_pin(); + { + std::lock_guard lk(p.mu); + if (!p.valid) { + p.before = before; + p.valid = true; + } + } + p.core.store(core, std::memory_order_release); +} + +inline void release_inherited_pin() { + HostPin& p = host_pin(); + const int core = p.core.load(std::memory_order_acquire); + if (core < 0) return; + cpu_set_t cur; + CPU_ZERO(&cur); + if (pthread_getaffinity_np(pthread_self(), sizeof cur, &cur) != 0) return; + if (CPU_COUNT(&cur) != 1 || !CPU_ISSET(core, &cur)) return; // not the host's pin: leave it + cpu_set_t to; + { + std::lock_guard lk(p.mu); + if (!p.valid) return; + to = p.before; + } + pthread_setaffinity_np(pthread_self(), sizeof to, &to); +} + +} // namespace strata::platform +#endif diff --git a/include/strata/prefill/gemm.hpp b/include/strata/prefill/gemm.hpp index 0740fd0d201..f2dc9679ca7 100644 --- a/include/strata/prefill/gemm.hpp +++ b/include/strata/prefill/gemm.hpp @@ -36,6 +36,10 @@ class Gemm { /// shape (so the caller falls back to bf16(), with the same ldx). bool bf16_hcd_exact(const uint16_t* X, int64_t ldx, const uint16_t* W, float* Y, int64_t T, int64_t N, int64_t K); + /// Y = X . W^T with both in FP32 (STRATA_FP16=load: the F32 gates). + void f32(const float* X, const float* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy = 0, + float beta = 0.0f); + /// Y = X . W^T with both in FP16 (bits). void f16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy = 0, float beta = 0.0f); @@ -67,6 +71,8 @@ class Gemm { void* workspace_ = nullptr; bool external_ = false; void* hipblaslt_state_ = nullptr; + uint16_t* w16_ = nullptr; ///< STRATA_PREFILL_F16: the BF16 weight converted to FP16 + int64_t w16_elems_ = 0; bool f16_io_ = false; /// cuBLAS 12 answers the default FP16 algorithm of a few (N, K) shapes with CUBLAS_STATUS_INTERNAL_ERROR once /// another CUDA device in the process has initialized its context (issue #1650, Turing + Pascal). A fixed diff --git a/include/strata/prefill/kernels.hpp b/include/strata/prefill/kernels.hpp index f12f34b94af..84bbb9a403e 100644 --- a/include/strata/prefill/kernels.hpp +++ b/include/strata/prefill/kernels.hpp @@ -12,6 +12,37 @@ namespace strata::prefill { +/// The 16-bit "BF16 images" below, by STRATA_FP16 (strata/core/fp16_mode.hpp): 0 = BF16 (off, the default); +/// 1 = the BF16-rounded activation in FP16 bits (prefill: the BF16 weights are converted to FP16 per GEMM, so the +/// products are BF16's and only cuBLAS's summation order differs); 2 = the activation rounded straight to FP16 (load: +/// the weights are FP16 from the GGUF). Both FP16 forms saturate at +-65504. +int prefill_f16_mode(); +/// While alive (this thread): BF16 images whatever STRATA_FP16 says - for the weights that stay BF16 under +/// STRATA_FP16=load (the MTP drafter's, a BF16 PLE key). +class Bf16Images { +public: + Bf16Images(); + ~Bf16Images(); + Bf16Images(const Bf16Images&) = delete; + Bf16Images& operator=(const Bf16Images&) = delete; +private: + int prev_; +}; +/// While alive (this thread): FP16 images (rounded straight from FP32), for the weights STRATA_FP16_MTP=1 holds in FP16. +class F16Images { +public: + F16Images(); + ~F16Images(); + F16Images(const F16Images&) = delete; + F16Images& operator=(const F16Images&) = delete; +private: + int prev_; +}; +/// STRATA_FP16=load, F32 inject weights: inj[t*4 + o] = sum_j xn[t, j] * w[o*10240 + j] in FP32, with xn recomputed +/// from R, the row scales rs and the norm weights exactly as gr_mix_r does (no 16-bit image). +void gr_inject_f32(const float* R, const float* rs, const float* w_norm, const float* w_inject, float* inj, int64_t T, + void* stream); + // ---- hyper-connection (n_embd 2560, hc 4, hc_lr 320) /// xn[t, c*2560 + d] = R[t,c,d] * rsqrt(mean_d R[t,c,:]^2 + eps) * w_norm[c*2560 + d]; also its BF16 image. /// `xn16_lo` (null: none) takes bf16(x - xn16): W.xn16 + W.xn16_lo is the product with ~16 mantissa bits of x. diff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hpp index b4f100df04e..9cf714e2099 100644 --- a/include/strata/prefill/prefill.hpp +++ b/include/strata/prefill/prefill.hpp @@ -161,6 +161,10 @@ class Prefill { void set_stage(int64_t layer_begin, int64_t layer_end, Prefill* next) { stage_lb_ = layer_begin; stage_le_ = layer_end; next_ = next; } + /// Wait until this stage's downstream calls (and theirs) have finished; the first stage's `run` does it before it + /// returns. A later stage's `run` (one chunk) returns once its own chunk is handed on, so every stage of a chain of + /// 3+ works on its own chunk at once instead of each waiting for the whole chain below it. + bool drain(std::string& err); /// LAYER SPLIT: `helper` is another stage's prompt path (another GPU). A prompt of one chunk runs the stages one /// after the other, so while this stage reads it the helper's GPU idles: it then streams a share of this stage's From 485792489e6ee9ecb7d7bf001bf938e21c8730e6 Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:01 +0200 Subject: [PATCH 02/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): root from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 1 file(s) under root -- 0 clean, 1 with conflict markers. Conflict-marked files -- a pull request with conflicts: resolve them, do not discard the work: CMakeLists.txt Original authors: Andy,BlueKingMuch,Chanyeong Lim,Daehun Original commits touching these paths (152 subject(s)): - #136: fused int8 tensor-core experts for the Q2_0 pack's prompt path, grouped on the GPU (opt-in STRATA_PF_FUSED=1) - #136: the fused int8 prompt experts for the native IQ packs (gate/up IQ2_XXS IQ2_XS IQ2_S IQ3_XXS IQ3_S IQ4_XS, down Q2_0 IQ4_NL; opt-in STRATA_PF_FUSED=1) - #290 follow-up: a BF16-row parity test (native_expert_parity --bf16-embd) - #296 follow-up: only the Q5_0 MMQ instance and case stay (Q4_K / Q5_1 come with S5's STRATA_MMQ_KQUANTS) - #296 follow-up: ple_q5_parity / q5_projection_parity build only with STRATA_BUILD_TESTS; the doc says what runs where - A fatal CUDA fault in the prompt path exits at once instead of hanging until the watchdog (#224); CMake warns when sm_120 is built with CUDA < 13.0 (#220, #224); iq_pack writes native_experts.txt under a temporary name, so a stop part-way leaves no partial pack (#172) - AMD HIP backend for gfx1100 (PR #121, squashed onto its 0.1.21 base) - AMD HIP: RDNA2 gfx1030 (tested), gfx(1031, 1032) untested. Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: root Fork-Replay-Automated: true --- CMakeLists.txt | 97 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e687af69b9..70484a9a08c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,13 @@ endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +# aarch64 (and other non-x86) GCC/Clang fuse a * b + c into one FMA by default; an x86 build without -mfma cannot. +# The host code's roundings (activation quantization, ggml's nearest_int) then differ at exact ties, so a non-x86 +# build would disagree with x86 and with ggml's reference (quantize_act_parity's Q8_K case). Off, as on x86; +# explicit FMA intrinsics (NEON vfmaq) are unaffected. +if(NOT MSVC AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|i[3-6]86|x86)$") + add_compile_options($<$:-ffp-contract=off>) +endif() if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) endif() @@ -377,7 +384,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) src/kernels/cuda/kv_q8.cu src/kernels/cuda/kv_stream.cu src/kernels/cuda/kv_q4.cu - src/kernels/cuda/fused_gr.cu + src/kernels/cuda/fused_gr.cu src/kernels/cuda/fused_expert.cu src/kernels/cuda/cvec.cu src/kernels/cuda/qsa_decode_attn.cu src/kernels/cuda/qsa_prompt_attn.cu @@ -483,9 +490,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) # about arithmetic, and this one knows the ORDER the arithmetic happens in. add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp src/core/remote_expert_opt.cu - src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp - src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/peer_experts.cpp - src/core/arch_defaults.cpp src/core/vmm.cpp src/core/conversation_file.cpp) + src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/load_converted.cpp src/core/verify.cpp src/core/mtp.cpp + src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/conversation_file.cpp src/core/peer_experts.cpp + src/core/arch_defaults.cpp src/core/vmm.cpp) target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target}) if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA AND NOT STRATA_HIP_GFX906) @@ -1208,12 +1215,25 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS) add_test(NAME batch_groups_test COMMAND batch_groups_test) endif() -add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp - src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp) +# The AVX kernels exist for x86 only; on aarch64 (DGX Spark / GB10, Grace) src/kernels/cpu/portable.cpp stands in +# for them and the native experts run on ggml-cpu's NEON dot products. +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|i[3-6]86|x86)$") + set(STRATA_CPU_X86 ON) +else() + set(STRATA_CPU_X86 OFF) +endif() +if(STRATA_CPU_X86) + add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp + src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp) +else() + add_library(strata_kernels_cpu STATIC src/kernels/cpu/portable.cpp src/kernels/cpu/pool.cpp + src/kernels/cpu/expert_layout.cpp) +endif() target_include_directories(strata_kernels_cpu PUBLIC include) # Plan v0.3 P6: only the kernel files get the AVX-512 / AVX2 flags; the pool and the probe run on any x86-64, # and the engine picks the kernels at run time (cpu_avx512_ok). -if(MSVC) +if(NOT STRATA_CPU_X86) +elseif(MSVC) set_source_files_properties(src/kernels/cpu/expert.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX512") set_source_files_properties(src/kernels/cpu/q2_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2") else() @@ -1278,6 +1298,10 @@ if(STRATA_NATIVE_EXPERTS) foreach(_f GGML_AVX GGML_AVX2 GGML_FMA GGML_F16C GGML_BMI2) set(${_f} ON CACHE BOOL "" FORCE) endforeach() + elseif(NOT STRATA_CPU_X86 AND GGML_CPU_ARM_ARCH) + # ARM with an explicit -march (gcc's -mcpu=native finds no features on big.LITTLE parts like GB10's + # Cortex-X925/A725, which leaves ggml-cpu without dotprod/i8mm): e.g. -DGGML_CPU_ARM_ARCH=armv8.6-a+dotprod+i8mm+fp16 + set(GGML_NATIVE OFF CACHE BOOL "" FORCE) elseif(STRATA_ISA_FLOOR) # the experimental older-CPU build (STRATA_ISA_FLOOR above): one ggml-cpu for the floor, ggml's "sandybridge" # feature set (SSE4.2 + AVX) or SSE4.2 alone - not the build host's, which may be newer than the PC it runs on. @@ -1297,6 +1321,7 @@ if(STRATA_NATIVE_EXPERTS) else() set(GGML_NATIVE ON CACHE BOOL "" FORCE) endif() +<<<<<<< this fork add_subdirectory(${STRATA_GGML_DIR}/ggml ${CMAKE_BINARY_DIR}/ggml EXCLUDE_FROM_ALL) # part of strata_kernels_cpu (the pool calls these and they call its CPU probe: one library, so GNU ld's # one-pass static linking has no cycle to resolve) @@ -1308,6 +1333,44 @@ if(STRATA_NATIVE_EXPERTS) if(MSVC) set_source_files_properties(src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/nvfp4_avx512.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX512") +======= + if(STRATA_GGML_DIR) + add_subdirectory(${STRATA_GGML_DIR}/ggml ${CMAKE_BINARY_DIR}/ggml EXCLUDE_FROM_ALL) + else() + include(FetchContent) + FetchContent_Declare(strata_llamacpp + GIT_REPOSITORY https://github.com/ggml-org/llama.cpp.git + GIT_TAG 3cf03257f219afbe7334045ff7c6a06ac68c627d + GIT_SHALLOW FALSE) + FetchContent_GetProperties(strata_llamacpp) + if(NOT strata_llamacpp_POPULATED) + FetchContent_Populate(strata_llamacpp) + endif() + add_subdirectory(${strata_llamacpp_SOURCE_DIR}/ggml ${CMAKE_BINARY_DIR}/ggml EXCLUDE_FROM_ALL) + endif() + # POWER: ggml-cpu's VSX code spells the AltiVec types `vector float`, a GNU keyword that the strict -std=c++17 this + # project's CMAKE_CXX_EXTENSIONS OFF gives the subdirectory turns off; upstream llama.cpp builds it as gnu++17 + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc|powerpc)") + set_target_properties(ggml-cpu PROPERTIES CXX_EXTENSIONS ON) + endif() + # part of strata_kernels_cpu (the pool calls these and they call its CPU probe: one library, so GNU ld's + # one-pass static linking has no cycle to resolve) + target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/native_expert.cpp) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64|powerpc64)") + # POWER9 VSX expert kernels (STRATA_VSX_EXPERTS=1); the half->float vector conversion needs ISA 3.0 + target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/kq_vsx.cpp) + set_source_files_properties(src/kernels/cpu/kq_vsx.cpp PROPERTIES COMPILE_OPTIONS "-mcpu=power9") + endif() + if(STRATA_CPU_X86) + target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/iq_avx2.cpp + src/kernels/cpu/kq_avx2.cpp + src/kernels/cpu/kq_avx1.cpp) + endif() + target_include_directories(strata_kernels_cpu PRIVATE third_party/ggml) + if(NOT STRATA_CPU_X86) + elseif(MSVC) + set_source_files_properties(src/kernels/cpu/iq_avx512.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX512") +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2") set_source_files_properties(src/kernels/cpu/kq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2") set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2") @@ -1362,6 +1425,7 @@ if(STRATA_NATIVE_EXPERTS) endif() # the AVX-2 expert kernels' variants (iq_avx2.cpp, q2_avx2.cpp) bit for bit against each other at 1..8 tokens and # against ggml-cpu's dot, on random blocks; --bench: ms per expert on one thread, the weights streamed from DRAM +<<<<<<< this fork add_executable(iq_avx2_parity src/kernels/cpu/iq_avx2_parity.cpp) target_link_libraries(iq_avx2_parity PRIVATE strata_kernels_cpu) # MODEL-UPDATE: the pool's expert rows per format on T threads against the RAM bandwidth (a benchmark, no test) @@ -1370,6 +1434,15 @@ if(STRATA_NATIVE_EXPERTS) if(STRATA_BUILD_TESTS) add_test(NAME iq_avx2_parity COMMAND iq_avx2_parity) add_test(NAME iq_avx2_parity_iq3s_mt1 COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ3S_MT1=1 $) +======= + if(STRATA_CPU_X86) # x86 only (cpuid.h, the AVX-2 kernels); non-x86 builds have portable.cpp instead + add_executable(iq_avx2_parity src/kernels/cpu/iq_avx2_parity.cpp) + target_link_libraries(iq_avx2_parity PRIVATE strata_kernels_cpu) + if(STRATA_BUILD_TESTS) + add_test(NAME iq_avx2_parity COMMAND iq_avx2_parity) + add_test(NAME iq_avx2_parity_iq3s_mt1 COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ3S_MT1=1 $) + endif() +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) endif() # the older CPUs' AVX1 expert rows (iq_avx1.cpp) against ggml-cpu's dots and against their own one-token # rows at every width; --bench: ms per expert, the weights streamed from DRAM. Runs on any AVX CPU: @@ -1667,10 +1740,12 @@ if(STRATA_BUILD_TESTS) target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact ${_strata_gpu_runtime_target}) endif() - add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp) - target_link_libraries(expert_multi_test PRIVATE strata_kernels_cpu) - add_test(NAME expert_multi_test COMMAND expert_multi_test) - set_tests_properties(expert_multi_test PROPERTIES SKIP_RETURN_CODE 77) # no AVX-512: skipped + if(STRATA_CPU_X86) # the Q2_0 expert kernel it checks is AVX-512 (portable.cpp refuses it elsewhere) + add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp) + target_link_libraries(expert_multi_test PRIVATE strata_kernels_cpu) + add_test(NAME expert_multi_test COMMAND expert_multi_test) + set_tests_properties(expert_multi_test PROPERTIES SKIP_RETURN_CODE 77) # no AVX-512: skipped + endif() # PR #706: the opt-in AVX-2 Q2_0 bit-plane kernel (STRATA_Q2_BITPLANE=1) against the legacy one, bitwise alone/window if(STRATA_BUILD_TESTS AND CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64|x86_64|x64|X86") From 9075a5d9486c48609b7fb6dc02c32129abadd455 Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:01 +0200 Subject: [PATCH 03/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): src/core from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 9 file(s) under src/core -- 4 clean, 5 with conflict markers. Conflict-marked files -- a pull request with conflicts: resolve them, do not discard the work: src/core/expert_source.cpp src/core/layer.cpp src/core/mtp.cpp src/core/pinned.cu src/core/verify.cpp Original authors: 4EverBuilder,Abhinand B,Andy,Anon Original commits touching these paths (158 subject(s)): - "The draft head does not fit" says what to change (#474) - #253: the 8 GiB arena pin cap only under WDDM; STRATA_ARENA_PIN_GIB; the split's prompt chunk steps down - #280 follow-up: the float64 RoPE table is opt-in (STRATA_ROPE_TABLE=1); a freed session's table is forgotten - #284 follow-up: wait for the async commit on an event, at every point that touches the session afterwards - #293 follow-up: the drafter stores rotated INT8 K/V as INT8; the rotation is part of the snapshot format - #465 engine: batch slots for one GPU - warn instead of refuse, windows over the active slots, slot conversation cache, prompts read beside decoding slots, BYIELD - #646 on 0.1.39b: the layer split and the CPU path as before, the resident plan for 8-token windows - A 6 GB card starts with a small expert cache, or says what is short (#496) Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: src/core Fork-Replay-Automated: true --- src/core/expert_cache.cpp | 1 + src/core/expert_source.cpp | 188 +++++++++++++++++++++++++++-- src/core/graph.cpp | 4 +- src/core/layer.cpp | 72 ++++++++---- src/core/load_converted.cpp | 182 +++++++++++++++++++++++++++++ src/core/mtp.cpp | 112 ++++++++++++++++-- src/core/pinned.cu | 133 ++++++++++++++++++++- src/core/session.cpp | 10 +- src/core/verify.cpp | 227 ++++++++++++++++++++++++++++-------- 9 files changed, 833 insertions(+), 96 deletions(-) create mode 100644 src/core/load_converted.cpp diff --git a/src/core/expert_cache.cpp b/src/core/expert_cache.cpp index 1920fdb7f9f..d6b7ddcfccb 100644 --- a/src/core/expert_cache.cpp +++ b/src/core/expert_cache.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp index 4878c111310..af555171900 100644 --- a/src/core/expert_source.cpp +++ b/src/core/expert_source.cpp @@ -1,5 +1,6 @@ // src/core/expert_source.cpp - the adapter. See the header for the three clauses of the contract. #include "strata/core/expert_source.hpp" +#include "strata/platform/thread_affinity.hpp" #include "strata/core/foresight_swap.hpp" #include "strata/core/on_device.hpp" #include "strata/platform/aux_cpus.hpp" @@ -45,9 +46,14 @@ #include #else #include +<<<<<<< this fork #if defined(__APPLE__) #include // host_statistics64: the available memory #endif +======= +#include +#include +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) #include #include #include @@ -987,8 +993,52 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64 return true; } +#if !defined(_WIN32) +namespace { +// The CPUs of NUMA node `node` (its sysfs cpulist), for a thread that must run there; false if none. +bool node_cpus(int node, cpu_set_t& set) { + CPU_ZERO(&set); + if (node < 0) return false; + char path[96]; + std::snprintf(path, sizeof path, "/sys/devices/system/node/node%d/cpulist", node); + std::FILE* f = std::fopen(path, "r"); + if (f == nullptr) return false; + int lo = 0, hi = 0; + char sep = 0; + while (std::fscanf(f, "%d", &lo) == 1) { + hi = lo; + if (std::fscanf(f, "%c", &sep) == 1 && sep == '-' && std::fscanf(f, "%d", &hi) == 1) (void) std::fscanf(f, "%c", &sep); + for (int c = lo; c <= hi && c < CPU_SETSIZE; ++c) CPU_SET(c, &set); + } + std::fclose(f); + return CPU_COUNT(&set) > 0; +} +} // namespace +#endif + +const uint8_t* FileExpertSource::complement_at(uint64_t offset) const { + for (const Segment& s : complement_segs_) + if (offset >= s.start && offset < s.end) return (const uint8_t*) s.arena + (size_t) (offset - s.start); + return complement_host_ == nullptr ? nullptr : complement_host_ + (size_t) offset; +} + +const uint8_t* FileExpertSource::complement_dev_at(uint64_t offset) const { + for (const Segment& s : complement_segs_) + if (offset >= s.start && offset < s.end) return s.device + (size_t) (offset - s.start); + return complement_device_ == nullptr ? nullptr : complement_device_ + (size_t) offset; +} + +const uint8_t* FileExpertSource::complement_blob(size_t index) const { + if (complement_host_ == nullptr || index >= complement_offsets_.size() || complement_offsets_[index] == kNoComplement) + return nullptr; + return complement_at(complement_offsets_[index]); +} + void FileExpertSource::close() { io_stop(); + for (const Segment& s : complement_segs_) (void) cudaFreeHost(s.arena); // set_arena_placement's, one per node + if (!complement_segs_.empty()) complement_arena_ = nullptr; + complement_segs_.clear(); inputs_.clear(); if (complement_arena_ != nullptr) { if (complement_pinned_ && !complement_registered_) (void) cudaFreeHost(complement_arena_); @@ -2229,10 +2279,14 @@ bool RouterLookahead::start(std::vector> routers, int64_t k_ = k < 1 ? 1 : k > (int) n_expert ? (int) n_expert : k; src_ = src; x_.assign((size_t) (8 * n_embd), 0.f); +<<<<<<< this fork thread_ = std::thread([this] { strata::aux_cpus::pin_current_thread(); run(); }); +======= + thread_ = std::thread([this] { strata::platform::release_inherited_pin(); run(); }); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) return true; } @@ -2376,13 +2430,22 @@ void RouterLookahead::run() { // one (kq_avx1.cpp) or, without AVX, the same sums in plain C++. From the Strata_Dirigo fork (rwkeyes): an // AVX-only Xeon E5-2687W died here (vpmovzxwd) on its first request. An estimate only (which experts to // prefetch); the order of the additions differs between the three, the output does not depend on it. +#if !(defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)) + // other CPUs: bf16_rows_dot_multi is portable.cpp's (VSX on POWER, NEON / plain C++ elsewhere) + if (true) { + strata::kernels::cpu::bf16_rows_dot_multi(routers_[(size_t) layer].data(), (int) n_expert_, (int) n_embd_, + x_.data(), (int) nt, logits.data()); + } else +#else if (strata::kernels::cpu::cpu_avx2_ok()) { strata::kernels::cpu::bf16_rows_dot_multi(routers_[(size_t) layer].data(), (int) n_expert_, (int) n_embd_, x_.data(), (int) nt, logits.data()); } else if (strata::kernels::cpu::cpu_avx1_ok()) { strata::kernels::cpu::bf16_rows_dot_multi_avx1(routers_[(size_t) layer].data(), (int) n_expert_, (int) n_embd_, x_.data(), (int) nt, logits.data()); - } else { + } else +#endif + { // a mul and an add, not std::fma: without an FMA instruction that is a libm call per element (74x // slower on a Xeon E5-2665, measured by the fork) const uint16_t* rw = routers_[(size_t) layer].data(); @@ -2667,6 +2730,11 @@ bool FileExpertSource::pin_cache_complement( if (slot >= 0 && slot < n_slots) slot_bytes[(size_t) slot] = layer_blob_bytes_[(size_t) layer]; } } + if (arena_whole_) { // set_arena_placement: every expert, the caches' too (see there) + primary_gpu_pairs.clear(); + pair_slot.clear(); + lend_from_slot = -1; + } std::vector offsets; uint64_t bytes = 0; if (!detail::make_cache_complement_plan(n_layers_, n_expert_, layer_blob_bytes_, primary_gpu_pairs, @@ -2731,6 +2799,15 @@ bool FileExpertSource::pin_cache_complement( at += b; ++held; } + if (arena_whole_) // a layer split's rank is CUDA0's layers only: the others after them, in layer order + for (size_t i = 0; i < offsets.size(); ++i) { + if (offsets[i] == kNoComplement || ranked[i] != kNoComplement) continue; + const uint64_t b = layer_blob_bytes_[i / (size_t) n_expert_]; + if (b > budget_bytes - at) continue; + ranked[i] = at; + at += b; + ++held; + } if (what_fits && held == 0) { // #467: nothing to keep - the caller's plain mmap fallback, not an empty copy err = "FileExpertSource: available memory cannot hold any expert of the complement"; return false; @@ -2791,6 +2868,7 @@ bool FileExpertSource::pin_cache_complement( } } +<<<<<<< this fork // A layer split: generate hands the copy every stage's lend region (`stage_lend_regions`, set before the // call). The prompt path borrows the tail slots of EVERY part's cache, and with `additional_gpu_pairs` // non-empty the single lend region above is off - so without this the borrowed experts sat in neither @@ -2820,6 +2898,28 @@ bool FileExpertSource::pin_cache_complement( if (stage_lent < 0) return false; stage_lent_bytes = bytes - before; } +======= + // set_arena_placement: the bytes grouped by the node of each layer's GPU (their order kept within a node), and + // where each node's range ends + std::vector> node_ranges; + if ((int64_t) arena_layer_node_.size() == n_layers_ && bytes > 0) { + std::vector idx; + for (size_t i = 0; i < offsets.size(); ++i) + if (offsets[i] != kNoComplement) idx.push_back(i); + auto node_of = [&](size_t i) { return arena_layer_node_[i / (size_t) n_expert_]; }; + std::stable_sort(idx.begin(), idx.end(), [&](size_t a, size_t b) { + return node_of(a) != node_of(b) ? node_of(a) < node_of(b) : offsets[a] < offsets[b]; + }); + uint64_t at = 0; + for (const size_t i : idx) { + if (node_ranges.empty() || node_ranges.back().first != node_of(i)) node_ranges.emplace_back(node_of(i), at); + offsets[i] = at; + at += layer_blob_bytes_[i / (size_t) n_expert_]; + node_ranges.back().second = at; + } + } + std::vector segs; +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) void* arena = nullptr; const uint8_t* host = nullptr; @@ -2831,6 +2931,9 @@ bool FileExpertSource::pin_cache_complement( uint64_t lock_off = 0; ///< where the working-set lock starts (after the registered prefix) std::string note; auto release = [&]() { + for (const Segment& s : segs) (void) cudaFreeHost(s.arena); + if (!segs.empty()) arena = nullptr; + segs.clear(); if (arena == nullptr) return; if (pinned_ok) (void) cudaFreeHost(arena); else { @@ -2893,7 +2996,51 @@ bool FileExpertSource::pin_cache_complement( } } #endif - if (pin && !paced) { +#if !defined(_WIN32) + if (pin && !paced && !node_ranges.empty()) { + // one page-locked, mapped allocation per node, made on that node's CPUs (see set_arena_placement) + int dev = 0; + cudaGetDevice(&dev); + uint64_t start = 0; + bool ok = true; + for (const auto& [node, end] : node_ranges) { + void* p = nullptr; + std::thread([&, node = node, end = end] { + cpu_set_t set; + if (node_cpus(node, set)) pthread_setaffinity_np(pthread_self(), sizeof set, &set); + cudaSetDevice(dev); + if (cudaHostAlloc(&p, (size_t) (end - start), cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) { + (void) cudaGetLastError(); + p = nullptr; + } + }).join(); + void* alias = nullptr; + if (p == nullptr || cudaHostGetDevicePointer(&alias, p, 0) != cudaSuccess || alias == nullptr) { + (void) cudaGetLastError(); + if (p != nullptr) (void) cudaFreeHost(p); + ok = false; + break; + } + segs.push_back({p, start, end, (const uint8_t*) alias, node}); + std::fprintf(stderr, "FileExpertSource: %.2f GiB of the arena on NUMA node %d (the GPUs there)\n", + (double) (end - start) / 1073741824.0, node); + start = end; + } + if (ok) { + arena = segs.front().arena; + device = segs.front().device; + pinned_ok = true; + char msg[96]; + std::snprintf(msg, sizeof msg, "page-locked and mapped, one allocation per NUMA node (%zu)", segs.size()); + note = msg; + } else { + for (const Segment& s : segs) (void) cudaFreeHost(s.arena); + segs.clear(); + note = "a per-node allocation failed: one arena"; + } + } +#endif + if (pin && !paced && arena == nullptr) { const cudaError_t allocated = cudaHostAlloc(&arena, (size_t) bytes, cudaHostAllocMapped | cudaHostAllocPortable); if (allocated == cudaSuccess) { @@ -3084,6 +3231,14 @@ bool FileExpertSource::pin_cache_complement( for (;;) { const int64_t layer = next_layer.fetch_add(1); if (layer >= n_layers_ || failed.load()) return; +#if !defined(_WIN32) + // set_arena_placement: written by the CPUs of the node it lives on - on a POWER9 a GPU reads memory the + // other socket's CPUs wrote at 45 GB/s, its own socket's at 68, though the pages are on its node either way + if (!segs.empty() && (size_t) layer < arena_layer_node_.size()) { + cpu_set_t set; + if (node_cpus(arena_layer_node_[(size_t) layer], set)) pthread_setaffinity_np(pthread_self(), sizeof set, &set); + } +#endif const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer]; std::vector batch; // #286, unbuffered: 32 blobs' reads in flight at once, merged where near auto flush = [&]() { @@ -3111,6 +3266,8 @@ bool FileExpertSource::pin_cache_complement( return; } uint8_t* dst = (uint8_t*) host + (size_t) offset; + for (const Segment& s : segs) // a layer split's arena: the segment on the stage's socket + if (offset >= s.start && offset < s.end) dst = (uint8_t*) s.arena + (size_t) (offset - s.start); if (!direct_.empty()) { batch.push_back({0, layer, expert, dst}); if (batch.size() == 32 && !flush()) { @@ -3162,8 +3319,9 @@ bool FileExpertSource::pin_cache_complement( { const int threads = (int) std::max(1, std::min(6, n_layers_)); std::vector pool; - for (int i = 1; i < threads; ++i) pool.emplace_back(worker); - worker(); + // with per-node segments the workers move between nodes (above): the calling thread keeps its own pin + for (int i = segs.empty() ? 1 : 0; i < threads; ++i) pool.emplace_back(worker); + if (segs.empty()) worker(); for (auto& t : pool) t.join(); } std::fflush(stderr); @@ -3200,6 +3358,7 @@ bool FileExpertSource::pin_cache_complement( } complement_arena_ = arena; + complement_segs_ = std::move(segs); complement_host_ = host; complement_device_ = device; complement_bytes_ = bytes; @@ -3253,6 +3412,8 @@ bool FileExpertSource::pin_cache_complement( const uint8_t* FileExpertSource::resident_blob(size_t index) const { if (exchange_storage_.active()) return exchange_storage_.resident(index).host; + // ac922: set_arena_placement's arena is one segment per NUMA node, not one block at complement_host_ + if (!complement_segs_.empty()) return complement_blob(index); return detail::cache_complement_blob_or_fallback(index, complement_offsets_, complement_host_, nullptr); } @@ -3286,7 +3447,7 @@ bool FileExpertSource::reserve_exchanges(int64_t n, std::string& err) { const bool requested = rotate_env && std::strcmp(rotate_env, "1") == 0; const bool uniform = std::all_of(layer_blob_bytes_.begin(), layer_blob_bytes_.end(), [blob](uint64_t b) { return b == blob; }); - const bool eligible = requested && uniform && complement_pinned_ && !complement_partial_ && + const bool eligible = requested && uniform && complement_pinned_ && !complement_partial_ && complement_segs_.empty() && complement_host_ && complement_device_ && complement_bytes_ > 0; const size_t total = (size_t) n * (size_t) blob; void* p = nullptr; @@ -3388,7 +3549,7 @@ int64_t FileExpertSource::commit_exchanges() { const uint64_t at = complement_offsets_[x.in]; if (src != nullptr && at != kNoComplement && at <= complement_bytes_ && x.bytes <= complement_bytes_ - at && complement_host_ != nullptr) { - std::memcpy((uint8_t*) complement_host_ + (size_t) at, src, (size_t) x.bytes); + std::memcpy((uint8_t*) complement_at(at), src, (size_t) x.bytes); if (detail::exchange_cache_complement(complement_offsets_, x.in, x.out)) ++n; } override_[x.out] = nullptr; @@ -3405,9 +3566,10 @@ void FileExpertSource::commit_copies() { for (const Exchange& x : staged_) { const uint8_t* src = override_.empty() ? nullptr : override_[x.out]; const uint64_t at = complement_offsets_[x.in]; + // complement_at: the per-socket segments too (set_arena_placement), as commit_exchanges if (src != nullptr && at != kNoComplement && at <= complement_bytes_ && x.bytes <= complement_bytes_ - at && - complement_host_ != nullptr) - std::memcpy((uint8_t*) complement_host_ + (size_t) at, src, (size_t) x.bytes); + complement_at(at) != nullptr) + std::memcpy((uint8_t*) complement_at(at), src, (size_t) x.bytes); } } @@ -3516,10 +3678,14 @@ const uint8_t* FileExpertSource::device_alias(int64_t layer, int64_t expert) con if (!pinned(layer, expert) || complement_device_ == nullptr) return nullptr; const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert; if (exchange_storage_.active()) return exchange_storage_.resident(index).device; +<<<<<<< this fork if (!soft_in_.empty() && soft_in_[index] != SIZE_MAX) // soft_commit_exchanges: out is read from its exchange buffer return override_.empty() || override_[index] == nullptr || xstage_dev_ == nullptr ? nullptr : xstage_dev_ + (size_t) (override_[index] - xstage_); return complement_device_ + (size_t) complement_offsets_[index]; +======= + return complement_dev_at(complement_offsets_[index]); // ac922: the per-socket segments (set_arena_placement) +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) } bool FileExpertSource::pcie_layer(int64_t layer) const { @@ -3887,6 +4053,8 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32 for (int64_t i = 0; i < n; ++i) if (kind[i] == -1) { any_cpu = true; break; } // -2: skipped (STRATA_ROUTE_TAIL_SKIP), no work const auto c1 = std::chrono::steady_clock::now(); + // the quantized activations feed only the CPU's jobs: a layer whose entries all went to VRAM, PCIe or a remote + // stage skips them (on POWER, ggml's Q8_K quantizer is the scalar reference: ~65 us a layer, every layer) if (any_cpu) { // #578 --remote-expert-opt: a token whose experts all run on a GPU (CUDA0 or a helper) needs no CPU activation const bool ep = d.remote_count > 0 && d.remote[0]->optimized_decode(); @@ -4451,9 +4619,9 @@ LoadStats load_experts_gguf(const std::string& gguf, uint8_t* dst, const strata: } } }; + // every worker on its own thread, on the arena's node (run_on_arena_node, pinned.hpp) std::vector pool; - for (int i = 1; i < threads; ++i) pool.emplace_back(worker); - worker(); + for (int i = 0; i < threads; ++i) pool.emplace_back([&] { run_on_arena_node(dst); worker(); }); for (auto& t : pool) t.join(); if (bad) { st.seconds = -1.0; diff --git a/src/core/graph.cpp b/src/core/graph.cpp index 2e9c7e70e8d..da2c50329e6 100644 --- a/src/core/graph.cpp +++ b/src/core/graph.cpp @@ -1,7 +1,7 @@ // src/core/graph.cpp - P2.S5: the GraphRegistry implementation. #include "strata/core/graph.hpp" -#include +#include "strata/platform/cpu_relax.hpp" #include #include @@ -97,7 +97,7 @@ bool CapturedGraph::wait_ms(int timeout_ms) const { if (q == cudaSuccess) return true; if (q != cudaErrorNotReady) return false; // a real error, not "not finished" if (std::chrono::steady_clock::now() >= deadline) return false; - for (int i = 0; i < 64; ++i) _mm_pause(); + for (int i = 0; i < 64; ++i) strata_cpu_pause(); } } diff --git a/src/core/layer.cpp b/src/core/layer.cpp index 1e6b3b93038..a0083cd8e6a 100644 --- a/src/core/layer.cpp +++ b/src/core/layer.cpp @@ -1,5 +1,6 @@ // src/core/layer.cpp - the GDN layer, composed. See the header for the operation order and its traps. +#include "strata/core/weight_form.hpp" #include "strata/core/layer.hpp" #include "strata/core/native_dense.hpp" #include "strata/core/native_head.hpp" @@ -94,11 +95,15 @@ constexpr int TPR = 32; bool native_bf16_projections = false; bool native_flash_attn_short = false; -void project_bf16(const float* x, const uint16_t* x_bf16, const uint16_t* weights, float* out, - int64_t n_in, int64_t n_out, bool split, void* stream) { +void project_bf16(const float* x, const uint16_t* x_bf16, const void* weights_v, float* out, + int64_t n_in, int64_t n_out, bool split, void* stream, + strata::kernels::WForm form = strata::kernels::WForm::Bf16) { using namespace strata::kernels; - if (native_bf16_projections) bf16_gemv_fp32_mmvf(x, weights, out, n_in, n_out, stream); - else if (split) bf16_gemv_split(x_bf16, weights, out, n_in, n_out, TPR, stream); + const uint16_t* weights = (const uint16_t*) weights_v; + if (native_bf16_projections) { gemv_fp32_mmvf(x, weights_v, form, out, n_in, n_out, stream); return; } + if (form != WForm::Bf16) + throw std::invalid_argument("project_bf16: FP16/F32 weights (STRATA_FP16=load) need the native BF16 projections"); + if (split) bf16_gemv_split(x_bf16, weights, out, n_in, n_out, TPR, stream); else bf16_gemv(x_bf16, weights, out, n_in, n_out, stream); } ///< threads per row for the row-split GEMVs. @@ -287,12 +292,12 @@ st_end(layer, 11, stream); // 36 GDN layers. st_begin(layer, 12, stream); if (fused_pre) { - fused_gdn_ab(mixed, (const uint16_t*) w_alpha->data, (const uint16_t*) w_beta->data, dt, ssm_a, b.gate, b.beta, - (int) g.n_embd, (int) g.ssm_v_heads, stream); + fused_gdn_ab(mixed, w_alpha->data, w_beta->data, dt, ssm_a, b.gate, b.beta, + (int) g.n_embd, (int) g.ssm_v_heads, stream, wform(w_alpha, "ssm_alpha.weight")); } else { if (!native_bf16_projections) f32_to_bf16_bulk(mixed, b.x_bf16, g.n_embd, stream); - project_bf16(mixed, b.x_bf16, (const uint16_t*) w_alpha->data, b.alpha, g.n_embd, g.ssm_v_heads, true, stream); - project_bf16(mixed, b.x_bf16, (const uint16_t*) w_beta->data, b.beta, g.n_embd, g.ssm_v_heads, true, stream); + project_bf16(mixed, b.x_bf16, w_alpha->data, b.alpha, g.n_embd, g.ssm_v_heads, true, stream, wform(w_alpha, "ssm_alpha.weight")); + project_bf16(mixed, b.x_bf16, w_beta->data, b.beta, g.n_embd, g.ssm_v_heads, true, stream, wform(w_beta, "ssm_beta.weight")); } if (!fused_pre) try { if (native_gdn_enabled()) { @@ -367,7 +372,7 @@ if (!native_bf16_projections) f32_to_bf16_bulk(x, b.x_bf16, g.n_embd, stream); // memory-bound floor and the largest single item left inside the MoE after the top-10 was fixed. // `bf16_gemv_split` exists for precisely this case; its own header says so. Measured here: // LEDGER L41 -> L42. -project_bf16(x, b.x_bf16, (const uint16_t*) w_router->data, b.logits, g.n_embd, g.n_expert, true, stream); +project_bf16(x, b.x_bf16, w_router->data, b.logits, g.n_embd, g.n_expert, true, stream, wform(w_router, "ffn_gate_inp.weight")); // ---- routing: softmax over ALL experts, stable descending argsort with ties by index, gather, renormalise // the native fused router is canonical-512x10 only; anything else takes the generic top-k kernel if (native_router_enabled() && g.n_expert == 512 && k == 10) { @@ -425,7 +430,7 @@ SForm f_gate, f_up, f_down; if (!sform_of(*w_sgate, f_gate, v.name("ffn_gate_ // re-rounded it to the engine form like any other BF16 tensor. Reading it as f32 is 5120 bytes past the // end of a 5120-byte tensor inside a 4.5 GiB arena, where nothing faults. Refuse here rather than trust // the layout check to have run: this function is reachable without it. -if (w_ginp->kind != WeightKind::Bf16InF32) { err = v.name("ffn_gate_inp_shexp.weight") + " is engine form " + std::to_string((int) w_ginp->kind) + " (" + std::to_string(w_ginp->bytes) + " B); the scalar gate reads it as bf16, so it must be form 1 (" + std::to_string((uint64_t) g.n_embd * 2) + " B)"; return false; } +if (w_ginp->kind != WeightKind::Bf16InF32 && w_ginp->kind != WeightKind::F16InF32 && w_ginp->kind != WeightKind::F32) { err = v.name("ffn_gate_inp_shexp.weight") + " is engine form " + std::to_string((int) w_ginp->kind) + " (" + std::to_string(w_ginp->bytes) + " B); the scalar gate reads it as bf16, so it must be form 1 (" + std::to_string((uint64_t) g.n_embd * 2) + " B)"; return false; } // THE SHARED EXPERT'S WEIGHTS ARE NOT ONE FAMILY: `ffn_gate_shexp` is Q2_0 on 21 layers and a K-quant on // the rest, `ffn_up_shexp` Q2_0 on 13, and `ffn_down_shexp` is LEGACY in every layer (IQ4_NL/Q4_0/Q5_0/ // Q8_0/Q2_0, `n_in` 640 so Q8_K is impossible). So BOTH quantized images of the activation are produced @@ -445,8 +450,8 @@ f32_to_bf16_bulk(x, b.x_bf16, g.n_embd, stream); try { shared_expert(b.x_q8_0, b.x_q8k, b.x_bf16, f_gate, p_gate.codes, p_gate.scales, p_gate.offset, f_up, p_up.codes, p_up.scales, p_up.offset, f_down, p_down.codes, p_down.scales, - p_down.offset, (const uint16_t*) w_ginp->data, b.sh_scratch, b.shared, - g.n_embd, g.n_ff, /*tpr=*/32, stream, x, &native); + p_down.offset, w_ginp->data, b.sh_scratch, b.shared, + g.n_embd, g.n_ff, /*tpr=*/32, stream, x, &native, wform(w_ginp, "ffn_gate_inp_shexp.weight")); } catch (const std::exception& error) { err = v.name("shared_expert") + ": " + error.what(); return false; @@ -1040,7 +1045,7 @@ const auto normalize_rotate = [&](float* data, const WeightRef* norm, int rows, struct Req { const char* suf; }; const WeightRef* w_idxk = v.get("indexer.k_proj.weight"); const WeightRef* w_attnq = v.get("attn_q.weight"); const WeightRef* w_attnk = v.get("attn_k.weight"); const WeightRef* w_attnv = v.get("attn_v.weight"); const WeightRef* w_attno = v.get("attn_output.weight"); const WeightRef* w_idxq = v.get("indexer.q_proj.weight"); const WeightRef* w_qn = v.get("attn_q_norm.weight"); const WeightRef* w_kn = v.get("attn_k_norm.weight"); const WeightRef* w_iqn = v.get("indexer.q_norm.weight"); const WeightRef* w_ikn = v.get("indexer.k_norm.weight"); const char* missing = !w_idxk ? "indexer.k_proj.weight" : !w_attnq ? "attn_q.weight" : !w_attnk ? "attn_k.weight" : !w_attnv ? "attn_v.weight" : !w_attno ? "attn_output.weight" : !w_idxq ? "indexer.q_proj.weight" : !w_qn ? "attn_q_norm.weight" : !w_kn ? "attn_k_norm.weight" : !w_iqn ? "indexer.q_norm.weight" : !w_ikn ? "indexer.k_norm.weight" : nullptr; if (missing) { err = v.name(missing) + " is missing"; return false; } if (pos < 0 || pos >= st.max_cells) { err = "qsa_layer: pos " + std::to_string(pos) + " is outside the state's 0.." + std::to_string(st.max_cells - 1); return false; } // THE TWO BF16 PROJECTIONS: the arena holds them re-rounded to 2 B/elem, which is what `bf16_gemv` wants. // Reading one as f32 would walk 2x its length inside the arena without faulting. -if (w_idxk->kind != WeightKind::Bf16InF32 || w_idxq->kind != WeightKind::Bf16InF32) { err = v.name("indexer.*_proj.weight") + " must be engine form 1 (bf16); they are " + std::to_string((int) w_idxk->kind) + " and " + std::to_string((int) w_idxq->kind); return false; } +if ((w_idxk->kind != WeightKind::Bf16InF32 && w_idxk->kind != WeightKind::F16InF32) || (w_idxq->kind != WeightKind::Bf16InF32 && w_idxq->kind != WeightKind::F16InF32)) { err = v.name("indexer.*_proj.weight") + " must be engine form 1 (bf16); they are " + std::to_string((int) w_idxk->kind) + " and " + std::to_string((int) w_idxq->kind); return false; } // ---- 1. the three activation formats, once each if (!w_attnk->native_data || !w_attnv->native_data || !w_attnq->native_data) { quantize_q8_K(x, b.x_q8k, g.n_embd, stream); @@ -1064,7 +1069,7 @@ if (!w_attnk->native_data || !w_attnv->native_data || !w_attnq->native_data) { } else if (cudaMemcpyAsync(st.step, st.host_step, qsa_step_bytes(), cudaMemcpyHostToDevice, (cudaStream_t) stream) != cudaSuccess || cudaMemcpyAsync(st.pos_dev, st.host_pos, (size_t) g.n_head * 4, cudaMemcpyHostToDevice, (cudaStream_t) stream) != cudaSuccess) { err = "qsa_layer: the step-state upload failed"; return false; } } // ---- 3. the indexer's RAW key: appended before any norm, pooled later once per block -project_bf16(x, b.x_bf16, (const uint16_t*) w_idxk->data, b.idx_raw, g.n_embd, g.idx_key_dim, false, stream); +project_bf16(x, b.x_bf16, w_idxk->data, b.idx_raw, g.n_embd, g.idx_key_dim, false, stream, wform(w_idxk, "indexer.k_proj.weight")); // ---- 4. K and V, in Q8_K, then norm and rotate K only SForm f_k, f_v, f_o, f_q; if (!sform_of(*w_attnk, f_k, v.name("attn_k.weight"), err)) return false; if (!sform_of(*w_attnv, f_v, v.name("attn_v.weight"), err)) return false; if (!sform_of(*w_attno, f_o, v.name("attn_output.weight"), err)) return false; if (!sform_of(*w_attnq, f_q, v.name("attn_q.weight"), err)) return false; Planes p_k, p_v, p_o, p_q; if (!plane_ptrs(*w_attnk, v.name("attn_k.weight"), p_k, err)) return false; if (!plane_ptrs(*w_attnv, v.name("attn_v.weight"), p_v, err)) return false; if (!plane_ptrs(*w_attno, v.name("attn_output.weight"), p_o, err)) return false; if (!plane_ptrs(*w_attnq, v.name("attn_q.weight"), p_q, err)) return false; // k and v, with the activation THIS layer's tensors ask for. Both are K-quants in every QSA layer of this @@ -1116,7 +1121,7 @@ if (!gemv_quantized(*w_attnq, p_q, f_q, b.x_q8_0, b.x_q8k, b.q_full, g.n_embd, g // case A) and needs no kernel. if (cudaMemcpy2DAsync(b.qcur, (size_t) g.head_dim * 4, b.q_full, (size_t) g.head_dim * 2 * 4, (size_t) g.head_dim * 4, (size_t) g.n_head, cudaMemcpyDeviceToDevice, (cudaStream_t) stream) != cudaSuccess) { err = "qsa_layer: the q/gate split failed"; return false; } if (!normalize_rotate(b.qcur, w_qn, (int) g.n_head, (int) g.head_dim)) return false; // ---- 7. the indexer's query: BF16, then norm and rotate -project_bf16(x, b.x_bf16, (const uint16_t*) w_idxq->data, b.q_idx, g.n_embd, g.idx_q_heads * g.idx_key_dim, false, stream); +project_bf16(x, b.x_bf16, w_idxq->data, b.q_idx, g.n_embd, g.idx_q_heads * g.idx_key_dim, false, stream, wform(w_idxq, "indexer.q_proj.weight")); if (!normalize_rotate(b.q_idx, w_iqn, (int) g.idx_q_heads, (int) g.idx_key_dim)) return false; // ---- 8. score, select, gather, attend. `max_blocks` and `cap` are CAPACITIES from the state, not this // token's counts: a grid or a shared-memory size that follows the sequence length is baked into a captured @@ -1254,15 +1259,16 @@ bool lm_head_mix(const WeightTable& tables, const ModelGeometry& g, const BlockB err = "lm_head: an output_hc_* weight is missing"; return false; } - if (wn->kind != WeightKind::F32 || wd->kind != WeightKind::Bf16InF32 || - wu->kind != WeightKind::Bf16InF32) { + if (wn->kind != WeightKind::F32 || wd->kind != wu->kind || + (wd->kind != WeightKind::Bf16InF32 && wd->kind != WeightKind::F16InF32)) { err = "lm_head: the output_hc_* weights have the wrong engine forms"; return false; } const strata::kernels::GrShapes gs{g.n_embd, g.hc, g.hc_lr}; - strata::kernels::gr_read(bb.R, (const float*) wn->data, (const uint16_t*) wd->data, - (const uint16_t*) wu->data, nullptr, RMS_EPS, gs, bb.gr, - bb.mixed, bb.inject, stream); + try { + strata::kernels::gr_read(bb.R, (const float*) wn->data, wd->data, wu->data, nullptr, RMS_EPS, gs, bb.gr, + bb.mixed, bb.inject, stream, wform(wd, "output_hc_down.weight")); + } catch (const std::exception& e) { err = std::string("lm_head: ") + e.what(); return false; } return true; } @@ -1391,7 +1397,7 @@ bool block_layer_pre(const WeightTable& tables, const ModelGeometry& g, int64_t const char* pre[2] = {"hc_attn_", "hc_ffn_"}; const WeightRef* w_norm[2]; const WeightRef* w_down[2]; const WeightRef* w_up[2]; const WeightRef* w_inject[2]; for (int h = 0; h < 2; ++h) { const std::string a = std::string(pre[h]) + "norm.weight"; const std::string d = std::string(pre[h]) + "down.weight"; const std::string u = std::string(pre[h]) + "up.weight"; const std::string i = std::string(pre[h]) + "inject.weight"; w_norm[h] = v.get(a.c_str()); w_down[h] = v.get(d.c_str()); w_up[h] = v.get(u.c_str()); w_inject[h] = v.get(i.c_str()); if (!w_norm[h] || !w_down[h] || !w_up[h] || !w_inject[h]) { err = v.name((std::string(pre[h]) + "{norm,down,up,inject}.weight").c_str()) + " is missing"; return false; } // The GR weights are BF16 and the arena holds them re-rounded to 2 B/elem. `gr_read` wants exactly // that; handing it f32 bytes would walk 2x the tensor inside the arena without faulting. -if (w_down[h]->kind != WeightKind::Bf16InF32 || w_up[h]->kind != WeightKind::Bf16InF32 || w_inject[h]->kind != WeightKind::Bf16InF32 || w_norm[h]->kind != WeightKind::F32) { err = v.name(pre[h]) + "has the wrong engine forms (norm must be F32, the other three bf16)"; return false; } } +const bool bf_ = w_down[h]->kind == WeightKind::Bf16InF32 && w_up[h]->kind == WeightKind::Bf16InF32 && w_inject[h]->kind == WeightKind::Bf16InF32; const bool ld_ = w_down[h]->kind == WeightKind::F16InF32 && w_up[h]->kind == WeightKind::F16InF32 && (w_inject[h]->kind == WeightKind::F16InF32 || w_inject[h]->kind == WeightKind::F32); if ((!bf_ && !ld_) || w_norm[h]->kind != WeightKind::F32) { err = v.name(pre[h]) + "has the wrong engine forms (norm must be F32, the other three bf16)"; return false; } } // ---- half 1: the mixer float* R = bb.R; // R0.11: WHICH STAGES TO RUN. `stage_prefix == 0` means "as `half` says", so every caller that predates @@ -1410,9 +1416,21 @@ st_begin(layer, 0, stream); fa.w_norm = (const float*) w_norm[0]->data; fa.w_down = NativeDense::hc_bf16_view(*w_down[0], stream, 0, 1); fa.w_up = NativeDense::hc_bf16_view(*w_up[0], stream, 1, 1); fa.w_inject = (const uint16_t*) w_inject[0]->data; fa.eps = RMS_EPS; fa.lo = bb.gr.lo; fa.rs = bb.gr_rs; fa.inject_out = bb.inject; fa.mixed = bb.mixed; + fa.w_f16 = w_down[0]->kind == WeightKind::F16InF32; + if (w_inject[0]->kind == WeightKind::F32) { + // STRATA_FP16=load: the one-token multi read leaves its FP32 normalized rows in bb.gr.xn for the F32 inject + fa.w_inject = nullptr; + strata::kernels::fused_gr_read_multi(&fa, 1, bb.gr.xn, stream); + strata::kernels::gemv_fp32_mmvf(bb.gr.xn, w_inject[0]->data, strata::kernels::WForm::F32, bb.inject, + g.hc * g.n_embd, g.hc, stream); + } else strata::kernels::fused_gr_read(fa, stream); } else { +<<<<<<< this fork gr_read(R, (const float*) w_norm[0]->data, NativeDense::hc_bf16_view(*w_down[0], stream, 0, 1), NativeDense::hc_bf16_view(*w_up[0], stream, 1, 1), (const uint16_t*) w_inject[0]->data, RMS_EPS, gs, bb.gr, bb.mixed, bb.inject, stream); +======= + gr_read(R, (const float*) w_norm[0]->data, w_down[0]->data, w_up[0]->data, w_inject[0]->data, RMS_EPS, gs, bb.gr, bb.mixed, bb.inject, stream, wform(w_down[0], "hc down"), wform(w_inject[0], "hc inject")); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) } st_end(layer, 0, stream); dump_half(bb, g, layer, bb.inject, 2 * g.n_embd, g.hc, stream); } if (run1) { @@ -1433,9 +1451,21 @@ st_begin(layer, 3, stream); fa.w_norm = (const float*) w_norm[1]->data; fa.w_down = NativeDense::hc_bf16_view(*w_down[1], stream, 0, 1); fa.w_up = NativeDense::hc_bf16_view(*w_up[1], stream, 1, 1); fa.w_inject = (const uint16_t*) w_inject[1]->data; fa.eps = RMS_EPS; fa.lo = bb.gr.lo; fa.rs = bb.gr_rs; fa.inject_out = bb.inject2; fa.mixed = bb.mixed; + fa.w_f16 = w_down[1]->kind == WeightKind::F16InF32; + if (w_inject[1]->kind == WeightKind::F32) { + // STRATA_FP16=load: the one-token multi read leaves its FP32 normalized rows in bb.gr.xn for the F32 inject + fa.w_inject = nullptr; + strata::kernels::fused_gr_read_multi(&fa, 1, bb.gr.xn, stream); + strata::kernels::gemv_fp32_mmvf(bb.gr.xn, w_inject[1]->data, strata::kernels::WForm::F32, bb.inject2, + g.hc * g.n_embd, g.hc, stream); + } else strata::kernels::fused_gr_read(fa, stream); } else { +<<<<<<< this fork gr_read(R, (const float*) w_norm[1]->data, NativeDense::hc_bf16_view(*w_down[1], stream, 0, 1), NativeDense::hc_bf16_view(*w_up[1], stream, 1, 1), (const uint16_t*) w_inject[1]->data, RMS_EPS, gs, bb.gr, bb.mixed, bb.inject, stream); +======= + gr_read(R, (const float*) w_norm[1]->data, w_down[1]->data, w_up[1]->data, w_inject[1]->data, RMS_EPS, gs, bb.gr, bb.mixed, bb.inject, stream, wform(w_down[1], "hc down"), wform(w_inject[1], "hc inject")); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) } st_end(layer, 3, stream); } if (run4) { diff --git a/src/core/load_converted.cpp b/src/core/load_converted.cpp new file mode 100644 index 00000000000..8d063bded49 --- /dev/null +++ b/src/core/load_converted.cpp @@ -0,0 +1,182 @@ +// src/core/load_converted.cpp - see include/strata/core/load_converted.hpp. +#include "strata/core/load_converted.hpp" +#include "strata/core/weights.hpp" +#include "strata/artifact/gguf_reader.hpp" +#include "strata/kernels/f16_bits.hpp" +#include "strata/kernels/bf16_bits.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace strata::core { + +bool load_converted_kind(const std::string& name, bool& gate) { + static const char* f16[] = {".hc_attn_down.weight", ".hc_attn_up.weight", ".hc_ffn_down.weight", + ".hc_ffn_up.weight", ".ple_value.weight", ".indexer.k_proj.weight", + ".indexer.q_proj.weight"}; + static const char* gates[] = {".ffn_gate_inp.weight", ".ffn_gate_inp_shexp.weight", ".hc_attn_inject.weight", + ".hc_ffn_inject.weight", ".ssm_alpha.weight", ".ssm_beta.weight"}; + gate = false; + if (name == "output_hc_down.weight" || name == "output_hc_up.weight") return true; + if (name.rfind("blk.", 0) != 0) return false; + for (const char* s : f16) if (name.ends_with(s)) return true; + for (const char* s : gates) if (name.ends_with(s)) { gate = true; return true; } + return false; +} + +namespace { +constexpr uint32_t kF32 = 0, kF16 = 1, kQ8_0 = 8, kBF16 = 30; +} + +bool LoadConverted::build(const std::vector& shards, bool gates_fp16, std::string& err) { + host_.clear(); + tensors_f16 = tensors_f32 = 0; + bytes = subnormal = flushed = 0; + try { + for (const auto& path : shards) { + strata::GgufFile gguf(path); + struct Job { const strata::TensorInfo* t; size_t h; }; + std::vector jobs; + for (const auto& t : gguf.tensors()) { + bool gate = false; + if (!load_converted_kind(t.name, gate)) continue; + if (t.type != kQ8_0 && t.type != kBF16 && t.type != kF16 && t.type != kF32) { + err = "STRATA_FP16=load: " + t.name + " is " + t.type_name() + " in the GGUF (Q8_0, BF16, F16 " + "or F32 expected)"; + return false; + } + if (t.shape.empty() || t.shape.size() > 2 || (t.type == kQ8_0 && t.shape[0] % 32 != 0)) { + err = "STRATA_FP16=load: " + t.name + " has an unexpected shape"; + return false; + } + Host h; + h.name = t.name; + h.ne0 = (int64_t) t.shape[0]; + h.ne1 = t.shape.size() > 1 ? (int64_t) t.shape[1] : 0; + h.f32 = gate && !gates_fp16 && t.type == kF32; + h.data.resize(t.elements() * (h.f32 ? 4 : 2)); + jobs.push_back(Job{&t, host_.size()}); + host_.push_back(std::move(h)); + } + // the conversions, in parallel over tensors (the GGUF stays mapped for this loop) + std::atomic next{0}; + std::atomic sub{0}, fl{0}, over{0}; + auto work = [&] { + for (size_t j; (j = next.fetch_add(1)) < jobs.size();) { + const strata::TensorInfo& t = *jobs[j].t; + Host& h = host_[jobs[j].h]; + const uint8_t* src = gguf.tensor_data(t); + const uint64_t n = t.elements(); + if (h.f32) { std::memcpy(h.data.data(), src, n * 4); continue; } + uint16_t* dst = reinterpret_cast(h.data.data()); + uint64_t s = 0, f = 0, o = 0; + auto put = [&](uint64_t i, float x) { + if (std::fabs(x) > 65504.0f) ++o; + const uint16_t v = strata::kernels::f16_from_f32(x); + if (x != 0.0f) { + if ((v & 0x7fffu) == 0) ++f; + else if ((v & 0x7c00u) == 0) ++s; + } + dst[i] = v; + }; + if (t.type == kQ8_0) { + for (uint64_t b = 0; b < n / 32; ++b) { + const uint8_t* blk = src + b * 34; + uint16_t dh; + std::memcpy(&dh, blk, 2); + const float d = strata::kernels::f32_from_f16(dh); + for (int k = 0; k < 32; ++k) put(b * 32 + k, d * (float) (int8_t) blk[2 + k]); + } + } else if (t.type == kBF16) { + for (uint64_t i = 0; i < n; ++i) { + uint16_t v; + std::memcpy(&v, src + 2 * i, 2); + put(i, strata::kernels::f32_from_bf16(v)); + } + } else if (t.type == kF16) { + std::memcpy(dst, src, n * 2); + } else { + for (uint64_t i = 0; i < n; ++i) { + float x; + std::memcpy(&x, src + 4 * i, 4); + put(i, x); + } + } + sub += s; fl += f; over += o; + } + }; + std::vector pool; + const unsigned nt = std::max(1u, std::min(32u, std::thread::hardware_concurrency())); + for (unsigned i = 0; i < nt; ++i) pool.emplace_back(work); + for (auto& th : pool) th.join(); + if (over) { + err = "STRATA_FP16=load: " + std::to_string((unsigned long long) over.load()) + + " weight values are beyond FP16's range (65504) in " + path; + return false; + } + subnormal += sub; + flushed += fl; + } + } catch (const std::exception& e) { + err = std::string("STRATA_FP16=load: ") + e.what(); + return false; + } + if (host_.empty()) { err = "STRATA_FP16=load: none of the BF16-form weights are in the GGUF shards"; return false; } + for (const auto& h : host_) { + (h.f32 ? tensors_f32 : tensors_f16)++; + bytes += h.data.size(); + } + return true; +} + +void LoadConverted::served_names(std::set& out) const { + for (const auto& h : host_) out.insert(h.name); +} + +bool LoadConverted::attach(WeightTable& table, std::vector& owner, std::string& err) const { + // ONE allocation per GPU for all of them: 483 cudaMallocs of 5 KB..13 MB each cost ~400 MiB of VRAM more than + // their bytes (measured: the expert cache lost ~700 slots on 4 GPUs) + auto align = [](uint64_t n) { return (n + 255) & ~(uint64_t) 255; }; + uint64_t total = 0; + for (const auto& h : host_) total += align(h.data.size()); + void* slab = nullptr; + if (const cudaError_t e = cudaMalloc(&slab, total); e != cudaSuccess) { + err = "STRATA_FP16=load: " + std::to_string((unsigned long long) (total >> 20)) + " MiB for the converted " + "weights: " + cudaGetErrorString(e); + return false; + } + owner.push_back(slab); + uint64_t off = 0; + for (const auto& h : host_) { + auto it = table.table_.find(h.name); + if (it == table.table_.end()) { err = "STRATA_FP16=load: " + h.name + " is not in the pack"; return false; } + WeightRef& r = it->second; + if (r.kind != WeightKind::Bf16InF32 || r.ne0 != h.ne0 || r.ne1 != h.ne1) { + err = "STRATA_FP16=load: " + h.name + " is not the pack's BF16 row of the same shape (form " + + std::to_string((int) r.kind) + ", " + std::to_string((long long) r.ne0) + " x " + + std::to_string((long long) r.ne1) + ")"; + return false; + } + void* p = static_cast(slab) + off; + off += align(h.data.size()); + const cudaError_t e = cudaMemcpy(p, h.data.data(), h.data.size(), cudaMemcpyHostToDevice); + if (e != cudaSuccess) { err = "STRATA_FP16=load: upload of " + h.name + ": " + cudaGetErrorString(e); return false; } + r.data = p; + r.bytes = h.data.size(); + r.kind = h.f32 ? WeightKind::F32 : WeightKind::F16InF32; + r.resident = true; + } + return true; +} + +void LoadConverted::free_all(std::vector& owner) { + for (void* p : owner) cudaFree(p); + owner.clear(); +} + +} // namespace strata::core diff --git a/src/core/mtp.cpp b/src/core/mtp.cpp index c36841c6ce2..510b7875068 100644 --- a/src/core/mtp.cpp +++ b/src/core/mtp.cpp @@ -1,4 +1,7 @@ // src/core/mtp.cpp - see include/strata/core/mtp.hpp. +#include "strata/kernels/bf16_bits.hpp" +#include "strata/kernels/f16_bits.hpp" +#include "strata/core/fp16_mode.hpp" #include "strata/core/mtp.hpp" #include "strata/core/coupled_draft.hpp" #include "strata/core/spec_prob.hpp" @@ -12,6 +15,7 @@ #include "strata/kernels/cpu/expert.hpp" #include "strata/kernels/elementwise.hpp" #include "strata/kernels/fused_gr.hpp" +#include "strata/kernels/iq_kernels.hpp" #include "strata/kernels/gr.hpp" #include "strata/kernels/kv_q4.hpp" #include "strata/kernels/kv_q8.hpp" @@ -154,6 +158,7 @@ MtpDrafter::~MtpDrafter() { if (sh_join_) cudaEventDestroy(sh_join_); if (owns_weights_ && dense_) cudaFree(dense_); if (owns_weights_ && experts_) cudaFree(experts_); + if (owns_weights_ && f16_) cudaFree(f16_); if (state_arena_) cudaFree(state_arena_); if (arena_) cudaFree(arena_); if (head_logits_) cudaFree(head_logits_); @@ -246,6 +251,11 @@ bool MtpDrafter::make_q4_head(std::string& err) { #endif } +const uint16_t* MtpDrafter::tensor_f16(const char* name) const { + for (const auto& [n, off] : f16_off_) if (n == name) return f16_ + off; + return nullptr; +} + bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, SessionState& ss, int max_t, std::string& err, int64_t window, const MtpDrafter* shared) { #if defined(STRATA_GPU_METAL) @@ -271,6 +281,9 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session dense_ = shared->dense_; experts_ = shared->experts_; tensors_ = shared->tensors_; + // ac922: the native-layout experts' types and size (STRATA_MTP_EXPERTS) and the FP16 copies (STRATA_FP16_MTP) + xgu_ = shared->xgu_; xd_ = shared->xd_; xblob_ = shared->xblob_; + f16_ = shared->f16_; f16_off_ = shared->f16_off_; owns_weights_ = false; owns_draft_head_ = false; } @@ -305,13 +318,80 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session vram_ += blob.size(); if (q4_ && !make_q4_dense(blob, err)) return false; } + // ---- STRATA_FP16_MTP=1: FP16 copies of the BF16 weights the prompt path's GEMMs read (exact: BF16 values in + // FP16's range; the count of values that became FP16 subnormals or zero is reported) + if (strata::fp16_mtp() && shared == nullptr) { // a slot drafter borrows the owner's copies (above) + static const char* names[] = {"attn_hyper_connection.input_mix_weight_down.weight", + "attn_hyper_connection.input_mix_weight_up.weight"}; + std::vector blob; + if (!read_file(rt_dir + "/dense.bin", blob)) { err = "mtp: cannot read dense.bin"; return false; } + std::vector host; + uint64_t sub = 0, zero = 0, over = 0; + for (const char* n : names) { + const Tensor* t = nullptr; + for (const auto& x : tensors_) if (x.name == n) t = &x; + if (t == nullptr || t->kind != "bf16") { err = std::string("mtp: STRATA_FP16_MTP=1: ") + n + " is not BF16"; return false; } + f16_off_.emplace_back(n, host.size()); + const uint16_t* src = reinterpret_cast(blob.data() + t->off); + for (uint64_t i = 0; i < t->bytes / 2; ++i) { + const float x = strata::kernels::f32_from_bf16(src[i]); + const uint16_t h = strata::kernels::f16_from_f32(x); + if (std::fabs(x) > 65504.0f) ++over; + else if (x != 0.0f && (h & 0x7fffu) == 0) ++zero; + else if (x != 0.0f && (h & 0x7c00u) == 0) ++sub; + host.push_back(h); + } + } + if (over) { err = "mtp: STRATA_FP16_MTP=1: " + std::to_string((unsigned long long) over) + " values beyond FP16's range"; return false; } + if (cudaMalloc((void**) &f16_, host.size() * 2) != cudaSuccess || + cudaMemcpy(f16_, host.data(), host.size() * 2, cudaMemcpyHostToDevice) != cudaSuccess) { + err = "mtp: STRATA_FP16_MTP=1: the FP16 copies do not fit in VRAM"; + return false; + } + vram_ += host.size() * 2; + std::fprintf(stderr, "strata mtp: STRATA_FP16_MTP=1: the prompt path's 2 hyper-connection weights as FP16 (%.1f MiB; " + "%llu values became FP16 subnormals, %llu zero)\n", (double) host.size() * 2 / 1048576.0, + (unsigned long long) sub, (unsigned long long) zero); + } // ---- the 512 routed experts, one blob each if (shared == nullptr) { - const uint64_t bytes = (uint64_t) g.n_expert * strata::kernels::cpu::BLOB; + // STRATA_MTP_EXPERTS=: the experts from an MTP GGUF in the native layout (tools: mtp_experts.py), run by + // the main model's grouped native kernels instead of the pack's Q2_0 blobs + uint64_t blob = strata::kernels::cpu::BLOB; + std::string xpath = rt_dir + "/experts.bin"; + long skip = 0; + if (const char* xp = std::getenv("STRATA_MTP_EXPERTS"); xp != nullptr && *xp) { + FILE* h = std::fopen(xp, "rb"); + char magic[8] = {}; + int32_t types[2] = {-1, -1}; + const bool hdr = h != nullptr && std::fread(magic, 1, 8, h) == 8 && std::fread(types, 4, 2, h) == 2 && + std::memcmp(magic, "SMTPEXP1", 8) == 0; + long size = -1; + if (h != nullptr) { std::fseek(h, 0, SEEK_END); size = std::ftell(h); std::fclose(h); } + if (!hdr) { err = std::string("mtp: STRATA_MTP_EXPERTS=") + xp + " is not an SMTPEXP1 file"; return false; } + if (!strata::kernels::native_expert_supported(types[0], types[1], g.n_embd, g.n_ff)) { + err = "mtp: STRATA_MTP_EXPERTS: no native kernels for gate/up type " + std::to_string(types[0]) + + ", down type " + std::to_string(types[1]); + return false; + } + const strata::kernels::NativeExpertLayout L = + strata::kernels::native_expert_layout(types[0], types[1], g.n_embd, g.n_ff); + if (size != (long) (16 + (uint64_t) g.n_expert * L.bytes)) { + err = std::string("mtp: STRATA_MTP_EXPERTS=") + xp + " has " + std::to_string(size) + " bytes, not " + + std::to_string(16 + (uint64_t) g.n_expert * L.bytes); + return false; + } + blob = L.bytes; xpath = xp; skip = 16; xgu_ = types[0]; xd_ = types[1]; + std::fprintf(stderr, "strata mtp: experts from %s (gate/up type %d, down type %d, %.2f GiB), native kernels\n", + xp, xgu_, xd_, (double) g.n_expert * blob / 1073741824.0); + } + xblob_ = blob; + const uint64_t bytes = (uint64_t) g.n_expert * blob; // Loader fix (0.1.15+loaderfix.2): each 64 MiB read below reached the disk as ~16k 4095-byte reads under // MSVC's `basic_filebuf::xsgetn`, which is what made 675 MiB of drafter experts take minutes. - FILE* f = std::fopen((rt_dir + "/experts.bin").c_str(), "rb"); - if (f == nullptr) { err = "mtp: cannot open experts.bin"; return false; } + FILE* f = std::fopen(xpath.c_str(), "rb"); + if (f == nullptr) { err = "mtp: cannot open " + xpath; return false; } + if (skip > 0) std::fseek(f, skip, SEEK_SET); struct Closer { FILE* f; ~Closer() { if (f != nullptr) std::fclose(f); } @@ -395,7 +475,7 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session qfull_ = b.take(T * NH * 2 * HD); qcur_ = b.take(T * NH * HD); kcur_ = b.take(T * NKV * HD); vcur_ = b.take(T * NKV * HD); attn_ = b.take(T * NH * HD); attn32_ = b.take(T * NH * HD); - attn_scratch_ = b.take((uint64_t) attn_scratch_floats_); // the full layer runs one row at a time + attn_scratch_ = b.take((uint64_t) attn_scratch_floats_ * T); // the batch call uses one stride per row logits_ = b.take(T * (uint64_t) g.n_expert); w_ = b.take(T * K); ids_ = b.take(T * K); shared_ = b.take(T * N); parts_ = b.take(T * K * N); y_ = b.take(T * N); sample_ = b.take(T * N); @@ -407,8 +487,10 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session hit_slot_ = b.take(T * K); hit_dst_ = b.take(T * K); hit_count_ = b.take(4); grp_ptr_ = b.take(T * K); grp_start_ = b.take(T * K + 1); grp_counts_ = b.take(4); - hit_xq_ = b.take(T * (N / 32) * 34); hit_xs_ = b.take(T * (N / 32)); - hit_scratch_ = b.take(strata::kernels::moe_hit_grouped_scratch_bytes((int64_t) (T * K), g.n_embd, g.n_ff)); + hit_xq_ = b.take(T * (N / 32) * 36); hit_xs_ = b.take(T * (N / 32)); // 36: Q8_1 (native) + hit_scratch_ = b.take(std::max( + strata::kernels::moe_hit_grouped_scratch_bytes((int64_t) (T * K), g.n_embd, g.n_ff), + xgu_ >= 0 ? strata::kernels::native_expert_scratch_bytes((int64_t) (T * K), g.n_ff) : 0)); sh_scratch_ = (float*) b.take(strata::kernels::shared_expert_scratch_bytes(g.n_ff)); x_bf16_ = b.take(N); out_ids_ = b.take(T + 4); @@ -494,9 +576,9 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session (double) vram_ / 1048576.0); } else { std::fprintf(stderr, "strata mtp: draft layer loaded, %.0f MiB of VRAM (experts %.0f, dense %.0f), files read in %.2f s (%.0f MiB/s)\n", - (double) vram_ / 1048576.0, (double) g.n_expert * strata::kernels::cpu::BLOB / 1048576.0, + (double) vram_ / 1048576.0, (double) g.n_expert * xblob_ / 1048576.0, (double) tensors_.back().off / 1048576.0, files_s, - files_s > 0 ? ((double) g.n_expert * strata::kernels::cpu::BLOB + (double) tensors_.back().off) / + files_s > 0 ? ((double) g.n_expert * xblob_ + (double) tensors_.back().off) / 1048576.0 / files_s : 0.0); } return true; @@ -1113,6 +1195,7 @@ bool MtpDrafter::record_rest(int step_row, cudaStream_t cs, std::string& err) { else router_top10(logits_ + t * g.n_expert, 1, (int) g.n_expert, (int) K, ids_ + t * K, w_ + t * K, cs); } } +<<<<<<< this fork if (rr_res_ != nullptr && rr_margin_ > 0.0f && g.n_expert == 512 && K == 10) { try { native_route_resident(logits_, ids_, w_, rr_res_, T, rr_margin_, rr_lo_, rr_hi_, nullptr, cs); @@ -1120,11 +1203,24 @@ bool MtpDrafter::record_rest(int step_row, cudaStream_t cs, std::string& err) { } g_mp.stamp(9, cs); moe_group_resident(ids_, (int) (T * K), (int) K, experts_, (int64_t) strata::kernels::cpu::BLOB, grp_ptr_, +======= + moe_group_resident(ids_, (int) (T * K), (int) K, experts_, (int64_t) xblob_, grp_ptr_, +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) grp_start_, grp_counts_, hit_dst_, hit_slot_, cs); + if (xgu_ >= 0) { // STRATA_MTP_EXPERTS: the native layout, Q8_1 activations + const strata::kernels::NativeExpertLayout L = strata::kernels::native_expert_layout(xgu_, xd_, g.n_embd, g.n_ff); + strata::kernels::quantize_q8_1_rows(mixed_, T, (int64_t) N, hit_xq_, cs); + strata::kernels::native_expert_grouped(L, grp_ptr_, grp_start_, grp_counts_, hit_dst_, hit_slot_, (int64_t) T * K, + (int64_t) T * K, hit_xq_, hit_scratch_, parts_, cs); + } else { quantize_q8_0_scaled(mixed_, hit_xq_, hit_xs_, (int64_t) T * N, cs); moe_grouped_s2(grp_ptr_, grp_start_, grp_counts_, hit_dst_, hit_slot_, (int64_t) T * K, (int64_t) T * K, hit_xq_, hit_xs_, hit_scratch_, parts_, cs); +<<<<<<< this fork g_mp.stamp(10, cs); +======= + } +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) if (branch_on) { if (cudaStreamWaitEvent(cs, sh_join_, 0) != cudaSuccess) { err = "mtp: the shared expert's join"; return false; } } else { diff --git a/src/core/pinned.cu b/src/core/pinned.cu index 42c18b76048..7f32bbe5cd0 100644 --- a/src/core/pinned.cu +++ b/src/core/pinned.cu @@ -29,6 +29,8 @@ #include #else #include +#include +#include #include #include #include @@ -384,6 +386,49 @@ bool sliced_pin_limit(uint64_t& limit, std::string& why) { } // namespace namespace { +#if !defined(_WIN32) +// The CPUs of NUMA node `node` (sysfs cpulist); false if none. +bool node_cpus(int node, cpu_set_t& set) { + CPU_ZERO(&set); + if (node < 0) return false; + char path[96]; + std::snprintf(path, sizeof path, "/sys/devices/system/node/node%d/cpulist", node); + std::FILE* f = std::fopen(path, "r"); + if (f == nullptr) return false; + int lo = 0, hi = 0; + char sep = 0; + while (std::fscanf(f, "%d", &lo) == 1) { + hi = lo; + if (std::fscanf(f, "%c", &sep) == 1 && sep == '-' && std::fscanf(f, "%d", &hi) == 1) (void) std::fscanf(f, "%c", &sep); + for (int c = lo; c <= hi && c < CPU_SETSIZE; ++c) CPU_SET(c, &set); + } + std::fclose(f); + return CPU_COUNT(&set) > 0; +} +// The NUMA node of the current CUDA device (its PCI device's numa_node), or -1. +int current_gpu_node() { + int dev = 0; + char bus[64] = {}; + unsigned dom = 0, b = 0, d = 0, fn = 0; + if (cudaGetDevice(&dev) != cudaSuccess || cudaDeviceGetPCIBusId(bus, (int) sizeof bus, dev) != cudaSuccess) { + (void) cudaGetLastError(); + return -1; + } + if (std::sscanf(bus, "%x:%x:%x.%x", &dom, &b, &d, &fn) != 4) return -1; + char path[128]; + std::snprintf(path, sizeof path, "/sys/bus/pci/devices/%04x:%02x:%02x.%x/numa_node", dom, b, d, fn); + int node = -1; + if (std::FILE* f = std::fopen(path, "r")) { + if (std::fscanf(f, "%d", &node) != 1) node = -1; + std::fclose(f); + } + return node; +} +// cudaHostAlloc'd arenas and their nodes, for run_on_arena_node +struct ArenaRange { const uint8_t* begin; const uint8_t* end; int node; }; +std::mutex g_arenas_mu; +std::vector g_arenas; +#endif std::vector uniform_bounds(uint64_t bytes, uint64_t slice) { std::vector b; if (slice == 0) return b; @@ -393,6 +438,40 @@ std::vector uniform_bounds(uint64_t bytes, uint64_t slice) { } } // namespace +cudaError_t cuda_host_alloc_near_gpu(void** p, size_t bytes, unsigned int flags) { +#if !defined(_WIN32) + int dev = 0; + if (cudaGetDevice(&dev) != cudaSuccess) return cudaHostAlloc(p, bytes, flags); + const int node = current_gpu_node(); + cpu_set_t set; + if (!node_cpus(node, set)) return cudaHostAlloc(p, bytes, flags); + cudaError_t e = cudaSuccess; + std::thread([&] { + pthread_setaffinity_np(pthread_self(), sizeof set, &set); + (void) cudaSetDevice(dev); + e = cudaHostAlloc(p, bytes, flags); + }).join(); + return e; +#else + return cudaHostAlloc(p, bytes, flags); +#endif +} + +void run_on_arena_node(const void* p) { +#if !defined(_WIN32) + int node = -1; + { + std::lock_guard lk(g_arenas_mu); + for (const ArenaRange& r : g_arenas) + if ((const uint8_t*) p >= r.begin && (const uint8_t*) p < r.end) node = r.node; + } + cpu_set_t set; + if (node_cpus(node, set)) pthread_setaffinity_np(pthread_self(), sizeof set, &set); +#else + (void) p; +#endif +} + PinnedArena::PinnedArena(uint64_t bytes, uint64_t slice) : PinnedArena(bytes, uniform_bounds(bytes, slice)) { if (slice_bytes) slice_bytes = slice; // sliced registration: record the uniform size } @@ -401,7 +480,42 @@ PinnedArena::PinnedArena(uint64_t bytes, const std::vector& bounds, uint64_t max_pinned_bytes, const std::string& shared_file, uint64_t shared_pack_hash) : capacity(bytes) { if (bytes == 0) return; +#if !defined(_WIN32) + // an IBM AC922 (POWER9 + V100, access-counter migration on): memory registered with cudaHostRegister reads at 1-25 GB/s, + // DMA and in place, the larger the worse; cudaHostAlloc's at 68-72 at any size. So the arena is the driver's, + // allocated by a thread on the NUMA node of the GPU it is for (the driver puts it on the allocating thread's + // node). STRATA_ARENA_HOSTALLOC=0: the registered mapping below, as before. + const char* ha = std::getenv("STRATA_ARENA_HOSTALLOC"); + if (shared_file.empty() && (ha == nullptr || std::string(ha) != "0")) { + int dev = 0; + (void) cudaGetDevice(&dev); + const int want = current_gpu_node(); + void* p = nullptr; + std::thread([&] { + cpu_set_t set; + if (node_cpus(want, set)) pthread_setaffinity_np(pthread_self(), sizeof set, &set); + (void) cudaSetDevice(dev); + if (cudaHostAlloc(&p, (size_t) bytes, cudaHostAllocPortable | cudaHostAllocMapped) != cudaSuccess) { + (void) cudaGetLastError(); + p = nullptr; + } + }).join(); + if (p != nullptr) { + base = p; + backing = PageBacking::PinnedByCuda; + node = want; + registered_bytes = bytes; + note = "cudaHostAlloc (page-locked by the driver) on NUMA node " + std::to_string(want); + std::lock_guard lk(g_arenas_mu); + g_arenas.push_back({(const uint8_t*) p, (const uint8_t*) p + bytes, want}); + return; + } + note = "cudaHostAlloc refused; "; + } +#endif + std::string why = note; base = reserve(bytes, backing, note, shared_file, shared_pack_hash, mapping_base, mapping_bytes); + note = why + note; if (base != nullptr && mapping_base == nullptr) { mapping_base = base; mapping_bytes = bytes; @@ -629,6 +743,7 @@ void PinnedArena::register_slices(std::atomic& ready) { } PinnedArena::~PinnedArena() { +<<<<<<< this fork if (base && by_need) { for (const auto& r : locked_ranges) strata::platform::unlock_resident((uint8_t*) base + r.first, r.second); for (const auto& r : pin_ranges) cudaHostUnregister((uint8_t*) base + r.first); @@ -638,6 +753,20 @@ PinnedArena::~PinnedArena() { mapping_bytes = 0; return; } +======= +#if !defined(_WIN32) + if (base && backing == PageBacking::PinnedByCuda) { + { + std::lock_guard lk(g_arenas_mu); + for (size_t i = 0; i < g_arenas.size(); ++i) + if (g_arenas[i].begin == (const uint8_t*) base) { g_arenas.erase(g_arenas.begin() + (ptrdiff_t) i); break; } + } + cudaFreeHost(base); + base = nullptr; + return; + } +#endif +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) if (base) { if (locked_bytes) strata::platform::unlock_resident((uint8_t*) base + (slice_bytes ? registered_bytes : 0), locked_bytes); if (slice_bytes) { @@ -903,9 +1032,9 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std:: copy_ns_sum.fetch_add(copy_ns); }; + // every worker on its own thread, on the arena's node (run_on_arena_node): the caller keeps its affinity std::vector pool; - for (int i = 1; i < threads; ++i) pool.emplace_back(worker); - worker(); + for (int i = 0; i < threads; ++i) pool.emplace_back([&] { run_on_arena_node(dst); worker(); }); for (auto& t : pool) t.join(); if (!err.empty()) { diff --git a/src/core/session.cpp b/src/core/session.cpp index 157c4cc4a9f..f8e0b9fff79 100644 --- a/src/core/session.cpp +++ b/src/core/session.cpp @@ -22,12 +22,8 @@ // `_mm_pause` for the doorbell spin. Guarded because it is x86-only; a target without it still builds, the // spin is just less polite to the pipeline. -#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) -#include -#define STRATA_SPIN_PAUSE() _mm_pause() -#else -#define STRATA_SPIN_PAUSE() ((void) 0) -#endif +#include "strata/platform/cpu_relax.hpp" +#define STRATA_SPIN_PAUSE() strata_cpu_pause() namespace strata::core { namespace { @@ -969,7 +965,7 @@ bool session_run_token(const ModelGeometry& g, int64_t pos, int32_t pos_base, Se progress_at("token: the CPU experts of layer", l); if (pool != nullptr) pool(user, s.db->h_x_f, s.db->h_ids, s.db->h_weights, g.n_embd, s.k, y_miss_host); std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); *flag = want; const auto t2 = Clock::now(); tg.ms_wait += std::chrono::duration(t1 - t0).count(); diff --git a/src/core/verify.cpp b/src/core/verify.cpp index a8791cc6714..fcddf975ffd 100644 --- a/src/core/verify.cpp +++ b/src/core/verify.cpp @@ -2,9 +2,13 @@ #include "strata/core/batch_rows.hpp" #include "strata/kernels/gfx_arch.hpp" #include "strata/core/verify.hpp" +<<<<<<< this fork #if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) #include "strata/core/host_graph_launch.hpp" #endif +======= +#include "strata/core/weight_form.hpp" +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) #include "strata/core/remote_expert_opt.hpp" #include "strata/core/dma_batch.hpp" #include "strata/core/spec_prob.hpp" @@ -65,7 +69,7 @@ #include #include #include -#include +#include "strata/platform/cpu_relax.hpp" #if defined(STRATA_GPU_METAL) // The Metal backend's decode window (metal/src/engine/window.cpp): this file's init/run/commit hand over to it. @@ -320,7 +324,8 @@ bool mapped(size_t bytes, void** h, void** d) { return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess; } #endif - if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false; // multi-GPU: portable only with a peer - the peer card writes its rows into them + // portable: a layer split's partner GPU (Verifier::set_partner) and a peer card read or write these too + if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) return false; std::memset(*h, 0, bytes); return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess; } @@ -390,6 +395,39 @@ const int64_t g_test_stall = [] { }(); } // namespace +bool Verifier::set_partner(int dev, std::string& err) { + (void) err; + if (const char* v = std::getenv("STRATA_FETCH_PARTNER"); v != nullptr && std::atoi(v) == 0) return true; + if (dev < 0 || dev == device_ || !fetch_overlap_ || sink_.pcie_mode != 2 || device_plan_) return true; + if ((le_ - lb_) * 2 > kMaxSteps) return true; + int can = 0; + if (cudaDeviceCanAccessPeer(&can, dev, device_) != cudaSuccess || !can) { cudaGetLastError(); return true; } + const auto& lay = strata::kernels::cpu::expert_layout(); + std::vector bytes((size_t) le_, 0); + for (int64_t l = 0; l < le_; ++l) bytes[(size_t) l] = (long long) lay.blob_bytes(l); + { + const OnDevice on(dev); + const cudaError_t pe = cudaDeviceEnablePeerAccess(device_, 0); + if (pe != cudaSuccess && pe != cudaErrorPeerAccessAlreadyEnabled) { cudaGetLastError(); return true; } + cudaGetLastError(); + if (cudaStreamCreateWithFlags(&partner_s_, cudaStreamNonBlocking) != cudaSuccess || + cudaMalloc((void**) &partner_arrive_, kMaxSteps * sizeof(unsigned)) != cudaSuccess || + cudaMalloc((void**) &partner_bytes_, bytes.size() * sizeof(long long)) != cudaSuccess || + cudaMemcpy(partner_bytes_, bytes.data(), bytes.size() * sizeof(long long), cudaMemcpyHostToDevice) != cudaSuccess) { + cudaGetLastError(); + return true; + } + } + { + const OnDevice on(device_); + if (!mapped(kMaxSteps * sizeof(uint32_t), (void**) &h_pdone_, (void**) &m_pdone_)) { cudaGetLastError(); return true; } + } + partner_ = dev; + std::fprintf(stderr, "strata verify: CUDA%d fetches half of CUDA%d's PCIe experts (peer, STRATA_FETCH_PARTNER)\n", + dev, device_); + return true; +} + bool Verifier::release_gpu_waits(int timeout_ms) { released_.store(true); trace_ev("RELEASE", -1, -1, timeout_ms); @@ -398,8 +436,10 @@ bool Verifier::release_gpu_waits(int timeout_ms) { // wait_flag_ge_or also returns on its flag.) A host function raising flag B later only raises. for (uint32_t* p : {h_flag_, h_flagA_, h_flagB_}) if (p != nullptr) *(volatile uint32_t*) p = UINT32_MAX; + if (h_pdone_ != nullptr) // the waits on the partner's share (the partner itself stops on flag A) + for (int k = 0; k < kMaxSteps; ++k) *(volatile uint32_t*) (h_pdone_ + k) = 1; std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); const OnDevice on_device(device_); const Clock::time_point t0 = Clock::now(); for (cudaStream_t s : {cs_, copy_}) { @@ -517,6 +557,15 @@ Verifier::~Verifier() { if (cs_ && cs_ != ext_stream_) cudaStreamDestroy(cs_); // set_stream: the stage's stream, shared, not ours if (sh_cs_) cudaStreamDestroy(sh_cs_); if (copy_) { cudaStreamSynchronize(copy_); cudaStreamDestroy(copy_); } + if (fetch_s_) { cudaStreamSynchronize(fetch_s_); cudaStreamDestroy(fetch_s_); } + if (partner_ >= 0) { + const OnDevice on(partner_); + if (partner_s_) { cudaStreamSynchronize(partner_s_); cudaStreamDestroy(partner_s_); } + if (partner_arrive_) cudaFree(partner_arrive_); + if (partner_bytes_) cudaFree(partner_bytes_); + } + if (ev_plan_) cudaEventDestroy(ev_plan_); + if (ev_fetched_) cudaEventDestroy(ev_fetched_); if (commit_done_) cudaEventDestroy(commit_done_); if (ev_done_) cudaEventDestroy(ev_done_); if (ev_commit_) cudaEventDestroy(ev_commit_); @@ -772,6 +821,23 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState& err = "verify: stream create failed"; return false; } + // The PCIe share's copy kernel (pcie_mode 2) reads the missed experts over the link into staging. On its own + // stream it starts once the plan is in place and runs while the VRAM experts are computed, instead of after them: + // on an IBM AC922 with two GPUs (~5 missed experts per layer-window, ~5 ms of link time per stage-window) the + // two were 7.0 + 2.8 ms of a 22 ms stage, one after the other. The PCIe groups wait for it. The buffers are + // disjoint (staging and the plan's second pointer list vs the VRAM groups' outputs), and the next layer's fetch + // forks after this layer's PCIe groups on the compute stream. STRATA_FETCH_OVERLAP=0: one stream, as before. + { + const char* v = std::getenv("STRATA_FETCH_OVERLAP"); + fetch_overlap_ = v == nullptr || std::atoi(v) != 0; + } + if (fetch_overlap_ && + (cudaStreamCreateWithFlags(&fetch_s_, cudaStreamNonBlocking) != cudaSuccess || + cudaEventCreateWithFlags(&ev_plan_, cudaEventDisableTiming) != cudaSuccess || + cudaEventCreateWithFlags(&ev_fetched_, cudaEventDisableTiming) != cudaSuccess)) { + (void) cudaGetLastError(); + fetch_overlap_ = false; + } if (cudaEventCreateWithFlags(&commit_done_, cudaEventDisableTiming) != cudaSuccess || cudaEventCreateWithFlags(&ev_fork_, cudaEventDisableTiming) != cudaSuccess || cudaEventCreateWithFlags(&ev_join_, cudaEventDisableTiming) != cudaSuccess) { @@ -1092,7 +1158,10 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { a.R = Rt(t); a.R_out = Rt(t); a.apply = apply; a.bo_prev = bo_ + t * N; a.inj_prev = inj_prev + t * HC; a.w_norm = (const float*) wn[half]->data; a.w_down = (const uint16_t*) wd[half]->data; - a.w_up = (const uint16_t*) wu[half]->data; a.w_inject = (const uint16_t*) wi[half]->data; + a.w_up = (const uint16_t*) wu[half]->data; + // STRATA_FP16=load: FP16 down/up; an F32 inject runs after the read, on its FP32 normalized rows + a.w_f16 = wd[half]->kind == WeightKind::F16InF32; + a.w_inject = wi[half]->kind == WeightKind::F32 ? nullptr : (const uint16_t*) wi[half]->data; a.q8_down = (const uint8_t*) wd[half]->hc_q8; a.q8_up = (const uint8_t*) wu[half]->hc_q8; a.q8_inject = (const uint8_t*) wi[half]->hc_q8; a.eps = EPS; a.lo = lo_ + t * g.hc_lr; a.rs = rs_ + t * HC; @@ -1105,6 +1174,9 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { } return fused_gr_read_multi(fa, n, xn_ + (size_t) tb * HC * N, cs, (prof_on_ && grp == 0) ? prof_ : nullptr, (int) (l * kProfPer + (half == 0 ? 27 : 30))); + if (wi[half]->kind == WeightKind::F32) + gemv_fp32_mmvf_multi(xn_ + (size_t) tb * HC * N, HC * N, wi[half]->data, strata::kernels::WForm::F32, + inj_out + tb * HC, HC, HC * N, HC, n, cs); }; const bool q8_attn = gr_read_group(0, pending, inj2_, inj_); // true: xq_ holds mixed's q8_1 (STRATA_QFUSE) { // measurement only: STRATA_DUMMY_NODES=k adds k empty kernel nodes per layer (what one node costs in a window) @@ -1149,11 +1221,12 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { float* gate = gate_L_ + (size_t) gi * MT * HV; float* beta = beta_L_ + (size_t) gi * MT * HV; if (!q8_attn) native_quantize_q8_1(xm, xq_, (int) N, n, cs); + if (wa->kind != wb->kind) { err = "verify: ssm_alpha and ssm_beta are in different forms"; return false; } if (br) { // STRATA_DF_BRANCH: a/b and z beside q/k/v and the conv; all of them read only this layer's input fork(0); - gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data, + gdn_ab_multi(xm, wa->data, wb->data, (const float*) wdt->data, (const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N, - (int) HV, n, df_side_[0]); + (int) HV, n, df_side_[0], wform(wa, "ssm_alpha.weight")); fork(1); native_mmvq(wg->native_type, wg->native_data, xq_, z_ + (size_t) tb * ZV, (int) N, (int) ZV, n, df_side_[1]); @@ -1186,10 +1259,14 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { join(0); join(1); } else { +<<<<<<< this fork if (!fuse_ab) gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data, +======= + gdn_ab_multi(xm, wa->data, wb->data, (const float*) wdt->data, +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) (const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N, (int) HV, - n, cs); + n, cs, wform(wa, "ssm_alpha.weight")); stamp(l, 4, grp); mm(wg, z_ + (size_t) tb * ZV, (int) N, (int) ZV); stamp(l, 5, grp); @@ -1257,6 +1334,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { }(); const bool qb = dec_batch && n > 1 && native_qsa_enabled() && native_rope_enabled() && (!st.kv_q4 || qb_q4); if (!q8_attn) native_quantize_q8_1(xm, xq_, (int) N, n, cs); + const strata::kernels::WForm fik = wform(wik, "indexer.k_proj.weight"); // STRATA_DF_BRANCH: the query side (q and its norm/RoPE/rotation on side 0, the indexer query on side 1) // beside the K/V side; the indexer query joins before the block scores, the query before attention const bool qbr = br && qb; @@ -1269,13 +1347,13 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { norm_rope_on(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH, df_side_[0]); if (st.kv_rot) fwht256_inplace_cuda(qcur_ + tb * NH * HD, (int64_t) n * NH, df_side_[0]); // = fork(1); - bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, - IQ * ID, N, IQ * ID, n, df_side_[1]); + gemv_fp32_mmvf_multi(mixed_ + tb * N, N, wiq->data, wform(wiq, "indexer.q_proj.weight"), + qidx_ + tb * IQ * ID, IQ * ID, N, IQ * ID, n, df_side_[1]); norm_rope_on(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ, df_side_[1]); } - if (qb) bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wik->data, idx_raw + tb * ID, ID, N, ID, n, cs); + if (qb) gemv_fp32_mmvf_multi(mixed_ + tb * N, N, wik->data, fik, idx_raw + tb * ID, ID, N, ID, n, cs); else for (int t = tb; t < te; ++t) - bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wik->data, idx_raw + t * ID, (int) N, (int) ID, cs); + gemv_fp32_mmvf(mixed_ + t * N, wik->data, fik, idx_raw + t * ID, (int) N, (int) ID, cs); stamp(l, 7, grp); mm(wk, kcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD)); mm(wv, vcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD)); @@ -1372,7 +1450,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { norm_rope(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH); } if (st.kv_rot) fwht256_inplace_cuda(qcur_ + tb * NH * HD, (int64_t) n * NH, cs); // = - bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID, + gemv_fp32_mmvf_multi(mixed_ + tb * N, N, wiq->data, wform(wiq, "indexer.q_proj.weight"), qidx_ + tb * IQ * ID, IQ * ID, N, IQ * ID, n, cs); norm_rope(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ); } else { @@ -1393,7 +1471,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { } for (int t = tb; t < te; ++t) { float* qx = qidx_ + t * IQ * ID; - bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), cs); + gemv_fp32_mmvf(mixed_ + t * N, wiq->data, wform(wiq, "indexer.q_proj.weight"), qx, (int) N, (int) (IQ * ID), cs); norm_rope(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH); } } @@ -1472,12 +1550,16 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { bool sg_ready = false; if (dec_batch && n > 1 && w_router != nullptr && native_router_enabled() && NE == 512 && K == 10) { try { - if (w_sgi != nullptr) + // upstream's fused router + shared-gate GEMV reads BF16 only: with STRATA_FP16=load's FP16 / F32 + // gates the router runs on its own (form-aware) and the shared expert computes its gate + const strata::kernels::WForm frt = wform(w_router, "ffn_gate_inp.weight"); + if (w_sgi != nullptr && frt == strata::kernels::WForm::Bf16 && + wform(w_sgi, "ffn_gate_inp_shexp.weight") == strata::kernels::WForm::Bf16) sg_ready = bf16_gemv_fp32_mmvf_multi_aux(mixed_ + tb * N, N, (const uint16_t*) w_router->data, logits_ + tb * NE, NE, N, NE, n, (const uint16_t*) w_sgi->data, sh_g_ + tb, 1, cs); if (!sg_ready) - bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) w_router->data, logits_ + tb * NE, NE, N, + gemv_fp32_mmvf_multi(mixed_ + tb * N, N, w_router->data, frt, logits_ + tb * NE, NE, N, NE, n, cs); native_router_top10_multi(logits_ + tb * NE, ids_ + tb * K, w_ + tb * K, n, cs); } catch (const std::exception& e) { err = "verify router: " + std::string(e.what()); return false; } @@ -1564,10 +1646,11 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { else for (int t = tb; t < te; ++t) f32_to_bf16_bulk(mixed_ + t * N, sh_bf16_ + t * N, N, sh_stream); } try { - shared_expert_multi(n, xm, sh_bf16_ + tb * N, nsw, (const uint16_t*) wgi->data, sh_gate_ + (size_t) tb * g.n_ff, + shared_expert_multi(n, xm, sh_bf16_ + tb * N, nsw, wgi->data, sh_gate_ + (size_t) tb * g.n_ff, sh_up_ + (size_t) tb * g.n_ff, sh_g_ + tb, shared_ + tb * N, N, g.n_ff, sh_stream, qdedup ? (const void*) (nat_xq_ + (size_t) tb * (N / 32) * 36) : nullptr, - (sg_ready ? 1 : 0) | (lfuse_on(n) && g_lfuse_pair() ? 2 : 0)); + (sg_ready ? 1 : 0) | (lfuse_on(n) && g_lfuse_pair() ? 2 : 0), + wform(wgi, "ffn_gate_inp_shexp.weight")); sg_gated_[grp] = sg_ready; } catch (const std::exception& e) { err = std::string("verify shared expert: ") + e.what(); @@ -1639,16 +1722,34 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { } } stamp(l, 19, grp); + // the PCIe share's fetch: on its own stream beside the VRAM experts (fetch_overlap_), and with a layer + // split's partner taking the odd blobs (set_partner) + auto fetch = [&](cudaStream_t s) { + if (device_plan_) wait_flag_ge_or(m_flagB_, ring, skip_ + grp, s); + else wait_flag_ge(m_flagB_, ring, s); // the PCIe share is in staging (DMA) or mapped + if (sink_.pcie_mode == 2) { // stage it with a copy kernel, then point at staging + const int64_t per = G == 2 ? kStagingBlobs / 2 : kStagingBlobs; + uint8_t* stage = staging_ + (size_t) (grp * per) * lay.max_blob; + if (partner_ >= 0) { // the even blobs here, the odd ones by the partner (set_partner) + fetch_blobs_strided(p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), 0, 2, s); + wait_and_clear(m_pdone_ + (ring - 1), s); + } else { + fetch_blobs(p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), (int) per, s); + } + rebase_ptrs((unsigned long long*) p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), s); + } + }; + const bool overlap = fetch_overlap_ && sink_.pcie_mode == 2; + if (overlap) { // the fetch beside the VRAM experts (init: fetch_s_), after the plan's copy above + cudaEventRecord(ev_plan_, cs); + cudaStreamWaitEvent(fetch_s_, ev_plan_, 0); + fetch(fetch_s_); + cudaEventRecord(ev_fetched_, fetch_s_); + } grouped(p_ptr, p_start, p_counts, 0, hit_out); stamp(l, 20, grp); - if (device_plan_) wait_flag_ge_or(m_flagB_, ring, skip_ + grp, cs); - else wait_flag_ge(m_flagB_, ring, cs); // the PCIe share is in staging (DMA) or mapped - if (sink_.pcie_mode == 2) { // stage it with a copy kernel, then point at staging - const int64_t per = G == 2 ? kStagingBlobs / 2 : kStagingBlobs; - uint8_t* stage = staging_ + (size_t) (grp * per) * lay.max_blob; - fetch_blobs(p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), (int) per, cs); - rebase_ptrs((unsigned long long*) p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), cs); - } + if (overlap) cudaStreamWaitEvent(cs, ev_fetched_, 0); + else fetch(cs); stamp(l, 21, grp); // the PCIe share is pcie_frac of the misses: a few groups when the cache is cold, usually none (always none at // pcie_frac 0), so its launch is kPcieGroupRows block rows striding over the groups, not cap of them @@ -1774,10 +1875,19 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { a.mixed = head_mixed_ + t * N; } fused_gr_read_multi(fa, T, xn_, cs); - } else if (head_mix_multi_enabled() && head_ != nullptr && head_->loaded()) { + } else if (head_mix_multi_enabled() && head_ != nullptr && head_->loaded() && + (hd->kind != WeightKind::F16InF32 || [] { + // ac922: with STRATA_FP16=load's FP16 hc weights the one-read mixer rounds a few logits + // differently from the per-token path (1e-4, 8 of 600 positions): opt-in, the default keeps + // the per-token mixer's outputs + static const bool on = [] { const char* v = std::getenv("STRATA_HEAD_MIX_F16"); return v && v[0] == '1'; }(); + return on; + }())) { // the final mixer, the window's tokens in one read (fused_gr_read_multi without the pending write: the // last layer's write was done above; its sums are gr_read's) - if (hn->kind != WeightKind::F32 || hd->kind != WeightKind::Bf16InF32 || hu->kind != WeightKind::Bf16InF32) { + // BF16, or FP16 under STRATA_FP16=load (the read takes either through w_f16, as the layers' hc reads do) + const bool hd16 = hd->kind == WeightKind::F16InF32; + if (hn->kind != WeightKind::F32 || (hd->kind != WeightKind::Bf16InF32 && !hd16) || hu->kind != hd->kind) { err = "verify: the output_hc_* weights have the wrong engine forms"; return false; } @@ -1785,7 +1895,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) { for (int t = 0; t < T; ++t) { fa[t].R = Rt(t); fa[t].R_out = Rt(t); fa[t].apply = false; fa[t].w_norm = (const float*) hn->data; fa[t].w_down = (const uint16_t*) hd->data; - fa[t].w_up = (const uint16_t*) hu->data; fa[t].eps = EPS; + fa[t].w_up = (const uint16_t*) hu->data; fa[t].eps = EPS; fa[t].w_f16 = hd16; fa[t].lo = lo_ + t * g.hc_lr; fa[t].rs = rs_ + t * HC; fa[t].mixed = head_mixed_ + t * N; } fused_gr_read_multi(fa, T, xn_, cs); @@ -2007,8 +2117,10 @@ bool Verifier::capture(int T, std::string& err) { if (!made) return false; const cudaError_t ue = cudaGraphUpload(exec_t, cs_); const cudaError_t us = cudaStreamSynchronize(cs_); - std::fprintf(stderr, "strata verify: captured the %d-token window (upload %s, sync %s)\n", T, - cudaGetErrorString(ue), cudaGetErrorString(us)); + size_t vfree = 0, vtotal = 0; + cudaMemGetInfo(&vfree, &vtotal); + std::fprintf(stderr, "strata verify: captured the %d-token window (upload %s, sync %s; %zu MiB of VRAM free)\n", T, + cudaGetErrorString(ue), cudaGetErrorString(us), vfree >> 20); return true; } @@ -2298,13 +2410,34 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool, const int G = groups_[T] > 0 ? groups_[T] : 1; const int gtb[2] = {0, (T + 1) / 2}, gte[2] = {G == 2 ? (T + 1) / 2 : T, T}; const int64_t steps = (le_ - lb_) * G; + if (partner_ >= 0 && steps > 0 && steps <= kMaxSteps) { // the peer's half of this window's PCIe experts (set_partner) + const OnDevice on(partner_); + const int64_t capx = (int64_t) max_t_ * ss.k; + strata::kernels::PartnerFetchArgs a{}; + a.flagA = h_flagA_; + a.plan = h_plan_; + a.plan_i32 = plan_i32_; + a.ptr_off = ((4 + (capx + 1) + 2 * capx) + 1) & ~1ll; + a.capx = capx; + a.G = G; + a.steps = (int) steps; + a.lb = (int) lb_; + a.blob_bytes = partner_bytes_; + a.staging = staging_; + a.per = G == 2 ? kStagingBlobs / 2 : kStagingBlobs; + a.max_blob = (long long) strata::kernels::cpu::expert_layout().max_blob; + a.done = m_pdone_; + a.arrive = partner_arrive_; + cudaMemsetAsync(partner_arrive_, 0, (size_t) steps * sizeof(unsigned), partner_s_); + strata::kernels::partner_fetch(a, 24, partner_s_); + } const bool test_stall = g_test_stall > 0 && windows + 1 == g_test_stall; // #267 test hook (off: false) if (ar_on() && !test_stall) { if (do_ple) { const Clock::time_point tp = Clock::now(); if (!ss.ple.table->gather_batch(ple_rows, (size_t) T, h_ple_, err)) return false; std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); *flag = 1; ms_host += ms_since(tp); } else { @@ -2320,7 +2453,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool, uint32_t spins = 0; progress_at("verify window: waiting for the GPU to reach layer", l); while (*seq < want) { - _mm_pause(); + strata_cpu_pause(); if ((++spins & 1023u) != 0) continue; const auto now = Clock::now(); if (now - last_flush > std::chrono::microseconds(2000)) { @@ -2376,7 +2509,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool, (int64_t) ms_since(b)); // aux: ms the CPU experts took progress_tick(); std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); if (*(volatile uint32_t*) h_flagA_ != want) { // the pool did not publish a plan: an empty one sink_.counts[0] = 0; sink_.counts[1] = 0; @@ -2395,7 +2528,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool, const Clock::time_point tp = Clock::now(); if (!ss.ple.table->gather_batch(ple_rows, (size_t) T, h_ple_, err)) return false; std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); ms_host += ms_since(tp); } if (!(test_stall && k + 1 == steps)) *flag = want; @@ -2565,7 +2698,7 @@ void Verifier::fetch_dma(void* ctx, const uint8_t* const* src, int n, size_t byt void Verifier::publish_plan(void* ctx) { Verifier* v = (Verifier*) ctx; - _mm_sfence(); + strata_store_fence(); *(volatile uint32_t*) v->h_flagA_ = v->cur_layer_ + 1; } @@ -2680,9 +2813,11 @@ bool Verifier::commit(int n_keep, std::string& err) { if (ct.on) { cudaEventRecord(ct.e, cs_); ct.pending = true; } if (le != cudaSuccess) { err = std::string("verify: commit launch: ") + cudaGetErrorString(le); return false; } // set_commit_async: no wait here - the next window runs on the same stream after it, and the drafter (its own - // stream) reads only this window's final rows and its own K/V. h_commit_ is next written after the next window's - // results are read, i.e. after this graph has run. Everything else waits on commit_done_ (wait_commit). - if (!g_commit_async || next_ != nullptr) { + // stream) reads only this window's final rows and its own K/V. A layer split's stages each launch theirs on their + // own stream the same way (ac922, 9098768: they used to wait one after the other, four launch + sync round trips + // a window). h_commit_ is next written after the next window's results are read, i.e. after this graph has run. + // Everything else waits on commit_done_ (wait_commit). + if (!g_commit_async) { const cudaError_t se = cudaStreamSynchronize(cs_); if (se != cudaSuccess) { err = std::string("verify: commit: ") + cudaGetErrorString(se); return false; } } else { @@ -3039,7 +3174,7 @@ bool Verifier::run_slot_rows(const int* rows, int S, const int32_t* tokens, cons if (ar_on()) { if (ss_->ple.ready() && ple_stage()) { std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); *flag = 1; } const cudaError_t se = cudaStreamSynchronize(cs_); @@ -3063,7 +3198,7 @@ bool Verifier::run_slot_rows(const int* rows, int S, const int32_t* tokens, cons uint32_t spins = 0; progress_at("verify batch: waiting for the GPU to reach layer", l); while (*seq < want) { - _mm_pause(); + strata_cpu_pause(); if ((++spins & 1023u) != 0) continue; const auto now = Clock::now(); if (now - last_flush > std::chrono::microseconds(2000)) { @@ -3091,7 +3226,7 @@ bool Verifier::run_slot_rows(const int* rows, int S, const int32_t* tokens, cons if (remote_opt_) remote_opt_->end(); progress_tick(); std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); if (*(volatile uint32_t*) h_flagA_ != want) { // the pool did not publish a plan: an empty one sink_.counts[0] = 0; sink_.counts[1] = 0; @@ -3215,7 +3350,7 @@ int Verifier::batch_poll(PoolMultiFn pool, void* user, std::string& err) { // doorbell graph needs the per-layer service below. if (ar_on() && ss_->ple.ready() && ple_stage()) { std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); *h_flag_ = 1; } while (!ar_on() && b_k_ < b_steps_) { @@ -3246,7 +3381,7 @@ int Verifier::batch_poll(PoolMultiFn pool, void* user, std::string& err) { if (remote_opt_) remote_opt_->end(); progress_tick(); std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); if (*(volatile uint32_t*) h_flagA_ != want) { // the pool did not publish a plan: an empty one sink_.counts[0] = 0; sink_.counts[1] = 0; @@ -3479,7 +3614,7 @@ int Verifier::service(PoolMultiFn pool, void* user, std::string& err) { const Clock::time_point tp = Clock::now(); if (!ss.ple.table->gather_batch(pl_ple_rows_.data(), (size_t) T, h_ple_, err)) return false; std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); ms_host += ms_since(tp); return true; }; @@ -3530,7 +3665,7 @@ int Verifier::service(PoolMultiFn pool, void* user, std::string& err) { (int64_t) ms_since(b)); progress_tick(); std::atomic_thread_fence(std::memory_order_seq_cst); - _mm_sfence(); + strata_store_fence(); if (*(volatile uint32_t*) h_flagA_ != want) { // the pool did not publish a plan: an empty one sink_.counts[0] = 0; sink_.counts[1] = 0; @@ -3604,7 +3739,7 @@ bool Verifier::pl_commit_async(int n_keep, std::string& err) { const Clock::time_point t0 = Clock::now(); if (commit_live_) { // its graph reads the words below when it starts: the previous one has (a window ago) cudaError_t q; - while ((q = cudaEventQuery(ev_commit_)) == cudaErrorNotReady) _mm_pause(); + while ((q = cudaEventQuery(ev_commit_)) == cudaErrorNotReady) strata_cpu_pause(); if (q != cudaSuccess) { err = std::string("verify: commit: ") + cudaGetErrorString(q); return false; } } h_commit_[0] = n_keep; From 8c736c181a70800d1c35834235b7e94a7b5ee815 Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:02 +0200 Subject: [PATCH 04/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): src/kernels from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 21 file(s) under src/kernels -- 14 clean, 7 with conflict markers. Conflict-marked files -- a pull request with conflicts: resolve them, do not discard the work: src/kernels/cpu/native_expert.cpp src/kernels/cpu/pool.cpp src/kernels/cuda/dequant_bf16.cu src/kernels/cuda/fused_gr.cu src/kernels/cuda/iq_kernels.cu src/kernels/cuda/verify_kernels.cu src/kernels/ngram.cpp Original authors: 4EverBuilder,Andy,BlueKingMuch,Chanyeong Lim Original commits touching these paths (151 subject(s)): - #152 as an opt-in (the owner's choice): STRATA_IQ_MT_MIN=1 makes CPU expert rounding independent of the drafting; the default stays 0.1.29's rule (IQ3_S decoded 1-3% slower with it on); docs; the width test runs the opt-in - #289 follow-up: a placeholder GPU name when the properties query fails; HIP builds compile (no device_code_error probe) - #315 follow-up: its variants and switch renamed (plain / split / staged, STRATA_HC_SPLIT) - no clash with STRATA_GR_V3 - #646 on 0.1.39b: the layer split and the CPU path as before, the resident plan for 8-token windows - #646: the single-matrix mmvq does not stage the IQ grids (10-20% slower on the RTX 5070 at 3-8 columns) - AMD HIP backend for gfx1100 (PR #121, squashed onto its 0.1.21 base) - AMD HIP: RDNA2 gfx1030 (tested), gfx(1031, 1032) untested. - C-1, C-2: the prompt path's QSA select grid reaches only the active blocks; the indexer appends a chunk at once Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: src/kernels Fork-Replay-Automated: true --- src/kernels/cpu/expert_layout.cpp | 31 ++- src/kernels/cpu/kq_vsx.cpp | 194 +++++++++++++ src/kernels/cpu/native_expert.cpp | 19 ++ src/kernels/cpu/pool.cpp | 31 ++- src/kernels/cpu/portable.cpp | 198 +++++++++++++ src/kernels/cpu/router_dot_parity.cpp | 2 + src/kernels/cuda/dequant_bf16.cu | 6 + src/kernels/cuda/elementwise.cu | 10 +- src/kernels/cuda/fused_expert.cu | 372 +++++++++++++++++++++++++ src/kernels/cuda/fused_gdn.cu | 49 +++- src/kernels/cuda/fused_gr.cu | 360 +++++++++++++++++------- src/kernels/cuda/gr.cu | 17 +- src/kernels/cuda/iq_kernels.cu | 66 ++++- src/kernels/cuda/native_bf16.cu | 121 ++++---- src/kernels/cuda/ple.cu | 3 +- src/kernels/cuda/qsa_prompt_attn.cu | 277 ++++++++++++++++++ src/kernels/cuda/qsa_select.cu | 141 ++++++++++ src/kernels/cuda/shared_expert.cu | 26 +- src/kernels/cuda/verify_kernels.cu | 267 +++++++++++++++--- src/kernels/ngram.cpp | 33 +++ src/kernels/qsa_prompt_attn_parity.cpp | 21 +- 21 files changed, 2019 insertions(+), 225 deletions(-) create mode 100644 src/kernels/cpu/kq_vsx.cpp create mode 100644 src/kernels/cpu/portable.cpp create mode 100644 src/kernels/cuda/fused_expert.cu diff --git a/src/kernels/cpu/expert_layout.cpp b/src/kernels/cpu/expert_layout.cpp index 001fa5d47cd..0f556dbaceb 100644 --- a/src/kernels/cpu/expert_layout.cpp +++ b/src/kernels/cpu/expert_layout.cpp @@ -6,10 +6,11 @@ #include #include #include +#include "strata/platform/cpu_relax.hpp" #if defined(_MSC_VER) #include #include -#else +#elif defined(STRATA_X86) #include #endif #include @@ -38,6 +39,32 @@ int cpu_isa_cap() { return cap; } +#if !defined(STRATA_X86) +// aarch64 / other: no AVX kernels exist in this build (CMake leaves their files out); the experts run on ggml-cpu's +// own (NEON) dot products. +bool cpu_avx512_ok() { return false; } +bool cpu_avx2_ok() { return false; } +bool cpu_avx1_ok() { return false; } +bool cpu_sse42_ok() { return false; } +// the AVX-2 gather decode (iq_avx2.cpp) does not exist here +int iq256_gather_setting() { return 0; } +bool cpu_gather_fast() { return false; } +bool cpu_gather_fast_here() { return false; } +std::string cpu_name() { + std::ifstream f("/proc/cpuinfo"); + std::string line; + while (std::getline(f, line)) + if (line.rfind("model name", 0) == 0 || line.rfind("Model name", 0) == 0) { + const size_t c = line.find(':'); + if (c != std::string::npos) return line.substr(line.find_first_not_of(' ', c + 1)); + } +#if defined(__aarch64__) + return "aarch64"; +#else + return "unknown"; +#endif +} +#else bool cpu_avx512_ok() { static const bool ok = [] { if (const char* f = std::getenv("STRATA_FORCE_AVX2"); f != nullptr && f[0] == '1') return false; @@ -353,6 +380,8 @@ void cpu_require_expert_support() { std::exit(1); } +#endif // STRATA_X86 + void q2_rows_any(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, int nt, float* const* out, int r0, int r1) { if (cpu_avx512_ok()) q2_0_gguf_rows_multi(w, row_bytes, nblocks, a, nt, out, r0, r1); diff --git a/src/kernels/cpu/kq_vsx.cpp b/src/kernels/cpu/kq_vsx.cpp new file mode 100644 index 00000000000..42ac75d8a4b --- /dev/null +++ b/src/kernels/cpu/kq_vsx.cpp @@ -0,0 +1,194 @@ +// src/kernels/cpu/kq_vsx.cpp - see kq_vsx.hpp. Measured on an IBM AC922 (one POWER9 core, 3.8 GHz, vs ggml-cpu's POWER9 +// vec_dot): 640 Q4_K rows 1.7x / 2.1x / 2.3x at 1 / 2 / 3 tokens, 2560 Q5_1 rows 4.3x / 6.5x / 7.7x. +#include "strata/kernels/cpu/kq_vsx.hpp" + +#include +#include +#include + +#if defined(__POWER9_VECTOR__) +#include +#undef vector +#undef bool +#undef pixel + +namespace strata::kernels::cpu { +namespace { +struct bq4K { uint16_t d, dmin; uint8_t scales[12]; uint8_t qs[128]; }; +struct bq8K { float d; int8_t qs[256]; int16_t bsums[16]; }; +struct bq51 { uint16_t d, m; uint8_t qh[4]; uint8_t qs[16]; }; +struct bq81 { uint16_t d, s; int8_t qs[32]; }; +static_assert(sizeof(bq4K) == 144 && sizeof(bq8K) == 292 && sizeof(bq51) == 24 && sizeof(bq81) == 36, "layout"); + +typedef __vector signed char vs8; +typedef __vector unsigned char vu8; +typedef __vector signed short vs16; +typedef __vector unsigned short vu16; +typedef __vector signed int vs32; +typedef __vector unsigned int vu32; +typedef __vector float vf32; +inline float h2f(uint16_t h) { + const vu16 v = {h, 0, 0, 0, 0, 0, 0, 0}; + return vec_extract_fp32_from_shorth(v)[0]; +} + +// four int32x4 partial-sum vectors -> one vector of their four totals +static inline vs32 reduce4(vs32 a, vs32 b, vs32 c, vs32 d) { + const vs32 ab = vec_add(vec_mergeh(a, b), vec_mergel(a, b)); // a0+a2, b0+b2, a1+a3, b1+b3 + const vs32 cd = vec_add(vec_mergeh(c, d), vec_mergel(c, d)); + const vs32 lo = (vs32) vec_mergeh((__vector unsigned long long) ab, (__vector unsigned long long) cd); // a0+a2 b0+b2 c0+c2 d0+d2 + const vs32 hi = (vs32) vec_mergel((__vector unsigned long long) ab, (__vector unsigned long long) cd); // a1+a3 b1+b3 c1+c3 d1+d3 + return vec_add(lo, hi); +} +static inline float hsum(vf32 v) { + v = vec_add(v, vec_sld(v, v, 8)); + v = vec_add(v, vec_sld(v, v, 4)); + return vec_extract(v, 0); +} + +// ---- Q4_K x Q8_K: rows [r0, r1) of a row-major Q4_K matrix (nsb superblocks per row) for nt tokens +constexpr int MT = 8; // MAXT +void q4k_rows(const uint8_t* W, size_t row_bytes, int nsb, const bq8K* const* y, int nt, float* const* out, int r0, int r1) { + const vu8 m4 = vec_splats((unsigned char) 0xF), s4 = vec_splats((unsigned char) 4); + const vs32 z = vec_splats(0); + for (int r = r0; r < r1; ++r) { + const bq4K* x = (const bq4K*) (W + (size_t) r * row_bytes); + vf32 acc[MT]; + for (int t = 0; t < nt; ++t) acc[t] = vec_splats(0.0f); + for (int b = 0; b < nsb; ++b) { + const bq4K& xb = x[b]; + // scales and mins: ggml's utmp unpacking in GPRs, then one vector load of the 16 bytes + uint32_t u[4]; + memcpy(u, xb.scales, 12); + u[3] = ((u[2] >> 4) & 0x0f0f0f0f) | (((u[1] >> 6) & 0x03030303) << 4); + const uint32_t ua = u[1] & 0x3f3f3f3f; + u[1] = (u[2] & 0x0f0f0f0f) | (((u[0] >> 6) & 0x03030303) << 4); + u[2] = ua; + u[0] &= 0x3f3f3f3f; + const vu8 utm = vec_xl(0, (const unsigned char*) u); // scales 0..7, mins 0..7 (all < 64) + const vs16 scs = (vs16) vec_mergeh(utm, vec_splats((unsigned char) 0)), mns = (vs16) vec_mergel(utm, vec_splats((unsigned char) 0)); + const vs32 SC0 = vec_unpackh(scs), SC1 = vec_unpackl(scs); + const vs16 MN0 = vec_mergeh(mns, mns), MN1 = vec_mergel(mns, mns); + const vu16 hd = {xb.d, xb.dmin, 0, 0, 0, 0, 0, 0}; + const vf32 fd = vec_extract_fp32_from_shorth(hd); + const float xd = fd[0], xm = fd[1]; + vu8 q[16]; // per 64-value chunk j: lo0, lo1, hi0, hi1 +#pragma GCC unroll 4 + for (int j = 0; j < 4; ++j) { + const vu8 a = vec_xl(0, xb.qs + 32 * j), c = vec_xl(16, xb.qs + 32 * j); + q[4 * j + 0] = vec_and(a, m4); q[4 * j + 1] = vec_and(c, m4); + q[4 * j + 2] = vec_sr(a, s4); q[4 * j + 3] = vec_sr(c, s4); + } + for (int t = 0; t < nt; ++t) { + const bq8K& yb = y[t][b]; + vs32 P[8]; +#pragma GCC unroll 4 + for (int j = 0; j < 4; ++j) { + const int8_t* yq = yb.qs + 64 * j; + P[2 * j] = vec_msum(vec_xl(16, yq), q[4 * j + 1], vec_msum(vec_xl(0, yq), q[4 * j + 0], z)); + P[2 * j + 1] = vec_msum(vec_xl(48, yq), q[4 * j + 3], vec_msum(vec_xl(32, yq), q[4 * j + 2], z)); + } + const vs32 R0 = reduce4(P[0], P[1], P[2], P[3]), R1 = reduce4(P[4], P[5], P[6], P[7]); + const vs32 isum = vec_add(vec_mul(R0, SC0), vec_mul(R1, SC1)); + const vs16 B0 = vec_xl(0, yb.bsums), B1 = vec_xl(16, yb.bsums); + const vs32 msum = vec_msum(MN1, B1, vec_msum(MN0, B0, z)); + const vf32 vd = vec_splats(xd * yb.d), vm = vec_splats(xm * yb.d); + acc[t] = vec_madd(vec_ctf(isum, 0), vd, acc[t]); + acc[t] = vec_nmsub(vec_ctf(msum, 0), vm, acc[t]); + } + } + for (int t = 0; t < nt; ++t) out[t][r] = hsum(acc[t]); + } +} + +// ---- Q5_1 x Q8_1: rows [r0, r1), nb blocks per row (a multiple of 4), nt tokens +void q51_rows(const uint8_t* W, size_t row_bytes, int nb, const bq81* const* y, int nt, float* const* out, int r0, int r1) { + const vu8 m4 = vec_splats((unsigned char) 0xF), s4 = vec_splats((unsigned char) 4), b10 = vec_splats((unsigned char) 0x10); + const vu8 bits = {1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128}; + const vu8 sel0 = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1}, sel1 = {2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3}; + const vs32 z = vec_splats(0); + // per token: the activation scales d and s as floats, four blocks to a vector + float yd[MT][32], ys[MT][32]; + for (int t = 0; t < nt; ++t) + for (int b = 0; b < nb; ++b) { yd[t][b] = h2f(y[t][b].d); ys[t][b] = h2f(y[t][b].s); } + for (int r = r0; r < r1; ++r) { + const bq51* x = (const bq51*) (W + (size_t) r * row_bytes); + vf32 acc[MT]; + for (int t = 0; t < nt; ++t) acc[t] = vec_splats(0.0f); + for (int b = 0; b < nb; b += 4) { + // the four 24-byte blocks as six vectors; d/m and qh gathered by permutes + const uint8_t* g = (const uint8_t*) (x + b); + const vu8 V0 = vec_xl(0, g), V1 = vec_xl(16, g), V2 = vec_xl(32, g), V3 = vec_xl(48, g), V4 = vec_xl(64, g), V5 = vec_xl(80, g); + const vu8 sp = {0, 1, 24, 25, 2, 3, 26, 27, 4, 5, 6, 7, 28, 29, 30, 31}; // d_a d_b m_a m_b qh_a qh_b + const vu8 P01 = vec_perm(V0, V1, sp), P23 = vec_perm(V3, V4, sp); + const vu8 sdm = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}; + const vu8 sqh = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31}; + const vu16 dm = (vu16) vec_perm(P01, P23, sdm); // d0 d1 d2 d3 m0 m1 m2 m3 + const vu8 QH = vec_perm(P01, P23, sqh); // qh0 qh1 qh2 qh3 (4 bytes each) + const vf32 D = vec_extract_fp32_from_shorth(dm), M = vec_extract_fp32_from_shortl(dm); + const vu8 smid = {8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}; + const vu8 QS[4] = {vec_perm(V0, V1, smid), V2, vec_perm(V3, V4, smid), V5}; + vu8 qa[4], qb[4]; +#pragma GCC unroll 4 + for (int k = 0; k < 4; ++k) { + // byte j of the expansion holds bit j of qh (lanes 0-15: bits 0-15, then bits 16-31), moved to 0x10 + const vu8 s0 = {(unsigned char) (4 * k), (unsigned char) (4 * k), (unsigned char) (4 * k), (unsigned char) (4 * k), + (unsigned char) (4 * k), (unsigned char) (4 * k), (unsigned char) (4 * k), (unsigned char) (4 * k), + (unsigned char) (4 * k + 1), (unsigned char) (4 * k + 1), (unsigned char) (4 * k + 1), (unsigned char) (4 * k + 1), + (unsigned char) (4 * k + 1), (unsigned char) (4 * k + 1), (unsigned char) (4 * k + 1), (unsigned char) (4 * k + 1)}; + const vu8 s1 = vec_add(s0, vec_splats((unsigned char) 2)); + const vu8 h0 = vec_and(vec_cmpeq(vec_and(vec_perm(QH, QH, s0), bits), bits), b10); + const vu8 h1 = vec_and(vec_cmpeq(vec_and(vec_perm(QH, QH, s1), bits), bits), b10); + qa[k] = vec_or(vec_and(QS[k], m4), h0); + qb[k] = vec_or(vec_sr(QS[k], s4), h1); + } + for (int t = 0; t < nt; ++t) { + vs32 S[4]; +#pragma GCC unroll 4 + for (int k = 0; k < 4; ++k) { + const int8_t* yq = y[t][b + k].qs; + S[k] = vec_msum(vec_xl(16, yq), qb[k], vec_msum(vec_xl(0, yq), qa[k], z)); + } + const vs32 tot = reduce4(S[0], S[1], S[2], S[3]); + acc[t] = vec_madd(vec_ctf(tot, 0), vec_mul(D, vec_xl(0, yd[t] + b)), acc[t]); + acc[t] = vec_madd(M, vec_xl(0, ys[t] + b), acc[t]); + } + } + for (int t = 0; t < nt; ++t) out[t][r] = hsum(acc[t]); + } +} + +} // namespace + +bool kq_vsx_on() noexcept { + static const bool on = [] { const char* v = std::getenv("STRATA_VSX_EXPERTS"); return v != nullptr && v[0] && v[0] != '0'; }(); + return on; +} +void kq_vsx_gu_rows(const uint8_t* blob, size_t gu_row, size_t up_off, int n_embd, const void* const* act, int nt, + float* const* ff, int r0, int r1) { + const bq8K* y[MT]; + for (int t = 0; t < nt; ++t) y[t] = (const bq8K*) act[t]; + float g[MT], u[MT]; + float* gp[MT]; + float* up[MT]; + for (int r = r0; r < r1; ++r) { + for (int t = 0; t < nt; ++t) { gp[t] = g + t - r; up[t] = u + t - r; } // out[t][r] lands in g[t] / u[t] + q4k_rows(blob, gu_row, n_embd / 256, y, nt, gp, r, r + 1); + q4k_rows(blob + up_off, gu_row, n_embd / 256, y, nt, up, r, r + 1); + for (int t = 0; t < nt; ++t) ff[t][r] = (g[t] / (1.f + std::exp(-g[t]))) * u[t]; + } +} +void kq_vsx_q51_rows(const uint8_t* W, size_t row_bytes, int n_ff, const void* const* hq, int nt, float* const* out, + int r0, int r1) { + const bq81* y[MT]; + for (int t = 0; t < nt; ++t) y[t] = (const bq81*) hq[t]; + q51_rows(W, row_bytes, n_ff / 32, y, nt, out, r0, r1); +} +} // namespace strata::kernels::cpu +#else +namespace strata::kernels::cpu { +bool kq_vsx_on() noexcept { return false; } +void kq_vsx_gu_rows(const uint8_t*, size_t, size_t, int, const void* const*, int, float* const*, int, int) { std::abort(); } +void kq_vsx_q51_rows(const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) { std::abort(); } +} // namespace strata::kernels::cpu +#endif diff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cpp index 1ab75b642bf..2b55fb7672f 100644 --- a/src/kernels/cpu/native_expert.cpp +++ b/src/kernels/cpu/native_expert.cpp @@ -7,8 +7,12 @@ #include "strata/kernels/cpu/iq_avx2.hpp" #include "strata/kernels/cpu/iq_avx1.hpp" #include "strata/kernels/cpu/kq_avx2.hpp" +<<<<<<< this fork #include "strata/kernels/cpu/nvfp4_avx2.hpp" #include "strata/kernels/cpu/nvfp4_avx512.hpp" +======= +#include "strata/kernels/cpu/kq_vsx.hpp" +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) #include "strata/kernels/cpu/expert_layout.hpp" #include "ggml.h" @@ -37,6 +41,7 @@ bool native_experts_available() noexcept { return true; } // compiled for AVX-512 and iq_avx2.cpp / kq_avx2.cpp for AVX2, and native_gu_rows asks these on every CPU before it // has checked what the CPU can run. A function in a wide-ISA file may use that ISA anywhere in its body, so an // answer that is only a comparison must not come from one (#795). +#if defined(__x86_64__) || defined(_M_X64) // ac922: elsewhere portable.cpp answers (none of these kernels exist) bool iq512_supported(int type) noexcept { return type == 16 || type == 17 || type == 18 || type == 21 || type == 22; } @@ -46,6 +51,7 @@ bool iq256_supported(int type) noexcept { } bool kq256_supported(int type) noexcept { return type == 12 || type == 7 || type == 8; } +#endif bool native_fmt(int gu_type, int d_type, int64_t n_embd, int64_t n_ff, NativeFmt& f, std::string& err) { init_once(); @@ -169,6 +175,11 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const* // size, no #152 rule). Opt-in, STRATA_KQ256=1: measured no faster in the engine (a window's expert groups hold // ~1.4 tokens and the weights stay in L1 across ggml's per-token calls; 1.01-1.13x in native_expert_parity). static const bool kq = [] { const char* v = std::getenv("STRATA_KQ256"); return cpu_avx2_ok() && v != nullptr && std::atoi(v) != 0; }(); + // POWER9 (STRATA_VSX_EXPERTS=1): Q4_K gate/up through the VSX multi-token kernel, any group size + if (f.gu_type == 12 && f.gu_act == 15 && kq_vsx_on()) { + kq_vsx_gu_rows(blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1); + return; + } if (kq && f.gu_type == 12 && nt >= 2) { // one token: ggml's own dot below (the same bits, less overhead) kq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1); return; @@ -234,9 +245,17 @@ void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const int r0, int r1) { // IQ4_NL down rows: the AVX-2 multi-token kernel decodes the nibbles and absolutises the weights once per // block instead of once per token; ggml-cpu's dot is single-token. STRATA_NO_IQ4NL falls back to it. +#if defined(__x86_64__) || defined(_M_X64) static const bool iq4nl_mt = std::getenv("STRATA_NO_IQ4NL") == nullptr; +#else + static const bool iq4nl_mt = false; // the multi-token kernel is AVX-2 (portable.cpp has none) +#endif static const int mt_min = [] { const char* e = std::getenv("STRATA_IQ_MT_MIN"); return e ? std::atoi(e) : 2; }(); static const bool kq = [] { const char* v = std::getenv("STRATA_KQ256"); return v != nullptr && std::atoi(v) != 0; }(); + if (f.d_type == 7 && f.d_act == 9 && f.n_ff % 128 == 0 && f.n_ff <= 1024 && kq_vsx_on()) { // POWER9: Q5_1 down + kq_vsx_q51_rows(blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1); + return; + } // Both multi-token kernels below are /arch:AVX2 translation units (kq_avx2.cpp and iq_avx2.cpp), // so a CPU without AVX2 has to reach ggml-cpu's vec_dot instead - same reasoning as the gate/up // rows above, where `avx512` tested cpu_avx512_ok() and `avx2` did not. diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cpp index 3bf8c93f651..e559446f8e3 100644 --- a/src/kernels/cpu/pool.cpp +++ b/src/kernels/cpu/pool.cpp @@ -8,10 +8,15 @@ #include #include #include +<<<<<<< this fork #include #if defined(_MSC_VER) #include // __cpuidex, __rdtsc (STRATA_POOL_UMWAIT) #endif +======= +#include "strata/platform/cpu_relax.hpp" +#include "strata/platform/thread_affinity.hpp" +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) #include #include @@ -338,7 +343,16 @@ static CpuTopology detect_cpu_topology_impl(bool skip_first, PoolAffinity affini [&](const CoreLinux& x, const CoreLinux& y) { return is_e(x) != is_e(y) ? !is_e(x) : x.cap > y.cap; }); - for (const auto& cl : all_cpus) { + // STRATA_POOL_SMT=k (opt-in): k hardware threads of each physical core, core by core (POWER9: a core's + // two halves each run one thread at full speed, measured 1.95x a core's expert throughput at k = 2) + static const int smt = [] { const char* e = std::getenv("STRATA_POOL_SMT"); return e ? (std::max)(1, std::atoi(e)) : 1; }(); + if (smt > 1) { + for (const auto& key : seen_phys) { + int taken = 0; + for (const auto& cl : all_cpus) + if (cl.pkg == key.first && cl.core == key.second && taken < smt) { topo.worker_cores.push_back(cl.cpu); ++taken; } + } + } else for (const auto& cl : all_cpus) { if (!cl.is_sibling) topo.worker_cores.push_back(cl.cpu); } if (skip_first && !topo.worker_cores.empty()) { @@ -420,6 +434,12 @@ ThreadAffinity pin_current_thread(int core) { return {}; } previous.valid = true; + { // the threads this one creates take the previous set back (platform::release_inherited_pin) + cpu_set_t prev; + CPU_ZERO(&prev); + std::memcpy(&prev, previous.mask.data(), std::min(sizeof prev, previous.mask.size() * sizeof(unsigned long))); + strata::platform::note_host_pin(prev, core); + } return previous; #endif } @@ -554,6 +574,7 @@ void ExpertPool::worker(int id) { #endif while (epoch_.load(std::memory_order_acquire) == seen) { if (stop_.load(std::memory_order_relaxed)) return; +<<<<<<< this fork #if defined(_MSC_VER) && defined(_M_X64) && !defined(__clang__) // MSVC proper: clang-cl (the Windows HIP build) has no WAITPKG intrinsics by default if (um != 0) { // STRATA_POOL_UMWAIT: wake on the publish's write to epoch_, or after `um` cycles _umonitor((void*) &epoch_); @@ -568,6 +589,10 @@ void ExpertPool::worker(int id) { _mm_pause(); if ((++spins & 1023u) != 0) continue; } +======= + strata_cpu_pause(); + if ((++spins & 1023u) != 0) continue; +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) if (std::chrono::steady_clock::now() - parked_at < spin_before_sleep_) continue; std::unique_lock lk(sleep_mu_); wstate_[(size_t) id].store(kSleeping, std::memory_order_relaxed); @@ -624,7 +649,7 @@ void ExpertPool::wait_parked(const char* what) { uint32_t spins = 0; std::chrono::steady_clock::time_point t0{}; while (parked_.load(std::memory_order_acquire) != (uint32_t) n_) { - _mm_pause(); + strata_cpu_pause(); if ((++spins & 1023u) != 0) continue; const auto now = std::chrono::steady_clock::now(); if (spins == 1024u) t0 = now; @@ -647,7 +672,7 @@ void ExpertPool::wait_done(int n) { for (;;) { const uint32_t d = done_.load(std::memory_order_acquire); if (d >= (uint32_t) n) return; // `>=`: never a wait that an overshoot outlives - _mm_pause(); + strata_cpu_pause(); if ((++spins & 1023u) != 0) continue; const auto now = std::chrono::steady_clock::now(); if (spins == 1024u || d != seen) { t0 = now; seen = d; } // progress restarts the clock diff --git a/src/kernels/cpu/portable.cpp b/src/kernels/cpu/portable.cpp new file mode 100644 index 00000000000..fcfb83ea8b1 --- /dev/null +++ b/src/kernels/cpu/portable.cpp @@ -0,0 +1,198 @@ +// src/kernels/cpu/portable.cpp - the CPU kernel symbols on a non-x86 build (aarch64: DGX Spark / GB10, Grace). +// +// Replaces expert.cpp, q2_avx2.cpp, iq_avx2.cpp, iq_avx512.cpp and kq_avx2.cpp, which are AVX-only. What a native +// (GGUF) pack needs runs here: the activation quantization (the scalar loop of expert.cpp, bitwise the same as its +// AVX-512 version) and the BF16 router dot. The i-quant / k-quant multi-token kernels report themselves +// unsupported, so native_expert.cpp takes ggml-cpu's vec_dot, which has NEON paths. The canonical Q2_0 pack (its +// own blob layout, and the Q2_0 GGUF down rows of ISTA-DASLab's files) is refused: its kernels are AVX-512 only. +#include "strata/kernels/cpu/expert.hpp" +#include "strata/kernels/cpu/iq_avx2.hpp" +#include "strata/kernels/cpu/iq_avx512.hpp" +#include "strata/kernels/cpu/kq_avx2.hpp" + +#include +#include +#include +#include +#include + +#if defined(__ARM_NEON) +#include +#elif defined(__VSX__) +#include // the types spelled __vector: a strict -std=c++20 has no `vector` keyword +#endif + +extern "C" void quantize_row_q8_K_ref(const float* x, void* y, int64_t k); // ggml-quants.h (ggml-base) + +namespace strata::kernels::cpu { +namespace { + +std::atomic oracle_q8_0{false}; + +[[noreturn]] void unsupported(const char* what) { + std::fprintf(stderr, + "strata: %s is x86-only (AVX2/AVX-512) and this build has no portable version.\n" + " On this CPU use a native GGUF pack whose experts are not Q2_0 (IQ2_S/IQ3_S/IQ4_NL/Q4_K/...).\n", + what); + std::abort(); +} + +inline float bf16f(uint16_t h) { + const uint32_t b = (uint32_t) h << 16; + float f; + std::memcpy(&f, &b, 4); + return f; +} + +} // namespace + +const char* CpuFeatures::reason() const { return usable() ? "ok" : "not an x86 CPU (no AVX-512)"; } + +CpuFeatures cpu_features() { return CpuFeatures{}; } + +void cpu_require_expert_support() { + std::fprintf(stderr, "strata: the canonical Q2_0 pack's expert kernel needs AVX512-VNNI/VBMI (x86).\n" + " On this CPU, use a native GGUF pack (tools/iq_pack.py) instead.\n"); + std::exit(1); +} + +void expert_set_oracle_q8_0(bool enabled) { oracle_q8_0.store(enabled, std::memory_order_relaxed); } +bool expert_oracle_q8_0_enabled() { return oracle_q8_0.load(std::memory_order_relaxed); } + +void act_quant_q8_1(const float* x, int n, ActQ& a) { + // expert.cpp's scalar loop: round half away from zero, clamp to +-127. + a.nchunks = n / QKA; + for (int k = 0; k < a.nchunks; ++k) { + const float* xb = x + k * QKA; + float amax = 0.f; + for (int j = 0; j < QKA; ++j) amax = std::fmax(amax, std::fabs(xb[j])); + const float s = amax > 0.f ? amax / 127.f : 0.f; + const float inv = s > 0.f ? 1.f / s : 0.f; + int32_t sum = 0; + int8_t* q = a.q + k * QKA; + for (int j = 0; j < QKA; ++j) { + const float t = xb[j] * inv; + const float r = t + (t >= 0.f ? 0.5f : -0.5f); + int v = (int) r; + v = v < -127 ? -127 : (v > 127 ? 127 : v); + q[j] = (int8_t) v; + sum += v; + } + a.scale[k] = s; + a.sum[k] = sum; + a.hx[k] = s * (float) sum; + } +} + +void act_quant_q8_1_avx2(const float* x, int n, ActQ& a) { act_quant_q8_1(x, n, a); } +// ggml's scalar Q8_K quantizer (the bytes q8k_quant_avx2 matches); not reached here, cpu_avx2_ok() is false +void q8k_quant_avx2(const float* x, void* y, int64_t k) { quantize_row_q8_K_ref(x, y, k); } + +void s2_expert_vnni(const uint8_t*, const float*, float*, ExpertScratch&) { unsupported("the Q2_0 expert kernel"); } +void s2_expert_vnni_q(const uint8_t*, const ActQ&, float*, ExpertScratch&) { unsupported("the Q2_0 expert kernel"); } +void s2_expert_gu_rows(const uint8_t*, const ActQ&, float*, int, int) { unsupported("the Q2_0 expert kernel"); } +void s2_expert_down_rows(const uint8_t*, const ActQ&, float*, int, int) { unsupported("the Q2_0 expert kernel"); } +void s2_expert_gu_rows_multi(const uint8_t*, const ActQ* const*, int, float* const*, int, int) { + unsupported("the Q2_0 expert kernel"); +} +void s2_expert_down_rows_multi(const uint8_t*, const ActQ* const*, int, float* const*, int, int) { + unsupported("the Q2_0 expert kernel"); +} +void s2_expert_vnni_multi(const uint8_t*, const ActQ* const*, int, float* const*, ExpertScratchMulti&) { + unsupported("the Q2_0 expert kernel"); +} +void q2_0_gguf_rows_multi(const uint8_t*, size_t, int, const ActQ* const*, int, float* const*, int, int) { + unsupported("the Q2_0 GGUF row kernel"); +} +void q2_0_gguf_rows_multi_avx2(const uint8_t*, size_t, int, const ActQ* const*, int, float* const*, int, int) { + unsupported("the Q2_0 GGUF row kernel"); +} +void s2_expert_scalar(const uint8_t*, const float*, float*, bool) { unsupported("the Q2_0 scalar oracle"); } + +bool iq512_supported(int) noexcept { return false; } +void iq512_gu_rows(int, const uint8_t*, size_t, size_t, int, const void* const*, int, float* const*, int, int) { + unsupported("the AVX-512 i-quant kernel"); +} +void iq512_rows(int, const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) { + unsupported("the AVX-512 i-quant kernel"); +} +bool iq256_supported(int) noexcept { return false; } +void iq256_gu_rows(int, const uint8_t*, size_t, size_t, int, const void* const*, int, float* const*, int, int) { + unsupported("the AVX-2 i-quant kernel"); +} +void iq256_rows(int, const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) { + unsupported("the AVX-2 i-quant kernel"); +} +void iq4nl256_down_rows(const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) { + unsupported("the AVX-2 IQ4_NL kernel"); +} +bool kq256_supported(int) noexcept { return false; } +void kq256_gu_rows(int, const uint8_t*, size_t, size_t, int, const void* const*, int, float* const*, int, int) { + unsupported("the AVX-2 k-quant kernel"); +} +void kq256_rows(int, const uint8_t*, size_t, int, const void* const*, int, float* const*, int, int) { + unsupported("the AVX-2 k-quant kernel"); +} + +void bf16_rows_dot(const uint16_t* w, int rows, int cols, const float* x, float* out) { + bf16_rows_dot_multi(w, rows, cols, x, 1, out); +} + +void bf16_rows_dot_multi(const uint16_t* w, int rows, int cols, const float* x, int nt, float* out) { +#if defined(__VSX__) && !defined(__ARM_NEON) + // POWER (VSX): as kq_avx2.cpp's AVX2 version, each row read once for all `nt` (<= 8) tokens, here as two 4-wide + // FMA accumulators per token. A BF16 is the high half of an F32: merging zeros below each u16 (little-endian) + // makes the F32 lanes. Its only caller is the router lookahead (a page prefetch hint), so the summation order + // differing from the scalar loop's changes no answer. + if (nt >= 1 && nt <= 8) { + const __vector unsigned short z = vec_splats((unsigned short) 0); + const int c8 = cols & ~7; + for (int r = 0; r < rows; ++r) { + const uint16_t* wr = w + (size_t) r * (size_t) cols; + __vector float a0[8], a1[8]; + for (int t = 0; t < nt; ++t) a0[t] = a1[t] = vec_splats(0.0f); + for (int c = 0; c < c8; c += 8) { + const __vector unsigned short h = vec_xl(0, wr + c); + const __vector float w0 = (__vector float) vec_mergeh(z, h); // lanes c .. c+3 + const __vector float w1 = (__vector float) vec_mergel(z, h); // lanes c+4 .. c+7 + for (int t = 0; t < nt; ++t) { + const float* xt = x + (size_t) t * cols + c; + a0[t] = vec_madd(w0, vec_xl(0, xt), a0[t]); + a1[t] = vec_madd(w1, vec_xl(16, xt), a1[t]); + } + } + for (int t = 0; t < nt; ++t) { + const __vector float a = vec_add(a0[t], a1[t]); + float s = (a[0] + a[1]) + (a[2] + a[3]); + const float* xt = x + (size_t) t * cols; + for (int c = c8; c < cols; ++c) s += bf16f(wr[c]) * xt[c]; + out[(size_t) t * rows + r] = s; + } + } + return; + } +#endif + for (int r = 0; r < rows; ++r) { + const uint16_t* wr = w + (size_t) r * (size_t) cols; + for (int t = 0; t < nt; ++t) { + const float* xt = x + (size_t) t * cols; + int c = 0; + float s = 0.f; +#if defined(__ARM_NEON) + float32x4_t acc0 = vdupq_n_f32(0.f), acc1 = vdupq_n_f32(0.f); + for (; c + 8 <= cols; c += 8) { + const uint16x8_t h = vld1q_u16(wr + c); + const float32x4_t w0 = vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(h), 16)); + const float32x4_t w1 = vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(h), 16)); + acc0 = vfmaq_f32(acc0, w0, vld1q_f32(xt + c)); + acc1 = vfmaq_f32(acc1, w1, vld1q_f32(xt + c + 4)); + } + s = vaddvq_f32(vaddq_f32(acc0, acc1)); +#endif + for (; c < cols; ++c) s += bf16f(wr[c]) * xt[c]; + out[(size_t) t * rows + r] = s; + } + } +} + +} // namespace strata::kernels::cpu diff --git a/src/kernels/cpu/router_dot_parity.cpp b/src/kernels/cpu/router_dot_parity.cpp index bd1341b9497..663a8f529c9 100644 --- a/src/kernels/cpu/router_dot_parity.cpp +++ b/src/kernels/cpu/router_dot_parity.cpp @@ -64,7 +64,9 @@ int main() { std::printf("router_dot_parity: %s (avx2 %d, avx %d)\n", cpu::cpu_name().c_str(), (int) cpu::cpu_avx2_ok(), (int) cpu::cpu_avx1_ok()); if (cpu::cpu_avx2_ok()) run("avx2", cpu::bf16_rows_dot_multi); +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) if (cpu::cpu_avx1_ok()) run("avx", cpu::bf16_rows_dot_multi_avx1); +#endif std::printf("router_dot_parity: %d failures\n", failures); return failures == 0 ? 0 : 1; } diff --git a/src/kernels/cuda/dequant_bf16.cu b/src/kernels/cuda/dequant_bf16.cu index 97759ae7c41..b070aac59c8 100644 --- a/src/kernels/cuda/dequant_bf16.cu +++ b/src/kernels/cuda/dequant_bf16.cu @@ -175,6 +175,7 @@ __global__ void dequant_kernel(const uint8_t* __restrict__ blocks, int64_t row_b group32(blocks + (row0 + r) * row_bytes, (int) gi, out + r * ld + gi * 32); } +<<<<<<< this fork // ---- the wide kernels: a thread per 8 values (Pascal, sm_61) -------------------------------------------------------- // dequant_kernel's thread reads its 32-value group a byte at a time and writes 32 two-byte values one by one, 64 bytes // from its neighbour's. On a Tesla P40 (sm_61) that is 7.2 ms for a 10240 x 2560 Q6_K weight (10 GB/s), and the IQ3_S @@ -281,6 +282,8 @@ bool dequant_wide_on() { return per_dev[dev] == 2; } +======= +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) // Q8_0 with four threads per block, 8 values each: the warp reads 8 consecutive blocks and writes 512 consecutive // bytes (a thread per block wrote 2-byte values 64 bytes apart: ~110 GB/s, a quarter of a prompt projection's time) template @@ -343,6 +346,7 @@ void launch(int type, const void* blocks, int64_t row0, int64_t rows, int64_t co if (e != cudaSuccess) { std::fprintf(stderr, "dequant launch: %s\n", cudaGetErrorString(e)); std::exit(1); } return; } +<<<<<<< this fork // the wide kernels: even block addresses, 16-byte stores at every row (contiguous or a stride of whole stores) if (wide_type(type) && (reinterpret_cast(blocks) & 1) == 0 && (reinterpret_cast(out) & 15) == 0 && ((size_t) ld * sizeof(T)) % 16 == 0 && 4 * total < (1ll << 31) && dequant_wide_on()) { @@ -359,6 +363,8 @@ void launch(int type, const void* blocks, int64_t row0, int64_t rows, int64_t co if (e != cudaSuccess) { std::fprintf(stderr, "dequant launch: %s\n", cudaGetErrorString(e)); std::exit(1); } return; } +======= +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) #define STRATA_DQ(TY) dequant_kernel<<>>(p, row_bytes, row0, rows, gpr, ld, out); break switch (type) { case 2: STRATA_DQ(2); diff --git a/src/kernels/cuda/elementwise.cu b/src/kernels/cuda/elementwise.cu index a67550a1f76..661fd0fb2ce 100644 --- a/src/kernels/cuda/elementwise.cu +++ b/src/kernels/cuda/elementwise.cu @@ -6,6 +6,7 @@ #include "strata/kernels/f16_bits.hpp" #include +#include #include #include @@ -311,7 +312,14 @@ void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream) { __global__ void doorbell_publish_kernel(const float* __restrict__ x, const int32_t* __restrict__ ids, const float* __restrict__ w, int n, int k, float* x_out, int32_t* ids_out, float* w_out, uint32_t* seq) { - for (int i = threadIdx.x; i < n; i += blockDim.x) x_out[i] = x[i]; + // 16-byte stores when both sides allow (a quarter of the writes across the link to mapped memory) + if ((n & 3) == 0 && (((uintptr_t) x | (uintptr_t) x_out) & 15u) == 0) { + const float4* x4 = reinterpret_cast(x); + float4* o4 = reinterpret_cast(x_out); + for (int i = threadIdx.x; i < n / 4; i += blockDim.x) o4[i] = x4[i]; + } else { + for (int i = threadIdx.x; i < n; i += blockDim.x) x_out[i] = x[i]; + } if ((int) threadIdx.x < k) { ids_out[threadIdx.x] = ids[threadIdx.x]; if (w_out != nullptr) w_out[threadIdx.x] = w[threadIdx.x]; diff --git a/src/kernels/cuda/fused_expert.cu b/src/kernels/cuda/fused_expert.cu new file mode 100644 index 00000000000..2739f2072fe --- /dev/null +++ b/src/kernels/cuda/fused_expert.cu @@ -0,0 +1,372 @@ +// Fused prefill expert products: see fused_expert.hpp. +#include "strata/kernels/fused_expert.hpp" + +#include +#include + +#include +#include +#include + +namespace strata::kernels { +namespace { + +// a block: 4 warps (2 x 2), TM tokens x BN weight rows, K in steps of BK; each warp a 32 x 32 tile, and its 16-row +// halves past the expert's last token are skipped. Two threads dequantize a weight row. +constexpr int TM = 64, BN = 64, BK = 64, LDE = BN + 4, NT = 128, TPR = NT / BN; +// the X and W tiles are [rows][BK] halves without padding, their 16-byte chunks XOR-swizzled by the row: the fragment +// loads of a quarter warp read rows r..r+3 and r+8..r+11 (or + 16, 24), which then fall in 8 distinct chunks +__device__ __forceinline__ int swz(int r) { return (r & 3) | (((r >> 3) & 1) << 2); } +__device__ __forceinline__ int soff(int r, int c) { return r * BK + ((c ^ swz(r)) << 3); } +constexpr int SMEM = (TM + BN) * BK * 2 > TM * LDE * 4 ? (TM + BN) * BK * 2 : TM * LDE * 4; +using Acc = float[8]; // an m8n8k4 accumulator + +struct Tiles { + const uint8_t* blob[kFusedExpertMax]; + int64_t row0[kFusedExpertMax]; + int ne[kFusedExpertMax], tile0[kFusedExpertMax + 1]; + int n; +}; + +// get_scale_min_k4 on the 12 scale bytes held in registers (s0..s2): a byte-indexed array would put the block header +// in local memory, and that store waits on the prefetch's global load +__device__ __forceinline__ int sbyte(uint32_t s0, uint32_t s1, uint32_t s2, int b) { + const uint32_t w = b < 4 ? s0 : (b < 8 ? s1 : s2); + return (int) ((w >> (8 * (b & 3))) & 0xFF); +} +__device__ __forceinline__ void scale_min_k4(int j, uint32_t s0, uint32_t s1, uint32_t s2, int& d, int& m) { + if (j < 4) { d = sbyte(s0, s1, s2, j) & 63; m = sbyte(s0, s1, s2, j + 4) & 63; } + else { + const int a = sbyte(s0, s1, s2, j + 4), lo = sbyte(s0, s1, s2, j - 4), hi = sbyte(s0, s1, s2, j); + d = (a & 0x0F) | ((lo >> 6) << 4); m = (a >> 4) | ((hi >> 6) << 4); + } +} +// SwiGLU as swiglu_il_kernel computes it (hf_sat included) +__device__ __forceinline__ __half silu_mul(float g, float u) { + const float h = g / (1.0f + __expf(-g)) * u; + return __float2half(isnan(h) ? h : fminf(fmaxf(h, -65504.0f), 65504.0f)); +} +__device__ __forceinline__ void locate(const Tiles& g, int nrt, int& e, int& rt, int& ne, int64_t& row0) { + const int t = (int) blockIdx.x; + e = 0; + while (e + 1 < g.n && g.tile0[e + 1] <= t) ++e; + const int tt = (t - g.tile0[e]) / nrt; + rt = (t - g.tile0[e]) % nrt; + ne = min(TM, g.ne[e] - tt * TM); + row0 = g.row0[e] + (int64_t) tt * TM; +} + +// the X tile [TM][BK] through registers (fetched one step ahead); rows >= ne are zero +struct XRegs { uint4 v[TM * BK / 8 / NT]; }; +__device__ __forceinline__ void fetch_x(XRegs& x, const __half* X, int64_t ld, int ne, int64_t k0) { +#pragma unroll + for (int j = 0; j < TM * BK / 8 / NT; ++j) { + const int i = threadIdx.x + NT * j, r = i / (BK / 8), c = (i % (BK / 8)) * 8; + x.v[j] = r < ne ? *(const uint4*) (X + r * ld + k0 + c) : make_uint4(0, 0, 0, 0); + } +} +__device__ __forceinline__ void commit_x(__half* Xs, const XRegs& x) { +#pragma unroll + for (int j = 0; j < TM * BK / 8 / NT; ++j) { + const int i = threadIdx.x + NT * j, r = i / (BK / 8); + *(uint4*) &Xs[soff(r, i % (BK / 8))] = x.v[j]; + } +} +__device__ __forceinline__ void mma884(Acc& d, uint32_t a0, uint32_t a1, uint32_t b0, uint32_t b1) { + asm("mma.sync.aligned.m8n8k4.row.col.f32.f16.f16.f32 {%0,%1,%2,%3,%4,%5,%6,%7}, {%8,%9}, {%10,%11}, " + "{%0,%1,%2,%3,%4,%5,%6,%7};\n" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]), "+f"(d[4]), "+f"(d[5]), "+f"(d[6]), "+f"(d[7]) + : "r"(a0), "r"(a1), "r"(b0), "r"(b1)); +} +// a warp's 32 x 32 tile as 4 x 4 blocks of 8 x 8; quad pair q computes M blocks (q & 1) + 2 i and N blocks +// 2 (q >> 1) + j, its lane t (0..7) holding row t of A and column t of B (4 k values each, one 8-byte piece) +__device__ __forceinline__ void qp_coords(int& q, int& t) { + const int lane = threadIdx.x & 31; + q = (lane >> 2) & 3; + t = (lane & 3) + ((lane >> 4) << 2); +} +// the step's products in fresh accumulators, then added to the running sum with ordinary FP32 adds: Volta's tensor +// cores truncate when they accumulate, and a 160-step chain (K = 2560) of truncations biases the sum toward zero. +// Rows past the expert's last token (ne) are skipped 16 at a time. +__device__ __forceinline__ void mma_tile(const __half* Xs, const __half* Ws, Acc (&tot)[2][2], int ne) { + const int w = threadIdx.x / 32, wm = w / 2, wn = w % 2; + if (32 * wm >= ne) return; + const bool f1 = 32 * wm + 16 < ne; // warp-uniform + int q, t; + qp_coords(q, t); + const int ra0 = 32 * wm + 8 * (q & 1) + t, ra1 = ra0 + 16, rb0 = 32 * wn + 16 * (q >> 1) + t, rb1 = rb0 + 8; + Acc acc[2][2]; +#pragma unroll + for (int i = 0; i < 2; ++i) +#pragma unroll + for (int j = 0; j < 2; ++j) +#pragma unroll + for (int e = 0; e < 8; ++e) acc[i][j][e] = 0.0f; + // independent accumulators back to back (a dependent HMMA waits out the previous one's latency) + if (f1) { +#pragma unroll + for (int c = 0; c < BK / 8; ++c) { // a 16-byte chunk: 8 k values, two k4 steps + const uint4 b0 = *(const uint4*) &Ws[soff(rb0, c)], b1 = *(const uint4*) &Ws[soff(rb1, c)]; + const uint4 a0 = *(const uint4*) &Xs[soff(ra0, c)], a1 = *(const uint4*) &Xs[soff(ra1, c)]; + mma884(acc[0][0], a0.x, a0.y, b0.x, b0.y); + mma884(acc[0][1], a0.x, a0.y, b1.x, b1.y); + mma884(acc[1][0], a1.x, a1.y, b0.x, b0.y); + mma884(acc[1][1], a1.x, a1.y, b1.x, b1.y); + mma884(acc[0][0], a0.z, a0.w, b0.z, b0.w); + mma884(acc[0][1], a0.z, a0.w, b1.z, b1.w); + mma884(acc[1][0], a1.z, a1.w, b0.z, b0.w); + mma884(acc[1][1], a1.z, a1.w, b1.z, b1.w); + } + } else { +#pragma unroll + for (int c = 0; c < BK / 8; ++c) { + const uint4 b0 = *(const uint4*) &Ws[soff(rb0, c)], b1 = *(const uint4*) &Ws[soff(rb1, c)]; + const uint4 a0 = *(const uint4*) &Xs[soff(ra0, c)]; + mma884(acc[0][0], a0.x, a0.y, b0.x, b0.y); + mma884(acc[0][1], a0.x, a0.y, b1.x, b1.y); + mma884(acc[0][0], a0.z, a0.w, b0.z, b0.w); + mma884(acc[0][1], a0.z, a0.w, b1.z, b1.w); + } + } +#pragma unroll + for (int i = 0; i < 2; ++i) +#pragma unroll + for (int j = 0; j < 2; ++j) +#pragma unroll + for (int e = 0; e < 8; ++e) tot[i][j][e] += acc[i][j][e]; +} +// the m8n8k4 FP32 accumulator layout: element e of lane l is row (l & 1) + (e & 2) (+ 4 for l >= 16), column +// (e & 4) + (l & 2) + (e & 1) +__device__ __forceinline__ void store_acc(float (*E)[LDE], Acc (&acc)[2][2]) { + const int w = threadIdx.x / 32, wm = w / 2, wn = w % 2, lane = threadIdx.x & 31; + int q, t; + qp_coords(q, t); + (void) t; +#pragma unroll + for (int i = 0; i < 2; ++i) +#pragma unroll + for (int j = 0; j < 2; ++j) +#pragma unroll + for (int e = 0; e < 8; ++e) { + const int r = 32 * wm + 8 * ((q & 1) + 2 * i) + (lane & 1) + (e & 2) + ((lane >> 4) << 2); + const int c = 32 * wn + 8 * (2 * (q >> 1) + j) + (e & 4) + (lane & 2) + (e & 1); + E[r][c] = acc[i][j][e]; + } +} + +// gate/up (Q4_K; interleaved row 2r = gate r, 2r + 1 = up r) and SwiGLU -> H +// GT: Q4_K (12) or Q5_K (13, the fifth bit of each value in qh[32]: bit 2 jj + hi of qh[l] adds 16) +template +__global__ void __launch_bounds__(NT) gu_kernel(Tiles g, FusedExpertLayout L, const __half* __restrict__ X, + __half* __restrict__ H) { + constexpr int BB = GT == 12 ? 144 : 176, QS = GT == 12 ? 16 : 48; // block bytes, quants offset + extern __shared__ __align__(16) unsigned char sm[]; + __half* Xs = (__half*) sm; + __half* Ws = (__half*) (sm + TM * BK * 2); + int e, rt, ne; + int64_t row0; + locate(g, (int) (2 * L.n_ff / BN), e, rt, ne, row0); + Acc acc[2][2]; +#pragma unroll + for (int i = 0; i < 2; ++i) +#pragma unroll + for (int j = 0; j < 2; ++j) +#pragma unroll + for (int e = 0; e < 8; ++e) acc[i][j][e] = 0.0f; + // TPR threads per weight row: the low (sub-block 2 jj) or high (2 jj + 1) nibbles of a 64-value chunk, NV 16-byte + // pieces of its 32 quant bytes each + constexpr int NV = 4 / TPR; + const int wr = threadIdx.x / TPR, hi = threadIdx.x % 2, v0 = (threadIdx.x % TPR) / 2 * NV, R = rt * BN + wr; + const uint8_t* rowp = g.blob[e] + ((R & 1) ? L.up_off : 0) + (size_t) (R >> 1) * L.gu_row; + const __half* Xb = X + row0 * L.n_embd; + XRegs xr; + uint4 qr[NV], hr, qhr[GT == 13 ? NV : 1]; + auto fetch_w = [&](int64_t k0) { + const uint8_t* b = rowp + (size_t) (k0 / 256) * BB; + const int jj = (int) (k0 % 256) / 64; + hr = *(const uint4*) b; // dm, scales[12] +#pragma unroll + for (int v = 0; v < NV; ++v) qr[v] = ((const uint4*) (b + QS + 32 * jj))[v0 + v]; + if constexpr (GT == 13) { +#pragma unroll + for (int v = 0; v < NV; ++v) qhr[v] = ((const uint4*) (b + 16))[v0 + v]; // qh[32] + } + }; + auto commit_w = [&](int64_t k0) { + const int jj = (int) (k0 % 256) / 64; + const float2 dm = __half22float2(*(const __half2*) &hr.x); + int s, m; + scale_min_k4(2 * jj + hi, hr.y, hr.z, hr.w, s, m); + // dq_q4_k's formulas: d1 = dall * sc, m1 = dmin * m, value d1 * q - m1 + const float d1 = dm.x * (uint8_t) s, m1 = dm.y * (uint8_t) m; + const int sh = 4 * hi; + +#pragma unroll + for (int v = 0; v < NV; ++v) { + const uint8_t* q = (const uint8_t*) &qr[v]; + __align__(16) __half o[16]; +#pragma unroll + for (int l = 0; l < 16; ++l) { + if constexpr (GT == 13) { // dq_q5_k's formula + const uint8_t* qh = (const uint8_t*) &qhr[v]; + const uint8_t hm = (uint8_t) (1u << (2 * jj + hi)); + o[l] = __float2half(d1 * (((q[l] >> sh) & 0xF) + (qh[l] & hm ? 16 : 0)) - m1); + } else { + o[l] = __float2half(d1 * ((q[l] >> sh) & 0xF) - m1); + } + } + *(uint4*) &Ws[soff(wr, hi * 4 + 2 * (v0 + v))] = *(const uint4*) &o[0]; + *(uint4*) &Ws[soff(wr, hi * 4 + 2 * (v0 + v) + 1)] = *(const uint4*) &o[8]; + } + }; + fetch_x(xr, Xb, L.n_embd, ne, 0); + fetch_w(0); + for (int64_t k0 = 0; k0 < L.n_embd; k0 += BK) { + commit_x(Xs, xr); + commit_w(k0); + __syncthreads(); + if (k0 + BK < L.n_embd) { fetch_x(xr, Xb, L.n_embd, ne, k0 + BK); fetch_w(k0 + BK); } + mma_tile(Xs, Ws, acc, ne); + __syncthreads(); + } + float (*E)[LDE] = (float (*)[LDE]) sm; + store_acc(E, acc); + __syncthreads(); + for (int i = threadIdx.x; i < TM * (BN / 2); i += NT) { + const int r = i / (BN / 2), c = i % (BN / 2); + if (r < ne) H[(row0 + r) * L.n_ff + rt * (BN / 2) + c] = silu_mul(E[r][2 * c], E[r][2 * c + 1]); + } +} + +// down (Q5_1 = 7 or Q8_0 = 8) -> D (FP32) +template +__global__ void __launch_bounds__(NT) down_kernel(Tiles g, FusedExpertLayout L, const __half* __restrict__ H, + float* __restrict__ D) { + extern __shared__ __align__(16) unsigned char sm[]; + __half* Xs = (__half*) sm; + __half* Ws = (__half*) (sm + TM * BK * 2); + int e, rt, ne; + int64_t row0; + locate(g, (int) (L.n_embd / BN), e, rt, ne, row0); + Acc acc[2][2]; +#pragma unroll + for (int i = 0; i < 2; ++i) +#pragma unroll + for (int j = 0; j < 2; ++j) +#pragma unroll + for (int e = 0; e < 8; ++e) acc[i][j][e] = 0.0f; + // TPR threads per weight row: one of the step's two 32-value blocks (hi), NI of its values (from i0) each + constexpr int NI = 32 / (TPR / 2); + const int wr = threadIdx.x / TPR, hi = threadIdx.x % 2, i0 = (threadIdx.x % TPR) / 2 * NI; + constexpr int BB = DT == 7 ? 24 : 34; // block bytes + const uint8_t* rowp = g.blob[e] + L.down_off + (size_t) (rt * BN + wr) * L.d_row; + const __half* Hb = H + row0 * L.n_ff; + XRegs xr; + uint2 w5[3]; // Q5_1: d, m, qh, qs[16] (8-byte aligned) + uint16_t w8[17]; // Q8_0: d, qs[32] (2-byte aligned) + auto fetch_w = [&](int64_t k0) { + const uint8_t* b = rowp + (size_t) (k0 / 32 + hi) * BB; + if constexpr (DT == 7) { + w5[0] = ((const uint2*) b)[0]; w5[1] = ((const uint2*) b)[1]; w5[2] = ((const uint2*) b)[2]; + } else { +#pragma unroll + for (int i = 0; i < 17; ++i) w8[i] = ((const uint16_t*) b)[i]; + } + }; + auto commit_w = [&]() { + __align__(16) __half y[NI]; + if constexpr (DT == 7) { + // dq_q5_1's formulas; value i and i + 16 come from byte i: this thread's bytes are i0 / 2 .. + NI / 2 + const float2 dm = __half22float2(*(const __half2*) &w5[0].x); + const uint32_t qh = w5[0].y; + const uint8_t* q = (const uint8_t*) &w5[1]; + const int b0 = i0 / 2; +#pragma unroll + for (int l = 0; l < NI / 2; ++l) { + const int i = b0 + l; + const int xh_0 = ((qh >> (i + 0)) << 4) & 0x10, xh_1 = ((qh >> (i + 12))) & 0x10; + y[l] = __float2half((float) ((q[i] & 0xf) | xh_0) * dm.x + dm.y); + y[l + NI / 2] = __float2half((float) ((q[i] >> 4) | xh_1) * dm.x + dm.y); + } +#pragma unroll + for (int l = 0; l < NI / 16; ++l) { + *(uint4*) &Ws[soff(wr, hi * 4 + b0 / 8 + l)] = *(const uint4*) &y[8 * l]; + *(uint4*) &Ws[soff(wr, hi * 4 + 2 + b0 / 8 + l)] = *(const uint4*) &y[NI / 2 + 8 * l]; + } + } else { + const float d = __half2float(*(const __half*) &w8[0]); + const int8_t* q = (const int8_t*) &w8[1]; +#pragma unroll + for (int i = 0; i < NI; ++i) y[i] = __float2half((float) q[i0 + i] * d); +#pragma unroll + for (int i = 0; i < NI / 8; ++i) *(uint4*) &Ws[soff(wr, hi * 4 + i0 / 8 + i)] = *(const uint4*) &y[8 * i]; + } + }; + fetch_x(xr, Hb, L.n_ff, ne, 0); + fetch_w(0); + for (int64_t k0 = 0; k0 < L.n_ff; k0 += BK) { + commit_x(Xs, xr); + commit_w(); + __syncthreads(); + if (k0 + BK < L.n_ff) { fetch_x(xr, Hb, L.n_ff, ne, k0 + BK); fetch_w(k0 + BK); } + mma_tile(Xs, Ws, acc, ne); + __syncthreads(); + } + float (*E)[LDE] = (float (*)[LDE]) sm; + store_acc(E, acc); + __syncthreads(); + for (int i = threadIdx.x; i < TM * BN; i += NT) { + const int r = i / BN, c = i % BN; + if (r < ne) D[(row0 + r) * L.n_embd + rt * BN + c] = E[r][c]; + } +} + +void check(const char* what) { + const cudaError_t e = cudaGetLastError(); + if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e)); std::exit(1); } +} +} // namespace + +bool fused_expert_supported(int gu_type, int d_type, int64_t n_embd, int64_t n_ff) noexcept { + return (gu_type == 12 || gu_type == 13) && (d_type == 7 || d_type == 8) && n_embd % 256 == 0 && n_ff % BN == 0 && n_ff % BK == 0; +} + +void fused_expert_run(const FusedExpertLayout& L, const FusedExpertGroup& in, const uint16_t* X, uint16_t* H, float* D, + void* stream) { + if (in.n <= 0) return; + if (!fused_expert_supported(L.gu_type, L.d_type, L.n_embd, L.n_ff) || in.n > kFusedExpertMax) { + std::fprintf(stderr, "fused_expert_run: types %d/%d, %d experts\n", L.gu_type, L.d_type, in.n); + std::exit(1); + } + static const bool once = [] { + cudaFuncSetAttribute(gu_kernel<12>, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM); + cudaFuncSetAttribute(gu_kernel<13>, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM); + cudaFuncSetAttribute(down_kernel<7>, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM); + cudaFuncSetAttribute(down_kernel<8>, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM); + return true; + }(); + (void) once; + Tiles gu{}, dn{}; + int n = 0; + for (int i = 0; i < in.n; ++i) { + if (in.ne[i] <= 0) continue; + gu.blob[n] = dn.blob[n] = in.blob[i]; + gu.row0[n] = dn.row0[n] = in.row0[i]; + gu.ne[n] = dn.ne[n] = in.ne[i]; + const int tt = (in.ne[i] + TM - 1) / TM; + gu.tile0[n + 1] = gu.tile0[n] + tt * (int) (2 * L.n_ff / BN); + dn.tile0[n + 1] = dn.tile0[n] + tt * (int) (L.n_embd / BN); + ++n; + } + if (n == 0) return; + gu.n = dn.n = n; + const cudaStream_t s = (cudaStream_t) stream; + if (L.gu_type == 13) gu_kernel<13><<>>(gu, L, (const __half*) X, (__half*) H); + else gu_kernel<12><<>>(gu, L, (const __half*) X, (__half*) H); + check("fused_expert gate/up"); + if (L.d_type == 7) down_kernel<7><<>>(dn, L, (const __half*) H, D); + else down_kernel<8><<>>(dn, L, (const __half*) H, D); + check("fused_expert down"); +} + +} // namespace strata::kernels diff --git a/src/kernels/cuda/fused_gdn.cu b/src/kernels/cuda/fused_gdn.cu index 7de1a3ea22d..5157db903f7 100644 --- a/src/kernels/cuda/fused_gdn.cu +++ b/src/kernels/cuda/fused_gdn.cu @@ -2,6 +2,7 @@ #include "strata/kernels/fused_gdn.hpp" #include "strata/core/geometry.hpp" +#include #include #include @@ -98,24 +99,43 @@ __global__ void __launch_bounds__(S) gdn_conv_l2_kernel(float* __restrict__ hist h[c] = y; } -__global__ void __launch_bounds__(256) gdn_ab_kernel(const float* __restrict__ x, const uint16_t* __restrict__ wa, - const uint16_t* __restrict__ wb, const float* __restrict__ dt, +// 8 weights of a row (16-byte chunk j of a 16-bit row; 32 bytes of an F32 one), widened to FP32 (STRATA_FP16=load) +template +__device__ __forceinline__ void ab_wload8(const void* base, int j, float* w) { + if constexpr (WF == 2) { + const float4* p = reinterpret_cast(base) + 2 * j; + const float4 a = __ldg(p), b = __ldg(p + 1); + w[0] = a.x; w[1] = a.y; w[2] = a.z; w[3] = a.w; w[4] = b.x; w[5] = b.y; w[6] = b.z; w[7] = b.w; + } else { + const uint4 v = __ldg(reinterpret_cast(base) + j); + const uint32_t u[4] = {v.x, v.y, v.z, v.w}; +#pragma unroll + for (int k = 0; k < 4; ++k) { + if constexpr (WF == 0) { w[2 * k] = __uint_as_float(u[k] << 16); w[2 * k + 1] = __uint_as_float(u[k] & 0xffff0000u); } + else { w[2 * k] = __half2float(__ushort_as_half((uint16_t) u[k])); w[2 * k + 1] = __half2float(__ushort_as_half((uint16_t) (u[k] >> 16))); } + } + } +} +template +__global__ void __launch_bounds__(256) gdn_ab_kernel(const float* __restrict__ x, const void* __restrict__ wa, + const void* __restrict__ wb, const float* __restrict__ dt, const float* __restrict__ ssm_a, float* __restrict__ gate, float* __restrict__ beta, int n, int h_v) { const int row = blockIdx.x * 8 + (threadIdx.x >> 5), lane = threadIdx.x & 31; if (row >= 2 * h_v) return; const bool is_beta = row >= h_v; const int r = is_beta ? row - h_v : row; - const uint4* w4 = reinterpret_cast((is_beta ? wb : wa) + (size_t) r * n); + const void* wrow = reinterpret_cast(is_beta ? wb : wa) + (size_t) r * n * (WF == 2 ? 4 : 2); float acc = 0.0f; for (int j = lane; j < n / 8; j += 32) { - const uint4 wv = __ldg(w4 + j); + float w[8]; + ab_wload8(wrow, j, w); const float4 xa = *reinterpret_cast(x + j * 8); const float4 xb = *reinterpret_cast(x + j * 8 + 4); - acc = fmaf(__uint_as_float(wv.x << 16), xa.x, acc); acc = fmaf(__uint_as_float(wv.x & 0xffff0000u), xa.y, acc); - acc = fmaf(__uint_as_float(wv.y << 16), xa.z, acc); acc = fmaf(__uint_as_float(wv.y & 0xffff0000u), xa.w, acc); - acc = fmaf(__uint_as_float(wv.z << 16), xb.x, acc); acc = fmaf(__uint_as_float(wv.z & 0xffff0000u), xb.y, acc); - acc = fmaf(__uint_as_float(wv.w << 16), xb.z, acc); acc = fmaf(__uint_as_float(wv.w & 0xffff0000u), xb.w, acc); + acc = fmaf(w[0], xa.x, acc); acc = fmaf(w[1], xa.y, acc); + acc = fmaf(w[2], xa.z, acc); acc = fmaf(w[3], xa.w, acc); + acc = fmaf(w[4], xb.x, acc); acc = fmaf(w[5], xb.y, acc); + acc = fmaf(w[6], xb.z, acc); acc = fmaf(w[7], xb.w, acc); } for (int o = 16; o > 0; o >>= 1) acc += __shfl_xor_sync(0xffffffffu, acc, o); if (lane != 0) return; @@ -141,14 +161,19 @@ void fused_gdn_conv_l2(float* history, const float* qkv, const float* conv_w, fl if (e != cudaSuccess) { std::fprintf(stderr, "fused_gdn_conv_l2: %s\n", cudaGetErrorString(e)); std::exit(1); } } -void fused_gdn_ab(const float* x, const uint16_t* w_alpha, const uint16_t* w_beta, const float* dt, const float* ssm_a, - float* gate, float* beta, int n_embd, int h_v, void* stream) { +void fused_gdn_ab(const float* x, const void* w_alpha, const void* w_beta, const float* dt, const float* ssm_a, + float* gate, float* beta, int n_embd, int h_v, void* stream, WForm form) { if (!x || !w_alpha || !w_beta || !dt || !ssm_a || !gate || !beta || n_embd % 8 != 0 || h_v <= 0) { std::fprintf(stderr, "fused_gdn_ab: invalid arguments\n"); std::exit(1); } - gdn_ab_kernel<<<(unsigned) ((2 * h_v + 7) / 8), 256, 0, (cudaStream_t) stream>>>(x, w_alpha, w_beta, dt, ssm_a, gate, - beta, n_embd, h_v); + const unsigned blocks = (unsigned) ((2 * h_v + 7) / 8); + if (form == WForm::Bf16) + gdn_ab_kernel<0><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v); + else if (form == WForm::F16) + gdn_ab_kernel<1><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v); + else + gdn_ab_kernel<2><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v); const cudaError_t e = cudaGetLastError(); if (e != cudaSuccess) { std::fprintf(stderr, "fused_gdn_ab: %s\n", cudaGetErrorString(e)); std::exit(1); } } diff --git a/src/kernels/cuda/fused_gr.cu b/src/kernels/cuda/fused_gr.cu index c3c2fc8d9d3..401b1646ccd 100644 --- a/src/kernels/cuda/fused_gr.cu +++ b/src/kernels/cuda/fused_gr.cu @@ -4,6 +4,8 @@ #include "strata/core/geometry.hpp" #include "strata/kernels/fused_gr.hpp" #include "strata/kernels/bf16_bits.hpp" + +#include #include "strata/kernels/verify_kernels.hpp" #include "strata/kernels/gfx_arch.hpp" #include "s26_tsum.cuh" @@ -46,37 +48,32 @@ __device__ __forceinline__ float warp_sum(float v) { } __device__ __forceinline__ float sigmoidf_(float x) { return 1.0f / (1.0f + __expf(-x)); } -// 8 bf16 packed in a uint4, unpacked to two float4 once when reused across tokens. -struct Bf16x8 { float4 a, b; }; -__device__ __forceinline__ Bf16x8 unpack8(const uint4 w) { - return { - make_float4(__uint_as_float(w.x << 16), __uint_as_float(w.x & 0xffff0000u), - __uint_as_float(w.y << 16), __uint_as_float(w.y & 0xffff0000u)), - make_float4(__uint_as_float(w.z << 16), __uint_as_float(w.z & 0xffff0000u), - __uint_as_float(w.w << 16), __uint_as_float(w.w & 0xffff0000u)) - }; -} -__device__ __forceinline__ float dot8u(const Bf16x8& w, const float4 x0, const float4 x1) { +// 8 bf16 packed in a uint4 against 8 floats. +__device__ __forceinline__ float dot8(const uint4 w, const float* x) { float acc = 0.0f; - acc = fmaf(w.a.x, x0.x, acc); - acc = fmaf(w.a.y, x0.y, acc); - acc = fmaf(w.a.z, x0.z, acc); - acc = fmaf(w.a.w, x0.w, acc); - acc = fmaf(w.b.x, x1.x, acc); - acc = fmaf(w.b.y, x1.y, acc); - acc = fmaf(w.b.z, x1.z, acc); - acc = fmaf(w.b.w, x1.w, acc); + const uint32_t v[4] = {w.x, w.y, w.z, w.w}; +#pragma unroll + for (int j = 0; j < 4; ++j) { + acc = fmaf(__uint_as_float(v[j] << 16), x[2 * j], acc); + acc = fmaf(__uint_as_float(v[j] & 0xffff0000u), x[2 * j + 1], acc); + } return acc; } -__device__ __forceinline__ float dot8u_ptr(const Bf16x8& w, const float* x) { - const float4* x4 = reinterpret_cast(x); - return dot8u(w, x4[0], x4[1]); -} -// 8 bf16 packed in a uint4 against 8 16-byte-aligned floats (same 8-FMA order). -__device__ __forceinline__ float dot8(const uint4 w, const float* x) { - return dot8u_ptr(unpack8(w), x); +// `dot8` for 8 FP16 weights (STRATA_FP16=load), the same fmaf order +__device__ __forceinline__ float dot8h(const uint4 w, const float* x) { + float acc = 0.0f; + const uint32_t v[4] = {w.x, w.y, w.z, w.w}; +#pragma unroll + for (int j = 0; j < 4; ++j) { + acc = fmaf(__half2float(__ushort_as_half((uint16_t) v[j])), x[2 * j], acc); + acc = fmaf(__half2float(__ushort_as_half((uint16_t) (v[j] >> 16))), x[2 * j + 1], acc); + } + return acc; } +// the weight form is uniform over a launch: `h` is FusedGrArgs::w_f16 +__device__ __forceinline__ float dot8w(bool h, const uint4 w, const float* x) { return h ? dot8h(w, x) : dot8(w, x); } + __global__ void __launch_bounds__(THREADS) gr_down_kernel(FusedGrArgs a) { __shared__ __align__(16) float xn[D]; @@ -125,7 +122,7 @@ __global__ void __launch_bounds__(THREADS) gr_down_kernel(FusedGrArgs a) { const uint4* w4 = reinterpret_cast(wrow); float acc = 0.0f; #pragma unroll 4 - for (int j = lane; j < D / 8; j += 32) acc += dot8(__ldg(w4 + j), xn + j * 8); + for (int j = lane; j < D / 8; j += 32) acc += dot8w(a.w_f16, __ldg(w4 + j), xn + j * 8); acc = warp_sum(acc); if (lane != 0) return; if (inject_block) { @@ -147,8 +144,8 @@ __global__ void __launch_bounds__(THREADS) gr_up_kernel(FusedGrArgs a) { for (int r = warp; r < HC * UP_COLS; r += WARPS) { const int c = r / UP_COLS, dd = r - c * UP_COLS, i = c * N + d0 + dd; const uint4* w4 = reinterpret_cast(a.w_up + (size_t) i * LR); - float acc = dot8(__ldg(w4 + lane), lo + lane * 8); - if (lane < LR / 8 - 32) acc += dot8(__ldg(w4 + 32 + lane), lo + (32 + lane) * 8); + float acc = dot8w(a.w_f16, __ldg(w4 + lane), lo + lane * 8); + if (lane < LR / 8 - 32) acc += dot8w(a.w_f16, __ldg(w4 + 32 + lane), lo + (32 + lane) * 8); acc = warp_sum(acc); if (lane == 0) { float rv = a.R[i]; @@ -297,7 +294,7 @@ __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) { const int j = lane + 32 * q; #pragma unroll for (int k = 0; k < MAX_T; ++k) - if (k < T) acc[k] += dot8(wv[q], tile + k * TILEV + j * 8); + if (k < T) acc[k] += dot8w(m.a[0].w_f16, wv[q], tile + k * TILEV + j * 8); } } if (!active) return; @@ -322,20 +319,19 @@ constexpr int UPM_BLOCKS = N / UPM_COLS; // 160 // `gr_up_kernel` for T tokens: each row of w_up read once; the T dots reduced by xor so every lane holds every // sum, and lane k runs token k's epilogue - the T epilogues in parallel instead of one after another. -template __global__ void __launch_bounds__(THREADS) gr_up_multi_kernel(GrMulti m) { - __shared__ __align__(16) float lo[MAX_T][LR]; - __shared__ float g[MAX_T][HC][UPM_COLS]; + __shared__ __align__(16) float lo[kFusedGrMaxT][LR]; + __shared__ float g[kFusedGrMaxT][HC][UPM_COLS]; const int t = threadIdx.x, lane = t & 31, warp = t >> 5; - const int T = EXACT_T ? MAX_T : m.T; + const int T = m.T; const int d0 = blockIdx.x * UPM_COLS; for (int i = t; i < T * LR; i += THREADS) lo[i / LR][i % LR] = m.a[i / LR].lo[i % LR]; __syncthreads(); for (int r = warp; r < HC * UPM_COLS; r += WARPS) { const int c = r / UPM_COLS, dd = r - c * UPM_COLS, i = c * N + d0 + dd; const uint4* w4 = reinterpret_cast(m.a[0].w_up + (size_t) i * LR); - const Bf16x8 wa = unpack8(__ldg(w4 + lane)); - const Bf16x8 wb = unpack8(lane < LR / 8 - 32 ? __ldg(w4 + 32 + lane) : make_uint4(0, 0, 0, 0)); + const uint4 wa = __ldg(w4 + lane); + const uint4 wb = lane < LR / 8 - 32 ? __ldg(w4 + 32 + lane) : make_uint4(0, 0, 0, 0); // the epilogue inputs of this lane's token, fetched while the dots run float rv = 0.0f, wn = 0.0f, rsc = 0.0f, bo = 0.0f, ip = 0.0f; bool apply = false; @@ -349,10 +345,10 @@ __global__ void __launch_bounds__(THREADS) gr_up_multi_kernel(GrMulti m) { } float mine = 0.0f; #pragma unroll - for (int k = 0; k < MAX_T; ++k) { - if (!EXACT_T && k >= T) break; - float acc = dot8u_ptr(wa, lo[k] + lane * 8); - if (lane < LR / 8 - 32) acc += dot8u_ptr(wb, lo[k] + (32 + lane) * 8); + for (int k = 0; k < kFusedGrMaxT; ++k) { + if (k >= T) break; + float acc = dot8w(m.a[0].w_f16, wa, lo[k] + lane * 8); + if (lane < LR / 8 - 32) acc += dot8w(m.a[0].w_f16, wb, lo[k] + (32 + lane) * 8); acc = warp_sum(acc); if (lane == k) mine = acc; } @@ -376,6 +372,79 @@ __global__ void __launch_bounds__(THREADS) gr_up_multi_kernel(GrMulti m) { if (m.a[0].q8_mixed != nullptr) gr_q8_tail(m, d0); // S26 STRATA_QFUSE } +// gr_up_multi_kernel compiled for TT tokens and one weight form, with the next row's weights loaded while this row's +// dots run: the same operations in the same order (the generic kernel waits on every row's load) +template +__global__ void __launch_bounds__(THREADS) gr_up_multi_t_kernel(GrMulti m) { + // lo with each 8-value chunk padded to 9 floats: lane l reads chunk l (stride 9, coprime with the 32 banks) + __shared__ float lo[TT][LR / 8 * 9]; + __shared__ float g[TT][HC][UPM_COLS]; + const int t = threadIdx.x, lane = t & 31, warp = t >> 5; + const int d0 = blockIdx.x * UPM_COLS; + for (int i = t; i < TT * LR; i += THREADS) lo[i / LR][(i % LR) / 8 * 9 + (i % 8)] = m.a[i / LR].lo[i % LR]; + // a row's loads: its weights (every lane) and its epilogue inputs (lane k: token k), issued a row ahead + struct Row { uint4 wa, wb; float rv, wn, rsc, bo, ip; bool apply; }; + auto load = [&](int r, Row& x) { + const int c = r / UPM_COLS, dd = r - c * UPM_COLS, i = c * N + d0 + dd; + const uint4* w4 = reinterpret_cast(m.a[0].w_up + (size_t) i * LR); + x.wa = __ldg(w4 + lane); + x.wb = lane < LR / 8 - 32 ? __ldg(w4 + 32 + lane) : make_uint4(0, 0, 0, 0); + x.rv = 0.0f; x.wn = 0.0f; x.rsc = 0.0f; x.bo = 0.0f; x.ip = 0.0f; x.apply = false; + if (lane < TT) { + const FusedGrArgs& a = m.a[lane]; + x.rv = a.R[i]; + x.wn = a.w_norm[i]; + x.rsc = a.rs[c]; + x.apply = a.apply; + if (x.apply) { x.bo = a.bo_prev[d0 + dd]; x.ip = a.inj_prev[c]; } + } + }; + Row cur, nxt{}; + load(warp, cur); + __syncthreads(); + for (int r = warp; r < HC * UPM_COLS; r += WARPS) { + if (r + WARPS < HC * UPM_COLS) load(r + WARPS, nxt); + const int c = r / UPM_COLS, dd = r - c * UPM_COLS, i = c * N + d0 + dd; + float mine = 0.0f; +#pragma unroll + for (int k = 0; k < TT; ++k) { + float acc = WH ? dot8h(cur.wa, lo[k] + lane * 9) : dot8(cur.wa, lo[k] + lane * 9); + if (lane < LR / 8 - 32) acc += WH ? dot8h(cur.wb, lo[k] + (32 + lane) * 9) : dot8(cur.wb, lo[k] + (32 + lane) * 9); + acc = warp_sum(acc); + if (lane == k) mine = acc; + } + if (lane < TT) { + float rv = cur.rv; + if (cur.apply) { + rv = fmaf(cur.bo, 2.0f * sigmoidf_(cur.ip / (float) HC), rv); + m.a[lane].R_out[i] = rv; + } + const float x = rv * cur.wn * cur.rsc; + g[lane][c][dd] = x * sigmoidf_(mine); + } + cur = nxt; + } + __syncthreads(); + for (int i = t; i < TT * UPM_COLS; i += THREADS) { + const int k = i / UPM_COLS, col = i - k * UPM_COLS; + float s = 0.0f; +#pragma unroll + for (int c = 0; c < HC; ++c) s += g[k][c][col]; + m.a[k].mixed[d0 + col] = s / (float) HC; + } + if (m.a[0].q8_mixed != nullptr) gr_q8_tail(m, d0); // S26 STRATA_QFUSE (as gr_up_multi_kernel) +} +#define STRATA_UP_T(TT) \ + case TT: if (m.a[0].w_f16) gr_up_multi_t_kernel<<>>(m); \ + else gr_up_multi_t_kernel<<>>(m); break; +void launch_up_multi_t(const GrMulti& m, cudaStream_t st) { + switch (m.T) { + STRATA_UP_T(1) STRATA_UP_T(2) STRATA_UP_T(3) STRATA_UP_T(4) + STRATA_UP_T(5) STRATA_UP_T(6) STRATA_UP_T(7) STRATA_UP_T(8) + default: gr_up_multi_kernel<<>>(m); + } +} +#undef STRATA_UP_T // 143-amdk (STRATA_QFUSE_HC): gr_up_multi_kernel with 32 columns a block (512 threads, 80 blocks, the same 8 rows a warp), so a block owns a whole // q8_1 group of `mixed` and writes its image itself. STRATA_QFUSE's version (16 columns a block) needs a counter, two fences and a last block @@ -530,10 +599,10 @@ __global__ void __launch_bounds__(THREADS) gr_down_v3_kernel(GrMulti m, float* _ #pragma unroll for (int q = 0; q < TQS; ++q) { const int j = lane + 32 * q; - const Bf16x8 wvq = unpack8(__ldg(w4 + j)); + const uint4 wvq = __ldg(w4 + j); #pragma unroll for (int k = 0; k < MAX_T; ++k) - if (EXACT_T || k < T) acc[k] += dot8u_ptr(wvq, xs + (size_t) k * SL + j * 8); + if (EXACT_T || k < T) acc[k] += dot8w(m.a[0].w_f16, wvq, xs + (size_t) k * SL + j * 8); } const int prow = inject_block ? LR + warp : row0 + r; #pragma unroll @@ -598,8 +667,8 @@ __global__ void __launch_bounds__(THREADS) gr_up_v3_kernel(GrMulti m, const floa const int r = warp + q * WARPS; const int c = r / UPM_COLS, dd = r - c * UPM_COLS, i = c * N + d0 + dd; const uint4* w4 = reinterpret_cast(m.a[0].w_up + (size_t) i * LR); - const Bf16x8 wa = unpack8(__ldg(w4 + lane)); - const Bf16x8 wb = unpack8(lane < LR / 8 - 32 ? __ldg(w4 + 32 + lane) : make_uint4(0, 0, 0, 0)); + const uint4 wa = __ldg(w4 + lane); + const uint4 wb = lane < LR / 8 - 32 ? __ldg(w4 + 32 + lane) : make_uint4(0, 0, 0, 0); float rv = 0.0f, wn = 0.0f, bo = 0.0f, ip = 0.0f; bool apply = false; if (lane < T) { @@ -613,8 +682,8 @@ __global__ void __launch_bounds__(THREADS) gr_up_v3_kernel(GrMulti m, const floa #pragma unroll for (int k = 0; k < MAX_T; ++k) { if (!EXACT_T && k >= T) break; - float acc = dot8u_ptr(wa, lo[k] + lane * 8); - if (lane < LR / 8 - 32) acc += dot8u_ptr(wb, lo[k] + (32 + lane) * 8); + float acc = dot8w(m.a[0].w_f16, wa, lo[k] + lane * 8); + if (lane < LR / 8 - 32) acc += dot8w(m.a[0].w_f16, wb, lo[k] + (32 + lane) * 8); acc = warp_sum(acc); if (lane == k) mine = acc; } @@ -800,6 +869,14 @@ __device__ __forceinline__ float dot8v(const uint4 w, const float4 x0, const flo return acc; } +__device__ __forceinline__ float dot8vh(const uint4 w, const float4 x0, const float4 x1) { + const float x[8] = {x0.x, x0.y, x0.z, x0.w, x1.x, x1.y, x1.z, x1.w}; + return dot8h(w, x); +} +__device__ __forceinline__ float dot8vw(bool h, const uint4 w, const float4 x0, const float4 x1) { + return h ? dot8vh(w, x0, x1) : dot8v(w, x0, x1); +} + // Stage tile `h` of every token into `buf`: [T][2 planes][160 chunks] float4, plane 0 = floats 0-3 of a chunk. __device__ __forceinline__ void stage_htile(const GrMulti& m, int T, int h, float4* buf, int t) { for (int i = t; i < T * (H_TILE / 4); i += THREADS) { @@ -809,11 +886,11 @@ __device__ __forceinline__ void stage_htile(const GrMulti& m, int T, int h, floa } } -template +template __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) { extern __shared__ __align__(16) float4 hbuf[]; // 2 buffers x [T][2][160] float4 const int t = threadIdx.x, lane = t & 31, warp = t >> 5; - const int T = EXACT_T ? MAX_T : m.T; + const int T = m.T; const bool inject_block = blockIdx.x == DOWN_BLOCKS; const int row = inject_block ? warp : blockIdx.x * WARPS + warp; const bool active = !(inject_block && (m.a[0].w_inject == nullptr || warp >= HC)); @@ -846,12 +923,11 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) { #pragma unroll for (int q = 0; q < HQ; ++q) { const int j = lane + 32 * q; - const Bf16x8 wvq = unpack8(wv[q]); #pragma unroll for (int k = 0; k < MAX_T; ++k) { - if (EXACT_T || k < T) { + if (k < T) { const float4* pk = cur + (size_t) k * (H_TILE / 4); - acc[k] += dot8u(wvq, pk[j], pk[H_TILE / 8 + j]); + acc[k] += dot8vw(m.a[0].w_f16, wv[q], pk[j], pk[H_TILE / 8 + j]); } } } @@ -867,11 +943,84 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) { if (!active) return; float s[MAX_T]; #pragma unroll - for (int k = 0; k < MAX_T; ++k) s[k] = (EXACT_T || k < T) ? warp_sum(acc[k]) : 0.0f; + for (int k = 0; k < MAX_T; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f; // lane k writes token k (every lane holds every sum after the xor reduction) #pragma unroll for (int k = 0; k < MAX_T; ++k) { - if ((!EXACT_T && k >= T) || lane != k) continue; + if (k >= T || lane != k) continue; + if (inject_block) { + m.a[k].inject_out[row] = s[k]; + } else { + const float x = s[k] / (float) HC; + m.a[k].lo[row] = x / (1.0f + __expf(-x)); + } + } +} + +// gr_down_staged_kernel compiled for TT tokens and one weight form (WH: FP16): the same operations in the same order, +// without the runtime token checks and form branch (the generic one is ~2.6k instructions: instruction-fetch bound) +template +__global__ void __launch_bounds__(THREADS) gr_down_staged_t_kernel(GrMulti m) { + extern __shared__ __align__(16) float4 hbuf[]; // 2 buffers x [T][2][160] float4 + const int t = threadIdx.x, lane = t & 31, warp = t >> 5; + constexpr int T = TT; + const bool inject_block = blockIdx.x == DOWN_BLOCKS; + const int row = inject_block ? warp : blockIdx.x * WARPS + warp; + const bool active = !(inject_block && (m.a[0].w_inject == nullptr || warp >= HC)); + const uint16_t* wrow = (inject_block ? m.a[0].w_inject : m.a[0].w_down) + (size_t) (active ? row : 0) * D; + const uint4* w4 = reinterpret_cast(wrow); + const size_t buf_f4 = (size_t) T * (H_TILE / 4); // float4 per buffer (the second one follows the first) + float acc[TT]; +#pragma unroll + for (int k = 0; k < TT; ++k) acc[k] = 0.0f; + uint4 wv[HQ], wnext[HQ]; + if (active) { +#pragma unroll + for (int q = 0; q < HQ; ++q) wv[q] = __ldg(w4 + lane + 32 * q); + } + stage_htile(m, T, 0, hbuf, t); + cp_async_commit(); + stage_htile(m, T, 1, hbuf + buf_f4, t); + cp_async_commit(); +#pragma unroll 1 + for (int h = 0; h < N_HTILES; ++h) { + if (active && h + 1 < N_HTILES) { +#pragma unroll + for (int q = 0; q < HQ; ++q) wnext[q] = __ldg(w4 + (h + 1) * (H_TILE / 8) + lane + 32 * q); + } + if (h + 1 < N_HTILES) cp_async_wait1(); // tile h has landed (h + 1 may still be on its way) + else cp_async_wait0(); + __syncthreads(); + const float4* cur = hbuf + (h & 1) * buf_f4; + if (active) { +#pragma unroll + for (int q = 0; q < HQ; ++q) { + const int j = lane + 32 * q; +#pragma unroll + for (int k = 0; k < TT; ++k) { + { + const float4* pk = cur + (size_t) k * (H_TILE / 4); + acc[k] += (WH ? dot8vh(wv[q], pk[j], pk[H_TILE / 8 + j]) : dot8v(wv[q], pk[j], pk[H_TILE / 8 + j])); + } + } + } + } + __syncthreads(); // every warp is done with buffer h & 1 + if (h + 2 < N_HTILES) stage_htile(m, T, h + 2, hbuf + (h & 1) * buf_f4, t); + cp_async_commit(); // an empty group at the end keeps the wait counts simple + if (h + 1 < N_HTILES) { +#pragma unroll + for (int q = 0; q < HQ; ++q) wv[q] = wnext[q]; + } + } + if (!active) return; + float s[TT]; +#pragma unroll + for (int k = 0; k < TT; ++k) s[k] = warp_sum(acc[k]); + // lane k writes token k (every lane holds every sum after the xor reduction) +#pragma unroll + for (int k = 0; k < TT; ++k) { + if (lane != k) continue; if (inject_block) { m.a[k].inject_out[row] = s[k]; } else { @@ -944,21 +1093,15 @@ int down_chunk(bool staged, int* tile_out) { } else { cudaFuncSetAttribute(gr_down_multi_kernel<2560>, cudaFuncAttributeMaxDynamicSharedMemorySize, want); } - auto set_staged_attr = [&](auto kernel, int max_t) { - int ws = (int) (2 * max_t * H_TILE * sizeof(float)); - if (optin > 0 && ws > optin) ws = optin; - if (ws > 48 * 1024) - cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, ws); - cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, 100); - }; - set_staged_attr(gr_down_staged_kernel<1, true>, 1); - set_staged_attr(gr_down_staged_kernel<2, true>, 2); - set_staged_attr(gr_down_staged_kernel<3, true>, 3); - set_staged_attr(gr_down_staged_kernel<4, true>, 4); - set_staged_attr(gr_down_staged_kernel<5, true>, 5); - set_staged_attr(gr_down_staged_kernel<6, true>, 6); - set_staged_attr(gr_down_staged_kernel<4, false>, 4); - set_staged_attr(gr_down_staged_kernel, kFusedGrMaxT); + int want_staged = (int) (2 * kFusedGrMaxT * H_TILE * sizeof(float)); + if (optin > 0 && want_staged > optin) want_staged = optin; + cudaFuncSetAttribute(gr_down_staged_kernel<>, cudaFuncAttributeMaxDynamicSharedMemorySize, want_staged); +#define STRATA_DOWN_ATTR(TT) \ + cudaFuncSetAttribute(gr_down_staged_t_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, want_staged); \ + cudaFuncSetAttribute(gr_down_staged_t_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, want_staged); + STRATA_DOWN_ATTR(1) STRATA_DOWN_ATTR(2) STRATA_DOWN_ATTR(3) STRATA_DOWN_ATTR(4) + STRATA_DOWN_ATTR(5) STRATA_DOWN_ATTR(6) STRATA_DOWN_ATTR(7) STRATA_DOWN_ATTR(8) +#undef STRATA_DOWN_ATTR cudaGetLastError(); // drop any error the attempt left behind // The opt-in is a promise a pre-Volta card does not keep: an sm_60 answers 65536 and accepts the // cudaFuncSetAttribute for 61440 B, then fails the LAUNCH with "invalid argument". What such a card @@ -987,6 +1130,17 @@ int down_chunk(bool staged, int* tile_out) { // many tokens as fit the card, the up projection; the profile's stamps after the norm and after the down projection. // kHcPlain is the default read exactly as before (never main's opt-in STRATA_GR_V3 path, which fused_gr_read_multi // takes first). +#define STRATA_DOWN_T(TT) \ + case TT: if (c.a[0].w_f16) gr_down_staged_t_kernel<<>>(c); \ + else gr_down_staged_t_kernel<<>>(c); break; +void launch_down_staged_t(const GrMulti& c, size_t smem, cudaStream_t st) { + switch (c.T) { + STRATA_DOWN_T(1) STRATA_DOWN_T(2) STRATA_DOWN_T(3) STRATA_DOWN_T(4) + STRATA_DOWN_T(5) STRATA_DOWN_T(6) STRATA_DOWN_T(7) STRATA_DOWN_T(8) + default: gr_down_staged_kernel<><<>>(c); + } +} +#undef STRATA_DOWN_T #if STRATA_GR_FAST_BUILD __global__ void __launch_bounds__(THREADS) gr_norm_fast_kernel(GrMulti m); // below, with the AMD fast path __global__ void __launch_bounds__(THREADS) gr_up_fast_kernel(GrMulti m); @@ -994,6 +1148,7 @@ __global__ void __launch_bounds__(THREADS) gr_up_fast_kernel(GrMulti m); static bool gr_fast(); namespace { #endif +<<<<<<< this fork // STRATA_GR_DOWN_MAX4=1 (opt-in): launches of up to 4 tokens hold 4 tokens' sums per thread instead of kFusedGrMaxT (the // same bits; checked bitwise on the 5070 for 0.1.40). Default off: a 10-pair interleaved decode A/B on the RTX 5070 (Q2_0, // default config) did not show the +2.8-3.5% of bench #832 (medians 0.97-0.99 of the off arm). @@ -1029,16 +1184,26 @@ bool qfuse_hc_local() { static const bool on = [] { const char* e = std::getenv("STRATA_QFUSE_HC_LOCAL"); return e == nullptr || std::atoi(e) != 0; }(); return on; } +======= +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long long* stamp_buf, int stamp_i0) { const int n_tok = m.T; #if STRATA_GR_FAST_BUILD // gfx906: the latency-hidden norm/up (STRATA_GR_FAST=0: off) - the same sums in the same order as the kernels +<<<<<<< this fork // they replace (gr_parity checks the multi-token read against single-token calls bitwise) const bool fast = gr_fast(); if (variant >= kHcSplit) { if (norm_fast_on()) gr_norm_split_fast_kernel<<>>(m); else gr_norm_split_kernel<<>>(m); } else if (fast) gr_norm_fast_kernel<<>>(m); +======= + // they replace (gr_parity checks the multi-token read against single-token calls bitwise). ac922: BF16 weights + // only - the fast kernels read them with dot8, so STRATA_FP16=load's FP16 weights keep the form-aware kernels + const bool fast = gr_fast() && !m.a[0].w_f16; + if (variant >= kHcSplit) gr_norm_split_kernel<<>>(m); + else if (fast) gr_norm_fast_kernel<<>>(m); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) else gr_norm_multi_kernel<<>>(m); #else if (variant >= kHcSplit) { @@ -1051,11 +1216,6 @@ void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long int tv = 2560; const int chunk_tok = down_chunk(staged, &tv); const size_t per_tok = (staged ? (size_t) 2 * H_TILE : (size_t) tv) * sizeof(float); - static const bool no_multi_gr = [] { - const char* no = std::getenv("STRATA_NO_MULTI_GR"); - return no != nullptr && no[0] != '\0' && no[0] != '0'; - }(); - const bool exact_t = !no_multi_gr; for (int c0 = 0; c0 < n_tok; c0 += chunk_tok) { const int ct = n_tok - c0 < chunk_tok ? n_tok - c0 : chunk_tok; GrMulti c{}; @@ -1067,20 +1227,16 @@ void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long for (int k = 0; k < ct; ++k) c.a[k] = m.a[c0 + k]; } const size_t smem = (size_t) ct * per_tok; - // #443, STRATA_GR_DOWN_MAX4: launches of up to 4 tokens hold 4 tokens' sums per thread instead of - // kFusedGrMaxT - the same tile, block size, accumulation order and plain/split/staged path, so the same bits - // (decided once per device: this runs per layer when decode is not captured) - const bool max4 = ct <= 4 && gr_down_max4(); - if (staged) { - // #783 PR-g (stuchapin909): a launch of exactly ct <= 6 tokens is its own instantiation, the loop bounds - // are compile-time (the same sums in the same order); STRATA_NO_MULTI_GR=1 keeps the generic kernels - if (exact_t && ct == 1) gr_down_staged_kernel<1, true><<>>(c); - else if (exact_t && ct == 2) gr_down_staged_kernel<2, true><<>>(c); - else if (exact_t && ct == 3) gr_down_staged_kernel<3, true><<>>(c); - else if (exact_t && ct == 4) gr_down_staged_kernel<4, true><<>>(c); - else if (exact_t && ct == 5) gr_down_staged_kernel<5, true><<>>(c); - else if (exact_t && ct == 6) gr_down_staged_kernel<6, true><<>>(c); - else if (max4) gr_down_staged_kernel<4><<>>(c); + // the staged read compiled per token count (launch_down_staged_t) first; STRATA_HC_DOWN_GENERIC=1 the + // generic kernels. #443, opt-in STRATA_GR_DOWN_MAX4=1: generic launches of up to 4 tokens hold 4 tokens' sums + // per thread instead of kFusedGrMaxT - the same bits (read once: this runs per layer when decode is not captured) + static const bool generic = [] { const char* v = std::getenv("STRATA_HC_DOWN_GENERIC"); return v && v[0] == '1'; }(); + static const bool max4_on = [] { const char* v = std::getenv("STRATA_GR_DOWN_MAX4"); return v && std::atoi(v) != 0; }(); + const bool max4 = max4_on && ct <= 4; + if (staged && !generic) { + launch_down_staged_t(c, smem, st); + } else if (staged) { + if (max4) gr_down_staged_kernel<4><<>>(c); else gr_down_staged_kernel<><<>>(c); } else if (tv == 1280) { if (max4) gr_down_multi_kernel<1280, 4><<>>(c); @@ -1095,6 +1251,7 @@ void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long if (fast) gr_up_fast_kernel<<>>(m); else #endif +<<<<<<< this fork #if !STRATA_GR_FAST_BUILD if (m.a[0].q8_mixed != nullptr && qfuse_hc_local()) { switch (no_multi_gr ? kFusedGrMaxT : n_tok) { @@ -1117,6 +1274,12 @@ void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long case 5: gr_up_multi_kernel<5, true><<>>(m); break; case 6: gr_up_multi_kernel<6, true><<>>(m); break; default: gr_up_multi_kernel<<>>(m); break; +======= + { + static const bool up_generic = [] { const char* v = std::getenv("STRATA_HC_DOWN_GENERIC"); return v && v[0] == '1'; }(); + if (up_generic) gr_up_multi_kernel<<>>(m); + else launch_up_multi_t(m, st); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) } } @@ -1964,7 +2127,7 @@ bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi const FusedGrArgs& x = a[t]; if (!x.R || !x.w_norm || !((x.w_down && x.w_up) || (x.q8_down && x.q8_up)) || !x.lo || !x.rs || !x.mixed || (x.w_inject && !x.inject_out) || (x.apply && (!x.bo_prev || !x.inj_prev || !x.R_out)) || x.w_down != a[0].w_down || x.w_up != a[0].w_up || - x.w_inject != a[0].w_inject || x.w_norm != a[0].w_norm || x.q8_down != a[0].q8_down || + x.w_inject != a[0].w_inject || x.w_norm != a[0].w_norm || x.w_f16 != a[0].w_f16 || x.q8_down != a[0].q8_down || x.q8_up != a[0].q8_up || x.q8_inject != a[0].q8_inject) { std::fprintf(stderr, "fused_gr_read_multi: invalid arguments for token %d\n", t); std::exit(1); @@ -1997,21 +2160,14 @@ bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi int optin = 0; cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev3); const int limit = optin > 0 ? optin : 48 * 1024; - const int need1 = (int) (kFusedGrMaxT * N * sizeof(float)), need6 = (int) (6 * N * sizeof(float)), - need5 = (int) (5 * N * sizeof(float)), need2 = need1 / 2; + const int need1 = (int) (kFusedGrMaxT * N * sizeof(float)), need6 = (int) (6 * N * sizeof(float)), need2 = need1 / 2; int split = -1; if (need1 <= limit && cudaFuncSetAttribute(gr_down_v3_kernel<1, kFusedGrMaxT, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, need1) == cudaSuccess && - cudaFuncSetAttribute(gr_down_v3_kernel<1, 6, true>, cudaFuncAttributeMaxDynamicSharedMemorySize, need6) == cudaSuccess && - cudaFuncSetAttribute(gr_down_v3_kernel<1, 5, true>, cudaFuncAttributeMaxDynamicSharedMemorySize, need5) == cudaSuccess) { - cudaFuncSetAttribute(gr_down_v3_kernel<1, 1, true>, cudaFuncAttributePreferredSharedMemoryCarveout, 100); - cudaFuncSetAttribute(gr_down_v3_kernel<1, 2, true>, cudaFuncAttributePreferredSharedMemoryCarveout, 100); - cudaFuncSetAttribute(gr_down_v3_kernel<1, 3, true>, cudaFuncAttributePreferredSharedMemoryCarveout, 100); + cudaFuncSetAttribute(gr_down_v3_kernel<1, 6, true>, cudaFuncAttributeMaxDynamicSharedMemorySize, need6) == cudaSuccess) { cudaFuncSetAttribute(gr_down_v3_kernel<1, 4, true>, cudaFuncAttributePreferredSharedMemoryCarveout, 100); - cudaFuncSetAttribute(gr_down_v3_kernel<1, 5, true>, cudaFuncAttributePreferredSharedMemoryCarveout, 100); - cudaFuncSetAttribute(gr_down_v3_kernel<1, 6, true>, cudaFuncAttributePreferredSharedMemoryCarveout, 100); cudaFuncSetAttribute(gr_down_v3_kernel<1, 4, false>, cudaFuncAttributePreferredSharedMemoryCarveout, 100); - cudaFuncSetAttribute(gr_down_v3_kernel<1, kFusedGrMaxT, false>, cudaFuncAttributePreferredSharedMemoryCarveout, 100); + cudaFuncSetAttribute(gr_down_v3_kernel<1, 2, true>, cudaFuncAttributePreferredSharedMemoryCarveout, 100); split = 1; } else if (need2 <= limit) // #375 (kenh0u): the S = 2 split (a 64 KB opt-in card: Turing) disagrees with itself in gr_parity @@ -2029,21 +2185,17 @@ bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi const size_t sm = (size_t) n_tok * (N / split) * sizeof(float); if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0, stream); if (split == 2) gr_down_v3_kernel<2><<>>(m, part, ssg); - else if (n_tok == 1) gr_down_v3_kernel<1, 1, true><<>>(m, part, ssg); else if (n_tok == 2) gr_down_v3_kernel<1, 2, true><<>>(m, part, ssg); - else if (n_tok == 3) gr_down_v3_kernel<1, 3, true><<>>(m, part, ssg); else if (n_tok == 4) gr_down_v3_kernel<1, 4, true><<>>(m, part, ssg); - else if (n_tok == 5) gr_down_v3_kernel<1, 5, true><<>>(m, part, ssg); else if (n_tok == 6) gr_down_v3_kernel<1, 6, true><<>>(m, part, ssg); + else if (n_tok < 4) gr_down_v3_kernel<1, 4, false><<>>(m, part, ssg); else gr_down_v3_kernel<1, kFusedGrMaxT, false><<>>(m, part, ssg); if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0 + 1, stream); if (split == 2) gr_up_v3_kernel<2><<>>(m, part, ssg); - else if (n_tok == 1) gr_up_v3_kernel<1, 1, true><<>>(m, part, ssg); else if (n_tok == 2) gr_up_v3_kernel<1, 2, true><<>>(m, part, ssg); - else if (n_tok == 3) gr_up_v3_kernel<1, 3, true><<>>(m, part, ssg); else if (n_tok == 4) gr_up_v3_kernel<1, 4, true><<>>(m, part, ssg); - else if (n_tok == 5) gr_up_v3_kernel<1, 5, true><<>>(m, part, ssg); else if (n_tok == 6) gr_up_v3_kernel<1, 6, true><<>>(m, part, ssg); + else if (n_tok < 4) gr_up_v3_kernel<1, 4, false><<>>(m, part, ssg); else gr_up_v3_kernel<1, kFusedGrMaxT, false><<>>(m, part, ssg); const cudaError_t e3 = cudaGetLastError(); if (e3 != cudaSuccess) { diff --git a/src/kernels/cuda/gr.cu b/src/kernels/cuda/gr.cu index 9081e9f9e0d..5a687c30d6d 100644 --- a/src/kernels/cuda/gr.cu +++ b/src/kernels/cuda/gr.cu @@ -342,10 +342,15 @@ size_t gr_workspace_init(const GrShapes& s, void* base, GrWorkspace& out) { return bytes; } -void gr_read(const float* R, const float* w_norm, const uint16_t* w_down, const uint16_t* w_up, - const uint16_t* w_inject, float eps, const GrShapes& s, const GrWorkspace& ws, float* mixed, - float* inject, void* stream) { +void gr_read(const float* R, const float* w_norm, const void* w_down_v, const void* w_up_v, + const void* w_inject_v, float eps, const GrShapes& s, const GrWorkspace& ws, float* mixed, + float* inject, void* stream, WForm f_down_up, WForm f_inject) { if (s.n_embd <= 0 || s.hc <= 0 || s.hc_lr <= 0) return; + const uint16_t* w_down = (const uint16_t*) w_down_v; + const uint16_t* w_up = (const uint16_t*) w_up_v; + const uint16_t* w_inject = (const uint16_t*) w_inject_v; + if ((f_down_up != WForm::Bf16 || (w_inject != nullptr && f_inject != WForm::Bf16)) && !native_mmvf) + throw std::invalid_argument("gr_read: FP16/F32 weights (STRATA_FP16=load) need the native MMVF read"); if (ws.xn == nullptr || ws.xq == nullptr || ws.lq == nullptr || ws.gated == nullptr || ws.lo == nullptr) { std::fprintf(stderr, "gr_read: GrWorkspace is not initialised (see gr_workspace_init)\n"); std::exit(1); @@ -366,9 +371,9 @@ void gr_read(const float* R, const float* w_norm, const uint16_t* w_down, const if ((hc_dim & 1) != 0 || (hc_lr & 1) != 0) throw std::invalid_argument("gr_read native MMVF requires even hc*n_embd and hc_lr"); native_gr_rms_norm_weighted(R, w_norm, ws.xn, n_embd, hc, eps, stream); - bf16_gemv_fp32_mmvf(ws.xn, w_down, ws.lo, hc_dim, hc_lr, stream); + gemv_fp32_mmvf(ws.xn, w_down, f_down_up, ws.lo, hc_dim, hc_lr, stream); native_gr_down_silu(ws.lo, hc_lr, hc, stream); - bf16_gemv_fp32_mmvf(ws.lo, w_up, ws.gated, hc_lr, hc_dim, stream); + gemv_fp32_mmvf(ws.lo, w_up, f_down_up, ws.gated, hc_lr, hc_dim, stream); // The existing null-injection contract identifies the final mixer. // Pinned qwen4exp uses fused HC pre for layers and the unfused graph for il=-1. native_gr_pre_gated(ws.xn, ws.gated, mixed, n_embd, hc, w_inject != nullptr, stream); @@ -390,7 +395,7 @@ void gr_read(const float* R, const float* w_norm, const uint16_t* w_down, const if (w_inject != nullptr) { const int nthreads = 32 * hc; if (use_native) - bf16_gemv_fp32_mmvf(ws.xn, w_inject, inject, hc_dim, hc, stream); + gemv_fp32_mmvf(ws.xn, w_inject, f_inject, inject, hc_dim, hc, stream); else if (use_fp32) gr_inject_kernel<<<1, nthreads, 0, st>>>(ws.xn, w_inject, hc_dim, hc, inject); else diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cu index 947952a0485..e70f3e17896 100644 --- a/src/kernels/cuda/iq_kernels.cu +++ b/src/kernels/cuda/iq_kernels.cu @@ -1877,6 +1877,22 @@ __device__ __forceinline__ void get_scale_min_k4(int j, const uint8_t* q, uint8_ m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); } } +// N consecutive values from registers in 16- or 8-byte stores (p aligned to the store); 2-byte stores left the +// prompt path's expert dequant at ~290 GB/s +template +__device__ __forceinline__ void store_vec(T* p, const T (&v)[N]) { + constexpr int B = (int) sizeof(T) * N; + if constexpr (B % 16 == 0) { +#pragma unroll + for (int i = 0; i < B / 16; ++i) ((uint4*) p)[i] = ((const uint4*) v)[i]; + } else if constexpr (B % 8 == 0) { +#pragma unroll + for (int i = 0; i < B / 8; ++i) ((uint2*) p)[i] = ((const uint2*) v)[i]; + } else { +#pragma unroll + for (int i = 0; i < N; ++i) p[i] = v[i]; + } +} template __device__ void dq_q4_k(const void* vx, int64_t ibs, dst_t* yy, int tid) { const block_q4_K* x = (const block_q4_K*) vx; @@ -1891,10 +1907,13 @@ __device__ void dq_q4_k(const void* vx, int64_t ibs, dst_t* yy, int tid) { const float d1 = dall * sc, m1 = dmin * m; get_scale_min_k4((int) is + 1, x[ibs].scales, sc, m); const float d2 = dall * sc, m2 = dmin * m; + __align__(16) dst_t lo[n], hi[n]; for (int l = 0; l < n; ++l) { - y[l + 0] = cvt(d1 * (q[l] & 0xF) - m1); - y[l + 32] = cvt(d2 * (q[l] >> 4) - m2); + lo[l] = cvt(d1 * (q[l] & 0xF) - m1); + hi[l] = cvt(d2 * (q[l] >> 4) - m2); } + store_vec(y, lo); + store_vec(y + 32, hi); } template __device__ void dq_q5_k(const void* vx, int64_t ibs, dst_t* yy, int tid) { @@ -1968,13 +1987,16 @@ __device__ void dq_q5_1(const void* vx, int64_t ibs, dst_t* yy, int tid) { uint32_t qh; memcpy(&qh, x[ib].qh, sizeof(qh)); dst_t* y = yy + 32 * ib; + __align__(16) dst_t lo[4], hi[4]; for (int j = 0; j < 4; ++j) { const int iqs = 4 * il + j; // llama.cpp's dequantize_q5_1 for value pairs iqs, iqs + 16 const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10; const int xh_1 = ((qh >> (iqs + 12))) & 0x10; - y[iqs] = cvt((float) ((x[ib].qs[iqs] & 0xf) | xh_0) * dm.x + dm.y); - y[iqs + 16] = cvt((float) ((x[ib].qs[iqs] >> 4) | xh_1) * dm.x + dm.y); + lo[j] = cvt((float) ((x[ib].qs[iqs] & 0xf) | xh_0) * dm.x + dm.y); + hi[j] = cvt((float) ((x[ib].qs[iqs] >> 4) | xh_1) * dm.x + dm.y); } + store_vec(y + 4 * il, lo); + store_vec(y + 4 * il + 16, hi); } template __device__ void dq_q6_k(const void* vx, int64_t ibs, dst_t* yy, int tid) { @@ -2001,8 +2023,9 @@ __device__ void dq_q8_0(const void* vx, int64_t ibs, dst_t* yy, int tid) { const block_q8_0* x = (const block_q8_0*) vx + ibs * (QK_K / QK8_0); const int ib = tid % 8, il = tid / 8; const float d = __half2float(x[ib].d); - dst_t* y = yy + 32 * ib + 8 * il; - for (int j = 0; j < 8; ++j) y[j] = cvt((float) x[ib].qs[8 * il + j] * d); + __align__(16) dst_t v[8]; + for (int j = 0; j < 8; ++j) v[j] = cvt((float) x[ib].qs[8 * il + j] * d); + store_vec(yy + 32 * ib + 8 * il, v); } // BF16 (the token embedding as the checkpoint ships it, tools/embd_bf16_pack.py): 8 values per thread. @@ -2057,11 +2080,20 @@ __device__ __forceinline__ void dq_dispatch(int ty, const void* vx, int64_t ibs, } } +// DQ_SB superblocks per block (one warp each): a 32-thread block per superblock kept a third of the SMs' warp slots +// idle (~290 GB/s on a Q4_K expert) +constexpr int DQ_SB = 8; // flat: superblock i -> y + 256 i template +<<<<<<< this fork __global__ void dequant_flat_kernel(int ty, const void* __restrict__ vx, dst_t* __restrict__ y, const float* __restrict__ scale) { const int64_t i = blockIdx.x; +======= +__global__ void dequant_flat_kernel(int ty, const void* __restrict__ vx, dst_t* __restrict__ y, int64_t nsb) { + const int64_t i = (int64_t) blockIdx.x * DQ_SB + threadIdx.y; + if (i >= nsb) return; +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) dq_dispatch(ty, vx, i, y + i * QK_K, threadIdx.x); if (scale != nullptr) { // NVFP4 down: s_down (the weights stay in FP16's normal range) __syncwarp(); @@ -2074,8 +2106,14 @@ __global__ void dequant_flat_kernel(int ty, const void* __restrict__ vx, dst_t* } // gate/up: superblock i of a role matrix (n_embd/256 per row) -> interleaved row 2r + parity __global__ void dequant_gu_kernel(int ty, const void* __restrict__ gate, const void* __restrict__ up, int64_t per_row, +<<<<<<< this fork __half* __restrict__ y, const float* __restrict__ tail) { const int64_t i = blockIdx.x; +======= + __half* __restrict__ y, int64_t nsb) { + const int64_t i = (int64_t) blockIdx.x * DQ_SB + threadIdx.y; + if (i >= nsb) return; +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) const int parity = blockIdx.y; const int64_t r = i / per_row, c = i % per_row; __half* dst = y + ((2 * r + parity) * per_row + c) * QK_K; @@ -2927,7 +2965,12 @@ void iq_mmvq(int t, const void* w, const void* x_q8_1, float* y, int n_in, int n void iq_dequant_f16(int t, const void* src, int64_t n, uint16_t* dst, void* stream, const float* scale) { if (n % 256 != 0 || !is_iq(t)) { std::fprintf(stderr, "iq_dequant_f16: bad arguments\n"); std::exit(1); } +<<<<<<< this fork dequant_flat_kernel<__half><<<(unsigned) (n / 256), 32, 0, (cudaStream_t) stream>>>(t, src, (__half*) dst, scale); +======= + dequant_flat_kernel<__half><<<(unsigned) ((n / 256 + DQ_SB - 1) / DQ_SB), dim3(32, DQ_SB), 0, (cudaStream_t) stream>>>( + t, src, (__half*) dst, n / 256); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) check("iq_dequant_f16"); } @@ -2952,7 +2995,12 @@ void iq_embed_rows(int t, const void* table, size_t row_bytes, const int32_t* to void iq_dequant_f32(int t, const void* src, int64_t n, float* dst, void* stream) { if (n % 256 != 0 || !embed_type_supported(t)) { std::fprintf(stderr, "iq_dequant_f32: bad arguments\n"); std::exit(1); } +<<<<<<< this fork dequant_flat_kernel<<<(unsigned) (n / 256), 32, 0, (cudaStream_t) stream>>>(t, src, dst, nullptr); +======= + dequant_flat_kernel<<<(unsigned) ((n / 256 + DQ_SB - 1) / DQ_SB), dim3(32, DQ_SB), 0, (cudaStream_t) stream>>>( + t, src, dst, n / 256); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) check("iq_dequant_f32"); } @@ -2961,8 +3009,14 @@ void iq_dequant_gu_f16(int t, const void* gate, const void* up, int64_t n_ff, in // checked like the other entry points: an unknown type used to leave `dst` unwritten, a wrong prompt and no error if (n_embd % 256 != 0 || !is_iq(t)) { std::fprintf(stderr, "iq_dequant_gu_f16: type %d / %lld\n", t, (long long) n_embd); std::exit(1); } const int64_t per_row = n_embd / 256; +<<<<<<< this fork dequant_gu_kernel<<>>(t, gate, up, per_row, (__half*) dst, tail); +======= + const int64_t nsb = n_ff * per_row; + dequant_gu_kernel<<>>( + t, gate, up, per_row, (__half*) dst, nsb); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) check("iq_dequant_gu_f16"); } diff --git a/src/kernels/cuda/native_bf16.cu b/src/kernels/cuda/native_bf16.cu index 1f4b98accfe..0afab0421a3 100644 --- a/src/kernels/cuda/native_bf16.cu +++ b/src/kernels/cuda/native_bf16.cu @@ -3,6 +3,7 @@ #include "strata/kernels/bf16_bits.hpp" #include "strata/kernels/pdl.hpp" +#include #include #include #include @@ -43,23 +44,39 @@ __device__ __forceinline__ float mmvf_warp_sum(float value) { return value; } -template -__global__ void bf16_f32_mmvf_kernel(const float* __restrict__ x, const uint16_t* __restrict__ w, +// weight pair `pair` of a row in form WF (0 BF16, 1 FP16, 2 F32), widened to FP32 +template +__device__ __forceinline__ void mmvf_pair(const void* row, int pair, float& w0, float& w1) { + if constexpr (WF == 2) { + const float2 v = reinterpret_cast(row)[pair]; + w0 = v.x; w1 = v.y; + } else { + const uint32_t v = reinterpret_cast(row)[pair]; + if constexpr (WF == 0) { w0 = f32_from_bf16((uint16_t) v); w1 = f32_from_bf16((uint16_t) (v >> 16)); } + else { w0 = __half2float(__ushort_as_half((uint16_t) v)); w1 = __half2float(__ushort_as_half((uint16_t) (v >> 16))); } + } +} +template __device__ __forceinline__ const void* mmvf_row(const void* w, int n_in) { + return reinterpret_cast(w) + (size_t) blockIdx.x * n_in * (WF == 2 ? 4 : 2); +} + +template +__global__ void bf16_f32_mmvf_kernel(const float* __restrict__ x, const void* __restrict__ w, float* __restrict__ y, int n_in) { const int t = threadIdx.x; - const uint16_t* row = w + (size_t) blockIdx.x * n_in; - const uint32_t* weights2 = reinterpret_cast(row); + const void* row = mmvf_row(w, n_in); const float2* inputs2 = reinterpret_cast(x); __shared__ float partials[32]; if (t < 32) partials[t] = 0.0f; __syncthreads(); float acc = 0.0f; for (int pair = t; pair < n_in / 2; pair += BLOCK_SIZE) { - const uint32_t weight = weights2[pair]; + float w0, w1; + mmvf_pair(row, pair, w0, w1); const float2 input = inputs2[pair]; // Match the two ordered multiply-adds in ggml_cuda_mad, not a pair sum followed by one add. - acc = __fmaf_rn(f32_from_bf16((uint16_t) weight), input.x, acc); - acc = __fmaf_rn(f32_from_bf16((uint16_t) (weight >> 16)), input.y, acc); + acc = __fmaf_rn(w0, input.x, acc); + acc = __fmaf_rn(w1, input.y, acc); } acc = mmvf_warp_sum(acc); if constexpr (BLOCK_SIZE > 32) { @@ -74,14 +91,13 @@ __global__ void bf16_f32_mmvf_kernel(const float* __restrict__ x, const uint16_t // the same kernel for up to 8 activation rows - the weight row is read ONCE and every // token keeps its own accumulator with exactly the single-row kernel's order (pairs, two ordered FMAs, the same warp // and block reductions), so each output is bit-identical to a bf16_f32_mmvf_kernel launch of its own. -template -__global__ void bf16_f32_mmvf_multi_kernel(const float* x_, int64_t ldx, const uint16_t* __restrict__ w, +template +__global__ void bf16_f32_mmvf_multi_kernel(const float* x_, int64_t ldx, const void* __restrict__ w, float* y_, int64_t ldy, int n_in, int n_tok) { const float* STRATA_PDL_RESTRICT x = x_; // __restrict__ below sm_70 only (pdl.hpp, #1469) float* STRATA_PDL_RESTRICT y = y_; const int t = threadIdx.x; - const uint16_t* row = w + (size_t) blockIdx.x * n_in; - const uint32_t* weights2 = reinterpret_cast(row); + const void* row = mmvf_row(w, n_in); __shared__ float partials[NT][32]; if (t < 32) { #pragma unroll @@ -91,19 +107,12 @@ __global__ void bf16_f32_mmvf_multi_kernel(const float* x_, int64_t ldx, const u float acc[NT]; #pragma unroll for (int k = 0; k < NT; ++k) acc[k] = 0.0f; - // PDL (pdl.hpp): this thread's first PRE weight pairs are loaded before waiting for the activations - constexpr int PRE = kPdlPrefetch ? 8 : 0; - uint32_t wpre[PRE > 0 ? PRE : 1]; -#pragma unroll - for (int q = 0; q < PRE; ++q) { - const int pair = t + q * BLOCK_SIZE; - wpre[q] = pair < n_in / 2 ? __ldg(weights2 + pair) : 0u; - } + // PDL (pdl.hpp): a no-op below sm_90 and outside the verify window; ac922 keeps the plain loop of every weight + // form here (upstream's prefetch of the first BF16 pairs before the wait is not taken) pdl_wait(); - int q = 0; - for (int pair = t; pair < n_in / 2; pair += BLOCK_SIZE, ++q) { - const uint32_t weight = q < PRE ? wpre[q] : __ldg(weights2 + pair); - const float w0 = f32_from_bf16((uint16_t) weight), w1 = f32_from_bf16((uint16_t) (weight >> 16)); + for (int pair = t; pair < n_in / 2; pair += BLOCK_SIZE) { + float w0, w1; + mmvf_pair(row, pair, w0, w1); #pragma unroll for (int k = 0; k < NT; ++k) { if (EXACT_T || k < n_tok) { @@ -294,62 +303,80 @@ bool bf16_gemv_fp32_mmvf_multi_aux(const float* x, int64_t ldx, const uint16_t* void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, float* y, int64_t ldy, int64_t n_in, int64_t n_out, int n_tok, void* stream) { - if (n_tok == 1 && ldy >= n_out) { bf16_gemv_fp32_mmvf(x, w, y, n_in, n_out, stream); return; } + gemv_fp32_mmvf_multi(x, ldx, w, WForm::Bf16, y, ldy, n_in, n_out, n_tok, stream); +} + +void bf16_gemv_fp32_mmvf(const float* x, const uint16_t* w, float* y, int64_t n_in, int64_t n_out, void* stream) { + gemv_fp32_mmvf(x, w, WForm::Bf16, y, n_in, n_out, stream); +} + +void gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const void* w, WForm f, float* y, int64_t ldy, + int64_t n_in, int64_t n_out, int n_tok, void* stream) { + if (n_tok == 1 && ldy >= n_out) { gemv_fp32_mmvf(x, w, f, y, n_in, n_out, stream); return; } if (n_tok < 1 || n_tok > 8 || n_in <= 0 || (n_in & 1) != 0 || n_out <= 0 || (ldx & 1) != 0 || x == nullptr || - w == nullptr || y == nullptr || (reinterpret_cast(x) & 7u) != 0) + w == nullptr || y == nullptr || (reinterpret_cast(x) & 7u) != 0 || + (f == WForm::F32 && (reinterpret_cast(w) & 7u) != 0)) throw std::invalid_argument("bf16_gemv_fp32_mmvf_multi: 1..8 rows, even n_in/ldx, aligned pointers"); const cudaStream_t st = (cudaStream_t) stream; + // STRATA_MMVF_ROWS=1 (upstream's opt-in row kernel) reads BF16 only: other weight forms (STRATA_FP16=load) below static const bool rows = [] { const char* v = std::getenv("STRATA_MMVF_ROWS"); return v && v[0] == '1'; }(); - if (rows && n_out >= 64) { + if (rows && n_out >= 64 && f == WForm::Bf16) { const int ni = (int) n_in, no = (int) n_out; + const uint16_t* wb = static_cast(w); switch (mmvf_block_size(n_in)) { - case 32: launch_rows<32>(x, ldx, w, y, ldy, ni, no, n_tok, st); break; - case 64: launch_rows<64>(x, ldx, w, y, ldy, ni, no, n_tok, st); break; - case 96: launch_rows<96>(x, ldx, w, y, ldy, ni, no, n_tok, st); break; - case 128: launch_rows<128>(x, ldx, w, y, ldy, ni, no, n_tok, st); break; - case 160: launch_rows<160>(x, ldx, w, y, ldy, ni, no, n_tok, st); break; - case 192: launch_rows<192>(x, ldx, w, y, ldy, ni, no, n_tok, st); break; - case 224: launch_rows<224>(x, ldx, w, y, ldy, ni, no, n_tok, st); break; - default: launch_rows<256>(x, ldx, w, y, ldy, ni, no, n_tok, st); break; + case 32: launch_rows<32>(x, ldx, wb, y, ldy, ni, no, n_tok, st); break; + case 64: launch_rows<64>(x, ldx, wb, y, ldy, ni, no, n_tok, st); break; + case 96: launch_rows<96>(x, ldx, wb, y, ldy, ni, no, n_tok, st); break; + case 128: launch_rows<128>(x, ldx, wb, y, ldy, ni, no, n_tok, st); break; + case 160: launch_rows<160>(x, ldx, wb, y, ldy, ni, no, n_tok, st); break; + case 192: launch_rows<192>(x, ldx, wb, y, ldy, ni, no, n_tok, st); break; + case 224: launch_rows<224>(x, ldx, wb, y, ldy, ni, no, n_tok, st); break; + default: launch_rows<256>(x, ldx, wb, y, ldy, ni, no, n_tok, st); break; } const cudaError_t result = cudaGetLastError(); if (result != cudaSuccess) throw std::runtime_error(std::string("bf16_gemv_fp32_mmvf_multi launch: ") + cudaGetErrorString(result)); return; } -#define STRATA_MMVF_M(N) case N: \ +#define STRATA_MMVF_MF(N, WF) \ switch (n_tok) { \ - case 1: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ - case 2: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ - case 3: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ - case 4: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ - case 5: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ - case 6: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ - default: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ - } break + case 1: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ + case 2: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ + case 3: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ + case 4: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ + case 5: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ + case 6: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ + default: launch_pdl(bf16_f32_mmvf_multi_kernel, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \ + } +#define STRATA_MMVF_M(N) case N: \ + if (f == WForm::Bf16) { STRATA_MMVF_MF(N, 0); } else if (f == WForm::F16) { STRATA_MMVF_MF(N, 1); } \ + else { STRATA_MMVF_MF(N, 2); } break switch (mmvf_block_size(n_in)) { STRATA_MMVF_M(32); STRATA_MMVF_M(64); STRATA_MMVF_M(96); STRATA_MMVF_M(128); STRATA_MMVF_M(160); STRATA_MMVF_M(192); STRATA_MMVF_M(224); STRATA_MMVF_M(256); } #undef STRATA_MMVF_M +#undef STRATA_MMVF_MF const cudaError_t result = cudaGetLastError(); if (result != cudaSuccess) throw std::runtime_error(std::string("bf16_gemv_fp32_mmvf_multi launch: ") + cudaGetErrorString(result)); } -void bf16_gemv_fp32_mmvf(const float* x, const uint16_t* w, float* y, - int64_t n_in, int64_t n_out, void* stream) { +void gemv_fp32_mmvf(const float* x, const void* w, WForm f, float* y, int64_t n_in, int64_t n_out, void* stream) { if (n_in <= 0 || (n_in & 1) != 0 || n_in > std::numeric_limits::max() || n_out <= 0 || n_out > std::numeric_limits::max()) throw std::invalid_argument("bf16_gemv_fp32_mmvf: require positive even n_in and positive n_out <= INT_MAX"); if (x == nullptr || w == nullptr || y == nullptr || (reinterpret_cast(x) & 7u) != 0 || - (reinterpret_cast(w) & 3u) != 0 || + (reinterpret_cast(w) & (f == WForm::F32 ? 7u : 3u)) != 0 || (reinterpret_cast(y) & 3u) != 0) throw std::invalid_argument("bf16_gemv_fp32_mmvf: null or misaligned pointer"); const cudaStream_t st = (cudaStream_t) stream; #define STRATA_MMVF_CASE(N) case N: \ - bf16_f32_mmvf_kernel<<<(unsigned) n_out, N, 0, st>>>(x, w, y, (int) n_in); break + if (f == WForm::Bf16) bf16_f32_mmvf_kernel<<<(unsigned) n_out, N, 0, st>>>(x, w, y, (int) n_in); \ + else if (f == WForm::F16) bf16_f32_mmvf_kernel<<<(unsigned) n_out, N, 0, st>>>(x, w, y, (int) n_in); \ + else bf16_f32_mmvf_kernel<<<(unsigned) n_out, N, 0, st>>>(x, w, y, (int) n_in); \ + break switch (mmvf_block_size(n_in)) { STRATA_MMVF_CASE(32); STRATA_MMVF_CASE(64); diff --git a/src/kernels/cuda/ple.cu b/src/kernels/cuda/ple.cu index f945ef3b8dd..e1d8256b102 100644 --- a/src/kernels/cuda/ple.cu +++ b/src/kernels/cuda/ple.cu @@ -325,8 +325,9 @@ void ple_block(const float* emb, const float* hidden, const float* hist_rows, co if (w.pre_value != nullptr) { cudaMemcpyAsync(d_value, w.pre_value, (size_t) n_embd * sizeof(float), cudaMemcpyDeviceToDevice, st); } else if (native_bf16) { - bf16_gemv_fp32_mmvf(emb, w.value_bf16, d_value, n_embd, n_embd, stream); + gemv_fp32_mmvf(emb, w.value_bf16, w.value_f16 ? WForm::F16 : WForm::Bf16, d_value, n_embd, n_embd, stream); } else { + if (w.value_f16) throw std::invalid_argument("ple: an FP16 value (STRATA_FP16=load) needs the native projection"); to_bf16_kernel<<<(n_embd + THREADS - 1) / THREADS, THREADS, 0, st>>>(emb, d_emb16, n_embd); bf16_gemv_kernel<<<(n_embd + THREADS - 1) / THREADS, THREADS, 0, st>>>(d_emb16, w.value_bf16, d_value, n_embd, n_embd); diff --git a/src/kernels/cuda/qsa_prompt_attn.cu b/src/kernels/cuda/qsa_prompt_attn.cu index bd99609db58..f11912b4420 100644 --- a/src/kernels/cuda/qsa_prompt_attn.cu +++ b/src/kernels/cuda/qsa_prompt_attn.cu @@ -744,6 +744,9 @@ bool launch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const // Volta card; the ready-made engine has none of it (no extra kernels to load, the same code as before). #if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60) // ---- sm_70 (Volta): v1's structure on mma.m8n8k4 ------------------------------------------------------------------ +// From Klaus Friedel's (fks) PR #600 to Niko1221/Strata, commit ee0843da07ee8da96fa0b5afebfdd8dde6c0b587 +// (https://github.com/fks/Strata/commit/ee0843da07ee8da96fa0b5afebfdd8dde6c0b587). ac922: opt-in (STRATA_ATTN_M884=1, +// int8 KV), and the hi / lo halves of the split operands accumulate in separate chains (see the scores below). // Volta's tensor cores only have m8n8k4 (FP16 in, FP32 accumulate): per warp FOUR independent 8x8x4 products, one per // quad-pair (QP q = lanes 4q..4q+3 and 4q+16..4q+19; thread j = (lane & 3) + 4 * (lane >> 4) within it). Layout, measured on // a V100 (tools-free microtest, and equal to turbomind's SM70_MMA_884): the thread holds row j of A (k 0..3 as two @@ -1450,6 +1453,271 @@ bool launch_wmma(const float* q, const QsaAttnPools& pools, const int32_t* ids, } #endif +// ---- P4: the int8-KV prompt attention on Volta tensor cores (sm_70; opt-in: STRATA_ATTN_WMMA=1). The RDNA4 kernel's +// design (above) with CUDA wmma 16x16x16 (FP16 in, FP32 accumulate): warp w owns dims [64w, 64w+64) (int8 scale group +// w) for q.k and p.v; q and p split into FP16 hi + lo, the int8 codes exact in FP16, scales applied in FP32, the four +// groups' q.k partials added in a fixed order, the online softmax in FP32. Volta's fragments are opaque, so the tiles +// go through shared memory: q once (its A fragments stay in registers), each warp's K slice (then its V slice, same +// buffer) and P per chunk; the p.v tile is stored and added to a per-thread FP32 accumulator with the rescale. +// FP32-level accuracy, deterministic, not bitwise equal to the FP32 kernel (another summation order). +#if !defined(__HIPCC__) +} // namespace +} // namespace strata::kernels +#include +namespace strata::kernels { +namespace { +constexpr int VCH = 32; // cells per chunk +constexpr int VLD = 72; // staged K/V row stride (halves): 64 dims + 8 (16-byte multiple, banks spread) +constexpr int PLD = 40; // P row stride (halves) +constexpr int SLD = 36; // q.k partial row stride (floats) +constexpr int OLD_ = 64; // p.v tile row stride (floats) +struct alignas(32) SmemV { + union alignas(32) { + __half q[2][16][HD]; // q hi / lo, read once into fragments + float part[4][16][SLD]; // q.k per dim group + float ot[4][16][OLD_]; // a chunk's p.v tile per warp + } u; + __half kv[4][VCH][VLD]; // per warp: the chunk's K slice, then its V slice + __half ph[4][16][PLD], pl[4][16][PLD]; // per warp: P (times the group's V scale) hi / lo + float p[16][VCH + 1]; + float ks[4][VCH], vs[4][VCH]; // K / V scale per (group, cell) + int valid[VCH]; + float qmax[4], alpha[16], lsum[16], mrow[16]; +}; + +__device__ __forceinline__ void i8x16_to_h(uint4 x, __half* dst) { // exact: |code| <= 128 + const uint32_t w[4] = {x.x, x.y, x.z, x.w}; +#pragma unroll + for (int i = 0; i < 4; ++i) +#pragma unroll + for (int b = 0; b < 4; ++b) dst[4 * i + b] = __int2half_rn((int) (int8_t) (w[i] >> (8 * b))); +} + +__global__ void __launch_bounds__(THREADS) prompt_attn_volta_kernel(const float* __restrict__ q, QsaAttnPools p, + const int32_t* __restrict__ ids, + const int32_t* __restrict__ steps, int n_kv_heads, + int page_size, float scale_log2, + float* __restrict__ attn, int cap) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700 + using namespace nvcuda; + extern __shared__ __align__(32) unsigned char smem_raw[]; + SmemV& S = *reinterpret_cast(smem_raw); + const int qi = blockIdx.x, kvh = blockIdx.y; + const int n_head = n_kv_heads * G; + q += (size_t) qi * n_head * HD + (size_t) kvh * G * HD; + attn += (size_t) qi * n_head * HD + (size_t) kvh * G * HD; + ids += (size_t) qi * cap; + const int n = __ldg(steps + (size_t) qi * kStepCount + kStepWidth); + const int t = threadIdx.x, lane = t & 31, warp = t >> 5; + const int dim0 = warp * 64; + + // q: the block's scale (as the RDNA4 kernel), then hi / lo in shared memory, rows 12..15 zero + float qm = 0.0f; + for (int i = t; i < G * HD; i += THREADS) qm = fmaxf(qm, fabsf(q[i])); +#pragma unroll + for (int o = 16; o > 0; o >>= 1) qm = fmaxf(qm, __shfl_xor_sync(0xffffffffu, qm, o)); + if (lane == 0) S.qmax[warp] = qm; + if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; } + __syncthreads(); + qm = fmaxf(fmaxf(S.qmax[0], S.qmax[1]), fmaxf(S.qmax[2], S.qmax[3])); + int qe = 0; + if (qm > 0.0f) frexpf(qm, &qe); + const float qup = ldexpf(1.0f, 14 - qe), qdown = ldexpf(scale_log2, qe - 14); + for (int i = t; i < 16 * HD; i += THREADS) { + const int r = i / HD, d = i % HD; + const float v = r < G ? q[(size_t) r * HD + d] * qup : 0.0f; + const __half hi = __float2half_rn(v); + S.u.q[0][r][d] = hi; + S.u.q[1][r][d] = __float2half_rn(v - __half2float(hi)); + } + __syncthreads(); + wmma::fragment qh[4], ql[4]; +#pragma unroll + for (int kk = 0; kk < 4; ++kk) { + wmma::load_matrix_sync(qh[kk], &S.u.q[0][0][dim0 + kk * 16], HD); + wmma::load_matrix_sync(ql[kk], &S.u.q[1][0][dim0 + kk * 16], HD); + } + float acc[16][2]; // this lane's p.v outputs: rows 0..15, dims dim0 + lane and dim0 + 32 + lane +#pragma unroll + for (int r = 0; r < 16; ++r) acc[r][0] = acc[r][1] = 0.0f; + + const int n_chunks = (n + VCH - 1) / VCH; + for (int ci = 0; ci < n_chunks; ++ci) { + const int c0 = ci * VCH; + __syncthreads(); // the union (q / part / ot) and S.p are free again + long long myrow = -1; // this lane's cell: its pool row, or -1 (past the selection, or non-resident: masked) + if (c0 + lane < n) { + const int cell = __ldg(ids + c0 + lane); + const long long page = (long long) __ldg(p.page_table + cell / page_size); + if (page >= 0) myrow = (page * n_kv_heads + kvh) * page_size + (cell % page_size); + } + { // this warp's K slice of the lane's cell, as FP16 + uint4 k4[4] = {make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0)}; + float ksc = 0.0f, vsc = 0.0f; + if (myrow >= 0) { + const uint4* src = reinterpret_cast(p.k_q + myrow * HD + dim0); +#pragma unroll + for (int i = 0; i < 4; ++i) k4[i] = __ldg(src + i); + ksc = __half2float(__ushort_as_half(__ldg(p.k_scale + myrow * (HD / KV_Q8_GROUP) + warp))); + vsc = __half2float(__ushort_as_half(__ldg(p.v_scale + myrow * (HD / KV_Q8_GROUP) + warp))); + } +#pragma unroll + for (int i = 0; i < 4; ++i) i8x16_to_h(k4[i], &S.kv[warp][lane][16 * i]); + S.ks[warp][lane] = ksc; + S.vs[warp][lane] = vsc; + if (warp == 0) S.valid[lane] = myrow >= 0; + } + __syncwarp(); + // q.k over this warp's 64 dims for two 16-cell tiles: B = K^T (k = dim, n = cell): column-major in kv +#pragma unroll + for (int nt = 0; nt < 2; ++nt) { + wmma::fragment s; + wmma::fill_fragment(s, 0.0f); +#pragma unroll + for (int kk = 0; kk < 4; ++kk) { + wmma::fragment b; + wmma::load_matrix_sync(b, &S.kv[warp][nt * 16][kk * 16], VLD); + wmma::mma_sync(s, qh[kk], b, s); + wmma::mma_sync(s, ql[kk], b, s); + } + wmma::store_matrix_sync(&S.u.part[warp][0][nt * 16], s, SLD, wmma::mem_row_major); + } + __syncwarp(); + { // now this warp's V slice into the same buffer + uint4 v4[4] = {make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0)}; + if (myrow >= 0) { + const uint4* src = reinterpret_cast(p.v_q + myrow * HD + dim0); +#pragma unroll + for (int i = 0; i < 4; ++i) v4[i] = __ldg(src + i); + } +#pragma unroll + for (int i = 0; i < 4; ++i) i8x16_to_h(v4[i], &S.kv[warp][lane][16 * i]); + } + __syncthreads(); + // online softmax over the four groups' sum (fixed order, each times its K scale): row t/8, 4 cells per thread + { + const int r = t >> 3, sub = t & 7; + float x[4], mx = -INFINITY; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int c = sub * 4 + j; + x[j] = S.valid[c] ? (((S.u.part[0][r][c] * S.ks[0][c] + S.u.part[1][r][c] * S.ks[1][c]) + + S.u.part[2][r][c] * S.ks[2][c]) + S.u.part[3][r][c] * S.ks[3][c]) * qdown + : -INFINITY; + mx = fmaxf(mx, x[j]); + } +#pragma unroll + for (int o = 1; o < 8; o <<= 1) mx = fmaxf(mx, __shfl_xor_sync(0xffffffffu, mx, o)); + const float m_old = S.mrow[r]; + const float m_new = fmaxf(m_old, mx); + float sum = 0.0f; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new); + S.p[r][sub * 4 + j] = e; + sum += e; + } +#pragma unroll + for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o); + if (sub == 0) { + const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new); + S.alpha[r] = a; + S.lsum[r] = fmaf(S.lsum[r], a, sum); + S.mrow[r] = m_new; + } + } + __syncthreads(); + // p.v over this warp's 64 dims: P (times the group's V scale, normalized to 16384) hi / lo + float vmax = S.vs[warp][lane]; +#pragma unroll + for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o)); + const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f, vdown = vmax * (1.0f / 16384.0f); + for (int i = lane; i < 16 * VCH; i += 32) { + const int r = i / VCH, c = i % VCH; + const float pv = S.p[r][c] * (S.vs[warp][c] * vup); + const __half hi = __float2half_rn(pv); + S.ph[warp][r][c] = hi; + S.pl[warp][r][c] = __float2half_rn(pv - __half2float(hi)); + } + __syncwarp(); +#pragma unroll + for (int j = 0; j < 4; ++j) { + wmma::fragment o; + wmma::fill_fragment(o, 0.0f); +#pragma unroll + for (int ks = 0; ks < 2; ++ks) { + wmma::fragment ah, al; + wmma::fragment b; + wmma::load_matrix_sync(ah, &S.ph[warp][0][ks * 16], PLD); + wmma::load_matrix_sync(al, &S.pl[warp][0][ks * 16], PLD); + wmma::load_matrix_sync(b, &S.kv[warp][ks * 16][j * 16], VLD); + wmma::mma_sync(o, ah, b, o); + wmma::mma_sync(o, al, b, o); + } + wmma::store_matrix_sync(&S.u.ot[warp][0][j * 16], o, OLD_, wmma::mem_row_major); + } + __syncwarp(); +#pragma unroll + for (int r = 0; r < 16; ++r) { + const float a = S.alpha[r]; + acc[r][0] = fmaf(acc[r][0], a, S.u.ot[warp][r][lane] * vdown); + acc[r][1] = fmaf(acc[r][1], a, S.u.ot[warp][r][32 + lane] * vdown); + } + } + __syncthreads(); +#pragma unroll + for (int r = 0; r < G; ++r) { + const float l = S.lsum[r]; + const float inv = l > 0.0f ? 1.0f / l : 0.0f; + attn[(size_t) r * HD + dim0 + lane] = acc[r][0] * inv; + attn[(size_t) r * HD + dim0 + 32 + lane] = acc[r][1] * inv; + } +#else + __trap(); +#endif +} + +bool volta_wmma_wanted() { + static const bool want = [] { const char* e = std::getenv("STRATA_ATTN_WMMA"); return e != nullptr && e[0] == '1'; }(); + return want; +} + +bool launch_volta(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap, + const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) { + static bool told[64] = {}, set[64] = {}; + int dev = 0; + cudaGetDevice(&dev); + const size_t smem = sizeof(SmemV); + if (dev >= 0 && dev < 64 && !set[dev]) { + set[dev] = true; + if (cudaFuncSetAttribute(prompt_attn_volta_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) smem) != + cudaSuccess) { + std::fprintf(stderr, "qsa_prompt_attn_batch (volta): %zu bytes of shared memory: %s\n", smem, + cudaGetErrorString(cudaGetLastError())); + std::exit(1); + } + } + if (dev >= 0 && dev < 64 && !told[dev]) { + told[dev] = true; + std::fprintf(stderr, "strata: STRATA_ATTN_WMMA: the prompt attention on Volta tensor cores (CUDA%d, %zu B shared)\n", + dev, smem); + } + const float scale_log2 = 1.4426950408889634f / sqrtf((float) HD); + for (int64_t q0 = 0; q0 < n_q; q0 += 65535) { + const int64_t nb = n_q - q0 < 65535 ? n_q - q0 : 65535; + prompt_attn_volta_kernel<<>>( + q + q0 * s.n_head * HD, pools, ids + q0 * cap, steps + q0 * kStepCount, (int) s.n_head_kv, + (int) s.page_size, scale_log2, attn + q0 * s.n_head * HD, (int) cap); + } + const cudaError_t e = cudaGetLastError(); + if (e != cudaSuccess) { + std::fprintf(stderr, "qsa_prompt_attn_batch (volta): %s\n", cudaGetErrorString(e)); + std::exit(1); + } + return true; +} +#endif + } // namespace bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, @@ -1478,6 +1746,15 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3 cc[dev] = w && (!std::strcmp(w, "1") || !std::strcmp(w, "attn")) ? 75 : 10 * strata::cc_major_of(major) + strata::cc_minor_of(minor); } +#if !defined(__HIPCC__) + // P4: Volta on its own wmma kernel, opt-in (STRATA_ATTN_WMMA=1), int8 KV only - ahead of the m8n8k4 kernel + // below, unless STRATA_ATTN_M884=1 asks for that one + static const bool m884 = [] { const char* e = std::getenv("STRATA_ATTN_M884"); return e && e[0] == '1'; }(); + if (!m884 && cc[dev] >= 70 && cc[dev] < 75 && volta_wmma_wanted() && pools.k_q != nullptr && pools.v_q != nullptr && + pools.k_scale != nullptr && pools.v_scale != nullptr && pools.k_q4 == nullptr && pools.v_q4 == nullptr && + s.head_dim == HD && s.n_head == (int64_t) G * s.n_head_kv && cap > 0 && ids && steps && pools.page_table) + return launch_volta(q, pools, ids, steps, cap, s, attn, n_q, (cudaStream_t) stream); +#endif if (cc[dev] < 70) return false; volta = cc[dev] < 75; // sm_70: the m8n8k4 kernel (STRATA_PA_VOLTA); STRATA_QSA_WARP=1 still forces the old one turing = cc[dev] < 80; diff --git a/src/kernels/cuda/qsa_select.cu b/src/kernels/cuda/qsa_select.cu index cb06396a567..7e4485face2 100644 --- a/src/kernels/cuda/qsa_select.cu +++ b/src/kernels/cuda/qsa_select.cu @@ -7,6 +7,10 @@ #include "strata/kernels/gfx_arch.hpp" #include +#if !defined(__HIPCC__) +#include +#include +#endif #include #include @@ -460,6 +464,118 @@ bool sel_gfx12_device() { } #endif // __HIPCC__ +#if !defined(__HIPCC__) +// Volta (sm_70, STRATA_SELECT_VOLTA=1): the scores on FP16 tensor cores. Each FP32 value is split into hi = fp16(x) +// and lo = fp16((x - hi) * 2^11); hi*hi + (hi*lo + lo*hi) / 2^11 (lo*lo dropped) is FP32-level (rms 1.0e-7 of the score +// against FP64, the warp kernel 6e-8) but not bitwise. hi*hi goes through a fresh fragment per 16-dim step, added with +// FP32 adds: Volta's tensor cores truncate when they accumulate. 16 queries per CTA share each key read. +namespace wmma = nvcuda::wmma; +constexpr int VQ = 16; // queries per CTA +constexpr int VITER = 4; // 16-block key tiles per warp +constexpr int VQS = IDX_DIM + 8; // halves per query row in LDS +constexpr int VKS = 16 + 8; // halves per key row of a warp's 16-dim slice +constexpr float LO_SCALE = 2048.0f; // the low parts are stored x 2^11 (no FP16 underflow); unscaled at the end +using AccF = wmma::fragment; + +__device__ __forceinline__ void split16(float x, __half& hi, __half& lo) { + hi = __float2half_rn(x); + lo = __float2half_rn((x - __half2float(hi)) * LO_SCALE); +} + +struct VSmem { + __half qh[IDX_HEADS][VQ][VQS], ql[IDX_HEADS][VQ][VQS]; // queries hi / lo, [head][query][dim] + __half kh[4][16][VKS], kl[4][16][VKS]; // per warp: the key slice hi / lo + float ep[4][16][16 + 4]; // per warp: the epilogue tile [block][query] + int nbid[VQ]; +}; + +__global__ void __launch_bounds__(128) scores_volta_kernel(const float* __restrict__ pooled, const float* __restrict__ q_idx, + const int32_t* __restrict__ steps, int64_t nq, int64_t max_blocks, + int64_t reach, float* __restrict__ out) { + extern __shared__ __align__(16) unsigned char smraw[]; + VSmem& S = *reinterpret_cast(smraw); + const int t = threadIdx.x, lane = t & 31, warp = t >> 5; + const int64_t q0 = (int64_t) blockIdx.y * VQ; + for (int i = t; i < VQ * IDX_HEADS * IDX_DIM / 4; i += 128) { + const int row = i / (IDX_DIM / 4), c = i % (IDX_DIM / 4); + const int qr = row / IDX_HEADS, h = row % IDX_HEADS; + float4 v = make_float4(0.f, 0.f, 0.f, 0.f); + if (q0 + qr < nq) v = reinterpret_cast(q_idx + (q0 + qr) * IDX_HEADS * IDX_DIM)[h * (IDX_DIM / 4) + c]; + const float x[4] = {v.x, v.y, v.z, v.w}; +#pragma unroll + for (int j = 0; j < 4; ++j) split16(x[j], S.qh[h][qr][c * 4 + j], S.ql[h][qr][c * 4 + j]); + } + if (t < VQ) S.nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : 0; + __syncthreads(); + int hi_nbid = 0; +#pragma unroll + for (int i = 0; i < VQ; ++i) hi_nbid = max(hi_nbid, S.nbid[i]); + for (int it = 0; it < VITER; ++it) { + const int64_t b0 = (((int64_t) blockIdx.x * VITER + it) * 4 + warp) * 16; + if (b0 >= reach || b0 >= hi_nbid) break; // warp-uniform + // this lane's share of the key slice: row lane/2, 8 dims at (lane%2)*8 + const int kr = lane >> 1, kc = (lane & 1) * 8; + const int64_t row = b0 + kr; + const bool rv = row < hi_nbid && row < max_blocks; + const float* kp = pooled + (rv ? row : 0) * IDX_DIM + kc; + AccF acc[IDX_HEADS], cor[IDX_HEADS]; +#pragma unroll + for (int h = 0; h < IDX_HEADS; ++h) { wmma::fill_fragment(acc[h], 0.f); wmma::fill_fragment(cor[h], 0.f); } + float4 n0 = rv ? *reinterpret_cast(kp) : make_float4(0.f, 0.f, 0.f, 0.f); + float4 n1 = rv ? *reinterpret_cast(kp + 4) : make_float4(0.f, 0.f, 0.f, 0.f); +#pragma unroll 1 + for (int kk = 0; kk < IDX_DIM / 16; ++kk) { + const float x[8] = {n0.x, n0.y, n0.z, n0.w, n1.x, n1.y, n1.z, n1.w}; + __syncwarp(); +#pragma unroll + for (int j = 0; j < 8; ++j) split16(x[j], S.kh[warp][kr][kc + j], S.kl[warp][kr][kc + j]); + __syncwarp(); + if (kk + 1 < IDX_DIM / 16 && rv) { // the next slice's loads behind this slice's products + n0 = *reinterpret_cast(kp + (kk + 1) * 16); + n1 = *reinterpret_cast(kp + (kk + 1) * 16 + 4); + } + wmma::fragment Ah, Al; + wmma::load_matrix_sync(Ah, &S.kh[warp][0][0], VKS); + wmma::load_matrix_sync(Al, &S.kl[warp][0][0], VKS); +#pragma unroll + for (int h = 0; h < IDX_HEADS; ++h) { + wmma::fragment Bh, Bl; + wmma::load_matrix_sync(Bh, &S.qh[h][0][kk * 16], VQS); + wmma::load_matrix_sync(Bl, &S.ql[h][0][kk * 16], VQS); + // hi*hi in a fresh fragment, added with FP32 adds (the tensor cores truncate when accumulating) + AccF st; + wmma::fill_fragment(st, 0.f); + wmma::mma_sync(st, Ah, Bh, st); +#pragma unroll + for (int e = 0; e < st.num_elements; ++e) acc[h].x[e] += st.x[e]; + wmma::mma_sync(cor[h], Ah, Bl, cor[h]); + wmma::mma_sync(cor[h], Al, Bh, cor[h]); + } + } + // relu per head, heads added in order; then masked write [query][block] + AccF sc; +#pragma unroll + for (int e = 0; e < sc.num_elements; ++e) { + float s = 0.f; +#pragma unroll + for (int h = 0; h < IDX_HEADS; ++h) { + const float d = acc[h].x[e] + cor[h].x[e] * (1.0f / LO_SCALE); + s += d > 0.f ? d : 0.f; + } + sc.x[e] = s; + } + wmma::store_matrix_sync(&S.ep[warp][0][0], sc, 16 + 4, wmma::mem_row_major); // [block][query] + __syncwarp(); +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int i = lane + 32 * j, qr = i / 16, br = i % 16; + const int64_t b = b0 + br; + if (q0 + qr < nq && b < S.nbid[qr] && b < max_blocks) out[(q0 + qr) * max_blocks + b] = S.ep[warp][br][qr]; + } + } +} +#endif + // the tail block n_bid of each query: exactly block_scores_kernel's arithmetic for that block __global__ void __launch_bounds__(32) block_scores_tail_kernel(const float* __restrict__ dead, const float* __restrict__ q_idx, @@ -1155,6 +1271,31 @@ bool qsa_block_scores_tc(const float* pooled, const float* dead, const float* q_ const char* w = std::getenv("STRATA_QSA_WARP"); cc_major[dev] = w && (!std::strcmp(w, "1") || !std::strcmp(w, "select")) ? 7 : strata::cc_major_of(major); } + // ac922: STRATA_SELECT_VOLTA=1, the wmma scorer (FP16 hi + lo); otherwise upstream's opt-in SIMT kernel below + static const bool volta_on = [] { + const char* v = std::getenv("STRATA_SELECT_VOLTA"); + return v != nullptr && v[0] != '\0' && v[0] != '0'; + }(); + if (cc_major[dev] == 7 && volta_on && nq <= 65535 * VQ) { + static bool vattr[64] = {}; + if (!vattr[dev]) { + if (cudaFuncSetAttribute(scores_volta_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + (int) sizeof(VSmem)) != cudaSuccess) { + cudaGetLastError(); + return false; + } + vattr[dev] = true; + } + const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks; + const int64_t per = (int64_t) 4 * VITER * 16; + const dim3 grid((unsigned) ((reach + per - 1) / per), (unsigned) ((nq + VQ - 1) / VQ)); + scores_volta_kernel<<>>(pooled, q_idx, steps, nq, max_blocks, + reach, scores); + block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks, scores); + const cudaError_t e = cudaGetLastError(); + if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores_tc (volta): %s\n", cudaGetErrorString(e)); std::exit(1); } + return true; + } if (cc_major[dev] < 8) { // no TF32 tensor cores: the FP32 tiled kernel, opt-in (STRATA_SELECT_SIMT=1): its sum order differs from the // warp kernel's, so the default stays the caller's warp kernel and the output bytes stay the same diff --git a/src/kernels/cuda/shared_expert.cu b/src/kernels/cuda/shared_expert.cu index 450fe24a991..e3dba7d3b5a 100644 --- a/src/kernels/cuda/shared_expert.cu +++ b/src/kernels/cuda/shared_expert.cu @@ -159,6 +159,7 @@ __global__ void moe_combine_kernel(const float* __restrict__ parts, const float* } // namespace void shared_expert_set_native_bf16(bool enabled) { native_bf16 = enabled; } +bool shared_expert_native_bf16() { return native_bf16; } bool shared_expert_native_bf16_enabled() { return native_bf16; } namespace { @@ -212,11 +213,13 @@ bool fused_swiglu_q81_enabled() { } // namespace void shared_expert_multi(int n_tok, const float* x, const uint16_t* x_bf16, const NativeSharedWeights& nw, - const uint16_t* gate_inp_bf16, float* gate, float* up, float* g, float* out, int64_t n_embd, - int64_t n_ff, void* stream, const void* x_q8_1_ready, int lfuse) { + const void* gate_inp_bf16, float* gate, float* up, float* g, float* out, int64_t n_embd, + int64_t n_ff, void* stream, const void* x_q8_1_ready, int lfuse, WForm gate_inp_form) { const bool gate_deferred = (lfuse & 1) != 0, pair = (lfuse & 2) != 0; if (n_tok < 1 || n_tok > 8 || !nw.q8_1 || !nw.gate_data || !nw.up_data || !nw.down_data || !stream) throw std::invalid_argument("shared_expert_multi: needs 1..8 tokens, native weights, scratch and a stream"); + if (gate_inp_form != WForm::Bf16 && !native_bf16) + throw std::invalid_argument("shared_expert_multi: an FP16/F32 scalar gate needs the native (FP32-activation) gate"); cudaStream_t cs = (cudaStream_t) stream; // the activation's q8_1: the caller's (STRATA_VERIFY_QDEDUP, x_q8_1_ready) or the descriptor's, else quantized here const void* x_q8_1 = x_q8_1_ready ? x_q8_1_ready : nw.x_q8_1; @@ -245,18 +248,19 @@ void shared_expert_multi(int n_tok, const float* x, const uint16_t* x_bf16, cons } static const bool batch = [] { const char* v = std::getenv("STRATA_DEC_BATCH"); return v == nullptr || std::atoi(v) != 0; }(); if (native_bf16 && batch && n_tok > 1) { // one gemv for all rows (outputs identical), fused sigmoid+scale - bf16_gemv_fp32_mmvf_multi(x, n_embd, gate_inp_bf16, g, 1, n_embd, 1, n_tok, stream); + gemv_fp32_mmvf_multi(x, n_embd, gate_inp_bf16, gate_inp_form, g, 1, n_embd, 1, n_tok, stream); launch_sigmoid_scale_rows(out, g, (int) n_embd, n_tok, cs); } else if (native_bf16 && n_tok == 1) { - bf16_gemv_fp32_mmvf(x, gate_inp_bf16, g, n_embd, 1, stream); + gemv_fp32_mmvf(x, gate_inp_bf16, gate_inp_form, g, n_embd, 1, stream); launch_sigmoid_scale_rows(out, g, (int) n_embd, 1, cs); } else { for (int t = 0; t < n_tok; ++t) { if (native_bf16) { - bf16_gemv_fp32_mmvf(x + (size_t) t * n_embd, gate_inp_bf16, g + t, n_embd, 1, stream); + gemv_fp32_mmvf(x + (size_t) t * n_embd, gate_inp_bf16, gate_inp_form, g + t, n_embd, 1, stream); native_scalar_sigmoid_kernel<<<1, 1, 0, cs>>>(g + t); } else { - scalar_gate_kernel<<<1, 256, 0, cs>>>(x_bf16 + (size_t) t * n_embd, gate_inp_bf16, g + t, (int) n_embd); + scalar_gate_kernel<<<1, 256, 0, cs>>>(x_bf16 + (size_t) t * n_embd, (const uint16_t*) gate_inp_bf16, g + t, + (int) n_embd); } } scale_rows_kernel<<>>( @@ -278,9 +282,9 @@ void shared_expert(const uint8_t* x_q8_0, const uint8_t* x_q8k, const uint16_t* const uint8_t* gate_codes, const float* gate_scales, const float* gate_off, const SForm& up_form, const uint8_t* up_codes, const float* up_scales, const float* up_off, const SForm& down_form, const uint8_t* down_codes, const float* down_scales, - const float* down_off, const uint16_t* gate_inp_bf16, float* scratch, float* out, + const float* down_off, const void* gate_inp_bf16, float* scratch, float* out, int64_t n_embd, int64_t n_ff, int tpr, void* stream, const float* x_f32, - const NativeSharedWeights* native) { + const NativeSharedWeights* native, WForm gate_inp_form) { if (n_embd <= 0 || n_ff <= 0) return; const bool use_native = native_bf16; const bool native_gate = native && native->gate_data && native_mmvq_supported(native->gate_type); @@ -379,10 +383,12 @@ void shared_expert(const uint8_t* x_q8_0, const uint8_t* x_q8k, const uint16_t* // `<<<1, 256>>>`: one block, because the output is ONE scalar and a second block would only add a global // round trip. 256 threads is the reduction's width, not the problem's size. if (use_native) { - bf16_gemv_fp32_mmvf(x_f32, gate_inp_bf16, g, n_embd, 1, stream); + gemv_fp32_mmvf(x_f32, gate_inp_bf16, gate_inp_form, g, n_embd, 1, stream); launch_sigmoid_scale_rows(out, g, (int) n_embd, 1, (cudaStream_t) stream); } else { - scalar_gate_kernel<<<1, 256, 0, (cudaStream_t) stream>>>(x_bf16, gate_inp_bf16, g, (int) n_embd); + if (gate_inp_form != WForm::Bf16) + throw std::invalid_argument("shared_expert: an FP16/F32 scalar gate (STRATA_FP16=load) needs the native gate"); + scalar_gate_kernel<<<1, 256, 0, (cudaStream_t) stream>>>(x_bf16, (const uint16_t*) gate_inp_bf16, g, (int) n_embd); scale_kernel<<>>(out, g, (int) n_embd); } diff --git a/src/kernels/cuda/verify_kernels.cu b/src/kernels/cuda/verify_kernels.cu index 39fdda2d6e3..37485b26ae2 100644 --- a/src/kernels/cuda/verify_kernels.cu +++ b/src/kernels/cuda/verify_kernels.cu @@ -90,6 +90,7 @@ __global__ void gdn_conv_commit_kernel(float* __restrict__ hist, const float* __ hist[c * 3 + 2] = seq[2]; } +<<<<<<< this fork template __device__ __forceinline__ void gdn_ab_body(const int bx, const float* __restrict__ x, const uint16_t* __restrict__ wa, const uint16_t* __restrict__ wb, @@ -97,36 +98,60 @@ __device__ __forceinline__ void gdn_ab_body(const int bx, const float* __restric const float* __restrict__ ssm_a, float* __restrict__ gate, float* __restrict__ beta, int n, int h_v, int T) { const int row = bx * 2 + (threadIdx.x >> 5), lane = threadIdx.x & 31; +======= +// 8 weights of row `base` (16-byte chunk j of a 16-bit row; 32 bytes of an F32 one), widened to FP32 +template +__device__ __forceinline__ void wload8(const void* base, int j, float* w) { + if constexpr (WF == 2) { + const float4* p = reinterpret_cast(base) + 2 * j; + const float4 a = __ldg(p), b = __ldg(p + 1); + w[0] = a.x; w[1] = a.y; w[2] = a.z; w[3] = a.w; w[4] = b.x; w[5] = b.y; w[6] = b.z; w[7] = b.w; + } else { + const uint4 v = __ldg(reinterpret_cast(base) + j); + const uint32_t u[4] = {v.x, v.y, v.z, v.w}; +#pragma unroll + for (int k = 0; k < 4; ++k) { + if constexpr (WF == 0) { w[2 * k] = __uint_as_float(u[k] << 16); w[2 * k + 1] = __uint_as_float(u[k] & 0xffff0000u); } + else { w[2 * k] = __half2float(__ushort_as_half((uint16_t) u[k])); w[2 * k + 1] = __half2float(__ushort_as_half((uint16_t) (u[k] >> 16))); } + } + } +} + +template +__global__ void __launch_bounds__(256) gdn_ab_multi_kernel(const float* __restrict__ x, const void* __restrict__ wa, + const void* __restrict__ wb, + const float* __restrict__ dt, + const float* __restrict__ ssm_a, float* __restrict__ gate, + float* __restrict__ beta, int n, int h_v, int T) { + const int row = blockIdx.x * 8 + (threadIdx.x >> 5), lane = threadIdx.x & 31; +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) if (row >= 2 * h_v) return; const bool is_beta = row >= h_v; const int r = is_beta ? row - h_v : row; - const uint4* w4 = reinterpret_cast((is_beta ? wb : wa) + (size_t) r * n); - float acc[MAX_T]; + const void* wrow = reinterpret_cast(is_beta ? wb : wa) + (size_t) r * n * (WF == 2 ? 4 : 2); + float acc[kVerifyMaxT]; #pragma unroll - for (int t = 0; t < MAX_T; ++t) acc[t] = 0.0f; + for (int t = 0; t < kVerifyMaxT; ++t) acc[t] = 0.0f; for (int j = lane; j < n / 8; j += 32) { - const uint4 wv = __ldg(w4 + j); - const float w0 = __uint_as_float(wv.x << 16), w1 = __uint_as_float(wv.x & 0xffff0000u); - const float w2 = __uint_as_float(wv.y << 16), w3 = __uint_as_float(wv.y & 0xffff0000u); - const float w4f = __uint_as_float(wv.z << 16), w5 = __uint_as_float(wv.z & 0xffff0000u); - const float w6 = __uint_as_float(wv.w << 16), w7 = __uint_as_float(wv.w & 0xffff0000u); + float w[8]; + wload8(wrow, j, w); #pragma unroll - for (int t = 0; t < MAX_T; ++t) { - if (!EXACT_T && t >= T) break; + for (int t = 0; t < kVerifyMaxT; ++t) { + if (t >= T) break; const float* xt = x + (size_t) t * n; const float4 xa = __ldg(reinterpret_cast(xt + j * 8)); const float4 xb = __ldg(reinterpret_cast(xt + j * 8 + 4)); float a = acc[t]; - a = fmaf(w0, xa.x, a); a = fmaf(w1, xa.y, a); - a = fmaf(w2, xa.z, a); a = fmaf(w3, xa.w, a); - a = fmaf(w4f, xb.x, a); a = fmaf(w5, xb.y, a); - a = fmaf(w6, xb.z, a); a = fmaf(w7, xb.w, a); + a = fmaf(w[0], xa.x, a); a = fmaf(w[1], xa.y, a); + a = fmaf(w[2], xa.z, a); a = fmaf(w[3], xa.w, a); + a = fmaf(w[4], xb.x, a); a = fmaf(w[5], xb.y, a); + a = fmaf(w[6], xb.z, a); a = fmaf(w[7], xb.w, a); acc[t] = a; } } #pragma unroll - for (int t = 0; t < MAX_T; ++t) { - if (!EXACT_T && t >= T) break; + for (int t = 0; t < kVerifyMaxT; ++t) { + if (t >= T) break; float a = acc[t]; #pragma unroll for (int o = 16; o > 0; o >>= 1) a += __shfl_xor_sync(0xffffffffu, a, o); @@ -141,6 +166,7 @@ __device__ __forceinline__ void gdn_ab_body(const int bx, const float* __restric } } +<<<<<<< this fork template __global__ void __launch_bounds__(64) gdn_ab_multi_kernel(const float* __restrict__ x, const uint16_t* __restrict__ wa, const uint16_t* __restrict__ wb, @@ -165,6 +191,54 @@ __global__ void __launch_bounds__(S) gdn_conv_ab_kernel(int conv_blocks, float* gdn_conv_l2_body((int) blockIdx.x - ab_blocks, (int) blockIdx.y, hist, qkv, w, h, C, qk_heads, eps, t_begin, commit); } else if (blockIdx.y == 0 && threadIdx.x < 64) { gdn_ab_body((int) blockIdx.x, x, wa, wb, dt, ssm_a, gate, beta, n, h_v, T); +======= +// gdn_ab_multi_kernel for TT tokens and n = 2560 (10 chunks per lane): the loops have fixed trip counts, so every +// chunk's weight and activation loads can be issued up front; the same FMAs in the same order +template +__global__ void __launch_bounds__(256) gdn_ab_multi_t_kernel(const float* __restrict__ x, const void* __restrict__ wa, + const void* __restrict__ wb, const float* __restrict__ dt, + const float* __restrict__ ssm_a, float* __restrict__ gate, + float* __restrict__ beta, int h_v) { + constexpr int n = 2560, CH = n / 8 / 32; + const int row = blockIdx.x * 8 + (threadIdx.x >> 5), lane = threadIdx.x & 31; + if (row >= 2 * h_v) return; + const bool is_beta = row >= h_v; + const int r = is_beta ? row - h_v : row; + const void* wrow = reinterpret_cast(is_beta ? wb : wa) + (size_t) r * n * (WF == 2 ? 4 : 2); + float acc[TT]; +#pragma unroll + for (int t = 0; t < TT; ++t) acc[t] = 0.0f; +#pragma unroll + for (int it = 0; it < CH; ++it) { + const int j = lane + 32 * it; + float w[8]; + wload8(wrow, j, w); +#pragma unroll + for (int t = 0; t < TT; ++t) { + const float* xt = x + (size_t) t * n; + const float4 xa = *reinterpret_cast(xt + j * 8); + const float4 xb = *reinterpret_cast(xt + j * 8 + 4); + float a = acc[t]; + a = fmaf(w[0], xa.x, a); a = fmaf(w[1], xa.y, a); + a = fmaf(w[2], xa.z, a); a = fmaf(w[3], xa.w, a); + a = fmaf(w[4], xb.x, a); a = fmaf(w[5], xb.y, a); + a = fmaf(w[6], xb.z, a); a = fmaf(w[7], xb.w, a); + acc[t] = a; + } + } +#pragma unroll + for (int t = 0; t < TT; ++t) { + float a = acc[t]; + for (int o = 16; o > 0; o >>= 1) a += __shfl_xor_sync(0xffffffffu, a, o); + if (lane != 0) continue; + if (is_beta) { + beta[(size_t) t * h_v + r] = 1.0f / (1.0f + __expf(-a)); + } else { + const float v = a + dt[r]; + const float sp = v > 20.0f ? v : log1pf(__expf(v)); + gate[(size_t) t * h_v + r] = sp * ssm_a[r]; + } +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) } } @@ -579,11 +653,103 @@ __global__ void copy_indexed_kernel(float* __restrict__ dst, const float* __rest __global__ void fetch_blobs_kernel(const unsigned long long* __restrict__ src, const int32_t* __restrict__ n, uint4* __restrict__ dst, long long per) { + // four independent 16-byte loads in flight per thread before their stores: the link's latency is covered with + // fewer blocks, which leaves SMs to the VRAM experts running beside this copy (STRATA_FETCH_BLOCKS) const long long total = (long long) *n * per; - for (long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; i < total; - i += (long long) gridDim.x * blockDim.x) { - const long long k = i / per, off = i - k * per; - dst[i] = ((const uint4*) src[k])[off]; + const long long S = (long long) gridDim.x * blockDim.x; + long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; + for (; i + 3 * S < total; i += 4 * S) { + uint4 v[4]; +#pragma unroll + for (int u = 0; u < 4; ++u) { + const long long j = i + u * S, k = j / per; + v[u] = ((const uint4*) src[k])[j - k * per]; + } +#pragma unroll + for (int u = 0; u < 4; ++u) dst[i + u * S] = v[u]; + } + for (; i < total; i += S) { + const long long k = i / per; + dst[i] = ((const uint4*) src[k])[i - k * per]; + } +} + +__global__ void fetch_blobs_strided_kernel(const unsigned long long* __restrict__ src, const int32_t* __restrict__ n, + uint8_t* __restrict__ dst, long long per, int q0, int qstep) { + const int nq = *n > q0 ? (*n - q0 + qstep - 1) / qstep : 0; + const long long total = (long long) nq * per; + const long long S = (long long) gridDim.x * blockDim.x; + for (long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += S) { + const long long j = i / per, off = i - j * per; + const int q = q0 + (int) j * qstep; + ((uint4*) (dst + (size_t) q * (size_t) per * 16))[off] = ((const uint4*) src[q])[off]; + } +} + +__global__ void wait_and_clear_kernel(uint32_t* flag) { + volatile uint32_t* f = flag; + while (*f == 0) __nanosleep(100); + __threadfence_system(); + *f = 0; +} + +__global__ void partner_fetch_kernel(PartnerFetchArgs a) { + __shared__ unsigned long long sp[32]; + __shared__ int s_n; + __shared__ int s_stop; + for (int k = 0; k < a.steps; ++k) { + const uint32_t ring = (uint32_t) k + 1; + if (threadIdx.x == 0) { + volatile const uint32_t* f = a.flagA; + uint32_t v; + while ((v = *f) < ring) __nanosleep(200); + s_stop = v == 0xffffffffu; + __threadfence_system(); + } + __syncthreads(); + if (s_stop) return; + const int grp = k % a.G, l = a.lb + k / a.G; + const volatile int32_t* pl = a.plan + (size_t) grp * (size_t) a.plan_i32; + if (threadIdx.x == 0) s_n = pl[2]; + __syncthreads(); + const int n2 = s_n < 32 ? s_n : 32; + const volatile unsigned long long* p2 = (const volatile unsigned long long*) (pl + a.ptr_off) + a.capx; + if ((int) threadIdx.x < n2) sp[threadIdx.x] = p2[threadIdx.x]; + __syncthreads(); + const long long per16 = a.blob_bytes[l] / 16; + uint8_t* stage = a.staging + (size_t) (grp * a.per) * (size_t) a.max_blob; + const int nodd = n2 / 2; // q = 1, 3, ... < n2 + const long long total = (long long) nodd * per16; + const long long S = (long long) gridDim.x * blockDim.x; + long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; + for (; i + 3 * S < total; i += 4 * S) { + uint4 v[4]; +#pragma unroll + for (int u = 0; u < 4; ++u) { + const long long x = i + u * S, j = x / per16; + v[u] = ((const uint4*) sp[2 * j + 1])[x - j * per16]; + } +#pragma unroll + for (int u = 0; u < 4; ++u) { + const long long x = i + u * S, j = x / per16; + ((uint4*) (stage + (size_t) (2 * j + 1) * (size_t) a.blob_bytes[l]))[x - j * per16] = v[u]; + } + } + for (; i < total; i += S) { + const long long j = i / per16; + ((uint4*) (stage + (size_t) (2 * j + 1) * (size_t) a.blob_bytes[l]))[i - j * per16] = + ((const uint4*) sp[2 * j + 1])[i - j * per16]; + } + __threadfence_system(); + __syncthreads(); + if (threadIdx.x == 0) { + const unsigned prev = atomicAdd(&a.arrive[k], 1u); + if (prev == gridDim.x - 1) { + __threadfence_system(); + *(volatile uint32_t*) (a.done + k) = 1u; + } + } + __syncthreads(); } } @@ -678,10 +844,37 @@ __global__ void force_token_kernel(int32_t* tok, const int32_t* force, int j) { void fetch_blobs(const unsigned long long* src, const int32_t* n, uint8_t* dst, int64_t blob_bytes, int cap, void* stream) { if (cap <= 0) return; if (blob_bytes % 16 != 0) { std::fprintf(stderr, "fetch_blobs: blob size must be a multiple of 16\n"); std::exit(1); } - fetch_blobs_kernel<<<48 * 8, 256, 0, (cudaStream_t) stream>>>(src, n, (uint4*) dst, (long long) (blob_bytes / 16)); + static const unsigned blocks = [] { + const char* v = std::getenv("STRATA_FETCH_BLOCKS"); + const int x = v ? std::atoi(v) : 0; + return x > 0 && x <= 4096 ? (unsigned) x : 24u; // 24 x 256 beside the VRAM experts (AC922, 2 GPUs: 43.7 ms/window vs 45.8) + }(); + static const unsigned threads = [] { + const char* v = std::getenv("STRATA_FETCH_THREADS"); + const int x = v ? std::atoi(v) : 0; + return x == 512 || x == 1024 ? (unsigned) x : 256u; + }(); + fetch_blobs_kernel<<>>(src, n, (uint4*) dst, (long long) (blob_bytes / 16)); check("fetch_blobs"); } +void fetch_blobs_strided(const unsigned long long* src, const int32_t* n, uint8_t* dst, int64_t blob_bytes, int q0, + int qstep, void* stream) { + if (blob_bytes % 16 != 0) { std::fprintf(stderr, "fetch_blobs: blob size must be a multiple of 16\n"); std::exit(1); } + fetch_blobs_strided_kernel<<<24, 256, 0, (cudaStream_t) stream>>>(src, n, dst, (long long) (blob_bytes / 16), q0, qstep); + check("fetch_blobs_strided"); +} + +void wait_and_clear(uint32_t* flag, void* stream) { + wait_and_clear_kernel<<<1, 1, 0, (cudaStream_t) stream>>>(flag); + check("wait_and_clear"); +} + +void partner_fetch(const PartnerFetchArgs& a, int blocks, void* stream) { + partner_fetch_kernel<<>>(a); + check("partner_fetch"); +} + void rebase_ptrs(unsigned long long* ptr, const int32_t* n, uint8_t* base, int64_t blob_bytes, void* stream) { rebase_ptrs_kernel<<<1, 128, 0, (cudaStream_t) stream>>>(ptr, n, (unsigned long long) base, (long long) blob_bytes); check("rebase_ptrs"); @@ -1016,6 +1209,7 @@ void gdn_conv_commit(float* history, const float* qkv, int channels, const int32 check("gdn_conv_commit"); } +<<<<<<< this fork bool gdn_commit_layers(float* state, int64_t state_stride, int64_t conv_off, const float* qkv, int64_t qkv_stride, const float* h, int64_t h_stride, const float* gate, const float* beta, int64_t gb_stride, int conv_channels, int h_k, int h_v, int n_max, int n_layers, const int32_t* n_keep, void* stream) { @@ -1041,21 +1235,32 @@ bool gdn_commit_layers(float* state, int64_t state_stride, int64_t conv_off, con void gdn_ab_multi(const float* x, const uint16_t* w_alpha, const uint16_t* w_beta, const float* dt, const float* ssm_a, float* gate, float* beta, int n_embd, int h_v, int n_tok, void* stream) { +======= +void gdn_ab_multi(const float* x, const void* w_alpha, const void* w_beta, const float* dt, const float* ssm_a, + float* gate, float* beta, int n_embd, int h_v, int n_tok, void* stream, WForm form) { +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) if (n_embd % 8 != 0 || n_tok < 1 || n_tok > kVerifyMaxT) { std::fprintf(stderr, "gdn_ab_multi: invalid arguments\n"); std::exit(1); } - const unsigned blocks = (unsigned) ((2 * h_v + 1) / 2); - cudaStream_t st = (cudaStream_t) stream; - switch (n_tok) { - case 1: gdn_ab_multi_kernel<1, true><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break; - case 2: gdn_ab_multi_kernel<2, true><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break; - case 3: gdn_ab_multi_kernel<3, true><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break; - case 4: gdn_ab_multi_kernel<4, true><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break; - case 5: gdn_ab_multi_kernel<5, true><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break; - case 6: gdn_ab_multi_kernel<6, true><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break; - default: gdn_ab_multi_kernel<<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break; + const unsigned blocks = (unsigned) ((2 * h_v + 7) / 8); + static const bool generic = [] { const char* v = std::getenv("STRATA_HC_DOWN_GENERIC"); return v && v[0] == '1'; }(); + if (n_embd == 2560 && !generic) { + const cudaStream_t st = (cudaStream_t) stream; +#define STRATA_AB_T(TT) case TT: \ + if (form == WForm::Bf16) gdn_ab_multi_t_kernel<0, TT><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, h_v); \ + else if (form == WForm::F16) gdn_ab_multi_t_kernel<1, TT><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, h_v); \ + else gdn_ab_multi_t_kernel<2, TT><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, h_v); \ + check("gdn_ab_multi"); return; + switch (n_tok) { STRATA_AB_T(1) STRATA_AB_T(2) STRATA_AB_T(3) STRATA_AB_T(4) STRATA_AB_T(5) STRATA_AB_T(6) STRATA_AB_T(7) STRATA_AB_T(8) default: break; } +#undef STRATA_AB_T } + if (form == WForm::Bf16) + gdn_ab_multi_kernel<0><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); + else if (form == WForm::F16) + gdn_ab_multi_kernel<1><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); + else + gdn_ab_multi_kernel<2><<>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); check("gdn_ab_multi"); } diff --git a/src/kernels/ngram.cpp b/src/kernels/ngram.cpp index 91de3705831..430c5e2d12e 100644 --- a/src/kernels/ngram.cpp +++ b/src/kernels/ngram.cpp @@ -20,6 +20,9 @@ #include #include #include +#include +#include +#include #include #if defined(_WIN32) @@ -545,6 +548,7 @@ bool PleTable::gather_batch(const uint32_t* rows, size_t n_tokens, float* out, s return true; } wait_prefetches(); +<<<<<<< this fork // #1984 #1971: a mapped table that is not in the file cache served this loop as serial demand faults, one random // 4 KiB page at a time (16 rows a token, 16 x the chunk's tokens for a prompt), and the first chunk of a prompt has // only layer 0 to hide it behind: a cold 27 GB table at ~0.2-1 ms a fault is minutes with no progress mark, which @@ -578,6 +582,35 @@ bool PleTable::gather_batch(const uint32_t* rows, size_t n_tokens, float* out, s const size_t e = std::min(n, w + kWin); for (size_t i = w; i < e; ++i) read_row(rows[i], out + i * PLE_HEAD_DIM); } +======= + // the prompt path's batch (16 rows a token, scattered over the table) on several threads: one thread decoded a + // 3,072-token chunk's 49,152 rows in ~375 ms on POWER9 and set the pace of a 4-GPU prompt (the first stage waited + // ~14 ms a chunk for it). Each row decodes into its own slot: the same output on any thread count. + // STRATA_PLE_GATHER_THREADS (default 8; 1 = the serial loop) + static const int nth = [] { + const char* v = std::getenv("STRATA_PLE_GATHER_THREADS"); + const int k = v ? std::atoi(v) : 8; + return k < 1 ? 1 : (k > 64 ? 64 : k); + }(); + if (nth > 1 && n >= 4096 && impl_->data != nullptr) { + std::vector th; + const size_t per = (n + (size_t) nth - 1) / (size_t) nth; + for (int k = 0; k < nth; ++k) { + const size_t i0 = (size_t) k * per, i1 = std::min(n, i0 + per); + if (i0 >= i1) break; + th.emplace_back([this, rows, out, i0, i1] { + for (size_t i = i0; i < i1; ++i) { + if (rows[i] >= impl_->n_rows) std::memset(out + i * PLE_HEAD_DIM, 0, (size_t) PLE_HEAD_DIM * sizeof(float)); + else impl_->decode(impl_->data + (size_t) rows[i] * impl_->rb, out + i * PLE_HEAD_DIM); + } + }); + } + for (auto& t : th) t.join(); + impl_->bytes_read += (uint64_t) n * impl_->rb; + return true; + } + for (size_t i = 0; i < n; ++i) read_row(rows[i], out + i * PLE_HEAD_DIM); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) return true; } diff --git a/src/kernels/qsa_prompt_attn_parity.cpp b/src/kernels/qsa_prompt_attn_parity.cpp index 3c8867c5695..cf9e4f57978 100644 --- a/src/kernels/qsa_prompt_attn_parity.cpp +++ b/src/kernels/qsa_prompt_attn_parity.cpp @@ -317,6 +317,19 @@ int main(int argc, char** argv) { setenv("STRATA_HIP_WMMA", "1", 1); #endif } +#endif +#if !defined(__HIP_PLATFORM_AMD__) + // P4: on Volta (sm_70) the kernel under test is the opt-in tensor-core one (STRATA_ATTN_WMMA), int8 KV only + bool volta = false; + { + int dev = 0, major = 0, minor = 0; + if (cudaGetDevice(&dev) == cudaSuccess && + cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess && + cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev) == cudaSuccess && major == 7 && minor < 5) { + volta = true; + setenv("STRATA_ATTN_WMMA", "1", 1); + } + } #endif const int64_t ctx = argc > 1 ? std::atoll(argv[1]) : 32768; const int64_t nq = argc > 2 ? std::atoll(argv[2]) : 2048; @@ -324,9 +337,11 @@ int main(int argc, char** argv) { int fails = 0; fails += run(1, ctx, nq, reps); #if !defined(__HIP_PLATFORM_AMD__) - fails += run(0, ctx, nq, reps); // FP16 KV: the RDNA4 kernel takes int8 KV only - fails += run(2, ctx, nq, reps); // Q4_0 KV (mode 4) - fails += run(2, 1500, std::min(nq, 1500), reps); + if (!volta) { // the RDNA4 and Volta kernels take int8 KV only + fails += run(0, ctx, nq, reps); // FP16 KV + fails += run(2, ctx, nq, reps); // Q4_0 KV (mode 4) + fails += run(2, 1500, std::min(nq, 1500), reps); + } #endif fails += run(1, 1500, std::min(nq, 1500), reps); // short context: the selection is every cell fails += run(1, 2100, std::min(nq, 256), reps); // the identity-to-sparse edge From a5bf5aabd542095cac8f9f67cbfb54042f0c7419 Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:02 +0200 Subject: [PATCH 05/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): src/ngram from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 1 file(s) under src/ngram -- 0 clean, 1 with conflict markers. Conflict-marked files -- a pull request with conflicts: resolve them, do not discard the work: src/ngram/ple_reader.cpp Original authors: BlueKingMuch,Eli Elgaev,Konstantinos Korres,Niko1221 Original commits touching these paths (5 subject(s)): - AMD HIP backend for gfx1100 (PR #121, squashed onto its 0.1.21 base) - Helper threads leave the host thread's core (Linux) - PLE: the n-gram table in FP8 E4M3, as Qwen ships it - Strata: Qwen3.8-Flash-Next on a 12-24 GB NVIDIA GPU + 64 GB RAM, one-click install - ple: keep the SSD awake while rows are read (STRATA_SSD_KEEPALIVE) Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: src/ngram Fork-Replay-Automated: true --- src/ngram/ple_reader.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ngram/ple_reader.cpp b/src/ngram/ple_reader.cpp index 9d2a05051e5..b093576e3df 100644 --- a/src/ngram/ple_reader.cpp +++ b/src/ngram/ple_reader.cpp @@ -1,6 +1,10 @@ // src/ngram/ple_reader.cpp - see include/strata/ngram/ple_reader.hpp. #include "strata/ngram/ple_reader.hpp" +<<<<<<< this fork #include "strata/platform/aux_cpus.hpp" +======= +#include "strata/platform/thread_affinity.hpp" +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) #include #include @@ -364,7 +368,11 @@ bool PleReader::open(const std::string& path, uint64_t table_offset, uint64_t n_ impl_->threaded = io_thread; if (io_thread) { try { +<<<<<<< this fork impl_->worker = std::thread([this] { strata::aux_cpus::pin_current_thread(); impl_->worker_loop(); }); +======= + impl_->worker = std::thread([this] { strata::platform::release_inherited_pin(); impl_->worker_loop(); }); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) } catch (const std::exception& e) { err = std::string("PleReader: cannot create I/O worker: ") + e.what(); close(); From 3a25730c73f6719bc556994f9a189a01b3299cb5 Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:02 +0200 Subject: [PATCH 06/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): src/platform from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 1 file(s) under src/platform -- 0 clean, 1 with conflict markers. Conflict-marked files -- a pull request with conflicts: resolve them, do not discard the work: src/platform/direct_file.cpp Original authors: Eli Elgaev,Niko1221 Original commits touching these paths (3 subject(s)): - C-4: the prompt's embedding in one gather, and the n-gram table read by several threads - Helper threads leave the host thread's core (Linux) - Strata: Qwen3.8-Flash-Next on a 12-24 GB NVIDIA GPU + 64 GB RAM, one-click install Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: src/platform Fork-Replay-Automated: true --- src/platform/direct_file.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/platform/direct_file.cpp b/src/platform/direct_file.cpp index b09536c28ee..0bb5b05812f 100644 --- a/src/platform/direct_file.cpp +++ b/src/platform/direct_file.cpp @@ -1,6 +1,10 @@ // src/platform/direct_file.cpp - see include/strata/platform/direct_file.hpp. #include "strata/platform/direct_file.hpp" +<<<<<<< this fork #include "strata/platform/aux_cpus.hpp" +======= +#include "strata/platform/thread_affinity.hpp" +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) #include #include @@ -185,7 +189,12 @@ bool DirectFile::open(const std::string& path, std::string& err) { // Completions of reads that finish synchronously are still queued to the port, so every issued read // produces exactly one packet; `wait` is the only completion path. const int n = io_threads(4); +<<<<<<< this fork for (int i = 0; i < n; ++i) impl_->pool.emplace_back([this] { strata::aux_cpus::pin_current_thread(); impl_->worker(); }); +======= + for (int i = 0; i < n; ++i) + impl_->pool.emplace_back([this] { strata::platform::release_inherited_pin(); impl_->worker(); }); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) return true; } @@ -317,7 +326,12 @@ bool DirectFile::open(const std::string& path, std::string& err) { if (fstat(impl_->fd, &st) != 0) { err = "DirectFile: cannot size " + path; close(); return false; } impl_->size = (uint64_t) st.st_size; const int n = io_threads(16); +<<<<<<< this fork for (int i = 0; i < n; ++i) impl_->pool.emplace_back([this] { strata::aux_cpus::pin_current_thread(); impl_->worker(); }); +======= + for (int i = 0; i < n; ++i) + impl_->pool.emplace_back([this] { strata::platform::release_inherited_pin(); impl_->worker(); }); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) return true; } From e9c0181f3b57035e7582bee787c1aada51aab3af Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:02 +0200 Subject: [PATCH 07/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): src/prefill from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 3 file(s) under src/prefill -- 2 clean, 1 with conflict markers. Conflict-marked files -- a pull request with conflicts: resolve them, do not discard the work: src/prefill/prefill.cpp Original authors: Andy,Anon,BlueKingMuch,Chen Bao Original commits touching these paths (123 subject(s)): - #136 P3: with the fused experts the prompt path's MoE buffers hold only what the fused kernels need - #136: fused int8 tensor-core experts for the Q2_0 pack's prompt path, grouped on the GPU (opt-in STRATA_PF_FUSED=1) - #136: the fused experts take a 1024-slot streamed ring by default (two layers' experts) - #136: the fused experts take a 512-slot streamed ring (a whole layer's experts) by default - #136: the fused int8 prompt experts for the native IQ packs (gate/up IQ2_XXS IQ2_XS IQ2_S IQ3_XXS IQ3_S IQ4_XS, down Q2_0 IQ4_NL; opt-in STRATA_PF_FUSED=1) - #253: the 8 GiB arena pin cap only under WDDM; STRATA_ARENA_PIN_GIB; the split's prompt chunk steps down - #280 follow-up: the rope kernels without the table are 0.1.31's code exactly (template ) - #281 follow-up: the saturated FP16 SwiGLU product keeps a NaN a NaN Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: src/prefill Fork-Replay-Automated: true --- src/prefill/gemm.cu | 41 ++++++ src/prefill/kernels.cu | 152 +++++++++++++++----- src/prefill/prefill.cpp | 301 +++++++++++++++++++++++++++++++++------- 3 files changed, 410 insertions(+), 84 deletions(-) diff --git a/src/prefill/gemm.cu b/src/prefill/gemm.cu index 300da53b7d3..81de65d3a95 100644 --- a/src/prefill/gemm.cu +++ b/src/prefill/gemm.cu @@ -1,6 +1,9 @@ // src/prefill/gemm.cu - see include/strata/prefill/gemm.hpp. #include "strata/prefill/gemm.hpp" #include "strata/kernels/dequant_bf16.hpp" +#include "strata/prefill/kernels.hpp" +#include "strata/core/fp16_mode.hpp" +#include #if defined(STRATA_PREFILL_MMQ) && defined(__HIPCC__) #include "strata/prefill/moe_mmq.hpp" #endif @@ -383,6 +386,7 @@ Gemm::~Gemm() { delete static_cast(hipblaslt_state_); #endif if (handle_) cublasDestroy((cublasHandle_t) handle_); + if (w16_) cudaFree(w16_); if (tc_w_) cudaFree(tc_w_); if (tc_x_) cudaFree(tc_x_); if (!external_) { @@ -459,6 +463,13 @@ bool Gemm::init(void* stream, int64_t scratch_elems, std::string& err) { return true; } +namespace { +// STRATA_FP16=prefill's weight conversion (grid-stride, no clamp; the FP16 path below has its own) +__global__ void bf16_to_f16_gs_kernel(const uint16_t* __restrict__ x, uint16_t* __restrict__ y, int64_t n) { + for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) + y[i] = __half_as_ushort(__float2half_rn(__uint_as_float((uint32_t) x[i] << 16))); +} +} // namespace #if !defined(__HIPCC__) // ---- below sm_80: BF16 GEMMs without BF16 tensor cores (PR #655, #540, #395) ---------------------------------------- // Turing and Volta have no BF16 tensor cores: cublasGemmEx on CUDA_R_16BF inputs falls back to a SIMT fp32 kernel @@ -624,6 +635,26 @@ bool Gemm::bf16_hcd_exact(const uint16_t* X, int64_t ldx, const uint16_t* W, flo void Gemm::bf16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy, float beta, int64_t ldx) { if (T <= 0 || N <= 0) return; + if (strata::fp16_mode() == strata::Fp16Mode::Prefill) { + // STRATA_FP16=prefill: X is already FP16 (the producers wrote FP16 images); W to FP16 here, every call: a + // weight's memory may be a staging buffer refilled under the same pointer (PLE), and the conversion is + // ~10 us per 6.5 MB weight. + const int64_t n = N * K; + if (n > w16_elems_) { + if (w16_) cudaFree(w16_); + w16_ = nullptr; + w16_elems_ = 0; + if (cudaMalloc((void**) &w16_, (size_t) n * 2) != cudaSuccess) { + std::fprintf(stderr, "prefill gemm: FP16 weight scratch of %lld elements\n", (long long) n); + std::exit(1); + } + w16_elems_ = n; + } + const int64_t blocks = (n + 255) / 256 < 4096 ? (n + 255) / 256 : 4096; + bf16_to_f16_gs_kernel<<<(unsigned) blocks, 256, 0, (cudaStream_t) stream_>>>(W, w16_, n); + f16(X, w16_, Y, T, N, K, ldy, beta); + return; + } if (ldy <= 0) ldy = N; if (ldx <= K) ldx = 0; #ifdef STRATA_USE_HIP @@ -725,6 +756,16 @@ void Gemm::bf16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64 STRATA_ABSORB_HIPBLAS_STICKY("cublasGemmEx"); } +void Gemm::f32(const float* X, const float* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy, float beta) { + if (T <= 0 || N <= 0) return; + if (ldy <= 0) ldy = N; + const float alpha = 1.0f; + ck(cublasGemmEx((cublasHandle_t) handle_, CUBLAS_OP_T, CUBLAS_OP_N, (int) N, (int) T, (int) K, &alpha, W, + CUDA_R_32F, (int) K, X, CUDA_R_32F, (int) K, &beta, Y, CUDA_R_32F, (int) ldy, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT), + "cublasGemmEx f32"); +} + void Gemm::f16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy, float beta) { if (T <= 0 || N <= 0) return; diff --git a/src/prefill/kernels.cu b/src/prefill/kernels.cu index f58a189c6e0..155eeb144c9 100644 --- a/src/prefill/kernels.cu +++ b/src/prefill/kernels.cu @@ -1,5 +1,6 @@ // src/prefill/kernels.cu - see include/strata/prefill/kernels.hpp. #include "strata/prefill/kernels.hpp" +#include "strata/core/fp16_mode.hpp" #include "strata/kernels/mrope.hpp" #include "strata/kernels/gfx_arch.hpp" #include "strata/kernels/router_top10.hpp" @@ -46,14 +47,27 @@ __device__ __forceinline__ uint16_t hf(float f) { return __half_as_ushort(__floa // NaN in the down projection (decode's q8_1 has room to ~8e6; FP16 ends at 65504). A NaN stays NaN (fminf/fmaxf // would make it -65504 and hide where it came from); finite values below 65504 round exactly as before. __device__ __forceinline__ uint16_t hf_sat(float f) { return hf(isnan(f) ? f : fminf(fmaxf(f, -65504.0f), 65504.0f)); } -// The prompt path's 16-bit activation image for the BF16-weight GEMMs: BF16, or FP16 where the GEMM library is fast -// only in FP16 (prompt_f16() in gemm.cu: rocBLAS on gfx103x). Set once per device before the first prompt. -#if defined(__HIPCC__) // HIP only (#835): the CUDA kernels stay exactly as they were, they never read the flag +// The 16-bit activation image the BF16-weight GEMMs read (STRATA_PREFILL_F16, see prefill_f16_mode()): +// M 0 = BF16 (the default); 1 = the BF16-rounded value in FP16 (exact below 65504); 2 = FP16 straight from FP32. +// 1 and 2 saturate at +-65504 like hf_sat. +// HIP (#835): M 0 is upstream's runtime choice - FP16 where the GEMM library is fast only in FP16 (prompt_f16() in +// gemm.cu: rocBLAS on gfx103x), set once per device by set_act_f16; CUDA's M 0 is always BF16. +#if defined(__HIPCC__) __device__ int g_act_f16 = 0; -__device__ __forceinline__ uint16_t act16(float f) { return g_act_f16 ? hf_sat(f) : bf(f); } +#endif +template __device__ __forceinline__ uint16_t act16(float f) { +#if defined(__HIPCC__) + if constexpr (M == 0) return g_act_f16 ? hf_sat(f) : bf(f); #else -__device__ __forceinline__ uint16_t act16(float f) { return bf(f); } + if constexpr (M == 0) return bf(f); #endif + else if constexpr (M == 1) return hf_sat(__uint_as_float((uint32_t) bf(f) << 16)); + else return hf_sat(f); +} +template __device__ __forceinline__ uint16_t act16_lo(float f, uint16_t h) { + if constexpr (M == 0) return bf_lo(f, h); + else return act16(f - __half2float(__ushort_as_half(h))); +} // block-wide sum for blockDim.x <= 1024, result broadcast __device__ float block_sum(float v, float* sh) { const int lane = threadIdx.x & 31, w = threadIdx.x >> 5; @@ -75,6 +89,7 @@ void check(const char* what) { unsigned blocks_for(int64_t n, int t = 256) { return (unsigned) ((n + t - 1) / t); } // ---------------------------------------------------------------- hyper-connection +template __global__ void gr_norm_kernel(const float* __restrict__ R, const float* __restrict__ w, float eps, float* __restrict__ xn, uint16_t* __restrict__ xn16, uint16_t* __restrict__ xn16_lo) { __shared__ float sh[32]; @@ -87,13 +102,14 @@ __global__ void gr_norm_kernel(const float* __restrict__ R, const float* __restr for (int d = threadIdx.x; d < N; d += blockDim.x) { const float v = r[d] * rs * w[c * N + d]; xn[row * N + d] = v; - const uint16_t h = act16(v); + const uint16_t h = act16(v); xn16[row * N + d] = h; - if (xn16_lo) xn16_lo[row * N + d] = bf_lo(v, h); + if (xn16_lo) xn16_lo[row * N + d] = act16_lo(v, h); } } // F-1: the row scale only (and the BF16 image); gr_mix_r_kernel recomputes r * rs * w itself, in the same order, // so the FP32 copy of the normalized rows (T x 10240 floats) is neither written nor read +template __global__ void gr_norm_rs_kernel(const float* __restrict__ R, const float* __restrict__ w, float eps, float* __restrict__ rs_out, uint16_t* __restrict__ xn16, uint16_t* __restrict__ xn16_lo, int64_t ldx) { @@ -108,11 +124,12 @@ __global__ void gr_norm_rs_kernel(const float* __restrict__ R, const float* __re if (threadIdx.x == 0) rs_out[row] = rs; for (int d = threadIdx.x; d < N; d += blockDim.x) { const float v = r[d] * rs * w[c * N + d]; - const uint16_t h = act16(v); + const uint16_t h = act16(v); xn16[xo + d] = h; - if (xn16_lo) xn16_lo[xo + d] = bf_lo(v, h); + if (xn16_lo) xn16_lo[xo + d] = act16_lo(v, h); } } +template __global__ void gr_mix_r_kernel(const float* __restrict__ R, const float* __restrict__ rs, const float* __restrict__ w, const float* __restrict__ g, float* __restrict__ mixed, uint16_t* __restrict__ mixed16, int64_t T, uint16_t* __restrict__ mixed_h, uint16_t* __restrict__ mixed16_lo) { @@ -129,9 +146,9 @@ __global__ void gr_mix_r_kernel(const float* __restrict__ R, const float* __rest s /= (float) HC; mixed[i] = s; if (mixed16) { - const uint16_t h = act16(s); + const uint16_t h = act16(s); mixed16[i] = h; - if (mixed16_lo) mixed16_lo[i] = bf_lo(s, h); + if (mixed16_lo) mixed16_lo[i] = act16_lo(s, h); } if (mixed_h) mixed_h[i] = hf(s); } @@ -139,6 +156,7 @@ __global__ void gr_mix_r_kernel(const float* __restrict__ R, const float* __rest // weights - the same thread-to-element mapping (256 threads, stride 256) and block_sum, so rs and the BF16 image are // the same bits, and R is not read back constexpr int GRW_PER = (N + 255) / 256; +template __global__ void __launch_bounds__(256) gr_write_norm_rs_kernel(float* __restrict__ R, const float* __restrict__ bo, const float* __restrict__ inj, int64_t inj_ld, const float* __restrict__ w, float eps, @@ -167,9 +185,9 @@ __global__ void __launch_bounds__(256) gr_write_norm_rs_kernel(float* __restrict #pragma unroll for (int d = threadIdx.x; d < N; d += 256, ++k) { const float x = v[k] * rs * w[c * N + d]; - const uint16_t h = act16(x); + const uint16_t h = act16(x); xn16[xo + d] = h; - if (xn16_lo) xn16_lo[xo + d] = bf_lo(x, h); + if (xn16_lo) xn16_lo[xo + d] = act16_lo(x, h); } } // S23 (opt-in STRATA_CVEC_FUSE=1): a steered layer's FFN write, its control vector and the next half's norm in one @@ -229,21 +247,23 @@ __global__ void __launch_bounds__(256) gr_write_cvec_norm_rs_kernel( #pragma unroll for (int d = threadIdx.x; d < N; d += 256, ++k) { const float y = x[k] * rs * w[c * N + d]; - const uint16_t h = act16(y); + const uint16_t h = act16<0>(y); xn16[t * ldx + (int64_t) c * N + d] = h; if (xn16_lo) xn16_lo[t * ldx + (int64_t) c * N + d] = bf_lo(y, h); } } +template __global__ void gr_silu_kernel(const float* __restrict__ lo, uint16_t* __restrict__ lo16, uint16_t* __restrict__ lo16_lo, int64_t n) { const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; if (i >= n) return; const float x = lo[i] / (float) HC; const float v = x / (1.0f + __expf(-x)); - const uint16_t h = act16(v); + const uint16_t h = act16(v); lo16[i] = h; - if (lo16_lo) lo16_lo[i] = bf_lo(v, h); + if (lo16_lo) lo16_lo[i] = act16_lo(v, h); } +template __global__ void gr_mix_kernel(const float* __restrict__ xn, const float* __restrict__ g, float* __restrict__ mixed, uint16_t* __restrict__ mixed16, int64_t T, uint16_t* __restrict__ mixed_h, uint16_t* __restrict__ mixed16_lo) { @@ -259,9 +279,9 @@ __global__ void gr_mix_kernel(const float* __restrict__ xn, const float* __restr s /= (float) HC; mixed[i] = s; if (mixed16) { - const uint16_t h = act16(s); + const uint16_t h = act16(s); mixed16[i] = h; - if (mixed16_lo) mixed16_lo[i] = bf_lo(s, h); + if (mixed16_lo) mixed16_lo[i] = act16_lo(s, h); } if (mixed_h) mixed_h[i] = hf(s); } @@ -272,6 +292,25 @@ __global__ void gr_write_kernel(float* __restrict__ R, const float* __restrict__ const int64_t t = i / D, c = (i % D) / N, d = i % N; R[i] = fmaf(bo[t * N + d], 2.0f * sigm(inj[t * inj_ld + c] / (float) HC), R[i]); } +// one block per token: the four inject outputs over the 10240 normalized values, FP32 throughout +__global__ void __launch_bounds__(256) gr_inject_f32_kernel(const float* __restrict__ R, const float* __restrict__ rs, + const float* __restrict__ w, const float* __restrict__ wi, + float* __restrict__ inj) { + __shared__ float sh[32]; + const int64_t t = blockIdx.x; + float a[HC] = {0.0f, 0.0f, 0.0f, 0.0f}; + for (int j = threadIdx.x; j < D; j += 256) { + const int c = j / N, d = j % N; + const float x = R[t * D + j] * rs[t * HC + c] * w[c * N + d]; // gr_mix_r_kernel's value, bit for bit +#pragma unroll + for (int o = 0; o < HC; ++o) a[o] = fmaf(x, wi[(int64_t) o * D + j], a[o]); + } +#pragma unroll + for (int o = 0; o < HC; ++o) { + const float s = block_sum(a[o], sh); + if (threadIdx.x == 0) inj[t * HC + o] = s; + } +} __global__ void gr_broadcast_kernel(const float* __restrict__ e, float* __restrict__ R, int64_t T) { const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; if (i >= T * D) return; @@ -1724,12 +1763,13 @@ __global__ void round_f16_kernel(const float* __restrict__ x, float* __restrict_ for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) y[i] = __half2float(__float2half_rn(x[i])); } +template __global__ void to_bf16_kernel(const float* __restrict__ x, uint16_t* __restrict__ y, uint16_t* __restrict__ ylo, int64_t n) { for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) { - const uint16_t h = act16(x[i]); + const uint16_t h = act16(x[i]); y[i] = h; - if (ylo) ylo[i] = bf_lo(x[i], h); + if (ylo) ylo[i] = act16_lo(x[i], h); } } @@ -1799,7 +1839,7 @@ __global__ void __launch_bounds__(256) gr_upmix_kernel(const uint16_t* __restric } s /= (float) HC; mixed[t * N + d] = s; - if (mixed16) mixed16[t * N + d] = act16(s); + if (mixed16) mixed16[t * N + d] = act16<0>(s); if (mixed_h) mixed_h[t * N + d] = hf(s); } } @@ -1815,6 +1855,24 @@ void kv_append(const float* K, const float* V, int64_t T, int64_t pos0, const in host ? *host : strata::kernels::KvHostPools{}, stage ? *stage : strata::kernels::KvHostPools{}); check("kv_append"); } +namespace { +thread_local int g_images_override = -1; +} +int prefill_f16_mode() { + if (g_images_override >= 0) return g_images_override; + const Fp16Mode m = fp16_mode(); + return m == Fp16Mode::Prefill ? 1 : m == Fp16Mode::Load ? 2 : 0; +} +Bf16Images::Bf16Images() : prev_(g_images_override) { g_images_override = 0; } +Bf16Images::~Bf16Images() { g_images_override = prev_; } +F16Images::F16Images() : prev_(g_images_override) { g_images_override = 2; } +F16Images::~F16Images() { g_images_override = prev_; } +void gr_inject_f32(const float* R, const float* rs, const float* w_norm, const float* w_inject, float* inj, int64_t T, + void* stream) { + if (T <= 0) return; + gr_inject_f32_kernel<<<(unsigned) T, 256, 0, (cudaStream_t) stream>>>(R, rs, w_norm, w_inject, inj); + check("gr_inject_f32"); +} void set_act_f16(bool on) { #if !defined(__HIPCC__) (void) on; // CUDA: the activation image is always BF16 @@ -1839,25 +1897,43 @@ void round_f16(const float* x, float* y, int64_t n, void* stream) { } void to_bf16(const float* x, uint16_t* y, int64_t n, void* stream, uint16_t* ylo) { if (n <= 0) return; - to_bf16_kernel<<<(unsigned) ((n + 255) / 256 < 4096 ? (n + 255) / 256 : 4096), 256, 0, (cudaStream_t) stream>>>(x, y, ylo, n); + switch (prefill_f16_mode()) { + case 1: to_bf16_kernel<1><<<(unsigned) ((n + 255) / 256 < 4096 ? (n + 255) / 256 : 4096), 256, 0, (cudaStream_t) stream>>>(x, y, ylo, n); break; + case 2: to_bf16_kernel<2><<<(unsigned) ((n + 255) / 256 < 4096 ? (n + 255) / 256 : 4096), 256, 0, (cudaStream_t) stream>>>(x, y, ylo, n); break; + default: to_bf16_kernel<0><<<(unsigned) ((n + 255) / 256 < 4096 ? (n + 255) / 256 : 4096), 256, 0, (cudaStream_t) stream>>>(x, y, ylo, n); + } check("to_bf16"); } void gr_norm(const float* R, const float* w_norm, float eps, float* xn, uint16_t* xn16, int64_t T, void* stream, uint16_t* xn16_lo) { - gr_norm_kernel<<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, xn, xn16, xn16_lo); + switch (prefill_f16_mode()) { + case 1: gr_norm_kernel<1><<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, xn, xn16, xn16_lo); break; + case 2: gr_norm_kernel<2><<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, xn, xn16, xn16_lo); break; + default: gr_norm_kernel<0><<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, xn, xn16, xn16_lo); + } check("gr_norm"); } void gr_norm_rs(const float* R, const float* w_norm, float eps, float* rs, uint16_t* xn16, int64_t T, void* stream, uint16_t* xn16_lo, int64_t ldx) { - gr_norm_rs_kernel<<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, rs, xn16, xn16_lo, - ldx > 0 ? ldx : D); + const int64_t lx = ldx > 0 ? ldx : D; + switch (prefill_f16_mode()) { + case 1: gr_norm_rs_kernel<1><<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, rs, xn16, xn16_lo, lx); break; + case 2: gr_norm_rs_kernel<2><<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, rs, xn16, xn16_lo, lx); break; + default: gr_norm_rs_kernel<0><<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, rs, xn16, xn16_lo, lx); + } check("gr_norm_rs"); } void gr_mix_r(const float* R, const float* rs, const float* w_norm, const float* gated, float* mixed, uint16_t* mixed16, int64_t T, void* stream, uint16_t* mixed_h, uint16_t* mixed16_lo) { - gr_mix_r_kernel<<>>(R, rs, w_norm, gated, mixed, mixed16, T, + switch (prefill_f16_mode()) { + case 1: gr_mix_r_kernel<1><<>>(R, rs, w_norm, gated, mixed, mixed16, T, + mixed_h, mixed16_lo); break; + case 2: gr_mix_r_kernel<2><<>>(R, rs, w_norm, gated, mixed, mixed16, T, + mixed_h, mixed16_lo); break; + default: gr_mix_r_kernel<0><<>>(R, rs, w_norm, gated, mixed, mixed16, T, mixed_h, mixed16_lo); + } check("gr_mix_r"); } bool gr_upmix(const uint16_t* lo16, const uint16_t* w_up, const float* R, const float* rs, const float* w_norm, @@ -1882,9 +1958,15 @@ bool gr_upmix(const uint16_t* lo16, const uint16_t* w_up, const float* R, const } void gr_write_norm_rs(float* R, const float* bo, const float* inj, int64_t inj_ld, const float* w_norm_next, float eps, float* rs, uint16_t* xn16, int64_t T, void* stream, uint16_t* xn16_lo, int64_t ldx) { - gr_write_norm_rs_kernel<<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, bo, inj, inj_ld, w_norm_next, eps, - rs, xn16, xn16_lo, - ldx > 0 ? ldx : D); + const int64_t lx = ldx > 0 ? ldx : D; + switch (prefill_f16_mode()) { + case 1: gr_write_norm_rs_kernel<1><<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, bo, inj, inj_ld, w_norm_next, eps, + rs, xn16, xn16_lo, lx); break; + case 2: gr_write_norm_rs_kernel<2><<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, bo, inj, inj_ld, w_norm_next, eps, + rs, xn16, xn16_lo, lx); break; + default: gr_write_norm_rs_kernel<0><<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, bo, inj, inj_ld, w_norm_next, eps, + rs, xn16, xn16_lo, lx); + } check("gr_write_norm_rs"); } void gr_write_cvec_norm_rs(float* R, const float* bo, const float* inj, int64_t inj_ld, const float* v_l, @@ -1895,12 +1977,20 @@ void gr_write_cvec_norm_rs(float* R, const float* bo, const float* inj, int64_t check("gr_write_cvec_norm_rs"); } void gr_silu(const float* lo, uint16_t* lo16, int64_t T, void* stream, uint16_t* lo16_lo) { - gr_silu_kernel<<>>(lo, lo16, lo16_lo, T * LR); + switch (prefill_f16_mode()) { + case 1: gr_silu_kernel<1><<>>(lo, lo16, lo16_lo, T * LR); break; + case 2: gr_silu_kernel<2><<>>(lo, lo16, lo16_lo, T * LR); break; + default: gr_silu_kernel<0><<>>(lo, lo16, lo16_lo, T * LR); + } check("gr_silu"); } void gr_mix(const float* xn, const float* gated, float* mixed, uint16_t* mixed16, int64_t T, void* stream, uint16_t* mixed_h, uint16_t* mixed16_lo) { - gr_mix_kernel<<>>(xn, gated, mixed, mixed16, T, mixed_h, mixed16_lo); + switch (prefill_f16_mode()) { + case 1: gr_mix_kernel<1><<>>(xn, gated, mixed, mixed16, T, mixed_h, mixed16_lo); break; + case 2: gr_mix_kernel<2><<>>(xn, gated, mixed, mixed16, T, mixed_h, mixed16_lo); break; + default: gr_mix_kernel<0><<>>(xn, gated, mixed, mixed16, T, mixed_h, mixed16_lo); + } check("gr_mix"); } void gr_write(float* R, const float* bo, const float* inj, int64_t inj_ld, int64_t T, void* stream) { diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp index 2fa71c7ce67..877adf58ce3 100644 --- a/src/prefill/prefill.cpp +++ b/src/prefill/prefill.cpp @@ -1,9 +1,13 @@ // src/prefill/prefill.cpp - see include/strata/prefill/prefill.hpp. #include "strata/prefill/prefill.hpp" +<<<<<<< this fork #include #if defined(__cpp_lib_atomic_wait) #define STRATA_ATOMIC_WAIT 1 // #1488: std::atomic::wait / notify_all are a GCC 11 library feature; GCC 10 spins instead #endif +======= +#include "strata/platform/thread_affinity.hpp" +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) #include "mmq_resident_sort.hpp" #include "wmma_gemm.h" #include "strata/core/mtp.hpp" @@ -21,6 +25,7 @@ #include "strata/kernels/ngram.hpp" #include "strata/kernels/ple.hpp" #include "strata/kernels/native_ple_postops.hpp" +#include "strata/kernels/fused_expert.hpp" #include "strata/kernels/iq_kernels.hpp" #include "strata/kernels/cpu/expert_layout.hpp" #include "strata/kernels/qsa.hpp" @@ -488,6 +493,7 @@ struct Stager { } } void work() { + strata::platform::release_inherited_pin(); cudaSetDevice(device); uint32_t seen = 0; for (;;) { @@ -853,6 +859,7 @@ struct Prefill::Impl { bool fused_bufs = false; std::unique_ptr stager; // the unpinned experts' host copies (step 4) cudaEvent_t copied[RING_MAX] = {}, used[RING_MAX] = {}; + cudaEvent_t ids_ready = nullptr; // the router's ids copied for the host grouping (it waits on this, not the stream) bool stage_live[RING_MAX] = {}; // the event that releases each ring slot: its own `used`, or - when an MMQ group is gathered in one launch - the // `used` of the last slot gathered with it, recorded once for all of them (a later record only waits longer) @@ -945,6 +952,7 @@ void Prefill::release() { if (impl_->copied[i]) cudaEventDestroy(impl_->copied[i]); if (impl_->used[i]) cudaEventDestroy(impl_->used[i]); } + if (impl_->ids_ready) { cudaEventDestroy(impl_->ids_ready); impl_->ids_ready = nullptr; } for (int b = 0; b < 2; ++b) { if (impl_->hand[b]) cudaFreeHost(impl_->hand[b]); if (impl_->ple_copied[b]) cudaEventDestroy(impl_->ple_copied[b]); @@ -1125,6 +1133,7 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, co // one-time: events, the stager, the host buffers (for the largest chunk), the identity page table for (int i = 0; i < ring_cap(); ++i) { if (cudaEventCreateWithFlags(&m.copied[i], cudaEventDisableTiming) != cudaSuccess) ok = false; + if (i == 0 && !m.ids_ready && cudaEventCreateWithFlags(&m.ids_ready, cudaEventDisableTiming) != cudaSuccess) ok = false; if (cudaEventCreateWithFlags(&m.used[i], cudaEventDisableTiming) != cudaSuccess) ok = false; } if (!m.stager) { @@ -1412,6 +1421,8 @@ bool Prefill::draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t const float* w_hn = mtp.tensor_f32("attn_hyper_connection.hc_norm.weight"); const uint16_t* w_dn = mtp.tensor_bf16("attn_hyper_connection.input_mix_weight_down.weight"); const uint16_t* w_up = mtp.tensor_bf16("attn_hyper_connection.input_mix_weight_up.weight"); + const uint16_t* w_dn16 = mtp.tensor_f16("attn_hyper_connection.input_mix_weight_down.weight"); + const uint16_t* w_up16 = mtp.tensor_f16("attn_hyper_connection.input_mix_weight_up.weight"); const void* w_k = mtp.tensor_q8("self_attn.k_proj.weight"); const void* w_v = mtp.tensor_q8("self_attn.v_proj.weight"); const float* w_kn = mtp.tensor_f32("self_attn.k_norm.weight"); @@ -1466,7 +1477,14 @@ bool Prefill::draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t if (!mtp.idle(err)) return false; // the drafter's own stream (its graph uploads) before this writes its K/V const double ms_idle = ms_since(ti0); const auto tl0 = Clock::now(); +<<<<<<< this fork strata::kernels::QsaShapes s = core::qsa_shapes(g); +======= + strata::kernels::QsaShapes s = strata::kernels::qsa_real_shapes(); + s.n_head = g.n_head; s.n_head_kv = g.n_head_kv; s.head_dim = g.head_dim; s.idx_n_head = g.idx_q_heads; + s.idx_dim = g.idx_key_dim; + if (st.kv_mode == 2 && n - r0 > st.n_slots * s.page_size) return false; // longer than the ring: token path +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) std::vector tk((size_t) B); if (q8) { if (!m.mmq_ctx) m.mmq_ctx = std::make_unique(); @@ -1521,11 +1539,22 @@ bool Prefill::draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t proj(hn, hn16, w_fh, h2, nb * g.hc, Nn, Nn, 1); // every stream through fc_hidden strata::kernels::add_streams_broadcast(h2, e2, Rm, Nn, (int) g.hc, (int) nb, m.cs); // the attention hyper-connection's read (its mixed input only: this pass writes nothing back) - gr_norm_rs(Rm, w_hn, EPS, grs, xn16, nb, m.cs); - m.gemm.bf16(xn16, w_dn, lo, nb, LR, HCN); - gr_silu(lo, lo16, nb, m.cs); - m.gemm.bf16(lo16, w_up, gated, nb, HCN, LR); - gr_mix_r(Rm, grs, w_hn, gated, mixed, nullptr, nb, m.cs, mixed_h); + if (w_dn16 != nullptr) { + // STRATA_FP16_MTP=1: the FP16 copies made at load, FP16 images + const F16Images keep; + gr_norm_rs(Rm, w_hn, EPS, grs, xn16, nb, m.cs); + m.gemm.f16(xn16, w_dn16, lo, nb, LR, HCN); + gr_silu(lo, lo16, nb, m.cs); + m.gemm.f16(lo16, w_up16, gated, nb, HCN, LR); + gr_mix_r(Rm, grs, w_hn, gated, mixed, nullptr, nb, m.cs, mixed_h); + } else { + const Bf16Images keep; // the drafter's weights stay BF16 under STRATA_FP16=load + gr_norm_rs(Rm, w_hn, EPS, grs, xn16, nb, m.cs); + m.gemm.bf16(xn16, w_dn, lo, nb, LR, HCN); + gr_silu(lo, lo16, nb, m.cs); + m.gemm.bf16(lo16, w_up, gated, nb, HCN, LR); + gr_mix_r(Rm, grs, w_hn, gated, mixed, nullptr, nb, m.cs, mixed_h); + } // K and V into the drafter's cache, as the prompt path's QSA layers append theirs proj(mixed, mixed_h, w_k, Kc, nb, KV, Nn, 0); proj(mixed, mixed_h, w_v, Vc, nb, KV, Nn, 0); @@ -1986,12 +2015,34 @@ bool native_proj(Gemm& gm, const core::WeightRef* w, const uint16_t* X, float* Y gm.native(X, w->native_type, w->native_data, Y, T, w->ne1, w->ne0, ldy, 0.0f, ldx); return true; } +// A BF16-form projection in whatever form the table holds it: BF16 (the pack's), or under STRATA_FP16=load FP16 +// (X and X_lo are then FP16 images) or F32 (the gates: X32, the FP32 activations, and no remainder GEMM). bool bf16_proj(Gemm& gm, const core::WeightRef* w, const uint16_t* X, float* Y, int64_t T, const std::string& name, - std::string& err, int64_t ldy = 0, const uint16_t* X_lo = nullptr, int64_t ldx = 0) { - if (w->kind != core::WeightKind::Bf16InF32 || !w->data) { err = "prefill: " + name + " is not a resident BF16 tensor"; return false; } - gm.bf16(X, (const uint16_t*) w->data, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 0.0f, ldx); - if (X_lo) gm.bf16(X_lo, (const uint16_t*) w->data, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 1.0f); - return true; + std::string& err, int64_t ldy = 0, const uint16_t* X_lo = nullptr, const float* X32 = nullptr, + int64_t ldx = 0) { + if (!w->data) { err = "prefill: " + name + " is not resident"; return false; } + const int64_t n = w->ne1 > 0 ? w->ne1 : 1; + if (ldx > 0 && w->kind != core::WeightKind::Bf16InF32) { // the padded xn16 (STRATA_PF_HCDOWN / HCD_EXACT): BF16 only + err = "prefill: " + name + ": a padded activation stride needs BF16 weights (not with STRATA_FP16=load)"; + return false; + } + switch (w->kind) { + case core::WeightKind::Bf16InF32: + gm.bf16(X, (const uint16_t*) w->data, Y, T, n, w->ne0, ldy, 0.0f, ldx); + if (X_lo) gm.bf16(X_lo, (const uint16_t*) w->data, Y, T, n, w->ne0, ldy, 1.0f); + return true; + case core::WeightKind::F16InF32: + gm.f16(X, (const uint16_t*) w->data, Y, T, n, w->ne0, ldy); + if (X_lo) gm.f16(X_lo, (const uint16_t*) w->data, Y, T, n, w->ne0, ldy, 1.0f); + return true; + case core::WeightKind::F32: + if (!X32) { err = "prefill: " + name + " is F32 and this call has no FP32 activations"; return false; } + gm.f32(X32, (const float*) w->data, Y, T, n, w->ne0, ldy); + return true; + default: + err = "prefill: " + name + " is engine form " + std::to_string((int) w->kind) + ", not BF16, FP16 or F32"; + return false; + } } } // namespace @@ -2157,9 +2208,24 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri const bool ple_batch = ple_on && ple_batch_env && strata::kernels::ple_native_postops_enabled() && (ss.ple.w.key_bf16 != nullptr || ss.ple.w.key_native_data != nullptr) && m.region_bytes / ((uint64_t) (3 * strata::kernels::NG_HC_DIM + N + 4) * 4 + (uint64_t) N * 2 + 4096) >= 64; + // STRATA_PREFILL_ADAPT=F (opt-in; other chunk boundaries than the fixed chunk: another summation order): on a + // layer split the first stage cuts the prompt into about F chunks per stage, so the later stages start sooner + // (the pipeline's fill and drain cost ~3 chunk-times a prompt on 4 stages) - at least STRATA_PREFILL_ADAPT_MIN + // tokens (2048), at most the buffers' chunk. A longer prompt keeps the full chunk. + int64_t step = m.T; + { + static const double adapt = [] { const char* v = std::getenv("STRATA_PREFILL_ADAPT"); return v ? std::atof(v) : 0.0; }(); + static const int64_t adapt_min = [] { const char* v = std::getenv("STRATA_PREFILL_ADAPT_MIN"); return v ? std::atoll(v) : 2048; }(); + int stages = 1; + for (const Prefill* q = next_; q != nullptr; q = q->next_) ++stages; + if (adapt > 0.0 && stage_lb_ == 0 && stages > 1) { + const int64_t want = (int64_t) std::ceil((double) n / (adapt * stages)); + step = std::min(m.T, std::max(adapt_min, (want + 255) / 256 * 256)); + } + } const int32_t prev0[2] = {prev[0], prev[1]}; - auto ple_gather = [&m, &ss, tokens, n, prev0](int64_t c0, int buf, std::string& e) -> bool { - const int64_t T = std::min(m.T, n - c0); + auto ple_gather = [&m, &ss, tokens, n, prev0, step](int64_t c0, int buf, std::string& e) -> bool { + const int64_t T = std::min(step, n - c0); auto at = [&](int64_t i) { return i < 2 ? prev0[i] : (int32_t) tokens[i - 2]; }; // prev0, then the tokens int32_t pv[2] = {at(c0), at(c0 + 1)}; for (int64_t t = 0; t < T; ++t) { @@ -2175,10 +2241,10 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri std::future ple_next; // declared after everything it reads: an early return waits for it int ple_buf = 0; - for (int64_t c0 = 0; c0 < n; c0 += m.T) { + for (int64_t c0 = 0; c0 < n; c0 += step) { if (should_stop && should_stop()) { err = "cancelled"; return false; } if (std::getenv("STRATA_TRACE")) { std::fprintf(stderr, "strata trace: prompt chunk %lld of %lld\n", (long long) c0, (long long) n); std::fflush(stderr); } - const int64_t T = std::min(m.T, n - c0), p0 = pos0 + c0; + const int64_t T = std::min(step, n - c0), p0 = pos0 + c0; core::progress_at("reading the prompt (batched): preparing the chunk from token", p0); // #251 ++stats_.chunks; pt.mark(kPfStart, cs); @@ -2280,14 +2346,19 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri err = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError()); return false; } - if (c0 + m.T < n) { + if (c0 + step < n) { // the other buffer's upload (a chunk ago) is done before the SSD thread refills it if (cudaEventSynchronize(m.ple_copied[ple_buf ^ 1]) != cudaSuccess) { err = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError()); return false; } +<<<<<<< this fork ple_next = std::async(std::launch::async, [&ple_gather, &ple_next_err, c1 = c0 + m.T, b = ple_buf ^ 1] { strata::aux_cpus::pin_current_thread(); +======= + ple_next = std::async(std::launch::async, [&ple_gather, &ple_next_err, c1 = c0 + step, b = ple_buf ^ 1] { + strata::platform::release_inherited_pin(); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) return ple_gather(c1, b, ple_next_err); }); } @@ -2473,7 +2544,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri const bool threaded_issue = stream_all && issuer_on; if (threaded_issue) { issuer = std::thread([&] { +<<<<<<< this fork strata::aux_cpus::pin_current_thread(); +======= + strata::platform::release_inherited_pin(); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) const core::OnDevice od(m.device); for (size_t idx = 0; idx < seq.size(); ++idx) { while (idx >= a_consumed.load(std::memory_order_acquire) + (size_t) m.ring) { @@ -2540,7 +2615,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri if (l > LB) pf_step("reading the prompt (batched, step sync): the experts and the rest of layer", l - 1); core::progress_at("reading the prompt (batched): layer", l, p0); // #251: a stall names layer and chunk const core::LayerView v(*m.wt, l); - if (l == std::max(LB, 1) && !ple_land()) return false; // the PLE rows, read from layer 1 on + // the PLE rows (read from layer 1 on) land at the stage's first layer: their upload, issued at layer 1, + // queued behind that layer's expert copies to the card (31 MB of a 3,072-token chunk took ~14 ms there) + if (l == LB && !ple_land()) return false; // ---- the PLE block at layer 1, token by token (its conv reads the previous tokens' rows) if (l == 1 && ple_on && ple_batch) { // the whole chunk at once, in sub-batches carved from the idle scratch region: the key and value @@ -2565,16 +2642,25 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri uint16_t* e16_lo = bf16x2(m.f16_io) ? (uint16_t*) carve_f((size_t) nb * N / 2) : nullptr; const float* emb = m.ple_emb + s0 * N; if (pw.key_bf16 != nullptr) { - to_bf16(emb, e16, nb * N, m.cs, e16_lo); - m.gemm.bf16(e16, pw.key_bf16, key, nb, HD, N); - if (e16_lo) m.gemm.bf16(e16_lo, pw.key_bf16, key, nb, HD, N, 0, 1.0f); + { + const Bf16Images keep; // the key stays BF16 under STRATA_FP16=load + to_bf16(emb, e16, nb * N, m.cs, e16_lo); + m.gemm.bf16(e16, pw.key_bf16, key, nb, HD, N); + if (e16_lo) m.gemm.bf16(e16_lo, pw.key_bf16, key, nb, HD, N, 0, 1.0f); + } + if (pw.value_f16) to_bf16(emb, e16, nb * N, m.cs, e16_lo); // the value's FP16 image } else { to_f16(emb, e16, nb * N, m.cs); m.gemm.native(e16, pw.key_native_type, pw.key_native_data, key, nb, HD, N); to_bf16(emb, e16, nb * N, m.cs, e16_lo); } - m.gemm.bf16(e16, pw.value_bf16, val, nb, N, N); - if (e16_lo) m.gemm.bf16(e16_lo, pw.value_bf16, val, nb, N, N, 0, 1.0f); + if (pw.value_f16) { + m.gemm.f16(e16, pw.value_bf16, val, nb, N, N); + if (e16_lo) m.gemm.f16(e16_lo, pw.value_bf16, val, nb, N, N, 0, 1.0f); + } else { + m.gemm.bf16(e16, pw.value_bf16, val, nb, N, N); + if (e16_lo) m.gemm.bf16(e16_lo, pw.value_bf16, val, nb, N, N, 0, 1.0f); + } try { strata::kernels::native_ple_postops_batch(key, m.R + s0 * D, val, ss.ple.hist, pw, qn, gated, gate, (int) nb, m.cs); @@ -2635,10 +2721,10 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri } hcd = true; } - if (!hcd && !hdown && !bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo, ldx != D ? ldx : 0)) return false; + if (!hcd && !hdown && !bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo, nullptr, ldx != D ? ldx : 0)) return false; gr_silu(m.lo, m.lo16, T, m.cs, m.lo16_lo); - bool upmixed = false; - if (hc_upmix() && T >= pf_switch_min_t() && !gr_unfused() && !m.lo16_lo && !m.mixed_bf_lo && + bool upmixed = false; // STRATA_HC_UPMIX (opt-in): BF16 weights only, so not under STRATA_FP16=load + if (hc_upmix() && prefill_f16_mode() == 0 && T >= pf_switch_min_t() && !gr_unfused() && !m.lo16_lo && !m.mixed_bf_lo && wu->kind == core::WeightKind::Bf16InF32 && wu->data && wu->ne0 == LR && wu->ne1 == D) { static int checks = [] { const char* e = std::getenv("STRATA_HC_UPMIX_CHECK"); return e ? std::atoi(e) : 0; }(); if (checks > 0) { // the default pair first, kept for the comparison @@ -2665,7 +2751,12 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri } } if (!upmixed && !bf16_proj(m.gemm, wu, m.lo16, m.gated, T, su, err, 0, m.lo16_lo)) return false; - if (!hcd && !bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo, ldx != D ? ldx : 0)) return false; + if (hcd) { // STRATA_PF_HCDOWN computed the inject with the down projection + } else if (wi->kind == core::WeightKind::F32 && !gr_unfused()) { + gr_inject_f32(m.R, m.grs, (const float*) wn->data, (const float*) wi->data, m.inj, T, m.cs); + } else if (!bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo, m.xn, ldx != D ? ldx : 0)) { + return false; + } if (upmixed) { } else if (gr_unfused()) { gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, m.cs, m.mixed_h, m.mixed_bf_lo); @@ -2691,8 +2782,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri float* conv = state + (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size; if (!native_proj(m.gemm, wqkv, m.mixed_h, m.qkv, T, v.name("attn_qkv.weight"), err)) return false; if (!native_proj(m.gemm, wg, m.mixed_h, m.z, T, v.name("attn_gate.weight"), err)) return false; - if (!bf16_proj(m.gemm, wa, m.mixed_bf, m.ab, T, v.name("ssm_alpha.weight"), err, 2 * HV, m.mixed_bf_lo)) return false; - if (!bf16_proj(m.gemm, wb, m.mixed_bf, m.ab + HV, T, v.name("ssm_beta.weight"), err, 2 * HV, m.mixed_bf_lo)) return false; + if (!bf16_proj(m.gemm, wa, m.mixed_bf, m.ab, T, v.name("ssm_alpha.weight"), err, 2 * HV, m.mixed_bf_lo, m.mixed)) return false; + if (!bf16_proj(m.gemm, wb, m.mixed_bf, m.ab + HV, T, v.name("ssm_beta.weight"), err, 2 * HV, m.mixed_bf_lo, m.mixed)) return false; pt.mark(kPfGdnConv, cs); // "gdn" is the projections in; the rest on their own lines gdn_gates(m.ab, (const float*) wdt->data, (const float*) wsa->data, m.gate, m.beta, T, m.cs); gdn_conv(conv, m.qkv, (const float*) wc->data, m.hbuf, T, EPS, m.cs); @@ -2943,7 +3034,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri *wsd = need(v, "ffn_down_shexp.weight", err); if (!wr || !wgi || !wsg || !wsu || !wsd) return false; pt.mark(kPfRouter, cs); - if (!bf16_proj(m.gemm, wr, m.mixed_bf, m.logits, T, v.name("ffn_gate_inp.weight"), err, 0, m.mixed_bf_lo)) return false; + if (!bf16_proj(m.gemm, wr, m.mixed_bf, m.logits, T, v.name("ffn_gate_inp.weight"), err, 0, m.mixed_bf_lo, m.mixed)) return false; route(m.logits, m.ids, m.w, T, m.g->n_expert, m.cs); // RESEARCH HOOK (141-research, opt-in, never on by default): STRATA_DBG_FEAT= dumps, per MoE layer and chunk, // the routed ids + weights (and for the layers in STRATA_DBG_FEAT_LAYERS the router-input hidden state, bf16); @@ -3005,15 +3096,77 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri } } } + // which path groups this layer's experts (see below): decided before the shared expert, so that the + // host grouping's copy of the ids goes ahead of it + const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout(); + const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l]; + const int mmq_gt = lay.native ? lay.fmt[(size_t) l].gu_type : 42; + const int mmq_dt = lay.native ? lay.fmt[(size_t) l].d_type : 42; + // --peer-device: MMQ only, whether or not the peer took the prompt path (set_peer can decline), as + // fused_ring() sized the ring and the buffers for it + const bool no_peer = !core::peer_portable(); + const bool fused_only = mmq_plan().any && mmq_plan().fo[(size_t) l]; + const bool fused_nat = (use_mmq || fused_only) && stream_all && no_peer && lay.native && fused::native_supported(mmq_gt, mmq_dt); + const bool fused_l = (use_mmq && stream_all && no_peer && !lay.native && fused::enabled()) || fused_nat; + // the host grouping needs only the ids: copied now, and the host waits for this event rather than + // the whole stream, so the shared expert below runs on the GPU while the host groups (it used to + // sit idle for the grouping, ~1 ms a layer) + const bool grp_mapped = m.grp_host != nullptr; + bool cpu_arm = true; // auto: this layer may share (false: the arm without) + int cpu_set = -1; // auto: the wall events' set, -1 when not measured + bool cpu_cold = false; // auto: one-time costs in the window (no reading) + const bool cpu_maybe = !fused_l && pool_hold.held && !stream_all && m.src != nullptr && + lay.native && !m.pp && !lay.fmt.empty(); + if (!fused_l) { + if (grp_mapped) copy_i32(m.grp_dev, m.ids, T * K, m.cs); + else cudaMemcpyAsync(m.ids_host.data(), m.ids, (size_t) T * K * 4, cudaMemcpyDeviceToHost, m.cs); + // the CPU experts' activations (STRATA_PREFILL_CPU_SHARE): before the event, which is all the host + // waits for before it reads them + // auto shares only while the layers that share cost less per non-resident expert than their + // neighbours that do not. #1282 (RX 7900 GRE + 5700X3D): every share lost, auto's 0.47 by + // 4.5%, the host's issue time per streamed expert rising 107 -> 200 us with the CPU busy - + // a cost g / (c + g) cannot see, as both sides are measured with the share on. The first + // eligible layers alternate, without first, until CPU_MIN_RATIOS ratios are in; then the + // cheaper arm, and every CPU_PROBE-th layer the other one (a prime: the probes move across + // the layers from one chunk to the next), so both stay measured. + if (cpu_maybe && cpu_share_env() < 0.0) { + constexpr int64_t CPU_PROBE = 29; + constexpr int CPU_MIN_RATIOS = 3; + const int64_t i = m.cpu_layers++; + cpu_arm = m.cpu_nratio < CPU_MIN_RATIOS ? (i & 1) != 0 : (i % CPU_PROBE == 0) != m.cpu_gate; + if (m.cpu_first_l < 0) m.cpu_first_l = l; + cpu_cold = l == m.cpu_first_l; + if (m.cpu_wall[0] == nullptr) + for (cudaEvent_t& e : m.cpu_wall) cudaEventCreate(&e); + cpu_set = (int) (i & 1); + cudaEventRecord(m.cpu_wall[2 * cpu_set], m.cs); + } + if (cpu_maybe && cpu_arm) { + const size_t want = (size_t) T * N; + if (m.cpu_x_n < want) { + cpu_cold = true; + if (m.cpu_x) cudaFreeHost(m.cpu_x); + m.cpu_x = nullptr; + m.cpu_x_n = 0; + if (cudaHostAlloc((void**) &m.cpu_x, want * sizeof(float), cudaHostAllocDefault) != cudaSuccess) { + err = "prefill: cannot allocate the CPU experts' activations"; + return false; + } + m.cpu_x_n = want; + } + cudaMemcpyAsync(m.cpu_x, m.mixed, want * sizeof(float), cudaMemcpyDeviceToHost, m.cs); + } + cudaEventRecord(m.ids_ready, m.cs); + } // the shared expert and its scalar gate auto shared_expert = [&]() -> bool { if (!native_proj(m.gemm, wsg, m.mixed_h, m.sgate, T, v.name("ffn_gate_shexp.weight"), err)) return false; if (!native_proj(m.gemm, wsu, m.mixed_h, m.sup, T, v.name("ffn_up_shexp.weight"), err)) return false; swiglu_pair(m.sgate, m.sup, m.sh_h, T, m.cs); if (!native_proj(m.gemm, wsd, m.sh_h, m.shared, T, v.name("ffn_down_shexp.weight"), err)) return false; - if (wgi->kind != core::WeightKind::Bf16InF32) { err = "prefill: shared gate is not BF16"; return false; } - m.gemm.bf16(m.mixed_bf, (const uint16_t*) wgi->data, m.sg, T, 1, N); - if (m.mixed_bf_lo) m.gemm.bf16(m.mixed_bf_lo, (const uint16_t*) wgi->data, m.sg, T, 1, N, 0, 1.0f); + if (wgi->ne1 > 1) { err = "prefill: the shared gate is not one row"; return false; } + if (!bf16_proj(m.gemm, wgi, m.mixed_bf, m.sg, T, v.name("ffn_gate_inp_shexp.weight"), err, 0, + m.mixed_bf_lo, m.mixed)) return false; return true; }; // Only the router needs to finish before readback. Queue the shared expert afterwards @@ -3027,16 +3180,6 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri // activations in Xq, the int8 H in H, the grouping tables in GU). Chunks below stream_all_min() // keep MMQ; without the variable nothing here runs. A native pack's layer takes the native kernels // (moe_fused_iq.hpp) where they cover its two formats, else MMQ (or the FP16 path: IQ1_M). - const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout(); - const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l]; - const int mmq_gt = lay.native ? lay.fmt[(size_t) l].gu_type : 42; - const int mmq_dt = lay.native ? lay.fmt[(size_t) l].d_type : 42; - // --peer-device: MMQ only, whether or not the peer took the prompt path (set_peer can decline), as - // fused_ring() sized the ring and the buffers for it - const bool no_peer = !core::peer_portable(); - const bool fused_only = mmq_plan().any && mmq_plan().fo[(size_t) l]; - const bool fused_nat = (use_mmq || fused_only) && stream_all && no_peer && lay.native && fused::native_supported(mmq_gt, mmq_dt); - const bool fused_l = (use_mmq && stream_all && no_peer && !lay.native && fused::enabled()) || fused_nat; // #583 / #954: the fused layout's GU/H/Xq hold the grouping tables and the int8 rows, sized for // stream_all_min() - 1 tokens of MMQ's rows, so a layer that takes MMQ or the FP16 path at the FULL // chunk would write T*K rows into a (stream_all_min() - 1)*K-row buffer: an illegal access, or a kernel @@ -3059,9 +3202,6 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri double cpu_ms = 0; // the thread's time (the measured share) std::future cpu_fut; int64_t rows_cpu = 0, n_cpu = 0, n_stream = 0; - bool cpu_arm = true; // auto: this layer may share (false: the arm without) - int cpu_set = -1; // auto: the wall events' set, -1 when not measured - bool cpu_cold = false; // auto: one-time costs in the window (no reading) if (fused_l) { if (static bool said = false; !said) { said = true; @@ -3123,12 +3263,13 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri } else { // group the (token, k) pairs by expert on the host pt.mark(kPfHostGroup, cs); - // (the sync below also orders this layer's writes of slot/src/bounds after the previous - // layer's kernels that read them) - const bool grp_mapped = m.grp_host != nullptr; + // (the ids' event also orders this layer's writes of slot/src/bounds after the previous + // layer's kernels that read them: those come before it on the stream). The phase timers fold + // completed marks only, so with them on the host still waits for the whole stream. int32_t* ids_h = grp_mapped ? m.grp_host : m.ids_host.data(); int32_t* slot_h = grp_mapped ? m.grp_host + m.grp_tk : m.slot_host.data(); int32_t* src_h = grp_mapped ? m.grp_host + 2 * m.grp_tk : m.src_host.data(); +<<<<<<< this fork if (grp_mapped) copy_i32(m.grp_dev, m.ids, T * K, m.cs); else if (m.up_ring.d2h_sync(m.ids_host.data(), m.ids, (size_t) T * K * 4, m.cs) != cudaSuccess) { err = "prefill: the routed ids' download failed"; // #1996 @@ -3182,11 +3323,14 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri } if (cpu_maybe) cudaMemcpyAsync(m.cpu_x, m.mixed, want * sizeof(float), cudaMemcpyDeviceToHost, m.cs); } +======= +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) // #579: a stall here is the GPU (this layer's attention and router, or the previous layer's // work), not the host: the watchdog's report says so (only its text changes) core::progress_at("reading the prompt (batched): waiting for the GPU (attention, router) at layer", l, p0); - cudaStreamSynchronize(m.cs); + if (pt.on || pe.on) cudaStreamSynchronize(m.cs); + else cudaEventSynchronize(m.ids_ready); core::progress_at("reading the prompt (batched): layer", l, p0); if (cpu_maybe) { // the rows buffer, once, at its largest (an expert takes at most MAXT tokens' rows) const size_t cap = (size_t) std::min(T * K, (int64_t) strata::kernels::cpu::MAXT * m.g->n_expert) * N; @@ -3768,6 +3912,40 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri p_release_to(l, (int32_t) m.g->n_expert, m.pp->s); cudaSetDevice(pd); } + // STRATA_FUSED_EXPERTS=1: a native layer of Q4_K gate/up and Q5_1 / Q8_0 down runs its experts in + // groups through fused_expert_run (no FP16 copy of the expert, one launch pair per group). A group + // holds its experts' ring slots until it is issued: their `used` events are recorded then, a slot + // it holds is flushed before stage_one refills it, and the streamed walk gives back no entry at or + // after the group's first one (so the issuer never waits on an event that is not queued yet). + static const bool fused_env = [] { const char* v = std::getenv("STRATA_FUSED_EXPERTS"); return v && v[0] && v[0] != '0'; }(); + strata::kernels::FusedExpertLayout fl; + bool fused = false; + if (fused_env && !use_mmq && lay.native) { + const auto& f = lay.fmt[(size_t) l]; + fused = strata::kernels::fused_expert_supported(f.gu_type, f.d_type, f.n_embd, f.n_ff); + if (fused) { + const auto nl = strata::kernels::native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff); + fl.gu_type = f.gu_type; fl.d_type = f.d_type; fl.n_embd = f.n_embd; fl.n_ff = f.n_ff; + fl.gu_row = nl.gu_row; fl.d_row = nl.d_row; fl.up_off = nl.up_off; fl.down_off = nl.down_off; + } + } + strata::kernels::FusedExpertGroup fg; + int fg_slot[strata::kernels::kFusedExpertMax] = {}; + size_t fg_k0 = SIZE_MAX; // the streamed walk: the group's first ring entry + auto fg_flush = [&]() { + if (fg.n == 0) return; + pt.mark(kPfGemmGU, cs); + strata::kernels::fused_expert_run(fl, fg, m.Xs, m.Hh, m.Dm, m.cs); + for (int i = 0; i < fg.n; ++i) + if (fg_slot[i] >= 0) { cudaEventRecord(m.used[fg_slot[i]], m.cs); m.used_of[fg_slot[i]] = fg_slot[i]; } + fg.n = 0; + if (fg_k0 != SIZE_MAX) { fg_k0 = SIZE_MAX; give_back(consumed); } + }; + auto fg_holds = [&](int sl) { + for (int i = 0; i < fg.n; ++i) if (fg_slot[i] == sl) return true; + return false; + }; + auto give = [&](size_t c) { give_back(fg_k0 != SIZE_MAX ? std::min(c, fg_k0) : c); }; // Stage ahead: the copy stream moves blobs host -> device while the compute stream works. int stage_next = 0; std::vector stage_of(order.size(), -1); @@ -3797,6 +3975,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri if (resident) return true; const int sl = stage_next; stage_next = (stage_next + 1) % STAGE; + if (fused && fg_holds(sl)) fg_flush(); const auto th = Clock::now(); const bool pinned = m.src->pinned(l, e); // pinned: never transient const uint8_t* b = pinned ? m.src->blob_stable(l, e) : nullptr; @@ -3869,6 +4048,12 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri // expert) is released once the blob is read auto compute = [&](size_t j, const uint8_t* blob_dev, int slot) -> bool { const int32_t e = order[j]; + if (fused) { + fg.blob[fg.n] = blob_dev; fg.row0[fg.n] = m.off[(size_t) e]; fg.ne[fg.n] = (int) m.cnt[(size_t) e]; + fg_slot[fg.n] = slot; + if (++fg.n == strata::kernels::kFusedExpertMax || j + 1 == order.size()) fg_flush(); + return true; + } pt.mark(kPfDequant, cs); if (use_mmq) { // gather the expert into its group slot (GGUF blocks, unchanged or converted) @@ -3986,7 +4171,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri cudaEventRecord(m.used[sl], m.cs); m.used_of[sl] = sl; consumed = ++k; - give_back(consumed); + give(consumed); } }; for (size_t j = 0; j < order.size(); ++j) { @@ -3994,6 +4179,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri release_to(e); if (k < kend && seq[k].e == e) { const int sl = (int) (k % (size_t) m.ring); + if (fused && fg_k0 != SIZE_MAX && k >= fg_k0 + (size_t) m.ring) fg_flush(); + if (fused && fg_k0 == SIZE_MAX) fg_k0 = k; wait_issued(k); if (!group_gather) { pt.mark(kPfWaitCopy, cs); @@ -4001,7 +4188,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri } if (!compute(j, m.stage_dev[sl], sl)) return false; consumed = ++k; - if (!group_gather || gg_nslots == 0) give_back(consumed); // its group was gathered + if (!group_gather || gg_nslots == 0) give(consumed); // its group was gathered } else { const bool r0 = m.host_res && m.cache && m.host_res[(size_t) l * m.g->n_expert + e] >= 0; const uint8_t* bp = r0 ? m.cache->device_slot(m.host_res[(size_t) l * m.g->n_expert + e]) @@ -4087,7 +4274,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri const bool steered = half == 1 && strata::kernels::cvec().covers(l); const float *cv_dir = nullptr, *cv_s = nullptr; const int* cv_on = nullptr; - const bool cv_fused = steered && cvec_fuse() && !gr_unfused() && nl < LE && !(nl == 1 && ple_on) && + const bool cv_fused = steered && cvec_fuse() && prefill_f16_mode() == 0 && !gr_unfused() && nl < LE && !(nl == 1 && ple_on) && strata::kernels::cvec().n_embd == N && strata::kernels::cvec().hc == HC && strata::kernels::cvec_tables(&cv_dir, &cv_s, &cv_on); const bool fuse = !gr_unfused() && nl < LE && !(half == 1 && nl == 1 && ple_on) && (!steered || cv_fused); @@ -4171,7 +4358,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri next_->hand_in_ = h; next_->single_chunk_ = single_chunk; next_run_ = std::async(std::launch::async, [this, tokens, c0, T, p0] { +<<<<<<< this fork strata::aux_cpus::pin_current_thread(); +======= + strata::platform::release_inherited_pin(); // a new thread starts on the host thread core (CPU 0) +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) return next_->run_impl(tokens + c0, T, p0, next_err_); }); hand_buf_ ^= 1; @@ -4276,8 +4467,12 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri std::snprintf(b, sizeof b, " %s %.0f (%.1f%%)", kPfNames[i], pt.ms[i], total > 0 ? 100.0 * pt.ms[i] / total : 0.0); line += b; } - std::fprintf(stderr, "strata prefill timing: %lld tokens, GPU timeline %.0f ms, wall %.0f ms, host staging %.0f ms:%s\n", - (long long) n, total, ms_since(t_start), stats_.ms_experts_host, line.c_str()); + int tdev = 0; + cudaGetDevice(&tdev); + const double t_end_ms = std::chrono::duration(Clock::now().time_since_epoch()).count(); + std::fprintf(stderr, "strata prefill timing: CUDA%d, %lld tokens, GPU timeline %.0f ms, wall %.0f ms (ended at %.0f), " + "host staging %.0f ms:%s\n", tdev, (long long) n, total, ms_since(t_start), std::fmod(t_end_ms, 1e7), + stats_.ms_experts_host, line.c_str()); std::fprintf(stderr, "strata prefill timing: host: chunk setup (PLE rows, the expert stream plan) %.0f ms, " "waiting for each chunk %.0f ms, after each chunk (the draft layer, progress) %.0f ms, " "PLE %.0f ms\n", host_setup_ms, host_sync_ms, host_chunk_ms, stats_.ms_ple); From 17dda1e6b223a2d1347788c48a9ec2dffe22bf22 Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:03 +0200 Subject: [PATCH 08/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): src/program from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 1 file(s) under src/program -- 0 clean, 1 with conflict markers. Conflict-marked files -- a pull request with conflicts: resolve them, do not discard the work: src/program/generate.cpp Original authors: Andy,Anon,BlueKingMuch,Constantin Original commits touching these paths (241 subject(s)): - #253: the 8 GiB arena pin cap only under WDDM; STRATA_ARENA_PIN_GIB; the split's prompt chunk steps down - #272 follow-up: --pool-affinity all (the pool's layout so far) is the default; auto and p-cores are opt-in - #282 follow-up: chunks above 8192 are opt-in (--prefill auto:16384 / auto:32768, or STRATA_PREFILL_AUTO_MAX) - #284 follow-up: wait for the async commit on an event, at every point that touches the session afterwards - #289 follow-up: a placeholder GPU name when the properties query fails; HIP builds compile (no device_code_error probe) - #293 follow-up: the drafter stores rotated INT8 K/V as INT8; the rotation is part of the snapshot format - #340 prototype: STRATA_SPLIT_SMALL_MAX=M - a split request of at most M prompt tokens reads in the stages' own S-token chunks (nothing lent) - #340 prototype: a layer split refills every stage's prompt loan concurrently (queue all, then wait); STRATA_REFILL_SERIAL=1: stage after stage; STRATA_TRACE times the loan and the refill Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: src/program Fork-Replay-Automated: true --- src/program/generate.cpp | 323 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 314 insertions(+), 9 deletions(-) diff --git a/src/program/generate.cpp b/src/program/generate.cpp index ba00f94d751..5bdeff1edc7 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -66,7 +66,13 @@ #include "strata/prefill/moe_mmq.hpp" #endif #include "strata/core/native_dense.hpp" +<<<<<<< this fork #include "strata/program/batch_read.hpp" +======= +#include "strata/core/fp16_mode.hpp" +#include "strata/platform/thread_affinity.hpp" +#include "strata/core/load_converted.hpp" +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) #include "strata/program/logits_selection.hpp" #include "strata/program/conv_cache.hpp" #include "strata/program/split_rules.hpp" @@ -130,6 +136,10 @@ namespace strata::metal_engine { void set_model_shard(const char* shard1); } / #include #include #include +#include +#if defined(__linux__) +#include +#endif #include #include #include @@ -533,6 +543,7 @@ struct Options { int vram_reserve_later_mib = -1; /// Plan v0.3 P5: batched prompt processing in chunks of this many tokens (0 = the token path). int64_t prefill_chunk = 0; + int prefill_ring = 0; ///< --prefill-ring N: the prompt path's streamed ring, N expert slots (0 = the engine's rule) /// `--prefill auto`: the largest chunk (up to 8192) whose buffers the expert cache can lend. Every expert a chunk /// routes to is streamed once per chunk, so a bigger chunk streams fewer bytes per token (the "ubatch" effect). bool prefill_auto = false; @@ -843,6 +854,9 @@ void usage() { " --prefill CHUNK batched prompt processing in chunks of CHUNK tokens (needs --native); auto =\n" " the largest chunk up to 8192 whose buffers the expert cache can lend;\n" " auto:16384 / auto:32768 (or STRATA_PREFILL_AUTO_MAX) allow bigger ones\n" + " --prefill-ring N the prompt path's streamed-expert ring: N slots (16..512; default: the engine's\n" + " rule, 96 or 384). Deeper prefetch, more cache slots lent during a prompt; the\n" + " same outputs. IBM AC922, 4x V100 layer split: 320 (65K prompt +12%)\n" " --no-pool skip the CPU expert pool (the GPU-only floor)\n" " --sync-every-layer debug: synchronise after every layer\n" " --ple-gguf PATH the n-gram/PLE shard. WITHOUT IT LAYER 1's PLE IS SILENTLY SKIPPED,\n" @@ -1058,6 +1072,7 @@ struct SplitDrive { const uint8_t* cache_base[kMax] = {}; const uint64_t* cache_slot_off[kMax] = {}; int pcie_num[kMax] = {}; + strata::kernels::cpu::ExpertPool* pool[kMax] = {}; ///< STRATA_POOL_PER_NODE: the pool on the stage's GPU's socket }; void drive_pool_split(void* user, const float* x_f, const int32_t* ids, int64_t n_tok, int64_t k, float* out, int64_t layer) { @@ -1069,6 +1084,7 @@ void drive_pool_split(void* user, const float* x_f, const int32_t* ids, int64_t d.d.cache_base = s->cache_base[st]; d.d.cache_slot_off = s->cache_slot_off[st]; d.d.pcie_num = s->pcie_num[st]; + if (s->pool[st] != nullptr) d.d.pool = s->pool[st]; drive_pool_multi(s->base, x_f, ids, n_tok, k, out, layer); } @@ -1082,6 +1098,7 @@ struct GpuStage { double link_gbps = 0.0; ///< this card's probed host->device bandwidth (PcieAuto's per-stage share) strata::core::WeightTable wt; strata::core::NativeDense dense; + std::vector converted; ///< STRATA_FP16=load: its copies of the converted weights strata::core::NativeHead head; strata::core::SessionState ss; cudaStream_t stream = nullptr; @@ -1651,6 +1668,53 @@ static bool pcie_auto_split_ok() { // The effective host->device bandwidth of the PCIe link: copies from pinned host memory, as the expert arena's // reads are. The native default share (0.55) was measured on x16 links (~26-28 GB/s); a x8 card in a x8 slot // carries about half of that. Returns < 0 when the probe cannot run (then the caller keeps the default). +// The NUMA node of CUDA device `dev` (its PCI device's numa_node in sysfs), or -1. +int gpu_numa_node(int dev) { +#if defined(_WIN32) + (void) dev; + return -1; +#else + char bus[64] = {}; + unsigned dom = 0, b = 0, d = 0, f = 0; + if (cudaDeviceGetPCIBusId(bus, (int) sizeof bus, dev) != cudaSuccess) { cudaGetLastError(); return -1; } + if (std::sscanf(bus, "%x:%x:%x.%x", &dom, &b, &d, &f) != 4) return -1; + char path[128]; + std::snprintf(path, sizeof path, "/sys/bus/pci/devices/%04x:%02x:%02x.%x/numa_node", dom, b, d, f); + int node = -1; + if (std::FILE* fp = std::fopen(path, "r")) { + if (std::fscanf(fp, "%d", &node) != 1) node = -1; + std::fclose(fp); + } + return node; +#endif +} + +// Timed copies from `h` (page-locked, at least `bytes`) to the current device: a layer split's stage probes the +// arena segment on its own socket with it (set_arena_placement). +double probe_h2d_from(const void* h, size_t bytes) { + constexpr int kIters = 4; + void* d = nullptr; + cudaEvent_t ev0, ev1; + if (cudaMalloc(&d, bytes) != cudaSuccess || cudaEventCreate(&ev0) != cudaSuccess || + cudaEventCreate(&ev1) != cudaSuccess) { + if (d != nullptr) cudaFree(d); + cudaGetLastError(); + return -1.0; + } + cudaMemcpyAsync(d, h, bytes, cudaMemcpyHostToDevice); // warmup: context up, copy engine primed + cudaEventRecord(ev0); + for (int i = 0; i < kIters; ++i) cudaMemcpyAsync(d, h, bytes, cudaMemcpyHostToDevice); + cudaEventRecord(ev1); + const bool ok = cudaEventSynchronize(ev1) == cudaSuccess; + float ms = 0.f; + const bool timed = ok && cudaEventElapsedTime(&ms, ev0, ev1) == cudaSuccess && ms > 0.01f; + const double bw = timed ? ((double) kIters * (double) bytes / (ms * 1e-3)) / 1e9 : -1.0; + cudaEventDestroy(ev0); + cudaEventDestroy(ev1); + cudaFree(d); + return bw; +} + // // #485: a single timed burst read an x16 PCIe 4 link (RTX A3000 laptop) at 18.5, 6.9 and 5.8 GB/s in three starts - // a link still in a low-power state, other DMA, a context not yet up to speed - and the low reading set the share. @@ -1722,7 +1786,12 @@ double probe_pcie_h2d_gbps(std::string* samples = nullptr) { // what the x16 share costs. Continuous (#485): before, 19.9 GB/s gave 0.42 and 20.0 the full 0.55 (and 4.0 GB/s // gave 0.08, 3.9 none), so a reading near either edge moved the share by a quarter of its range. From 20 GB/s up // (an x16 PCIe 4/5 link: ~26-28 GB/s) the share is unchanged. +// A coherent link far past PCIe - NVLink 2.0 to a POWER9, 72 GB/s probed on an IBM AC922 - carries a missed expert +// faster than that CPU computes it: all of them. Measured there, one V100, Unsloth UD-Q4_K_XL with a 71 GiB RAM +// budget, decode: share 0 24.9 tok/s, 0.55 33.7, 0.8 37.3, 1.0 38.7. From 60 GB/s, so a PCIe 5.0 x16 link (~50-55 +// GB/s), not measured with a larger share, keeps the default. double pcie_frac_for_gbps(double gbps, double base) { + if (gbps >= 60.0) return 1.0; return gbps <= 0.0 ? base : base * std::min(1.0, gbps / 20.0); } @@ -2065,6 +2134,13 @@ int main(int argc, char** argv) { v.c_str(), (long long) strata::platform::host_limits::kWinHipAutoChunkCap); } } + else if (a == "--prefill-ring") { + o.prefill_ring = std::atoi(next("--prefill-ring")); + if (o.prefill_ring < 16 || o.prefill_ring > 512) { + std::fprintf(stderr, "strata generate: --prefill-ring takes 16 to 512 slots\n"); + return 2; + } + } else if (a == "--no-split-rows") o.no_split_rows = true; else if (a == "--no-prefill-borrow") o.no_prefill_borrow = true; else if (a == "--vram-elastic") o.vram_elastic = true; @@ -2761,6 +2837,7 @@ int main(int argc, char** argv) { // ... and nothing runs on a CPU without AVX2: every CPU expert kernel is AVX2 at least (the AVX-512 ones are // chosen above it), and so is ggml-cpu in the release build, which the native pack's layout load initializes // next. Refused here, by name, rather than an illegal instruction in the first expert. +#if defined(__x86_64__) || defined(_M_X64) // The experimental older-CPU build (STRATA_ISA_FLOOR=avx|none, compiled on that PC; #394 #595 #623) has ggml-cpu // for that floor, so a native pack's experts run there on ggml-cpu (every AVX2 kernel is behind cpu_avx2_ok). const char* isa_floor = strata::kernels::cpu::isa_floor_build(); @@ -2794,6 +2871,7 @@ int main(int argc, char** argv) { std::fprintf(stderr, "strata generate: this is the older-CPU build (ggml-cpu for %s); this CPU has AVX2, " "and the normal build is faster on it\n", isa_floor); } +#endif std::string err; if (!o.native_head_gguf.empty() && !o.stream_token) { @@ -2900,6 +2978,7 @@ int main(int argc, char** argv) { else if (!strata::kernels::cpu::cpu_avx512_ok()) std::fprintf(stderr, "strata generate: this CPU has no AVX-512: the expert kernels run on %s " "(multi-token for the i-quant gate/up rows)\n", +<<<<<<< this fork !strata::kernels::cpu::cpu_avx2_ok() ? (strata::kernels::cpu::cpu_avx1_ok() && std::getenv("STRATA_NO_IQ128") == nullptr ? "AVX1 128-bit (iq_avx1, the older-CPU build)" @@ -2909,6 +2988,16 @@ int main(int argc, char** argv) { // pruned variant (GSQ-RCO Coder, fewer than 512 experts) differs from strata::core::ModelGeometry g; int64_t K = g.n_expert_used; +======= +#if defined(__x86_64__) || defined(_M_X64) + !strata::kernels::cpu::cpu_avx2_ok() ? "ggml-cpu vec_dot (no AVX2: the older-CPU build)" + : std::getenv("STRATA_NO_IQ256") == nullptr ? "AVX-2" : "ggml-cpu vec_dot (STRATA_NO_IQ256 set)"); +#else + "ggml-cpu vec_dot (not an x86 CPU)"); +#endif + strata::core::ModelGeometry g; // canonical defaults; the model file overrides the MoE shape below + int64_t K = 10; +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) // THE ROPE CONFIG RESOLVES HERE, BEFORE ANY WEIGHT MOVES - the CLI and the model file have both spoken, // and `session_init` below builds the rope table from it and captures the kernels reading its constants // (rope_scaling.hpp); the only hard constraint is "set before that", and dying on a bad rope key beats @@ -3064,6 +3153,27 @@ int main(int argc, char** argv) { if (!o.native_dense_gguf.empty() && !multi_gpu && strata::core::NativeDense::hc_q8_requested()) strata::core::NativeDense::add_hc_skip(o.pack, skip); } + // STRATA_FP16=load: the BF16-form weights come from the GGUF (FP16; the gates F32), so the pack's rows are skipped + strata::core::LoadConverted converted; + std::vector converted_main; + if (strata::fp16_mode() == strata::Fp16Mode::Load) { + if (o.native_shards.empty()) { + std::fprintf(stderr, "strata generate: STRATA_FP16=load reads the weights from the GGUF: it needs --native\n"); + return 1; + } + const auto t0 = std::chrono::steady_clock::now(); + if (!converted.build(o.native_shards, strata::fp16_gates(), err)) { + std::fprintf(stderr, "strata generate: %s\n", err.c_str()); + return 1; + } + converted.served_names(skip); + std::fprintf(stderr, "strata generate: STRATA_FP16=load: %zu tensors as FP16, %zu gates as F32%s, %.1f MiB, from " + "the GGUF in %.1f s (%llu values became FP16 subnormals, %llu became zero)\n", + converted.tensors_f16, converted.tensors_f32, strata::fp16_gates() ? " (STRATA_FP16_GATES=1)" : "", + (double) converted.bytes / 1048576.0, + std::chrono::duration(std::chrono::steady_clock::now() - t0).count(), + (unsigned long long) converted.subnormal, (unsigned long long) converted.flushed); + } // Layer split: every GPU holds only the dense weights of ITS layers (the PLE tensors stay everywhere). // Without this each card keeps a full copy (~3.4 GB for the Coder) that its stage never reads - VRAM the // expert cache wants. Opt-in, STRATA_STAGE_TRIM=1, on HIP and CUDA alike: measured on 2x MI50 (PR #639), @@ -3173,6 +3283,10 @@ int main(int argc, char** argv) { std::fprintf(stderr, "strata generate: %zu native projection matrices, %.2f MiB of weights, in %.1f s\n", native_dense.tensor_count(), (double) native_dense.weight_bytes() / (1024.0 * 1024.0), load_s()); } + if (strata::fp16_mode() == strata::Fp16Mode::Load && !converted.attach(wt, converted_main, err)) { + std::fprintf(stderr, "strata generate: %s\n", err.c_str()); + return 1; + } strata::kernels::gr_set_fp32_activations(o.gr_fp32_activations); strata::kernels::gr_set_native_mmvf(o.gr_native_mmvf); @@ -3396,6 +3510,7 @@ int main(int argc, char** argv) { ss.ple.w.key_native_q8_1 = wk->native_q8_1; } ss.ple.w.value_bf16 = (const uint16_t*) wv->data; + ss.ple.w.value_f16 = wv->kind == strata::core::WeightKind::F16InF32; ss.ple.w.norm_key = (const float*) wnk->data; ss.ple.w.norm_query = (const float*) wnq->data; ss.ple.w.norm_conv = (const float*) wnc->data; @@ -3488,7 +3603,9 @@ int main(int argc, char** argv) { } } } - strata::core::Verifier::set_commit_async(!multi_gpu); // see Verifier::set_commit_async + // a layer split commits asynchronously too (ac922, 9098768), except with --pipeline-windows (below), which was + // built and measured on the synchronous commit + strata::core::Verifier::set_commit_async(true); // see Verifier::set_commit_async (a layer split too) // --pipeline-windows (opt-in): one conversation's windows with the two stages of a layer split overlapped. Decided // here, before any stage sizes its expert cache (the second verifier per stage and the snapshots are allocated // after the caches, so their room is kept out of them). What it does not support turns it off, said once. @@ -3542,6 +3659,7 @@ int main(int argc, char** argv) { o.pipeline_windows = 0; } } + if (o.pipeline_windows > 0) strata::core::Verifier::set_commit_async(!multi_gpu); // ---- layer split across GPUs: each later stage's own copy of the dense weights, its session and (the last) the // head, made on its device before the host arena is mapped (as the drafter below, for the same WDDM reason) std::vector> stages; @@ -3579,6 +3697,14 @@ int main(int argc, char** argv) { err.c_str()); return false; } + // STRATA_FP16=load: this stage's copies of the FP16-form weights, on its device + if (strata::fp16_mode() == strata::Fp16Mode::Load) { + const strata::core::OnDevice on(st.dev); + if (!converted.attach(st.wt, st.converted, err)) { + std::fprintf(stderr, "strata generate: layer split, CUDA%d: %s\n", st.dev, err.c_str()); + return false; + } + } return true; }; for (size_t i = 0; multi_gpu && i < split_devs.size(); ++i) { @@ -3885,7 +4011,10 @@ int main(int argc, char** argv) { // Up to 3 GPUs the 0.1.39 curve (r+1)^-1.2 is kept, so 2- and 3-GPU placements do not move (the new curve is a // fit to a 4-way rig and was not measured on them); STRATA_SPLIT_COVER_B asks for the coverage curve at any // GPU count. - const bool coverage_curve = ns >= 4 || std::getenv("STRATA_SPLIT_COVER_B") != nullptr; + // ac922: STRATA_SPLIT_CURVE=0139 keeps the 0.1.39 curve on 4 GPUs too - there the coverage curve put UD-Q4_K_XL at + // 11/24/36 (a 13-layer stage paces the prompt pipeline) where 12/24/36 measured balanced and fastest + static const bool curve_0139 = [] { const char* v = std::getenv("STRATA_SPLIT_CURVE"); return v && std::string(v) == "0139"; }(); + const bool coverage_curve = (ns >= 4 && !curve_0139) || std::getenv("STRATA_SPLIT_COVER_B") != nullptr; for (size_t r = 0; r < n_ranked; ++r) { if (!coverage_curve) { total_mass += (mass[r] = std::pow((double) r + 1.0, -1.2)); @@ -3966,6 +4095,11 @@ int main(int argc, char** argv) { return m; }; const double tie_eps = 1e-6; + // ac922: STRATA_SPLIT_BALANCE=PCT widens the tie to PCT% of the best window, so a placement predicted only a + // little faster for decode does not unbalance the prompt pipeline (4x V100, UD-Q4_K_XL: 11/24/36 was predicted + // ahead of 12/24/36 by its cache coverage, and its 13-layer stage paces every prompt). Unset: upstream's 1e-6 ms. + static const double bal_tol = [] { const char* v = std::getenv("STRATA_SPLIT_BALANCE"); return v ? std::max(0.0, std::atof(v)) / 100.0 : 0.0; }(); + auto tie_at = [&](double ref) { return std::max(tie_eps, ref < 1e29 ? ref * bal_tol : 0.0); }; // ---- LAYER SPLIT STARTABILITY GATE (LAYER 1). The search must not price placements only by decode time: // a split is usable only if the prompt path can start, which requires (serve, `fits_one` 5625-5636 + own_fits // 5855-5869): every participant must lend a chunk, or else afford its own prompt buffers with the serve-time @@ -4096,7 +4230,7 @@ int main(int argc, char** argv) { ++tried; if (ns <= 3) priced.emplace_back(ms, at); const double mx = stage_max(at); - if (ms < best_ms - tie_eps || (ms <= best_ms + tie_eps && mx < best_max - tie_eps)) { + if (ms < best_ms - tie_at(best_ms) || (ms <= best_ms + tie_at(best_ms) && mx < best_max - tie_eps)) { // feasibility gate: mirrors the serve path (`fits_one` + own_fits) for the 512-token last resort; // a placement the prompt path cannot start is not chosen (#1094) - unless nothing can start, below if (gate_on && !can_start(parts_of(at))) return; @@ -4187,7 +4321,7 @@ int main(int argc, char** argv) { strata::program::split_rules::smallest_stage({k1, k2, k3}, L) < min_stage_layers) continue; ++tried; const double mx = stage_max({k1, k2, k3}); - if (!(ms < best_ms - tie_eps || (ms <= best_ms + tie_eps && mx < best_max - tie_eps))) continue; + if (!(ms < best_ms - tie_at(best_ms) || (ms <= best_ms + tie_at(best_ms) && mx < best_max - tie_eps))) continue; if (gate_on) { // the same startability gate as `consider`, over the memoised fills const int64_t lbs[4] = {0, k1, k2, k3}; const int64_t les[4] = {k1, k2, k3, L}; @@ -5031,6 +5165,7 @@ int main(int argc, char** argv) { #endif srcp = &arena_src; } +<<<<<<< this fork #if defined(__linux__) // 0.1.43.1: on by default (STRATA_ADAPT_CALLBACK_WARMUP=0 turns it off). Linux threads inherit the creator's CPU mask, and // the host pins itself to one CPU later; a callback thread the driver creates lazily after that would run on the @@ -5045,6 +5180,45 @@ int main(int argc, char** argv) { } else { std::fprintf(stderr, "strata generate: GPU host-callback thread started before host-core pinning\n"); } +======= + // STRATA_POOL_PER_NODE=1 (Linux, opt-in): the pool on the main GPU's NUMA node's CPUs, and a second pool on the + // other node's, which the layer-split stages whose GPU sits there use (their experts are in that socket's arena + // segment). A pool takes its cores from the creating thread's CPU set, so each is built under that node's mask. + static const bool per_node = [] { const char* v = std::getenv("STRATA_POOL_PER_NODE"); return v && v[0] && v[0] != '0'; }(); + std::unique_ptr pool_far; + int pool_near_node = -1, pool_far_node = -1; +#if defined(__linux__) + cpu_set_t pool_saved_mask; + CPU_ZERO(&pool_saved_mask); + auto node_mask = [](int node, cpu_set_t& m) -> bool { + CPU_ZERO(&m); + char path[96]; + std::snprintf(path, sizeof path, "/sys/devices/system/node/node%d/cpulist", node); + std::FILE* f = std::fopen(path, "r"); + if (!f) return false; + char buf[512] = {}; + const bool ok = std::fgets(buf, sizeof buf, f) != nullptr; + std::fclose(f); + if (!ok) return false; + int n = 0; + for (char* t = std::strtok(buf, ",\n"); t; t = std::strtok(nullptr, ",\n")) { + int a = -1, b = -1; + if (std::sscanf(t, "%d-%d", &a, &b) == 2) for (int c = a; c <= b; ++c) { CPU_SET(c, &m); ++n; } + else if (std::sscanf(t, "%d", &a) == 1) { CPU_SET(a, &m); ++n; } + } + return n > 0; + }; + if (per_node) { + int pool_dev = 0; + cudaGetDevice(&pool_dev); + pool_near_node = gpu_numa_node(pool_dev); + cpu_set_t m; + if (pool_near_node >= 0 && sched_getaffinity(0, sizeof pool_saved_mask, &pool_saved_mask) == 0 && + node_mask(pool_near_node, m)) + sched_setaffinity(0, sizeof m, &m); + else + pool_near_node = -1; +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) } #endif strata::kernels::cpu::ExpertPool pool(o.pool_workers, /*pin=*/true, /*host_works=*/!o.no_host_worker, @@ -5053,6 +5227,21 @@ int main(int argc, char** argv) { o.pool_tasks ? o.pool_tasks : 3 * (pool.workers() + (pool.host_works() ? 1 : 0)), o.pool_tasks ? " (capped by rows)" : " (automatic)", pool.workers() + (pool.host_works() ? 1 : 0)); +#if defined(__linux__) + if (per_node && pool_near_node >= 0) { + for (int node = 0; node < 256 && !pool_far; ++node) { + cpu_set_t m; + if (node == pool_near_node || !node_mask(node, m)) continue; + sched_setaffinity(0, sizeof m, &m); + pool_far = std::make_unique(o.pool_workers, true, !o.no_host_worker, o.pool_affinity, + o.pool_tasks); + pool_far_node = node; + } + sched_setaffinity(0, sizeof pool_saved_mask, &pool_saved_mask); + std::fprintf(stderr, "strata generate: expert pools per NUMA node: %d workers on node %d, %d on node %d\n", + pool.workers(), pool_near_node, pool_far ? pool_far->workers() : 0, pool_far_node); + } +#endif if (pool.is_hybrid() && pool.affinity() != strata::kernels::cpu::PoolAffinity::All) { const char* aff_str = pool.affinity() == strata::kernels::cpu::PoolAffinity::PCores ? "p-cores" : pool.affinity() == strata::kernels::cpu::PoolAffinity::All ? "all" : "auto"; @@ -5553,14 +5742,21 @@ int main(int argc, char** argv) { GpuStage& st = *stp; const auto& lay = strata::kernels::cpu::expert_layout(); // the drafter and the head are already allocated by now (they load above, before this), so what is left - // to hold back is the windows - and `free_b` has already lost the drafter. - const int64_t room = stage_room(st.dev, true, false); + // to hold back is the windows - and `free_b` has already lost the drafter. Not yet the draft head and + // logits: MtpDrafter::bind allocates them on the last stage after this (a 106K-token draft vocabulary over + // the Q5_K head: 276 MiB), so they come out of its room here, as the one-GPU cache books them (#199) - + // without, they took the reserve and the window graphs (~13 MiB each on a 24-layer stage) ran it out + const int64_t draft_bind = (&st == last_st && !o.mtp.empty() && st.head.loaded()) + ? (int64_t) mtp.bind_bytes(st.head.row_bytes(), n_vocab) : 0; + const int64_t room = std::max(stage_room(st.dev, true, false) - draft_bind, 0); const strata::core::OnDevice on(st.dev); { size_t fb = 0, tb = 0; cudaMemGetInfo(&fb, &tb); - std::fprintf(stderr, "strata generate: layer split, CUDA%d: %.2f GiB free of %.2f, room for experts %.2f GiB\n", - st.dev, (double) fb / 1073741824.0, (double) tb / 1073741824.0, (double) room / 1073741824.0); + std::fprintf(stderr, "strata generate: layer split, CUDA%d: %.2f GiB free of %.2f, room for experts %.2f GiB" + "%s\n", st.dev, (double) fb / 1073741824.0, (double) tb / 1073741824.0, + (double) room / 1073741824.0, + draft_bind > 0 ? (" (the draft head's " + std::to_string(draft_bind >> 20) + " MiB booked)").c_str() : ""); } std::vector sized; int64_t used = 0; @@ -7017,11 +7213,14 @@ int main(int argc, char** argv) { const int64_t k = plan_lend(chunk); if (k > 0) lend_from = xcache.slots() - k; } - // a layer split: the experts every later stage's cache holds are left out of the RAM copy, as CUDA0's are + // ac922: a layer split with a RAM budget (--resident-budget-gib) keeps the whole model in one arena, each + // stage's layers on its GPU's socket, and no lend region (a lent slot's expert is in RAM anyway). Otherwise + // upstream's split: the experts every later stage's cache holds are left out of the RAM copy, as CUDA0's are // (with them the copy keeps no lend region: pin_cache_complement turns the loan off) // (true of the single lend region only: the stage_lend_regions block below hands every part's // borrowed tail to the copy as well, so under a split the copy keeps a lend region per part again) std::vector> stage_pairs; +<<<<<<< this fork for (auto& st : stages) for (int64_t l = st->lb; l < st->le; ++l) for (int64_t e = 0; e < g.n_expert; ++e) @@ -7081,6 +7280,21 @@ int main(int argc, char** argv) { if (!region.empty()) stage_lends.push_back(std::move(region)); } src.stage_lend_regions(std::move(stage_lends)); +======= + if (!stages.empty() && o.resident_budget > 0) { + int dev0 = 0; + cudaGetDevice(&dev0); + std::vector layer_node((size_t) g.n_layers, gpu_numa_node(dev0)); + for (const auto& st : stages) + for (int64_t l = st->lb; l < st->le && l < g.n_layers; ++l) layer_node[(size_t) l] = gpu_numa_node(st->dev); + src.set_arena_placement(std::move(layer_node), true); + lend_from = -1; + } else { + for (auto& st : stages) + for (int64_t l = st->lb; l < st->le; ++l) + for (int64_t e = 0; e < g.n_expert; ++e) + if (st->cache.slot_of(l, e) >= 0) stage_pairs.emplace_back((int32_t) l, (int32_t) e); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) } const std::vector>& rank_all = profile_all.empty() ? profile : profile_all; bool resident_ok = src.pin_cache_complement(xcache, err, o.resident_pin, stage_pairs, lend_from, @@ -7138,6 +7352,20 @@ int main(int argc, char** argv) { std::fprintf(stderr, "strata generate: CPU expert residency: %s\n", err.c_str()); return 1; } + // a layer split: each stage's share of the misses from its link to the arena segment on its own socket + // (the probe above copies from wherever the driver put its buffer) + for (auto& st : stages) { + const uint8_t* seg = nullptr; + uint64_t seg_bytes = 0; + if (pcie_given || !src.arena_segment(gpu_numa_node(st->dev), seg, seg_bytes) || seg_bytes < (64ull << 20)) + continue; + const strata::core::OnDevice on(st->dev); + const double bw = probe_h2d_from(seg, 64ull << 20); + if (bw <= 0.0) continue; + st->pcie_frac = pcie_frac_for_gbps(bw, 0.55); + std::fprintf(stderr, "strata generate: layer split: CUDA%d from its socket's arena: %.1f GB/s -> pcie_frac " + "%.2f\n", st->dev, bw, st->pcie_frac); + } std::fprintf(stderr, "strata generate: resident RAM mode: %.2f GiB of experts in RAM (%s), %lld in the GPU " "cache; adaptive swaps %s\n", (double) src.resident_bytes() / 1073741824.0, @@ -7330,8 +7558,29 @@ int main(int argc, char** argv) { // the 384-slot ring (sized for a card that streams nearly every expert of a chunk) only makes every stage's // loan bigger: 96 slots (0.1.30's ring here) when >= 75% of the (layer, expert) pairs are resident. One GPU // keeps the pinned-share rule. STRATA_SPLIT_RING=N: N slots on a split; 0: the pinned-share rule. +<<<<<<< this fork // (decided above, before the lend regions were sized: a no-op here unless the resident RAM mode skipped it) decide_split_ring(); +======= + if (multi_gpu && !host_res.empty()) { + int64_t res_n = 0; + for (const int32_t r : host_res) res_n += r >= 0; + const double res_share = (double) res_n / (double) host_res.size(); + const char* v = std::getenv("STRATA_SPLIT_RING"); + const int ring = v ? std::atoi(v) : (res_share >= 0.75 ? 96 : 0); + if (ring > 0) { + strata::prefill::Prefill::set_ring_override(ring); + std::fprintf(stderr, "strata serve: layer split: %.0f%% of the experts resident, the prompt path's " + "streamed ring %d slots\n", 100.0 * res_share, ring); + } + } + // --prefill-ring: the ring as asked, before the loans below are sized (STRATA_PREFILL_RING still wins, A/B) + if (o.prefill_ring > 0) { + strata::prefill::Prefill::set_ring_override(o.prefill_ring); + std::fprintf(stderr, "strata serve: the prompt path's streamed ring %d slots (--prefill-ring)\n", + o.prefill_ring); + } +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) std::vector pf_parts; // a cache too small to lend the prompt path its buffers would make it allocate them on top - on a card // whose cache already filled its reserve, that is the over-subscription the auto sizing avoids - so the @@ -7842,6 +8091,10 @@ int main(int argc, char** argv) { split_drive.cache_base[st] = drive.d.cache_base; split_drive.cache_slot_off[st] = drive.d.cache_slot_off; split_drive.pcie_num[st] = pcie_num_of(o.pcie_frac); + int sdev = 0; + if (st == 0) cudaGetDevice(&sdev); + else sdev = stages[(size_t) st - 1]->dev; + split_drive.pool[st] = pool_far && gpu_numa_node(sdev) == pool_far_node ? pool_far.get() : &pool; } for (int st = 1; st < n_stages; ++st) { bool ok_s = false; @@ -7991,6 +8244,23 @@ int main(int argc, char** argv) { stage_ver(st).set_pcie_mode(o.pcie_mode == "dma" ? 0 : o.pcie_mode == "direct" ? 1 : 2); } } + // the missed-expert fetch: each stage's same-socket peer GPU (an AC922's NVLink pair) fetches half of it + if (n_stages > 1 && !split_same) { + ver.set_pcie_mode(o.pcie_mode == "dma" ? 0 : o.pcie_mode == "direct" ? 1 : 2); + int dev0 = 0; + cudaGetDevice(&dev0); + auto dev_of = [&](int st) { return st == 0 ? dev0 : stages[(size_t) st - 1]->dev; }; + for (int st = 0; st < n_stages; ++st) { + int peer = -1; + for (int o2 = 0; o2 < n_stages && peer < 0; ++o2) + if (o2 != st && gpu_numa_node(dev_of(o2)) >= 0 && gpu_numa_node(dev_of(o2)) == gpu_numa_node(dev_of(st))) + peer = dev_of(o2); + if (peer >= 0 && !stage_ver(st).set_partner(peer, err)) { + std::fprintf(stderr, "strata serve: %s\n", err.c_str()); + return 1; + } + } + } if (pipe) { // the odd windows' pool routing: the same as the even ones', with their plans split_drive_b = split_drive; for (int st = 0; st < n_stages; ++st) { @@ -8508,6 +8778,7 @@ int main(int argc, char** argv) { sp.on_chunk = [&](const float* R_rows, int64_t T, int64_t p0, std::string& e) -> bool { std::vector nxt((size_t) T); for (int64_t t = 0; t < T; ++t) nxt[(size_t) t] = (int32_t) cur[(size_t) (p0 + t + 1)]; +<<<<<<< this fork // E-9: batched through the prompt path when it can (a layer split: only with STRATA_SPLIT_MTP_BATCH=1) bool batched = false; const strata::program::draft_kv::Plan dkv = strata::program::draft_kv::plan( @@ -8529,6 +8800,14 @@ int main(int argc, char** argv) { } } } +======= + // E-9: batched through the prompt path when it can. A layer split moves this callback to the last stage + // (below), whose prompt path is on the drafter's GPU and idle here (its chunk is done): batch through it + // (STRATA_MTP_BATCH_SPLIT=0: the token path, as before) + static const bool split_batch = [] { const char* v = std::getenv("STRATA_MTP_BATCH_SPLIT"); return !v || v[0] != '0'; }(); + strata::prefill::Prefill& dsp = multi_gpu ? stages.back()->sp : sp; + const bool batched = use_mtp && (!multi_gpu || split_batch) && dsp.draft_kv(mtp, R_rows, nxt.data(), T, p0, e); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) if (!e.empty() || (use_mtp && !batched && !mtp.prefill(R_rows, nxt.data(), T, p0, e))) return false; if (use_mtp && std::getenv("STRATA_SNAPSHOT_VERIFY") != nullptr) std::fprintf(stderr, "strata serve: DRAFT_PREFILL path=%s mode=%d cells=%lld\n", @@ -9510,6 +9789,12 @@ int main(int argc, char** argv) { // and a page-in while the verify graph spins on a host flag stalls the request for good size_t free_b = 0, total_b = 0; cudaMemGetInfo(&free_b, &total_b); + // a layer split: the fullest card counts (the last stage carries the head, the drafter and its graphs) + for (const auto& stp : stages) { + const strata::core::OnDevice on(stp->dev); + size_t f2 = 0, t2 = 0; + if (cudaMemGetInfo(&f2, &t2) == cudaSuccess && f2 < free_b) free_b = f2; + } // below ~256 MiB a later allocation (a first-used window's buffers, the desktop, another program) can make // the driver page GPU memory, and a verify graph spinning on a host flag then never finishes const int64_t free_mib = (int64_t) (free_b >> 20); @@ -12688,12 +12973,24 @@ int main(int argc, char** argv) { bool adapt_ok = true; // (--adapt-async 1: the asynchronous tier above instead, ticked before the window) if (!drive.d.usage.empty() && !ajob && ((rounds + 1) % o.adapt_every) == 0) +<<<<<<< this fork adapt_thr = std::thread([&] { strata::aux_cpus::pin_current_thread(); adapt_ok = adapt(); }); if (!ver.commit(a + 1, err)) { +======= + adapt_thr = std::thread([&] { strata::platform::release_inherited_pin(); adapt_ok = adapt(); }); + // the outputs this window hands out: up to max_new, through the first end of turn. Only their + // positions are committed, so the session holds exactly what the client saw (a follow-up that sends + // the answer back resumes from it); an accepted draft past them would be in no prompt + int emit = 0; + for (bool end = false; emit <= a && produced_n + emit < max_new && !end; ++emit) + end = std::find(o.eos_ids.begin(), o.eos_ids.end(), (int64_t) outv[(size_t) emit]) != o.eos_ids.end(); + if (!ver.commit(emit, err)) { +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) if (adapt_thr.joinable()) adapt_thr.join(); std::printf("ERR %s\n", err.c_str()); return 1; } +<<<<<<< this fork if (nonfinite_hit(ver, first_window ? "first decode" : "decode")) { // #879: none of its tokens goes out if (adapt_thr.joinable()) adapt_thr.join(); if (!adapt_ok) { @@ -12705,6 +13002,10 @@ int main(int argc, char** argv) { } // the window's first a + 1 tokens are in the session now (the last output is not: it is next x) for (int i = 0; i <= a; ++i) consumed.push_back(window[(size_t) i]); +======= + // the window's first `emit` tokens are in the session now (the last output is not: it is next x) + for (int i = 0; i < emit; ++i) consumed.push_back(window[(size_t) i]); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) draft_offered += T - 1; draft_accepted += a; first_window = false; @@ -13861,7 +14162,11 @@ int main(int argc, char** argv) { std::thread adapt_thr; bool adapt_ok = true; if (!drive.d.usage.empty() && ((rounds + 1) % o.adapt_every) == 0) +<<<<<<< this fork adapt_thr = std::thread([&] { strata::aux_cpus::pin_current_thread(); adapt_ok = adapt(); }); +======= + adapt_thr = std::thread([&] { strata::platform::release_inherited_pin(); adapt_ok = adapt(); }); +>>>>>>> their fork (githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922) if (!ver.commit(a + 1, err)) { if (adapt_thr.joinable()) adapt_thr.join(); std::fprintf(stderr, "strata generate: %s\n", err.c_str()); From 1cd2237256e7b52053a532c64e55a4ea3c342f63 Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:03 +0200 Subject: [PATCH 09/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): tools from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 1 file(s) under tools -- 1 clean, 0 with conflict markers. Original authors: Eli Elgaev Original commits touching these paths (3 subject(s)): - STRATA_MTP_EXPERTS=: the drafter experts from an MTP GGUF, native kernels - tidy: no host name or private path in the sources - tools/mtp_experts.py: the BF16 reference directory is an argument, no private path Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: tools Fork-Replay-Automated: true --- tools/mtp_experts.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tools/mtp_experts.py diff --git a/tools/mtp_experts.py b/tools/mtp_experts.py new file mode 100644 index 00000000000..9ae696a6aa4 --- /dev/null +++ b/tools/mtp_experts.py @@ -0,0 +1,54 @@ +# mtp_experts.py GGUF OUT [TENSORS]: an Unsloth MTP head's 512 routed experts in the engine's native blob layout +# (per expert: gate rows, up rows, down rows - native_expert_layout), behind a 16-byte header +# "SMTPEXP1" int32 gate/up ggml type, int32 down type. TENSORS (optional): the directory of Strata's BF16 MTP tensors +# (mtp/tensors in the model's data directory); a few experts are checked against them. +import sys, struct, pathlib, numpy as np +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) # tools/: gguf_reader.py +from gguf_reader import GGUFFile +src, out = sys.argv[1], sys.argv[2] +tensors = sys.argv[3] if len(sys.argv) > 3 else None +H, FF, NE = 2560, 640, 512 +g = GGUFFile(pathlib.Path(src)); mm = np.memmap(src, dtype=np.uint8, mode="r") +T = {t.name: t for t in g.tensors} +gate, up, down = T["blk.48.ffn_gate_exps.weight"], T["blk.48.ffn_up_exps.weight"], T["blk.48.ffn_down_exps.weight"] +assert gate.shape == [H, FF, NE] and up.shape == [H, FF, NE] and down.shape == [FF, H, NE], (gate.shape, down.shape) +assert gate.type_id == up.type_id +BLK = {8: (32, 34), 12: (256, 144)} # Q8_0, Q4_K: values and bytes per block +def row_bytes(t, n): q, b = BLK[t]; return n // q * b +gu_row, d_row = row_bytes(gate.type_id, H), row_bytes(down.type_id, FF) +e_gu, e_d = FF * gu_row, H * d_row +def part(t, e, nb): o = g.data_start + t.offset + e * nb; return mm[o:o + nb] +with open(out, "wb") as f: + f.write(b"SMTPEXP1" + struct.pack("> 6) << 4), (sc[:, j + 4] >> 4) | ((sc[:, j] >> 6) << 4) + for k in range(4): + s1, m1 = sm(2 * k); s2, m2 = sm(2 * k + 1); q = qs[:, 32 * k:32 * k + 32] + y[:, 64 * k:64 * k + 32] = (d * s1)[:, None] * (q & 0xF) - (dm * m1)[:, None] + y[:, 64 * k + 32:64 * k + 64] = (d * s2)[:, None] * (q >> 4) - (dm * m2)[:, None] + return y.reshape(-1)[:n] +if tensors is None: sys.exit(0) +S = str(pathlib.Path(tensors) / "mtp.layers.0.mlp.experts.") +gu_bf = np.memmap(S + "gate_up_proj.bin", dtype=np.uint16, mode="r").reshape(NE, 2 * FF, H) +dn_bf = np.memmap(S + "down_proj.bin", dtype=np.uint16, mode="r").reshape(NE, H, FF) +def bf(a): return (a.astype(np.uint32) << 16).view(np.float32) +blob = np.memmap(out, dtype=np.uint8, mode="r", offset=16).reshape(NE, -1) +for e in (0, 7, 300, 511): + gq = deq(gate.type_id, blob[e, :e_gu], FF * H).reshape(FF, H) + uq = deq(gate.type_id, blob[e, e_gu:2 * e_gu], FF * H).reshape(FF, H) + dq = deq(down.type_id, blob[e, 2 * e_gu:], H * FF).reshape(H, FF) + for nm, a, r in (("gate", gq, bf(gu_bf[e, :FF])), ("up", uq, bf(gu_bf[e, FF:])), ("down", dq, bf(dn_bf[e]))): + rel = np.sqrt(((a - r) ** 2).mean() / (r ** 2).mean()); c = np.corrcoef(a.ravel(), r.ravel())[0, 1] + print(f" expert {e:3d} {nm:4s}: rel RMS error {rel:.4f} corr {c:.5f}") From 04a2991c6a79a323e910d226ba5a20272408b343 Mon Sep 17 00:00:00 2001 From: k0valik Date: Thu, 15 Oct 2026 04:54:03 +0200 Subject: [PATCH 10/10] replay(githubproxy.fjygbaifeng.eu.org-eelgaev-Strata-AC922-ac922): tools/vision from ac922 Squashed from 920 commit(s) fb58e0dbc839..1f029aadc044 on https://github.com/eelgaev/Strata-AC922, restricted to the paths this fork still mirrors: 1 file(s) under tools/vision -- 1 clean, 0 with conflict markers. Original authors: Eli Elgaev,Niko1221 Original commits touching these paths (3 subject(s)): - Merge upstream 0.1.40 part 12: through e2de691 (0.1.40.1 follow-ups) - Strata: Qwen3.8-Flash-Next on a 12-24 GB NVIDIA GPU + 64 GB RAM, one-click install - Support aarch64 and NVIDIA DGX Spark (GB10, unified memory) Review artifact produced by fork-replay.sh (a spike adapted from sync-upstream.sh): a filtered, squashed replay for reading -- not built, not tested, and not evidence that any of it works. Fork-Replay-Source: https://github.com/eelgaev/Strata-AC922 Fork-Replay-Branch: ac922 Fork-Replay-Base: fb58e0dbc8399662c0e47c76578c6e878b14f6cf Fork-Replay-Tip: 1f029aadc0441ebc07ab00f411f2537359ceace3 Fork-Replay-Group: tools/vision Fork-Replay-Automated: true --- tools/vision/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/vision/CMakeLists.txt b/tools/vision/CMakeLists.txt index d6518c71daf..87a8f5c6737 100644 --- a/tools/vision/CMakeLists.txt +++ b/tools/vision/CMakeLists.txt @@ -41,6 +41,8 @@ if(STRATA_PORTABLE) set(${_f} OFF CACHE BOOL "" FORCE) endforeach() message(STATUS "strata-vision: portable build (AVX2 baseline, runs on other PCs)") +elseif(GGML_CPU_ARM_ARCH) # aarch64 with explicit flags (setup's arm_cpu_arch): -mcpu=native finds none on GB10 + set(GGML_NATIVE OFF CACHE BOOL "" FORCE) else() # this PC's instruction set; a build folder that was portable before gets a fresh native build's settings back set(GGML_NATIVE ON CACHE BOOL "" FORCE)