diff --git a/CMakeLists.txt b/CMakeLists.txt index aff47134fd5..e78d0f508e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -650,6 +650,10 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS) add_executable(bf16_bits_test src/kernels/bf16_bits_test.cpp) target_include_directories(bf16_bits_test PRIVATE include) add_test(NAME bf16_bits_test COMMAND bf16_bits_test) + # the layer split's preparation (layer_split.hpp, header-only, no library to link) + add_executable(layer_split_test src/program/layer_split_test.cpp) + target_include_directories(layer_split_test PRIVATE include) + add_test(NAME layer_split_test COMMAND layer_split_test) endif() add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp diff --git a/docs/MULTI_GPU.md b/docs/MULTI_GPU.md index 61ec5291edb..bcce82f41ef 100644 --- a/docs/MULTI_GPU.md +++ b/docs/MULTI_GPU.md @@ -58,19 +58,22 @@ The engine flags behind it: `--layer-split K1[,K2..]|auto` and `--split-device D devices; default the next visible ones). `--layer-split K --split-device 0` runs both stages on one card sharing everything - the bit-exact check of the hand-off, not a speed mode. -**auto** tries every placement (all of them for two or three cards; proportional to the free VRAM beyond that) and -keeps the one whose caches would hold the most of the expert profile, hottest pairs weighted most; ties go to the -placement that leaves the fullest card the most room. The startup log prints the choice: +**auto** tries every placement (all of them for two or three cards; in proportion to speed beyond that) and keeps the +one with the shortest predicted decode window: each card's layers at its per-layer time (from SMs x clock), plus the +routed mass no cache would hold (each card's cache takes its own layers' profiled pairs, hottest first, in the VRAM +its layer range's session leaves), plus one hand-off per later card (`include/strata/program/layer_split.hpp`). The +startup log prints the choice: ``` -strata generate: layer split auto: K=19 - the caches hold 11767 of 12288 profiled pairs (fullest device 100%) -strata serve: layer split: layers 0-18 (CUDA0), 19-47 (CUDA1), one hand-off per window +strata generate: layer split auto: K=25 - predicted 39.9 ms per decode window; the caches hold 8852 of 24576 profiled pairs (~97.0% of the routed mass) +strata serve: layer split: layers 0-24 (CUDA0), 25-47 (CUDA1), one hand-off per window ``` ## What each card holds -- **every card**: a copy of the dense weights (~3.4 GB for the Coder), its own session state (the KV cache of the full - context), its verify window and its prompt-path buffers, and an expert cache for its layers filled from the profile; +- **every card**: a copy of the dense weights (~3.4 GB for the Coder), the session state of its own layers (their KV + cache for the full context), its verify window, and an expert cache for its layers filled from the profile. While a + prompt is read, the top slots of that cache hold the card's prompt buffers; - **the last card**: also the output head and the draft layer (~0.8 GB); - **host RAM**: the expert arena once, shared by all cards (the CPU pool computes whatever no card holds). @@ -92,11 +95,11 @@ into the card that owns the layer. as it did without a split: its per-layer round trip costs more than the CPU pool needs for those experts. - `--mmap-experts` needs a canonical pack (`experts.bin`), with or without a split; a native (IQ) pack says so at start. -- The prompt path has its own buffers on every card (1.5 GB each at the default 2048-token chunk; `--prefill 1024` - halves that) instead of borrowing cache slots as one card does. An explicit `--expert-cache` on the first card is - capped to leave room for them. +- Every card borrows its prompt buffers from its own cache, and all cards read the same chunk: with `--prefill auto`, + the largest one every card can lend. Without an expert profile, or with `--no-prefill-borrow`, every card keeps its + own buffers for the session instead (2048-token chunks, ~1.5 GB a card). - On Windows only 8 GiB of the expert arena is pinned (more, mapped into two GPU contexts, leaves WDDM refusing - allocations); the PCIe share covers those layers. + allocations); the PCIe share covers those layers. Linux pins all of it, as with one card. - Every card needs compute capability 7.5 (RTX 20 or newer). The pre-sm_80 QSA scorer path is fp32 FMAs, so a Turing card runs the same kernels instead of the tensor-core prompt attention. @@ -116,6 +119,18 @@ The Coder on an RTX 5080 + RTX 3090 (Ryzen 9 9950X3D), 32K context; details in expert, the per-layer GPU time decides. - **Correctness:** one GPU is byte-identical to 0.1.20, and the hand-off itself is bit-exact. +Qwen3.8-Flash-Next IQ2_XS (33 GiB of experts, more than both cards hold together) on 2 x RTX 2080 Ti (11 GB, PCIe +gen3 x16, Core i7-7700K), Linux, 32K context, `--spec 4` with the MTP drafter: + +| | Prompt 1.9K / 8K / 16K / 29K tok/s | Decode tok/s | Decode cache hits | +|---|---|---|---| +| one 2080 Ti | 466 / 695 / 840 / 836 | 36.7 | 66% | +| 2 x 2080 Ti, auto (K=25) | 485-490 / 774-781 / 1,116-1,131 / 1,278-1,302 | 48.4-50.3 | 80-81% | +| same, arena pinned only to 8 GiB | 442 / 696 / 921 / 999 | 44.7 | - | + +- **Decode +32-37%.** On this 4-core CPU, decode waits on the CPU pool; the second cache holds ~3,800 more experts. +- **Prompts gain with length** (+4-5% at 1.9K, +53-56% at 29K): the more chunks, the more the two cards overlap. + **Which cards and in what order:** - Put the fastest card first; auto gives it as many layers as its cache allows. - Leave out a much slower card when two already hold the model. An RTX 2080 Ti as a third card made the 5080 + diff --git a/include/strata/program/layer_split.hpp b/include/strata/program/layer_split.hpp new file mode 100644 index 00000000000..3942d0c6ad9 --- /dev/null +++ b/include/strata/program/layer_split.hpp @@ -0,0 +1,135 @@ +// include/strata/program/layer_split.hpp - the layer split's arithmetic (docs/MULTI_GPU.md), apart from the devices, +// so src/program/layer_split_test.cpp can check it. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace strata::program::layer_split { + +/// The most of the host expert arena to pin (0 = all of it). Under WDDM, pinning all of it into several contexts +/// left the driver refusing later allocations, so there it stays at 8 GiB. +inline uint64_t arena_pin_cap(bool several_contexts, bool wddm) { + return several_contexts && wddm ? (8ull << 30) : 0; +} + +// --layer-split auto: the predicted time of one decode window, fitted on an RTX 5080 + RTX 3090 +// (bench/results/2026-09-29-layer-split). Each card's layers cost layer_ms each, every later card a hand-off, and +// the routed mass no cache holds miss_ms per unit, profile rank r weighing (r+1)^-mass_exp. A cache takes its layers' +// pairs hottest first, up to the first that does not fit (as the engine fills it), in the room its session leaves. + +/// A card's time per layer and window, from SMs x clock: 0.33 ms on an RTX 5080 (84 SMs, 2.62 GHz). +inline double layer_ms_estimate(int sms, double ghz) { + const double speed = (double) sms * ghz; + return 0.33 * (84.0 * 2.617) / (speed > 1.0 ? speed : 1.0); +} + +/// One card of a placement: the VRAM its expert cache and its session may take, and what a layer costs it per window. +struct Card { + int64_t cap_bytes = 0; + double layer_ms = 0; +}; + +struct Costs { + double miss_ms = 190.0; ///< a unit of routed mass no cache holds (STRATA_SPLIT_MISS_MS) + double handoff_ms = 1.0; ///< one card handing a window to the next (~1.1 ms on 2 x RTX 2080 Ti, PCIe gen3) + double mass_exp = 1.2; ///< the routed mass of profile rank r: (r+1)^-mass_exp +}; + +struct Placement { + std::vector at; ///< the first layer of every later card (empty: one card) + double ms = 0; ///< the predicted decode window + double held_mass = 0; ///< the share of the routed mass the caches hold + int64_t held = 0; ///< the profiled pairs they hold +}; + +/// The VRAM a card running layers [lb, le) needs for their session (the carve), which its expert cache cannot use. +using RangeBytes = std::function; + +class Planner { +public: + /// `profile`: the ranked (layer, expert) pairs, hottest first; `slot_bytes[l]`: the VRAM one expert of layer l + /// takes in a cache; `range_bytes`: what a card's layer range takes out of its `cap_bytes` (empty: nothing). + Planner(int64_t n_layers, std::vector> profile, std::vector slot_bytes, Costs c, + RangeBytes range_bytes = {}) + : n_layers_(n_layers), profile_(std::move(profile)), slot_bytes_(std::move(slot_bytes)), costs_(c), + range_bytes_(std::move(range_bytes)), mass_(profile_.size()) { + for (size_t r = 0; r < profile_.size(); ++r) total_mass_ += (mass_[r] = std::pow((double) r + 1.0, -c.mass_exp)); + } + + /// The window time of `cards` running from the layers `at` on (at.size() == cards.size() - 1). + Placement predict(const std::vector& cards, const std::vector& at) const { + Placement p; + p.at = at; + const size_t ns = cards.size(); + auto lb = [&](size_t i) { return i == 0 ? 0 : at[i - 1]; }; + auto le = [&](size_t i) { return i + 1 < ns ? at[i] : n_layers_; }; + std::vector room(ns), used(ns, 0); + for (size_t i = 0; i < ns; ++i) room[i] = cards[i].cap_bytes - (range_bytes_ ? range_bytes_(lb(i), le(i)) : 0); + std::vector full(ns, false); + double mass = 0; + for (size_t r = 0; r < profile_.size(); ++r) { + const int64_t l = profile_[r].first; + size_t st = 0; + while (st + 1 < ns && l >= at[st]) ++st; + if (full[st]) continue; + const int64_t b = slot_bytes_[(size_t) l]; + if (used[st] + b > room[st]) { full[st] = true; continue; } + used[st] += b; + mass += mass_[r]; + ++p.held; + } + p.held_mass = total_mass_ > 0 ? mass / total_mass_ : 1.0; + p.ms = costs_.miss_ms * (1.0 - p.held_mass) + costs_.handoff_ms * (double) (ns > 0 ? ns - 1 : 0); + for (size_t i = 0; i < ns; ++i) p.ms += (double) (le(i) - lb(i)) * cards[i].layer_ms; + return p; + } + + /// The fastest placement: every one for two or three cards (a later card starts at layer 2 or later), the layers + /// in proportion to speed beyond. Ties keep the earlier split point. + Placement best(const std::vector& cards) const { + const size_t ns = cards.size(); + const int64_t L = n_layers_; + Placement best; + best.ms = 1e300; + std::vector at(ns > 0 ? ns - 1 : 0); + auto consider = [&]() { + Placement p = predict(cards, at); + if (p.ms < best.ms) best = std::move(p); + }; + if (ns <= 1) { + consider(); + } else if (ns == 2) { + for (int64_t k = 2; k < L; ++k) { at[0] = k; consider(); } + } else if (ns == 3) { + for (int64_t k1 = 2; k1 + 1 < L; ++k1) + for (int64_t k2 = k1 + 1; k2 < L; ++k2) { at[0] = k1; at[1] = k2; consider(); } + } else { + double total = 0, acc = 0; + for (const Card& c : cards) total += 1.0 / c.layer_ms; + for (size_t i = 0; i + 1 < ns; ++i) { + acc += 1.0 / cards[i].layer_ms; + const int64_t lo = i == 0 ? 2 : at[i - 1] + 1, hi = L - (int64_t) (ns - 1 - i); + at[i] = std::clamp((int64_t) std::llround(acc / total * (double) L), lo, hi); + } + consider(); + } + return best; + } + +private: + int64_t n_layers_; + std::vector> profile_; + std::vector slot_bytes_; + Costs costs_; + RangeBytes range_bytes_; + std::vector mass_; + double total_mass_ = 0; +}; + +} // namespace strata::program::layer_split diff --git a/src/program/generate.cpp b/src/program/generate.cpp index 75e874b004f..b0589627069 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -49,6 +49,7 @@ #include "strata/core/native_dense.hpp" #include "strata/program/logits_selection.hpp" #include "strata/program/conv_cache.hpp" +#include "strata/program/layer_split.hpp" #include "strata/spec/draft_policy.hpp" #include "strata/spec/suffix_drafter.hpp" #include "strata/kernels/cvec.hpp" @@ -93,6 +94,8 @@ #include #include +namespace layer_split = strata::program::layer_split; + namespace { // perf-review D-4: the lent slots are refilled with queued copies and one wait; STRATA_REFILL_BLOCKING=1 waits on each bool refill_blocking() { @@ -1986,19 +1989,8 @@ int main(int argc, char** argv) { std::fprintf(stderr, "strata generate: profile %s: %zu ranked pairs, built for %lld slots\n", o.expert_profile.c_str(), profile.size(), (long long) pslots); } - // ---- layer split across GPUs: "auto" places the split points by a cost model of one decode window, measured on - // the 5080 + 3090 rig (bench/results/2026-09-29-layer-split): - // - every layer costs its GPU a time inversely proportional to SMs x clock (0.33 ms on an RTX 5080, 0.50 on a - // 3090: the per-layer round trip and kernels, not the bytes - both cards have ~950 GB/s); - // - an expert no cache holds costs ~190 ms per unit of routed mass: the CPU pool in decode and the PCIe stream - // in prompts (fitted: the sweep's best K, 26-28, is where one more layer on the faster card stops paying - // for the ~0.1% of the mass it pushes out of its cache); - // - which experts a cache holds: its layers' profiled pairs, hottest first, until its free VRAM (less the - // reserve, the prompt path's buffers and, on a later GPU, 1 GiB for its windows and the drafter) is used; - // the routed mass of rank r is taken as (r+1)^-1.2 (fits the sweep's hit rates: K=24/26/28 predicted - // 99.53/99.34/99.15%, measured 99.5/99.4/99.0%). - // Up to 3 GPUs every placement is tried; beyond, the layers are shared in proportion to speed. - // STRATA_SPLIT_MISS_MS tunes the miss cost (a slower CPU: higher). + // ---- layer split across GPUs: "auto" places the split points by layer_split::Planner's model of one decode + // window (layer_split.hpp); STRATA_SPLIT_MISS_MS, STRATA_SPLIT_HANDOFF_MS and STRATA_SPLIT_MASS_EXP tune it. // THE PROMPT PATH'S BUFFERS ARE BORROWED FROM THE CACHE, NOT WITHHELD BESIDE IT. With borrowing the cache // is sized first and at full size, and the prompt path is laid out in the tail of it (`Prefill::relayout`), // so it withholds no VRAM of its own and this reserve is zero. Only without borrowing - no profile to fill @@ -2029,93 +2021,38 @@ int main(int argc, char** argv) { if (multi_gpu && split_auto) { const auto& lay = strata::kernels::cpu::expert_layout(); const int ns = (int) stages.size() + 1; - std::vector cap((size_t) ns), used((size_t) ns); - std::vector layer_ms((size_t) ns); + std::vector cards((size_t) ns); for (int i = 0; i < ns; ++i) { const int dev = i == 0 ? 0 : stages[(size_t) i - 1]->dev; - cap[(size_t) i] = stage_room(i == 0 ? -1 : dev, i > 0, i + 1 == ns); int sms = 0, khz = 0; cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev); if (cudaDeviceGetAttribute(&khz, cudaDevAttrClockRate, dev) != cudaSuccess || khz <= 0) khz = 1800000; cudaGetLastError(); - const double speed = std::max(1.0, (double) sms * (double) khz / 1e6); // SMs x GHz - layer_ms[(size_t) i] = 0.33 * (84.0 * 2.617) / speed; + cards[(size_t) i] = {stage_room(i == 0 ? -1 : dev, i > 0, i + 1 == ns), layer_split::layer_ms_estimate(sms, khz / 1e6)}; std::fprintf(stderr, "strata generate: layer split auto: CUDA%d %d SMs at %.2f GHz -> %.2f ms per layer, " - "%.2f GiB free before its session carve\n", dev, sms, khz / 1e6, layer_ms[(size_t) i], - (double) cap[(size_t) i] / 1073741824.0); - } - const double miss_ms = std::getenv("STRATA_SPLIT_MISS_MS") ? std::atof(std::getenv("STRATA_SPLIT_MISS_MS")) : 190.0; - std::vector mass(profile.size()); - double total_mass = 0; - for (size_t r = 0; r < profile.size(); ++r) total_mass += (mass[r] = std::pow((double) r + 1.0, -1.2)); - auto cost = [&](int64_t l) -> int64_t { - return native_pack ? ((int64_t) lay.blob_bytes(l) + 255) / 256 * 256 : (int64_t) lay.max_blob; - }; - // the predicted window time (ms) of a placement, and the routed mass its caches hold - auto predict = [&](const std::vector& at, double& held_mass, int64_t& held) -> double { - // THE CARVE, PRICED: a placement gives stage i the layers [lb, le), and that range's session is a - // real cost on its device - subtracted here so the search knows what it leaves for experts. This - // is why the sessions are allocated after the search: `session_bytes` is pure arithmetic. - std::vector capr((size_t) ns); - for (int i = 0; i < ns; ++i) { - const int64_t lb = i == 0 ? 0 : at[(size_t) i - 1]; - const int64_t le = i + 1 < ns ? at[(size_t) i] : g.n_layers; - capr[(size_t) i] = cap[(size_t) i] - (int64_t) strata::core::session_bytes(g, o.max_context, K, lb, le); - } - std::fill(used.begin(), used.end(), 0); - held_mass = 0; - held = 0; - std::vector full((size_t) ns, false); - for (size_t r = 0; r < profile.size(); ++r) { - const int64_t l = profile[r].first; - int st = 0; - while (st + 1 < ns && l >= at[(size_t) st]) ++st; - if (full[(size_t) st]) continue; - if (used[(size_t) st] + cost(l) > capr[(size_t) st]) { full[(size_t) st] = true; continue; } // as the fill - used[(size_t) st] += cost(l); - held_mass += mass[r]; - ++held; - } - held_mass /= std::max(total_mass, 1e-9); - double ms = miss_ms * (1.0 - held_mass); - for (int i = 0; i < ns; ++i) { - const int64_t lb = i == 0 ? 0 : at[(size_t) i - 1], le = i + 1 < ns ? at[(size_t) i] : g.n_layers; - ms += (double) (le - lb) * layer_ms[(size_t) i]; - } - return ms; - }; - std::vector best, at((size_t) ns - 1); - double best_ms = 1e30, best_mass = 0; - int64_t best_held = 0; - auto consider = [&]() { - double hm = 0; - int64_t held = 0; - const double ms = predict(at, hm, held); - if (ms < best_ms) { best = at; best_ms = ms; best_mass = hm; best_held = held; } - }; - const int64_t L = g.n_layers; - if (ns == 2) { - for (int64_t k = 2; k < L; ++k) { at[0] = k; consider(); } - } else if (ns == 3) { - for (int64_t k1 = 2; k1 + 1 < L; ++k1) - for (int64_t k2 = k1 + 1; k2 < L; ++k2) { at[0] = k1; at[1] = k2; consider(); } - } else { - double total = 0; - for (const double c : layer_ms) total += 1.0 / c; - double acc = 0; - for (int i = 0; i + 1 < ns; ++i) { - acc += 1.0 / layer_ms[(size_t) i]; - at[(size_t) i] = std::clamp((int64_t) std::llround(acc / total * (double) L), - i == 0 ? 2 : at[(size_t) i - 1] + 1, L - (ns - 1 - i)); - } - consider(); - } - split_at = best; + "%.2f GiB free before its session carve\n", dev, sms, khz / 1e6, cards[(size_t) i].layer_ms, + (double) cards[(size_t) i].cap_bytes / 1073741824.0); + } + auto env_or = [](const char* name, double v) { const char* s = std::getenv(name); return s ? std::atof(s) : v; }; + layer_split::Costs costs; + costs.miss_ms = env_or("STRATA_SPLIT_MISS_MS", costs.miss_ms); + costs.handoff_ms = env_or("STRATA_SPLIT_HANDOFF_MS", costs.handoff_ms); + costs.mass_exp = env_or("STRATA_SPLIT_MASS_EXP", costs.mass_exp); + std::vector slot_bytes((size_t) g.n_layers); + for (int64_t l = 0; l < g.n_layers; ++l) + slot_bytes[(size_t) l] = native_pack ? ((int64_t) lay.blob_bytes(l) + 255) / 256 * 256 : (int64_t) lay.max_blob; + // session_bytes prices each placement's sessions: they are allocated after the search + const layer_split::Planner planner(g.n_layers, profile, std::move(slot_bytes), costs, + [&](int64_t lb, int64_t le) -> int64_t { + return (int64_t) strata::core::session_bytes(g, o.max_context, K, lb, le); + }); + const layer_split::Placement best = planner.best(cards); + split_at = best.at; std::string ks; for (const int64_t k : split_at) ks += (ks.empty() ? "" : ",") + std::to_string(k); std::fprintf(stderr, "strata generate: layer split auto: K=%s - predicted %.1f ms per decode window; the caches " - "hold %lld of %zu profiled pairs (~%.1f%% of the routed mass)\n", ks.c_str(), best_ms, - (long long) best_held, profile.size(), 100.0 * best_mass); + "hold %lld of %zu profiled pairs (~%.1f%% of the routed mass)\n", ks.c_str(), best.ms, + (long long) best.held, profile.size(), 100.0 * best.held_mass); } for (size_t i = 0; i < split_at.size(); ++i) if (split_at[i] >= g.n_layers) { @@ -2278,7 +2215,12 @@ int main(int argc, char** argv) { // Unregistered layers remain in the resident arena and use the CPU expert path. // (a layer split across GPUs too: pinning all of it into two contexts leaves WDDM refusing every later // allocation - measured on the 5080 + 3090 rig: cudaMemGetInfo and the next cudaMalloc fail) - const uint64_t pin_limit = (o.expert_cache_remote[0] > 0 || multi_gpu) ? (8ull << 30) : 0; +#ifdef _WIN32 + constexpr bool wddm = true; +#else + constexpr bool wddm = false; +#endif + const uint64_t pin_limit = layer_split::arena_pin_cap(o.expert_cache_remote[0] > 0 || multi_gpu, wddm); if (!arena_src.open(o.pack, g.n_layers, g.n_expert, /*threads=*/6, err, pin_limit, o.shared_expert_arena)) { std::fprintf(stderr, "strata generate: %s\n", err.c_str()); diff --git a/src/program/layer_split_test.cpp b/src/program/layer_split_test.cpp new file mode 100644 index 00000000000..7b0662b9685 --- /dev/null +++ b/src/program/layer_split_test.cpp @@ -0,0 +1,112 @@ +// src/program/layer_split_test.cpp - checks include/strata/program/layer_split.hpp. +#include "strata/program/layer_split.hpp" + +#include +#include +#include +#include +#include + +namespace ls = strata::program::layer_split; + +namespace { +int g_fail = 0; +void check(bool ok, const char* what) { + std::printf(" %-74s %s\n", what, ok ? "ok" : "FAIL"); + if (!ok) ++g_fail; +} +bool near(double a, double b, double eps = 1e-9) { return std::fabs(a - b) <= eps; } +constexpr uint64_t GiB = 1ull << 30; +// the toy model: 8 layers x 4 experts, the layers interleaved in the ranking, one byte a slot +constexpr int64_t kL = 8, kE = 4; +std::vector> toy_profile() { + std::vector> p; + for (int32_t r = 0; r < kL * kE; ++r) p.emplace_back(r % kL, r / kL); + return p; +} +double toy_mass(int64_t n, double exp) { // the share of the routed mass the n hottest ranks hold + double a = 0, t = 0; + for (int64_t r = 0; r < kL * kE; ++r) { + const double m = std::pow((double) r + 1.0, -exp); + t += m; + if (r < n) a += m; + } + return a / t; +} +} // namespace + +int main() { + std::printf("layer_split_test\n"); + { + check(ls::arena_pin_cap(false, true) == 0, "one context, WDDM: the whole arena"); + check(ls::arena_pin_cap(false, false) == 0, "one context, Linux: the whole arena"); + check(ls::arena_pin_cap(true, true) == 8 * GiB, "several contexts under WDDM: 8 GiB"); + check(ls::arena_pin_cap(true, false) == 0, "several contexts, Linux: the whole arena"); + } + { + check(near(ls::layer_ms_estimate(84, 2.617), 0.33, 1e-6), "per-layer time: RTX 5080 (84 SMs, 2.62 GHz) 0.33 ms"); + check(near(ls::layer_ms_estimate(68, 1.545), 0.6905, 1e-3), "per-layer time: RTX 2080 Ti (68 SMs, 1.55 GHz) 0.69 ms"); + } + const std::vector ones((size_t) kL, 1); + ls::Costs costs; + costs.miss_ms = 100.0; + costs.handoff_ms = 0.0; + costs.mass_exp = 1.0; + { + const ls::Planner pl(kL, toy_profile(), ones, costs); + const ls::Placement all = pl.predict({{100, 1.0}}, {}); + check(all.held == kL * kE && near(all.held_mass, 1.0) && near(all.ms, 8.0), + "one card holding every pair: 8 layers x 1 ms, nothing missed"); + const ls::Placement half = pl.predict({{16, 1.0}}, {}); + check(half.held == 16 && near(half.held_mass, toy_mass(16, 1.0)) && + near(half.ms, 8.0 + 100.0 * (1.0 - toy_mass(16, 1.0))), + "one card holding half: the missed mass costs miss_ms per unit"); + ls::Costs flat = costs; + flat.mass_exp = 0.5; + const ls::Planner pf(kL, toy_profile(), ones, flat); + check(pf.predict({{16, 1.0}}, {}).held_mass < half.held_mass, "a flatter routing curve: the same cache holds less"); + } + { + std::vector bytes = ones; + bytes[1] = 5; // layer 1's experts take 5 bytes: rank 1 does not fit a 3-byte cache after rank 0 + const ls::Planner pl(kL, toy_profile(), bytes, costs); + check(pl.predict({{3, 1.0}}, {}).held == 1, "the fill stops at the first pair that does not fit"); + } + { + ls::Costs h = costs; + h.handoff_ms = 0.5; + const ls::Planner pl(kL, toy_profile(), ones, h); + check(near(pl.predict({{100, 1.0}, {100, 1.0}}, {4}).ms, 8.5), "two cards: one hand-off per window"); + check(near(pl.predict({{100, 1.0}, {100, 1.0}, {100, 1.0}}, {3, 6}).ms, 9.0), "three cards: two hand-offs"); + } + { + const ls::Planner pl(kL, toy_profile(), ones, costs); + const ls::Placement a = pl.best({{100, 0.5}, {100, 1.0}}); + check(a.at == std::vector{7} && near(a.ms, 4.5), "ample caches, faster first card: it takes 7 layers"); + const ls::Placement b = pl.best({{100, 1.0}, {100, 0.5}}); + check(b.at == std::vector{2} && near(b.ms, 5.0), "ample caches, faster second card: it takes 6 layers"); + const ls::Placement c = pl.best({{8, 1.0}, {24, 1.0}}); + check(c.at == std::vector{2} && c.held == kL * kE, + "equal cards, tight caches: K=2 lets both hold their layers' pairs"); + const ls::Placement d = pl.best({{100, 0.5}, {100, 1.0}, {100, 1.0}}); + check(d.at == (std::vector{6, 7}) && near(d.ms, 5.0), "three cards: every placement tried"); + const ls::Placement e = pl.best({{100, 1.0}, {100, 1.0}, {100, 1.0}, {100, 1.0}}); + check(e.at == (std::vector{2, 4, 6}), "four equal cards: layers in proportion to speed"); + } + { + // a session of 10 bytes a layer: one card running all 8 layers has 100 - 80 = 20 bytes left for experts + const ls::Planner pl(kL, toy_profile(), ones, costs, [](int64_t lb, int64_t le) { return 10 * (le - lb); }); + const ls::Placement one = pl.predict({{100, 1.0}}, {}); + check(one.held == 20 && near(one.ms, 8.0 + 100.0 * (1.0 - toy_mass(20, 1.0))), + "the carve comes off the card's room: 20 of 32 pairs held"); + // 12 bytes a layer plus its 4 pairs: the faster card fits 6 layers in 100, so K=6, not 7 as without it + const ls::Planner pc(kL, toy_profile(), ones, costs, [](int64_t lb, int64_t le) { return 12 * (le - lb); }); + const ls::Placement k7 = pc.predict({{100, 0.5}, {100, 1.0}}, {7}); + check(k7.held == 20 && k7.ms > 15.0, "the carve priced: K=7 holds 20 pairs (16 on the faster card)"); + const ls::Placement b = pc.best({{100, 0.5}, {100, 1.0}}); + check(b.at == std::vector{6} && b.held == kL * kE && near(b.ms, 5.0), + "the carve priced: the faster card takes the 6 layers it can hold (K=6)"); + } + std::printf(g_fail ? "layer_split_test: %d FAILED\n" : "layer_split_test: all passed\n", g_fail); + return g_fail ? 1 : 0; +}